diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 2d9f724413169..917a0db62da28 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -244,7 +244,6 @@ static TransportVersion def(int id) { public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO = def(9_039_0_00); public static final TransportVersion PROJECT_ID_IN_SNAPSHOT = def(9_040_0_00); public static final TransportVersion INDEX_STATS_AND_METADATA_INCLUDE_PEAK_WRITE_LOAD = def(9_041_0_00); - public static final TransportVersion REPOSITORIES_METADATA_AS_PROJECT_CUSTOM = def(9_042_0_00); public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_043_0_00); public static final TransportVersion REMOTE_EXCEPTION = def(9_044_0_00); public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00); diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java index 82f11748bf6f3..647839f435599 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.lucene.util.SetOnce; import org.elasticsearch.TransportVersions; import org.elasticsearch.cluster.ClusterState; import org.elasticsearch.cluster.Diff; @@ -53,7 +52,6 @@ import java.io.IOException; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; import java.util.EnumSet; import java.util.HashMap; @@ -997,12 +995,11 @@ private MetadataDiff(StreamInput in) throws IOException { multiProject = null; } else { fromNodeBeforeMultiProjectsSupport = false; - // Repositories metadata is sent as Metadata#customs diff from old node. We need to - // 1. Split it from the Metadata#customs diff - // 2. Merge it into the default project's ProjectMetadataDiff - final var bwcCustoms = maybeReadBwcCustoms(in); - clusterCustoms = bwcCustoms.v1(); - final var defaultProjectCustoms = bwcCustoms.v2(); + clusterCustoms = DiffableUtils.readImmutableOpenMapDiff( + in, + DiffableUtils.getStringKeySerializer(), + CLUSTER_CUSTOM_VALUE_SERIALIZER + ); reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff( in, @@ -1011,61 +1008,15 @@ private MetadataDiff(StreamInput in) throws IOException { ); singleProject = null; - multiProject = readMultiProjectDiffs(in, defaultProjectCustoms); - } - } - - private static - Tuple< - MapDiff>, - MapDiff>> - maybeReadBwcCustoms(StreamInput in) throws IOException { - if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) { - return readBwcCustoms(in); - } else { - return new Tuple<>( - DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CLUSTER_CUSTOM_VALUE_SERIALIZER), - null + multiProject = DiffableUtils.readJdkMapDiff( + in, + PROJECT_ID_SERIALIZER, + ProjectMetadata::readFrom, + ProjectMetadata.ProjectMetadataDiff::new ); } } - @SuppressWarnings("unchecked") - private static MapDiff> readMultiProjectDiffs( - StreamInput in, - MapDiff> defaultProjectCustoms - ) throws IOException { - final var multiProject = DiffableUtils.readJdkMapDiff( - in, - PROJECT_ID_SERIALIZER, - ProjectMetadata::readFrom, - ProjectMetadata.ProjectMetadataDiff::new - ); - - // If the defaultProjectCustoms has content, the diff is read from an old node. We need to merge it into the - // default project's ProjectMetadataDiff - if (defaultProjectCustoms != null && defaultProjectCustoms.isEmpty() == false) { - return DiffableUtils.updateDiffsAndUpserts(multiProject, ProjectId.DEFAULT::equals, (k, v) -> { - assert ProjectId.DEFAULT.equals(k) : k; - assert v instanceof ProjectMetadata.ProjectMetadataDiff : v; - final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v; - return projectMetadataDiff.withCustoms( - DiffableUtils.merge( - projectMetadataDiff.customs(), - defaultProjectCustoms, - DiffableUtils.getStringKeySerializer(), - BWC_CUSTOM_VALUE_SERIALIZER - ) - ); - }, (k, v) -> { - assert ProjectId.DEFAULT.equals(k) : k; - return ProjectMetadata.builder(v).clearCustoms().customs(defaultProjectCustoms.apply(v.customs())).build(); - }); - } else { - return multiProject; - } - } - @SuppressWarnings("unchecked") private static Tuple< @@ -1109,105 +1060,16 @@ public void writeTo(StreamOutput out) throws IOException { buildUnifiedCustomDiff().writeTo(out); buildUnifiedReservedStateMetadataDiff().writeTo(out); } else { - final var multiProjectToWrite = multiProject != null - ? multiProject - : DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER); - - if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) { - writeDiffWithRepositoriesMetadataAsClusterCustom(out, clusterCustoms, multiProjectToWrite, reservedStateMetadata); + clusterCustoms.writeTo(out); + reservedStateMetadata.writeTo(out); + if (multiProject != null) { + multiProject.writeTo(out); } else { - clusterCustoms.writeTo(out); - reservedStateMetadata.writeTo(out); - multiProjectToWrite.writeTo(out); + DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER).writeTo(out); } } } - @SuppressWarnings({ "rawtypes", "unchecked" }) - private static void writeDiffWithRepositoriesMetadataAsClusterCustom( - StreamOutput out, - MapDiff> clusterCustoms, - MapDiff> multiProject, - MapDiff> reservedStateMetadata - ) throws IOException { - assert out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT) - && out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion(); - - // For old nodes, RepositoriesMetadata needs to be sent as a cluster custom. This is possible when (a) the repositories - // are defined only for the default project or (b) no repositories at all. What we need to do are: - // 1. Iterate through the multi-project's MapDiff to extract the RepositoriesMetadata of the default project - // 2. Throws if any repositories are found for non-default projects - // 3. Merge default project's RepositoriesMetadata into Metadata#customs - final var combineClustersCustoms = new SetOnce>>(); - final var updatedMultiProject = DiffableUtils.updateDiffsAndUpserts(multiProject, ignore -> true, (k, v) -> { - assert v instanceof ProjectMetadata.ProjectMetadataDiff : v; - final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v; - final var bwcCustoms = DiffableUtils.split( - projectMetadataDiff.customs(), - RepositoriesMetadata.TYPE::equals, - PROJECT_CUSTOM_VALUE_SERIALIZER, - type -> RepositoriesMetadata.TYPE.equals(type) == false, - PROJECT_CUSTOM_VALUE_SERIALIZER - ); - // Simply return if RepositoriesMetadata is not found - if (bwcCustoms.v1().isEmpty()) { - return projectMetadataDiff; - } - // RepositoriesMetadata can only be defined for the default project. Otherwise throw exception. - if (ProjectId.DEFAULT.equals(k) == false) { - throwForVersionBeforeRepositoriesMetadataMigration(out); - } - // RepositoriesMetadata is found for the default project as a diff, merge it into the Metadata#customs - combineClustersCustoms.set( - DiffableUtils.>merge( - clusterCustoms, - bwcCustoms.v1(), - DiffableUtils.getStringKeySerializer() - ) - ); - return projectMetadataDiff.withCustoms(bwcCustoms.v2()); - }, (k, v) -> { - final ProjectCustom projectCustom = v.customs().get(RepositoriesMetadata.TYPE); - // Simply return if RepositoriesMetadata is not found - if (projectCustom == null) { - return v; - } - // RepositoriesMetadata can only be defined for the default project. Otherwise throw exception. - if (ProjectId.DEFAULT.equals(k) == false) { - throwForVersionBeforeRepositoriesMetadataMigration(out); - } - // RepositoriesMetadata found for the default project as an upsert, package it as MapDiff and merge into Metadata#customs - combineClustersCustoms.set( - DiffableUtils.>merge( - clusterCustoms, - DiffableUtils.singleUpsertDiff(RepositoriesMetadata.TYPE, projectCustom, DiffableUtils.getStringKeySerializer()), - DiffableUtils.getStringKeySerializer() - ) - ); - return ProjectMetadata.builder(v).removeCustom(RepositoriesMetadata.TYPE).build(); - }); - - if (combineClustersCustoms.get() != null) { - combineClustersCustoms.get().writeTo(out); - } else { - clusterCustoms.writeTo(out); - } - - reservedStateMetadata.writeTo(out); - updatedMultiProject.writeTo(out); - } - - private static void throwForVersionBeforeRepositoriesMetadataMigration(StreamOutput out) { - assert out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion(); - throw new UnsupportedOperationException( - "Serialize a diff with repositories defined for multiple projects requires version on or after [" - + TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM - + "], but got [" - + out.getTransportVersion() - + "]" - ); - } - @SuppressWarnings("unchecked") private Diff> buildUnifiedCustomDiff() { assert multiProject == null : "should only be used for single project metadata"; @@ -1361,17 +1223,7 @@ public static Metadata readFrom(StreamInput in) throws IOException { builder.put(ReservedStateMetadata.readFrom(in)); } } else { - List defaultProjectCustoms = List.of(); - if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) { - // Extract the default project's repositories metadata from the Metadata#customs from an old node - defaultProjectCustoms = new ArrayList<>(); - readBwcCustoms(in, builder, defaultProjectCustoms::add); - assert defaultProjectCustoms.size() <= 1 - : "expect only a single default project custom for repository metadata, but got " - + defaultProjectCustoms.stream().map(ProjectCustom::getWriteableName).toList(); - } else { - readClusterCustoms(in, builder); - } + readClusterCustoms(in, builder); int reservedStateSize = in.readVInt(); for (int i = 0; i < reservedStateSize; i++) { @@ -1379,16 +1231,11 @@ public static Metadata readFrom(StreamInput in) throws IOException { } builder.projectMetadata(in.readMap(ProjectId::readFrom, ProjectMetadata::readFrom)); - defaultProjectCustoms.forEach(c -> builder.getProject(ProjectId.DEFAULT).putCustom(c.getWriteableName(), c)); } return builder.build(); } private static void readBwcCustoms(StreamInput in, Builder builder) throws IOException { - readBwcCustoms(in, builder, projectCustom -> builder.putProjectCustom(projectCustom.getWriteableName(), projectCustom)); - } - - private static void readBwcCustoms(StreamInput in, Builder builder, Consumer projectCustomConsumer) throws IOException { final Set clusterScopedNames = in.namedWriteableRegistry().getReaders(ClusterCustom.class).keySet(); final Set projectScopedNames = in.namedWriteableRegistry().getReaders(ProjectCustom.class).keySet(); final int count = in.readVInt(); @@ -1404,9 +1251,9 @@ private static void readBwcCustoms(StreamInput in, Builder builder, Consumer combinedCustoms = new ArrayList<>(customs.size() + 1); - combinedCustoms.addAll(customs.values()); - final ProjectCustom custom = getProject(ProjectId.DEFAULT).custom(RepositoriesMetadata.TYPE); - if (custom != null) { - combinedCustoms.add(custom); - } - VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms); - } else { - throw new UnsupportedOperationException( - "Serialize metadata with repositories defined for multiple projects requires version on or after [" - + TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM - + "], but got [" - + out.getTransportVersion() - + "]" - ); - } - } else { - VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); - } + VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); out.writeCollection(reservedStateMetadata.values()); out.writeMap(projectMetadata, StreamOutput::writeWriteable, StreamOutput::writeWriteable); } } - /** - * @return {@code true} iff no repositories are defined for non-default-projects. - */ - private static boolean hasNoNonDefaultProjectRepositories(Collection projects) { - return projects.stream() - .allMatch(project -> ProjectId.DEFAULT.equals(project.id()) || project.custom(RepositoriesMetadata.TYPE) == null); - } - public static Builder builder() { return new Builder(); } diff --git a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java index 480172ef3aace..fddd27b7f1d01 100644 --- a/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java +++ b/server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java @@ -1530,11 +1530,6 @@ public Builder removeCustomIf(BiPredicate customs) { customs.forEach((key, value) -> Objects.requireNonNull(value, key)); this.customs.putAllFromMap(customs); @@ -2266,17 +2261,7 @@ public void writeTo(StreamOutput out) throws IOException { indexMetadata.writeTo(out, true); } out.writeCollection(templates.values()); - Collection filteredCustoms = customs.values(); - if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) { - // RepositoriesMetadata is sent as part of Metadata#customs for version before RepositoriesMetadata migration - // So we exclude it from the project level customs - if (custom(RepositoriesMetadata.TYPE) != null) { - assert ProjectId.DEFAULT.equals(id) - : "Only default project can have repositories metadata. Otherwise the code should have thrown before it reaches here"; - filteredCustoms = filteredCustoms.stream().filter(custom -> custom instanceof RepositoriesMetadata == false).toList(); - } - } - VersionedNamedWriteable.writeVersionedWriteables(out, filteredCustoms); + VersionedNamedWriteable.writeVersionedWriteables(out, customs.values()); out.writeCollection(reservedStateMetadata.values()); if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) { @@ -2409,12 +2394,6 @@ public ProjectMetadata apply(ProjectMetadata part) { builder.settings = settingsDiff.apply(part.settings); return builder.build(true); } - - ProjectMetadataDiff withCustoms( - DiffableUtils.MapDiff> customs - ) { - return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata, settingsDiff); - } } @Override diff --git a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java index bfffd99efb338..5bf619e964e0b 100644 --- a/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java +++ b/server/src/test/java/org/elasticsearch/cluster/metadata/MetadataRepositoriesMetadataTests.java @@ -96,22 +96,10 @@ public void testRepositoriesMetadataDiffSerialization() throws IOException { } public void testRepositoriesMetadataSerializationBwc() throws IOException { - { - final var oldVersion = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MULTI_PROJECT, - TransportVersionUtils.getPreviousVersion(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) - ); - final Metadata orig = randomMetadata(between(0, 5), -1); - doTestRepositoriesMetadataSerializationBwc(orig, oldVersion); - } - - { - final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); - // Before multi-project, BWC is possible for a single project - final Metadata orig = randomMetadata(0, -1); - doTestRepositoriesMetadataSerializationBwc(orig, oldVersion); - } + final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); + // Before multi-project, BWC is possible for a single project + final Metadata orig = randomMetadata(0, -1); + doTestRepositoriesMetadataSerializationBwc(orig, oldVersion); } private void doTestRepositoriesMetadataSerializationBwc(Metadata orig, TransportVersion oldVersion) throws IOException { @@ -138,22 +126,10 @@ private void simulateReadOnOldNodeAndAssert(BytesReference bytesReference, Trans } public void testRepositoriesMetadataDiffSerializationBwc() throws IOException { - { - final var oldVersion = TransportVersionUtils.randomVersionBetween( - random(), - TransportVersions.MULTI_PROJECT, - TransportVersionUtils.getPreviousVersion(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) - ); - final Tuple tuple = randomMetadataAndUpdate(between(0, 5), -1); - doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion); - } - - { - final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); - // Before multi-project, BWC is possible for a single project - final Tuple tuple = randomMetadataAndUpdate(0, -1); - doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion); - } + final var oldVersion = TransportVersionUtils.getPreviousVersion(TransportVersions.MULTI_PROJECT); + // Before multi-project, BWC is possible for a single project + final Tuple tuple = randomMetadataAndUpdate(0, -1); + doTestRepositoriesMetadataDiffSerializationBwc(tuple, oldVersion); } private void doTestRepositoriesMetadataDiffSerializationBwc(Tuple tuple, TransportVersion oldVersion)