Skip to content

Remove obsolete Metadata BWC for repositories #129685

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jun 20, 2025
Merged

Remove obsolete Metadata BWC for repositories #129685

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
88df0a4
Remove obsolete Metadata BWC for repositories
ywangd Jun 19, 2025
5ac0b90
Merge remote-tracking branch 'origin/main' into remove-obsolete-metad…
ywangd Jun 19, 2025
764ee60
Merge branch 'main' into remove-obsolete-metadata-bwc-for-repositories
elasticmachine Jun 19, 2025
d61bdb8
Merge branch 'main' into remove-obsolete-metadata-bwc-for-repositories
elasticmachine Jun 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion server/src/main/java/org/elasticsearch/TransportVersions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,7 +244,6 @@ static TransportVersion def(int id) {
public static final TransportVersion RESCORE_VECTOR_ALLOW_ZERO = def(9_039_0_00);
public static final TransportVersion PROJECT_ID_IN_SNAPSHOT = def(9_040_0_00);
public static final TransportVersion INDEX_STATS_AND_METADATA_INCLUDE_PEAK_WRITE_LOAD = def(9_041_0_00);
public static final TransportVersion REPOSITORIES_METADATA_AS_PROJECT_CUSTOM = def(9_042_0_00);
public static final TransportVersion BATCHED_QUERY_PHASE_VERSION = def(9_043_0_00);
public static final TransportVersion REMOTE_EXCEPTION = def(9_044_0_00);
public static final TransportVersion ESQL_REMOVE_AGGREGATE_TYPE = def(9_045_0_00);
Expand Down
220 changes: 19 additions & 201 deletions server/src/main/java/org/elasticsearch/cluster/metadata/Metadata.java
View file
Open in desktop
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Change to this file is mostly reverting the changes in #125398. It's not an actual revert because a small number of xcontent changes are still needed.

Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@

import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.lucene.util.SetOnce;
import org.elasticsearch.TransportVersions;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.Diff;
Expand Down Expand Up @@ -53,7 +52,6 @@

import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
Expand Down Expand Up @@ -997,12 +995,11 @@ private MetadataDiff(StreamInput in) throws IOException {
multiProject = null;
} else {
fromNodeBeforeMultiProjectsSupport = false;
// Repositories metadata is sent as Metadata#customs diff from old node. We need to
// 1. Split it from the Metadata#customs diff
// 2. Merge it into the default project's ProjectMetadataDiff
final var bwcCustoms = maybeReadBwcCustoms(in);
clusterCustoms = bwcCustoms.v1();
final var defaultProjectCustoms = bwcCustoms.v2();
clusterCustoms = DiffableUtils.readImmutableOpenMapDiff(
in,
DiffableUtils.getStringKeySerializer(),
CLUSTER_CUSTOM_VALUE_SERIALIZER
);

reservedStateMetadata = DiffableUtils.readImmutableOpenMapDiff(
in,
Expand All @@ -1011,61 +1008,15 @@ private MetadataDiff(StreamInput in) throws IOException {
);

singleProject = null;
multiProject = readMultiProjectDiffs(in, defaultProjectCustoms);
}
}

private static
Tuple<
MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>>,
MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>>>
maybeReadBwcCustoms(StreamInput in) throws IOException {
if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
return readBwcCustoms(in);
} else {
return new Tuple<>(
DiffableUtils.readImmutableOpenMapDiff(in, DiffableUtils.getStringKeySerializer(), CLUSTER_CUSTOM_VALUE_SERIALIZER),
null
multiProject = DiffableUtils.readJdkMapDiff(
in,
PROJECT_ID_SERIALIZER,
ProjectMetadata::readFrom,
ProjectMetadata.ProjectMetadataDiff::new
);
}
}

@SuppressWarnings("unchecked")
private static MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> readMultiProjectDiffs(
StreamInput in,
MapDiff<String, ProjectCustom, ImmutableOpenMap<String, ProjectCustom>> defaultProjectCustoms
) throws IOException {
final var multiProject = DiffableUtils.readJdkMapDiff(
in,
PROJECT_ID_SERIALIZER,
ProjectMetadata::readFrom,
ProjectMetadata.ProjectMetadataDiff::new
);

// If the defaultProjectCustoms has content, the diff is read from an old node. We need to merge it into the
// default project's ProjectMetadataDiff
if (defaultProjectCustoms != null && defaultProjectCustoms.isEmpty() == false) {
return DiffableUtils.updateDiffsAndUpserts(multiProject, ProjectId.DEFAULT::equals, (k, v) -> {
assert ProjectId.DEFAULT.equals(k) : k;
assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
return projectMetadataDiff.withCustoms(
DiffableUtils.merge(
projectMetadataDiff.customs(),
defaultProjectCustoms,
DiffableUtils.getStringKeySerializer(),
BWC_CUSTOM_VALUE_SERIALIZER
)
);
}, (k, v) -> {
assert ProjectId.DEFAULT.equals(k) : k;
return ProjectMetadata.builder(v).clearCustoms().customs(defaultProjectCustoms.apply(v.customs())).build();
});
} else {
return multiProject;
}
}

@SuppressWarnings("unchecked")
private static
Tuple<
Expand Down Expand Up @@ -1109,105 +1060,16 @@ public void writeTo(StreamOutput out) throws IOException {
buildUnifiedCustomDiff().writeTo(out);
buildUnifiedReservedStateMetadataDiff().writeTo(out);
} else {
final var multiProjectToWrite = multiProject != null
? multiProject
: DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER);

if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
writeDiffWithRepositoriesMetadataAsClusterCustom(out, clusterCustoms, multiProjectToWrite, reservedStateMetadata);
clusterCustoms.writeTo(out);
reservedStateMetadata.writeTo(out);
if (multiProject != null) {
multiProject.writeTo(out);
} else {
clusterCustoms.writeTo(out);
reservedStateMetadata.writeTo(out);
multiProjectToWrite.writeTo(out);
DiffableUtils.singleEntryDiff(DEFAULT_PROJECT_ID, singleProject, PROJECT_ID_SERIALIZER).writeTo(out);
}
}
}

@SuppressWarnings({ "rawtypes", "unchecked" })
private static void writeDiffWithRepositoriesMetadataAsClusterCustom(
StreamOutput out,
MapDiff<String, ClusterCustom, ImmutableOpenMap<String, ClusterCustom>> clusterCustoms,
MapDiff<ProjectId, ProjectMetadata, Map<ProjectId, ProjectMetadata>> multiProject,
MapDiff<String, ReservedStateMetadata, ImmutableOpenMap<String, ReservedStateMetadata>> reservedStateMetadata
) throws IOException {
assert out.getTransportVersion().onOrAfter(TransportVersions.MULTI_PROJECT)
&& out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();

// For old nodes, RepositoriesMetadata needs to be sent as a cluster custom. This is possible when (a) the repositories
// are defined only for the default project or (b) no repositories at all. What we need to do are:
// 1. Iterate through the multi-project's MapDiff to extract the RepositoriesMetadata of the default project
// 2. Throws if any repositories are found for non-default projects
// 3. Merge default project's RepositoriesMetadata into Metadata#customs
final var combineClustersCustoms = new SetOnce<MapDiff<String, MetadataCustom, Map<String, MetadataCustom>>>();
final var updatedMultiProject = DiffableUtils.updateDiffsAndUpserts(multiProject, ignore -> true, (k, v) -> {
assert v instanceof ProjectMetadata.ProjectMetadataDiff : v;
final var projectMetadataDiff = (ProjectMetadata.ProjectMetadataDiff) v;
final var bwcCustoms = DiffableUtils.split(
projectMetadataDiff.customs(),
RepositoriesMetadata.TYPE::equals,
PROJECT_CUSTOM_VALUE_SERIALIZER,
type -> RepositoriesMetadata.TYPE.equals(type) == false,
PROJECT_CUSTOM_VALUE_SERIALIZER
);
// Simply return if RepositoriesMetadata is not found
if (bwcCustoms.v1().isEmpty()) {
return projectMetadataDiff;
}
// RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
if (ProjectId.DEFAULT.equals(k) == false) {
throwForVersionBeforeRepositoriesMetadataMigration(out);
}
// RepositoriesMetadata is found for the default project as a diff, merge it into the Metadata#customs
combineClustersCustoms.set(
DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
clusterCustoms,
bwcCustoms.v1(),
DiffableUtils.getStringKeySerializer()
)
);
return projectMetadataDiff.withCustoms(bwcCustoms.v2());
}, (k, v) -> {
final ProjectCustom projectCustom = v.customs().get(RepositoriesMetadata.TYPE);
// Simply return if RepositoriesMetadata is not found
if (projectCustom == null) {
return v;
}
// RepositoriesMetadata can only be defined for the default project. Otherwise throw exception.
if (ProjectId.DEFAULT.equals(k) == false) {
throwForVersionBeforeRepositoriesMetadataMigration(out);
}
// RepositoriesMetadata found for the default project as an upsert, package it as MapDiff and merge into Metadata#customs
combineClustersCustoms.set(
DiffableUtils.<String, MetadataCustom, ClusterCustom, ProjectCustom, Map<String, MetadataCustom>>merge(
clusterCustoms,
DiffableUtils.singleUpsertDiff(RepositoriesMetadata.TYPE, projectCustom, DiffableUtils.getStringKeySerializer()),
DiffableUtils.getStringKeySerializer()
)
);
return ProjectMetadata.builder(v).removeCustom(RepositoriesMetadata.TYPE).build();
});

if (combineClustersCustoms.get() != null) {
combineClustersCustoms.get().writeTo(out);
} else {
clusterCustoms.writeTo(out);
}

reservedStateMetadata.writeTo(out);
updatedMultiProject.writeTo(out);
}

private static void throwForVersionBeforeRepositoriesMetadataMigration(StreamOutput out) {
assert out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM) : out.getTransportVersion();
throw new UnsupportedOperationException(
"Serialize a diff with repositories defined for multiple projects requires version on or after ["
+ TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
+ "], but got ["
+ out.getTransportVersion()
+ "]"
);
}

@SuppressWarnings("unchecked")
private Diff<ImmutableOpenMap<String, ?>> buildUnifiedCustomDiff() {
assert multiProject == null : "should only be used for single project metadata";
Expand Down Expand Up @@ -1361,34 +1223,19 @@ public static Metadata readFrom(StreamInput in) throws IOException {
builder.put(ReservedStateMetadata.readFrom(in));
}
} else {
List<ProjectCustom> defaultProjectCustoms = List.of();
if (in.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
// Extract the default project's repositories metadata from the Metadata#customs from an old node
defaultProjectCustoms = new ArrayList<>();
readBwcCustoms(in, builder, defaultProjectCustoms::add);
assert defaultProjectCustoms.size() <= 1
: "expect only a single default project custom for repository metadata, but got "
+ defaultProjectCustoms.stream().map(ProjectCustom::getWriteableName).toList();
} else {
readClusterCustoms(in, builder);
}
readClusterCustoms(in, builder);

int reservedStateSize = in.readVInt();
for (int i = 0; i < reservedStateSize; i++) {
builder.put(ReservedStateMetadata.readFrom(in));
}

builder.projectMetadata(in.readMap(ProjectId::readFrom, ProjectMetadata::readFrom));
defaultProjectCustoms.forEach(c -> builder.getProject(ProjectId.DEFAULT).putCustom(c.getWriteableName(), c));
}
return builder.build();
}

private static void readBwcCustoms(StreamInput in, Builder builder) throws IOException {
readBwcCustoms(in, builder, projectCustom -> builder.putProjectCustom(projectCustom.getWriteableName(), projectCustom));
}

private static void readBwcCustoms(StreamInput in, Builder builder, Consumer<ProjectCustom> projectCustomConsumer) throws IOException {
final Set<String> clusterScopedNames = in.namedWriteableRegistry().getReaders(ClusterCustom.class).keySet();
final Set<String> projectScopedNames = in.namedWriteableRegistry().getReaders(ProjectCustom.class).keySet();
final int count = in.readVInt();
Expand All @@ -1404,9 +1251,9 @@ private static void readBwcCustoms(StreamInput in, Builder builder, Consumer<Pro
if (custom instanceof PersistentTasksCustomMetadata persistentTasksCustomMetadata) {
final var tuple = persistentTasksCustomMetadata.split();
builder.putCustom(tuple.v1().getWriteableName(), tuple.v1());
projectCustomConsumer.accept(tuple.v2());
builder.putProjectCustom(tuple.v2().getWriteableName(), tuple.v2());
} else {
projectCustomConsumer.accept(custom);
builder.putProjectCustom(custom.getWriteableName(), custom);
}
} else {
throw new IllegalArgumentException("Unknown custom name [" + name + "]");
Expand Down Expand Up @@ -1473,42 +1320,13 @@ public void writeTo(StreamOutput out) throws IOException {
combinedMetadata.addAll(singleProject.reservedStateMetadata().values());
out.writeCollection(combinedMetadata);
} else {
if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
if (isSingleProject() || hasNoNonDefaultProjectRepositories(projects().values())) {
// Repositories metadata must be sent as Metadata#customs for old nodes
final List<VersionedNamedWriteable> combinedCustoms = new ArrayList<>(customs.size() + 1);
combinedCustoms.addAll(customs.values());
final ProjectCustom custom = getProject(ProjectId.DEFAULT).custom(RepositoriesMetadata.TYPE);
if (custom != null) {
combinedCustoms.add(custom);
}
VersionedNamedWriteable.writeVersionedWriteables(out, combinedCustoms);
} else {
throw new UnsupportedOperationException(
"Serialize metadata with repositories defined for multiple projects requires version on or after ["
+ TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM
+ "], but got ["
+ out.getTransportVersion()
+ "]"
);
}
} else {
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
}
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());

out.writeCollection(reservedStateMetadata.values());
out.writeMap(projectMetadata, StreamOutput::writeWriteable, StreamOutput::writeWriteable);
}
}

/**
* @return {@code true} iff no repositories are defined for non-default-projects.
*/
private static boolean hasNoNonDefaultProjectRepositories(Collection<ProjectMetadata> projects) {
return projects.stream()
.allMatch(project -> ProjectId.DEFAULT.equals(project.id()) || project.custom(RepositoriesMetadata.TYPE) == null);
}

public static Builder builder() {
return new Builder();
}
Expand Down
23 changes: 1 addition & 22 deletions server/src/main/java/org/elasticsearch/cluster/metadata/ProjectMetadata.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1530,11 +1530,6 @@ public Builder removeCustomIf(BiPredicate<String, ? super Metadata.ProjectCustom
return this;
}

public Builder clearCustoms() {
customs.clear();
return this;
}

public Builder customs(Map<String, Metadata.ProjectCustom> customs) {
customs.forEach((key, value) -> Objects.requireNonNull(value, key));
this.customs.putAllFromMap(customs);
Expand Down Expand Up @@ -2266,17 +2261,7 @@ public void writeTo(StreamOutput out) throws IOException {
indexMetadata.writeTo(out, true);
}
out.writeCollection(templates.values());
Collection<Metadata.ProjectCustom> filteredCustoms = customs.values();
if (out.getTransportVersion().before(TransportVersions.REPOSITORIES_METADATA_AS_PROJECT_CUSTOM)) {
// RepositoriesMetadata is sent as part of Metadata#customs for version before RepositoriesMetadata migration
// So we exclude it from the project level customs
if (custom(RepositoriesMetadata.TYPE) != null) {
assert ProjectId.DEFAULT.equals(id)
: "Only default project can have repositories metadata. Otherwise the code should have thrown before it reaches here";
filteredCustoms = filteredCustoms.stream().filter(custom -> custom instanceof RepositoriesMetadata == false).toList();
}
}
VersionedNamedWriteable.writeVersionedWriteables(out, filteredCustoms);
VersionedNamedWriteable.writeVersionedWriteables(out, customs.values());
out.writeCollection(reservedStateMetadata.values());

if (out.getTransportVersion().onOrAfter(TransportVersions.PROJECT_METADATA_SETTINGS)) {
Expand Down Expand Up @@ -2409,12 +2394,6 @@ public ProjectMetadata apply(ProjectMetadata part) {
builder.settings = settingsDiff.apply(part.settings);
return builder.build(true);
}

ProjectMetadataDiff withCustoms(
DiffableUtils.MapDiff<String, Metadata.ProjectCustom, ImmutableOpenMap<String, Metadata.ProjectCustom>> customs
) {
return new ProjectMetadataDiff(indices, templates, customs, reservedStateMetadata, settingsDiff);
}
}

@Override
Expand Down
Loading