From d1622d1b4b70c3fe7f0ee7c99aa90d0217b826ec Mon Sep 17 00:00:00 2001 From: Ray Mattingly Date: Fri, 13 Dec 2024 18:39:56 -0500 Subject: [PATCH] HBASE-28513 The StochasticLoadBalancer should support discrete evaluations --- .../master/balancer/BalancerClusterState.java | 39 ++-- .../master/balancer/BalancerConditionals.java | 205 ++++++++++++++++ .../DistributeReplicasConditional.java | 172 ++++++++++++++ .../MetaTableIsolationCandidateGenerator.java | 29 +++ .../MetaTableIsolationConditional.java | 88 +++++++ .../balancer/RegionPlanConditional.java | 32 +++ .../balancer/StochasticLoadBalancer.java | 78 ++++++- ...ystemTableIsolationCandidateGenerator.java | 31 +++ .../SystemTableIsolationConditional.java | 83 +++++++ .../TableIsolationCandidateGenerator.java | 88 +++++++ .../balancer/TestBalancerConditionals.java | 141 +++++++++++ .../TestDistributeReplicasConditional.java | 136 +++++++++++ .../TestMetaTableIsolationConditional.java | 157 +++++++++++++ .../TestSystemTableIsolationConditional.java | 104 +++++++++ .../BalancerConditionalsTestUtil.java | 221 ++++++++++++++++++ ...TestLargerClusterBalancerConditionals.java | 149 ++++++++++++ ...eplicaDistributionBalancerConditional.java | 123 ++++++++++ ...temTableIsolationBalancerConditionals.java | 128 ++++++++++ 18 files changed, 1980 insertions(+), 24 deletions(-) create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java create mode 100644 hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java create mode 100644 hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java create mode 100644 hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDistributeReplicasConditional.java create mode 100644 hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationConditional.java create mode 100644 hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestSystemTableIsolationConditional.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/BalancerConditionalsTestUtil.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestLargerClusterBalancerConditionals.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestReplicaDistributionBalancerConditional.java create mode 100644 hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestSystemTableIsolationBalancerConditionals.java diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java index 4b3809c107cb..bc5bde1b0d70 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerClusterState.java @@ -33,12 +33,15 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.RegionReplicaUtil; import org.apache.hadoop.hbase.master.RackManager; +import org.apache.hadoop.hbase.master.RegionPlan; import org.apache.hadoop.hbase.net.Address; import org.apache.hadoop.hbase.util.Pair; import org.apache.yetus.audience.InterfaceAudience; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList; + /** * An efficient array based implementation similar to ClusterState for keeping the status of the * cluster in terms of region assignment and distribution. LoadBalancers, such as @@ -705,7 +708,7 @@ enum LocalityType { RACK } - public void doAction(BalanceAction action) { + public List doAction(BalanceAction action) { switch (action.getType()) { case NULL: break; @@ -715,17 +718,20 @@ public void doAction(BalanceAction action) { AssignRegionAction ar = (AssignRegionAction) action; regionsPerServer[ar.getServer()] = addRegion(regionsPerServer[ar.getServer()], ar.getRegion()); - regionMoved(ar.getRegion(), -1, ar.getServer()); - break; + return ImmutableList.of(regionMoved(ar.getRegion(), -1, ar.getServer())); case MOVE_REGION: assert action instanceof MoveRegionAction : action.getClass(); MoveRegionAction mra = (MoveRegionAction) action; - regionsPerServer[mra.getFromServer()] = - removeRegion(regionsPerServer[mra.getFromServer()], mra.getRegion()); - regionsPerServer[mra.getToServer()] = - addRegion(regionsPerServer[mra.getToServer()], mra.getRegion()); - regionMoved(mra.getRegion(), mra.getFromServer(), mra.getToServer()); - break; + try { + regionsPerServer[mra.getFromServer()] = + removeRegion(regionsPerServer[mra.getFromServer()], mra.getRegion()); + regionsPerServer[mra.getToServer()] = + addRegion(regionsPerServer[mra.getToServer()], mra.getRegion()); + return ImmutableList + .of(regionMoved(mra.getRegion(), mra.getFromServer(), mra.getToServer())); + } catch (Exception e) { + throw e; + } case SWAP_REGIONS: assert action instanceof SwapRegionsAction : action.getClass(); SwapRegionsAction a = (SwapRegionsAction) action; @@ -733,12 +739,12 @@ public void doAction(BalanceAction action) { replaceRegion(regionsPerServer[a.getFromServer()], a.getFromRegion(), a.getToRegion()); regionsPerServer[a.getToServer()] = replaceRegion(regionsPerServer[a.getToServer()], a.getToRegion(), a.getFromRegion()); - regionMoved(a.getFromRegion(), a.getFromServer(), a.getToServer()); - regionMoved(a.getToRegion(), a.getToServer(), a.getFromServer()); - break; + return ImmutableList.of(regionMoved(a.getFromRegion(), a.getFromServer(), a.getToServer()), + regionMoved(a.getToRegion(), a.getToServer(), a.getFromServer())); default: - throw new RuntimeException("Uknown action:" + action.getType()); + throw new RuntimeException("Unknown action:" + action.getType()); } + return Collections.emptyList(); } /** @@ -822,7 +828,7 @@ void doAssignRegion(RegionInfo regionInfo, ServerName serverName) { doAction(new AssignRegionAction(region, server)); } - void regionMoved(int region, int oldServer, int newServer) { + RegionPlan regionMoved(int region, int oldServer, int newServer) { regionIndexToServerIndex[region] = newServer; if (initialRegionIndexToServerIndex[region] == newServer) { numMovedRegions--; // region moved back to original location @@ -853,6 +859,11 @@ void regionMoved(int region, int oldServer, int newServer) { updateForLocation(serverIndexToRackIndex, regionsPerRack, colocatedReplicaCountsPerRack, oldServer, newServer, primary, region); } + + // old server name can be null + ServerName oldServerName = oldServer == -1 ? null : servers[oldServer]; + + return new RegionPlan(regions[region], oldServerName, servers[newServer]); } /** diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java new file mode 100644 index 000000000000..d0101c42119a --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/BalancerConditionals.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.lang.reflect.Constructor; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.util.ReflectionUtils; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +/** + * Balancer conditionals supplement cost functions in the {@link StochasticLoadBalancer}. Cost + * functions are insufficient and difficult to work with when making discrete decisions; this is + * because they operate on a continuous scale, and each cost function's multiplier affects the + * relative importance of every other cost function. So it is difficult to meaningfully and clearly + * value many aspects of your region distribution via cost functions alone. Conditionals allow you + * to very clearly define discrete rules that your balancer would ideally follow. To clarify, a + * conditional violation will not block a region assignment because we would prefer to have uptime + * than have perfectly intentional balance. But conditionals allow you to, for example, define that + * a region's primary and secondary should not live on the same rack. Another example, conditionals + * make it easy to define that system tables will ideally be isolated on their own RegionServer. + */ +@InterfaceAudience.Private +public final class BalancerConditionals { + + private static final Logger LOG = LoggerFactory.getLogger(BalancerConditionals.class); + + static final BalancerConditionals INSTANCE = new BalancerConditionals(); + public static final String ISOLATE_SYSTEM_TABLES_KEY = + "hbase.master.balancer.stochastic.conditionals.isolateSystemTables"; + public static final boolean ISOLATE_SYSTEM_TABLES_DEFAULT = false; + + public static final String ISOLATE_META_TABLE_KEY = + "hbase.master.balancer.stochastic.conditionals.isolateMetaTable"; + public static final boolean ISOLATE_META_TABLE_DEFAULT = false; + + public static final String DISTRIBUTE_REPLICAS_CONDITIONALS_KEY = + "hbase.master.balancer.stochastic.conditionals.distributeReplicas"; + public static final boolean DISTRIBUTE_REPLICAS_CONDITIONALS_DEFAULT = false; + + public static final String ADDITIONAL_CONDITIONALS_KEY = + "hbase.master.balancer.stochastic.additionalConditionals"; + + // when this count is low, we'll be more likely to trigger a subsequent balancer run + private static final AtomicInteger BALANCE_COUNT_WITHOUT_IMPROVEMENTS = new AtomicInteger(0); + private static final int BALANCE_COUNT_WITHOUT_IMPROVEMENTS_CEILING = 10; + + private Set> conditionalClasses = Collections.emptySet(); + private Set conditionals = Collections.emptySet(); + private Configuration conf; + + private BalancerConditionals() { + } + + protected boolean isTableIsolationEnabled() { + return conditionalClasses.contains(SystemTableIsolationConditional.class) + || conditionalClasses.contains(MetaTableIsolationConditional.class); + } + + protected boolean shouldRunBalancer() { + return BALANCE_COUNT_WITHOUT_IMPROVEMENTS.get() < BALANCE_COUNT_WITHOUT_IMPROVEMENTS_CEILING; + } + + protected int getConsecutiveBalancesWithoutImprovement() { + return BALANCE_COUNT_WITHOUT_IMPROVEMENTS.get(); + } + + protected void incConsecutiveBalancesWithoutImprovement() { + if (BALANCE_COUNT_WITHOUT_IMPROVEMENTS.get() == Integer.MAX_VALUE) { + return; + } + this.BALANCE_COUNT_WITHOUT_IMPROVEMENTS.getAndIncrement(); + LOG.trace("Set balanceCountWithoutImprovements={}", + this.BALANCE_COUNT_WITHOUT_IMPROVEMENTS.get()); + } + + protected void resetConsecutiveBalancesWithoutImprovement() { + this.BALANCE_COUNT_WITHOUT_IMPROVEMENTS.set(0); + LOG.trace("Set balanceCountWithoutImprovements=0"); + } + + protected Set> getConditionalClasses() { + return Set.copyOf(conditionalClasses); + } + + protected boolean shouldSkipSloppyServerEvaluation() { + return conditionals.stream() + .anyMatch(conditional -> conditional instanceof SystemTableIsolationConditional + || conditional instanceof MetaTableIsolationConditional); + } + + protected void loadConf(Configuration conf) { + this.conf = conf; + ImmutableSet.Builder> conditionalClasses = + ImmutableSet.builder(); + + boolean isolateSystemTables = + conf.getBoolean(ISOLATE_SYSTEM_TABLES_KEY, ISOLATE_SYSTEM_TABLES_DEFAULT); + if (isolateSystemTables) { + conditionalClasses.add(SystemTableIsolationConditional.class); + } + + boolean isolateMetaTable = conf.getBoolean(ISOLATE_META_TABLE_KEY, ISOLATE_META_TABLE_DEFAULT); + if (isolateMetaTable) { + conditionalClasses.add(MetaTableIsolationConditional.class); + } + + boolean distributeReplicas = conf.getBoolean(DISTRIBUTE_REPLICAS_CONDITIONALS_KEY, + DISTRIBUTE_REPLICAS_CONDITIONALS_DEFAULT); + if (distributeReplicas) { + conditionalClasses.add(DistributeReplicasConditional.class); + } + + Class[] classes = conf.getClasses(ADDITIONAL_CONDITIONALS_KEY); + for (Class clazz : classes) { + if (!RegionPlanConditional.class.isAssignableFrom(clazz)) { + LOG.warn("Class {} is not a RegionPlanConditional", clazz.getName()); + continue; + } + conditionalClasses.add(clazz.asSubclass(RegionPlanConditional.class)); + } + this.conditionalClasses = conditionalClasses.build(); + } + + protected void loadClusterState(BalancerClusterState cluster) { + conditionals = conditionalClasses.stream().map(clazz -> createConditional(clazz, conf, cluster)) + .collect(Collectors.toSet()); + } + + protected int getConditionalViolationChange(List regionPlans) { + if (conditionals.isEmpty()) { + incConsecutiveBalancesWithoutImprovement(); + return 0; + } + int conditionalViolationChange = 0; + for (RegionPlan regionPlan : regionPlans) { + conditionalViolationChange += getConditionalViolationChange(conditionals, regionPlan); + } + return conditionalViolationChange; + } + + private static int getConditionalViolationChange(Set conditionals, + RegionPlan regionPlan) { + RegionPlan inverseRegionPlan = new RegionPlan(regionPlan.getRegionInfo(), + regionPlan.getDestination(), regionPlan.getSource()); + int currentConditionalViolationCount = + getConditionalViolationCount(conditionals, inverseRegionPlan); + int newConditionalViolationCount = getConditionalViolationCount(conditionals, regionPlan); + int violationChange = newConditionalViolationCount - currentConditionalViolationCount; + if (violationChange < 0) { + LOG.trace("Should move region {}_{} from {} to {}", regionPlan.getRegionName(), + regionPlan.getRegionInfo().getReplicaId(), regionPlan.getSource().getServerName(), + regionPlan.getDestination().getServerName()); + } + return violationChange; + } + + private static int getConditionalViolationCount(Set conditionals, + RegionPlan regionPlan) { + int regionPlanConditionalViolationCount = 0; + for (RegionPlanConditional regionPlanConditional : conditionals) { + if (regionPlanConditional.isViolating(regionPlan)) { + regionPlanConditionalViolationCount++; + } + } + return regionPlanConditionalViolationCount; + } + + private RegionPlanConditional createConditional(Class clazz, + Configuration conf, BalancerClusterState cluster) { + try { + Constructor ctor = + clazz.getDeclaredConstructor(Configuration.class, BalancerClusterState.class); + return ReflectionUtils.instantiate(clazz.getName(), ctor, conf, cluster); + } catch (NoSuchMethodException e) { + LOG.warn("Cannot find constructor with Configuration and " + + "BalancerClusterState parameters for class '{}': {}", clazz.getName(), e.getMessage()); + } + return null; + } +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java new file mode 100644 index 000000000000..78db528154de --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/DistributeReplicasConditional.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionReplicaUtil; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * If enabled, this class will help the balancer ensure that replicas aren't placed on the same + * servers or racks as their primary. Configure this via + * {@link BalancerConditionals#DISTRIBUTE_REPLICAS_CONDITIONALS_KEY} + */ +@InterfaceAudience.Private +public class DistributeReplicasConditional extends RegionPlanConditional { + + /** + * Local mini cluster tests are only run on one host/rack by design. If enabled, this will + * pretend that localhost RegionServer threads are actually running on separate hosts/racks. This + * should only be used in unit tests. + */ + public static final String TEST_MODE_ENABLED_KEY = + "hbase.replica.distribution.conditional.testModeEnabled"; + + private static final Logger LOG = LoggerFactory.getLogger(DistributeReplicasConditional.class); + + private final BalancerClusterState cluster; + private final boolean isTestModeEnabled; + + public DistributeReplicasConditional(Configuration conf, BalancerClusterState cluster) { + super(conf, cluster); + this.cluster = cluster; + this.isTestModeEnabled = conf.getBoolean(TEST_MODE_ENABLED_KEY, false); + } + + @Override + boolean isViolating(RegionPlan regionPlan) { + if (!cluster.hasRegionReplicas) { + return false; + } + + Integer destinationServerIndex = + cluster.serversToIndex.get(regionPlan.getDestination().getAddress()); + if (destinationServerIndex == null) { + LOG.warn("Could not find server index for {}", regionPlan.getDestination().getHostname()); + return false; + } + + int regionIndex = cluster.regionsToIndex.get(regionPlan.getRegionInfo()); + if (regionIndex == -1) { + LOG.warn("Region {} not found in the cluster state", regionPlan.getRegionInfo()); + return false; + } + + int primaryRegionIndex = cluster.regionIndexToPrimaryIndex[regionIndex]; + if (primaryRegionIndex == -1) { + LOG.warn("No primary index found for region {}", regionPlan.getRegionInfo()); + return false; + } + + if ( + checkViolation(cluster.regions, regionPlan.getRegionInfo(), destinationServerIndex, + cluster.serversPerHost, cluster.serverIndexToHostIndex, cluster.regionsPerServer, + primaryRegionIndex, "host", isTestModeEnabled) + ) { + return true; + } + + if ( + checkViolation(cluster.regions, regionPlan.getRegionInfo(), destinationServerIndex, + cluster.serversPerRack, cluster.serverIndexToRackIndex, cluster.regionsPerServer, + primaryRegionIndex, "rack", isTestModeEnabled) + ) { + return true; + } + + return false; + } + + /** + * Checks if placing a region replica on a location (host/rack) violates distribution rules. + * @param destinationServerIndex Index of the destination server. + * @param serversPerLocation Array mapping locations (hosts/racks) to servers. + * @param serverToLocationIndex Array mapping servers to their location index. + * @param regionsPerServer Array mapping servers to their assigned regions. + * @param primaryRegionIndex Index of the primary region. + * @param locationType Type of location being checked ("Host" or "Rack"). + * @return True if a violation is found, false otherwise. + */ + static boolean checkViolation(RegionInfo[] regions, RegionInfo regionToBeMoved, + int destinationServerIndex, int[][] serversPerLocation, int[] serverToLocationIndex, + int[][] regionsPerServer, int primaryRegionIndex, String locationType, + boolean isTestModeEnabled) { + + if (isTestModeEnabled) { + // Take the flat serversPerLocation, like {0: [0, 1, 2, 3, 4]} + // and pretend it is multi-location, like {0: [1], 1: [2] ...} + int numServers = serversPerLocation[0].length; + // Create a new serversPerLocation array where each server gets its own "location" + int[][] simulatedServersPerLocation = new int[numServers][]; + for (int i = 0; i < numServers; i++) { + simulatedServersPerLocation[i] = new int[] { serversPerLocation[0][i] }; + } + // Adjust serverToLocationIndex to map each server to its simulated location + int[] simulatedServerToLocationIndex = new int[numServers]; + for (int i = 0; i < numServers; i++) { + simulatedServerToLocationIndex[serversPerLocation[0][i]] = i; + } + LOG.trace("Test mode enabled: Simulated {} locations for servers.", numServers); + // Use the simulated arrays for test mode + serversPerLocation = simulatedServersPerLocation; + serverToLocationIndex = simulatedServerToLocationIndex; + } + + if (serversPerLocation == null) { + LOG.trace("{} violation check skipped: serversPerLocation is null", locationType); + return false; + } + + if (serversPerLocation.length == 1) { + LOG.warn("{} violation inevitable: serversPerLocation has only 1 entry. " + + "You probably should not be using read replicas.", locationType); + return true; + } + + int destinationLocationIndex = serverToLocationIndex[destinationServerIndex]; + LOG.trace("Checking {} violations for destination server index {} at location index {}", + locationType, destinationServerIndex, destinationLocationIndex); + + // For every RegionServer on host/rack + for (int serverIndex : serversPerLocation[destinationLocationIndex]) { + // For every Region on RegionServer + for (int hostedRegion : regionsPerServer[serverIndex]) { + RegionInfo targetRegion = regions[hostedRegion]; + if (targetRegion.getEncodedName().equals(regionToBeMoved.getEncodedName())) { + // The balancer state will already show this region as having moved. + // A region's replicas will also have unique encoded names. + // So we should skip this check if the encoded name is the same. + continue; + } + boolean isReplicaForSameRegion = + RegionReplicaUtil.isReplicasForSameRegion(targetRegion, regionToBeMoved); + if (isReplicaForSameRegion) { + LOG.trace("{} violation detected: region {} on {} {}", locationType, primaryRegionIndex, + locationType, destinationLocationIndex); + return true; + } + } + } + return false; + } + +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..1ef9f18cdb84 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationCandidateGenerator.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class MetaTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator { + @Override + boolean shouldBeIsolated(RegionInfo regionInfo) { + return regionInfo.isMetaRegion(); + } +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java new file mode 100644 index 000000000000..3eacb3668470 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/MetaTableIsolationConditional.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.yetus.audience.InterfaceAudience; + +/** + * If enabled, this class will help the balancer ensure that the meta table lives on its own + * RegionServer. Configure this via {@link BalancerConditionals#ISOLATE_META_TABLE_KEY} + */ +@InterfaceAudience.Private +class MetaTableIsolationConditional extends RegionPlanConditional { + + private final Set emptyServers = new HashSet<>(); + private final Set serversHostingMeta = new HashSet<>(); + + public MetaTableIsolationConditional(Configuration conf, BalancerClusterState cluster) { + super(conf, cluster); + + for (int i = 0; i < cluster.servers.length; i++) { + ServerName server = cluster.servers[i]; + boolean hasMeta = false; + boolean hasOtherRegions = false; + + for (int region : cluster.regionsPerServer[i]) { + RegionInfo regionInfo = cluster.regions[region]; + if (regionInfo.getTable().equals(TableName.META_TABLE_NAME)) { + hasMeta = true; + } else { + hasOtherRegions = true; + } + } + + if (hasMeta) { + serversHostingMeta.add(server); + } else if (!hasOtherRegions) { + emptyServers.add(server); + } + } + } + + @Override + public boolean isViolating(RegionPlan regionPlan) { + return checkViolation(regionPlan, serversHostingMeta, emptyServers); + } + + /** + * Checks if the placement of `hbase:meta` adheres to isolation rules. + * @param regionPlan The region plan being evaluated. + * @param serversHostingMeta Servers currently hosting `hbase:meta`. + * @param emptyServers Servers with no regions. + * @return True if the placement violates isolation, false otherwise. + */ + protected static boolean checkViolation(RegionPlan regionPlan, Set serversHostingMeta, + Set emptyServers) { + boolean isMeta = regionPlan.getRegionInfo().getTable().equals(TableName.META_TABLE_NAME); + ServerName destination = regionPlan.getDestination(); + if (isMeta) { + // meta must go to an empty server or a server already hosting meta + return !(serversHostingMeta.contains(destination) || emptyServers.contains(destination)); + } else { + // Non-meta regions must not go to servers hosting meta + return serversHostingMeta.contains(destination); + } + } +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java new file mode 100644 index 000000000000..db80e12070d0 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/RegionPlanConditional.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.yetus.audience.InterfaceAudience; +import org.apache.yetus.audience.InterfaceStability; + +@InterfaceAudience.Public +@InterfaceStability.Evolving +public abstract class RegionPlanConditional { + public RegionPlanConditional(Configuration conf, BalancerClusterState cluster) { + } + + abstract boolean isViolating(RegionPlan regionPlan); +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java index e5cd5446c5c8..726eccb4ddf2 100644 --- a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java @@ -163,9 +163,13 @@ public enum GeneratorType { RANDOM, LOAD, LOCALITY, - RACK + RACK, + SYSTEM_TABLE_ISOLATION, + META_TABLE_ISOLATION, } + private final BalancerConditionals balancerConditionals = BalancerConditionals.INSTANCE; + /** * The constructor that pass a MetricsStochasticBalancer to BaseLoadBalancer to replace its * default MetricsBalancer @@ -224,6 +228,10 @@ protected List createCandidateGenerators() { candidateGenerators.add(GeneratorType.LOCALITY.ordinal(), localityCandidateGenerator); candidateGenerators.add(GeneratorType.RACK.ordinal(), new RegionReplicaRackCandidateGenerator()); + candidateGenerators.add(GeneratorType.SYSTEM_TABLE_ISOLATION.ordinal(), + new SystemTableIsolationCandidateGenerator()); + candidateGenerators.add(GeneratorType.META_TABLE_ISOLATION.ordinal(), + new MetaTableIsolationCandidateGenerator()); return candidateGenerators; } @@ -269,6 +277,8 @@ protected void loadConf(Configuration conf) { curFunctionCosts = new double[costFunctions.size()]; tempFunctionCosts = new double[costFunctions.size()]; + balancerConditionals.loadConf(conf); + LOG.info("Loaded config; maxSteps=" + maxSteps + ", runMaxSteps=" + runMaxSteps + ", stepsPerRegion=" + stepsPerRegion + ", maxRunningTime=" + maxRunningTime + ", isByTable=" + isByTable + ", CostFunctions=" + Arrays.toString(getCostFunctionNames()) @@ -372,12 +382,17 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { return true; } - if (sloppyRegionServerExist(cs)) { + if (!balancerConditionals.shouldSkipSloppyServerEvaluation() && sloppyRegionServerExist(cs)) { LOG.info("Running balancer because cluster has sloppy server(s)." + " function cost={}", functionCost()); return true; } + if (balancerConditionals.shouldRunBalancer()) { + LOG.info("Running balancer because conditional violations existed and improved recently"); + return true; + } + double total = 0.0; for (CostFunction c : costFunctions) { if (!c.isNeeded()) { @@ -394,11 +409,13 @@ boolean needsBalance(TableName tableName, BalancerClusterState cluster) { costFunctions); LOG.info( "{} - skipping load balancing because weighted average imbalance={} <= " - + "threshold({}). If you want more aggressive balancing, either lower " + + "threshold({}) and we have not improved balancer conditionals in {} " + + "consecutive balancer runs. If you want more aggressive balancing, either lower " + "hbase.master.balancer.stochastic.minCostNeedBalance from {} or increase the relative " + "multiplier(s) of the specific cost function(s). functionCost={}", isByTable ? "Table specific (" + tableName + ")" : "Cluster wide", total / sumMultiplier, - minCostNeedBalance, minCostNeedBalance, functionCost()); + minCostNeedBalance, balancerConditionals.getConsecutiveBalancesWithoutImprovement(), + minCostNeedBalance, functionCost()); } else { LOG.info("{} - Calculating plan. may take up to {}ms to complete.", isByTable ? "Table specific (" + tableName + ")" : "Cluster wide", maxRunningTime); @@ -512,6 +529,10 @@ protected List balanceTable(TableName tableName, calculatedMaxSteps, maxSteps); } } + + // Update conditionals to reflect current balancer state + balancerConditionals.loadClusterState(cluster); + LOG.info( "Start StochasticLoadBalancer.balancer, initial weighted average imbalance={}, " + "functionCost={} computedMaxSteps={}", @@ -521,6 +542,7 @@ protected List balanceTable(TableName tableName, // Perform a stochastic walk to see if we can get a good fit. long step; + boolean improvedConditionals = false; for (step = 0; step < computedMaxSteps; step++) { BalanceAction action = nextAction(cluster); @@ -528,14 +550,24 @@ protected List balanceTable(TableName tableName, continue; } - cluster.doAction(action); + List regionPlans = cluster.doAction(action); + int conditionalViolationsChange = + balancerConditionals.getConditionalViolationChange(regionPlans); updateCostsAndWeightsWithAction(cluster, action); - newCost = computeCost(cluster, currentCost); - // Should this be kept? - if (newCost < currentCost) { + boolean conditionalsImproved = conditionalViolationsChange < 0; + if (conditionalsImproved) { + improvedConditionals = true; + } + boolean conditionalsSimilarCostsImproved = + (newCost < currentCost && conditionalViolationsChange == 0); + // Our first priority is to reduce conditional violations + // Our second priority is to reduce balancer cost + // change, regardless of cost change + if (conditionalsImproved || conditionalsSimilarCostsImproved) { currentCost = newCost; + balancerConditionals.loadClusterState(cluster); // save for JMX curOverallCost = currentCost; @@ -552,11 +584,16 @@ protected List balanceTable(TableName tableName, break; } } + if (improvedConditionals) { + balancerConditionals.resetConsecutiveBalancesWithoutImprovement(); + } else { + balancerConditionals.incConsecutiveBalancesWithoutImprovement(); + } long endTime = EnvironmentEdgeManager.currentTime(); metricsBalancer.balanceCluster(endTime - startTime); - if (initCost > currentCost) { + if (improvedConditionals || initCost > currentCost) { updateStochasticCosts(tableName, curOverallCost, curFunctionCosts); List plans = createRegionPlans(cluster); LOG.info( @@ -571,7 +608,8 @@ protected List balanceTable(TableName tableName, } LOG.info( "Could not find a better moving plan. Tried {} different configurations in " - + "{} ms, and did not find anything with an imbalance score less than {}", + + "{} ms, and did not find anything with an imbalance score less than {} " + + "and could not improve conditional violations", step, endTime - startTime, initCost / sumMultiplier); return null; } @@ -752,6 +790,7 @@ void initCosts(BalancerClusterState cluster) { c.prepare(cluster); c.updateWeight(weightsOfGenerators); } + updateConditionalGeneratorWeights(cluster); } /** @@ -770,6 +809,25 @@ void updateCostsAndWeightsWithAction(BalancerClusterState cluster, BalanceAction c.updateWeight(weightsOfGenerators); } } + updateConditionalGeneratorWeights(cluster); + } + + private void updateConditionalGeneratorWeights(BalancerClusterState cluster) { + if (balancerConditionals.isTableIsolationEnabled()) { + CandidateGenerator systemTableIsolationGenerator = + candidateGenerators.get(GeneratorType.SYSTEM_TABLE_ISOLATION.ordinal()); + if (systemTableIsolationGenerator instanceof SystemTableIsolationCandidateGenerator) { + weightsOfGenerators[GeneratorType.SYSTEM_TABLE_ISOLATION.ordinal()] = + ((SystemTableIsolationCandidateGenerator) systemTableIsolationGenerator) + .getWeight(cluster); + } + CandidateGenerator metaTableIsolationGenerator = + candidateGenerators.get(GeneratorType.META_TABLE_ISOLATION.ordinal()); + if (metaTableIsolationGenerator instanceof MetaTableIsolationCandidateGenerator) { + weightsOfGenerators[GeneratorType.META_TABLE_ISOLATION.ordinal()] = + ((MetaTableIsolationCandidateGenerator) metaTableIsolationGenerator).getWeight(cluster); + } + } } /** diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..4ac4f7cf3071 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationCandidateGenerator.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; + +@InterfaceAudience.Private +public class SystemTableIsolationCandidateGenerator extends TableIsolationCandidateGenerator { + + @Override + boolean shouldBeIsolated(RegionInfo regionInfo) { + return regionInfo.getTable().isSystemTable(); + } + +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java new file mode 100644 index 000000000000..e8fd96449108 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/SystemTableIsolationConditional.java @@ -0,0 +1,83 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.master.RegionPlan; + +/** + * If enabled, this class will help the balancer ensure that system tables live on their own + * RegionServer. System tables will share one RegionServer! This conditional can be used in tandem + * with {@link MetaTableIsolationConditional} to add a second RegionServer specifically for meta + * table hosting. Configure this via {@link BalancerConditionals#ISOLATE_SYSTEM_TABLES_KEY} + */ +class SystemTableIsolationConditional extends RegionPlanConditional { + + private final Set serversHostingSystemTables = new HashSet<>(); + private final Set metaOnlyServers = new HashSet<>(); + + public SystemTableIsolationConditional(Configuration conf, BalancerClusterState cluster) { + super(conf, cluster); + + // If meta is isolating, then don't count it here + boolean isolatingMeta = conf.getBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, false); + + for (int i = 0; i < cluster.regions.length; i++) { + RegionInfo regionInfo = cluster.regions[i]; + if (isolatingMeta && regionInfo.isMetaRegion()) { + // Exclude meta if we're separately isolating it + metaOnlyServers.add(cluster.servers[cluster.regionIndexToServerIndex[i]]); + } else if (regionInfo.getTable().isSystemTable()) { + serversHostingSystemTables.add(cluster.servers[cluster.regionIndexToServerIndex[i]]); + } + } + } + + @Override + public boolean isViolating(RegionPlan regionPlan) { + return checkViolation(regionPlan, serversHostingSystemTables, metaOnlyServers); + } + + protected static boolean checkViolation(RegionPlan regionPlan, + Set serversHostingSystemTables, Set metaOnlyServers) { + if (!metaOnlyServers.isEmpty() && regionPlan.getRegionInfo().isMetaRegion()) { + return metaOnlyServers.contains(regionPlan.getDestination()); + } + + boolean isSystemTable = regionPlan.getRegionInfo().getTable().isSystemTable(); + if (isSystemTable) { + // Approve if we are currently on a disallowed server (ie, contaminating the meta servers) + if ( + metaOnlyServers.contains(regionPlan.getSource()) + && !metaOnlyServers.contains(regionPlan.getDestination()) + ) { + return false; + } + // Otherwise, approve if only system tables exist where we're going, and the server is allowed + return !serversHostingSystemTables.contains(regionPlan.getDestination()) + && !metaOnlyServers.contains(regionPlan.getDestination()); + } else { + // Ensure the destination server has no system tables + return serversHostingSystemTables.contains(regionPlan.getDestination()); + } + } +} diff --git a/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java new file mode 100644 index 000000000000..8a538887f8b3 --- /dev/null +++ b/hbase-balancer/src/main/java/org/apache/hadoop/hbase/master/balancer/TableIsolationCandidateGenerator.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.yetus.audience.InterfaceAudience; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@InterfaceAudience.Private +public abstract class TableIsolationCandidateGenerator extends CandidateGenerator { + + private static final Logger LOG = LoggerFactory.getLogger(TableIsolationCandidateGenerator.class); + + abstract boolean shouldBeIsolated(RegionInfo regionInfo); + + double getWeight(BalancerClusterState cluster) { + if (generateCandidate(cluster, true) != BalanceAction.NULL_ACTION) { + // If this generator has something to do, then it's important + return Double.MAX_VALUE; + } else { + return 0; + } + } + + @Override + BalanceAction generate(BalancerClusterState cluster) { + return generateCandidate(cluster, false); + } + + private BalanceAction generateCandidate(BalancerClusterState cluster, boolean isWeighing) { + if (!BalancerConditionals.INSTANCE.isTableIsolationEnabled()) { + return BalanceAction.NULL_ACTION; + } + + int fromServer = -1; + int lastMovableRegion = -1; + + // Find regions that need to move + for (int serverIndex = 0; serverIndex < cluster.servers.length; serverIndex++) { + boolean hasRegionToBeIsolated = false; + lastMovableRegion = -1; + for (int regionIndex : cluster.regionsPerServer[serverIndex]) { + RegionInfo regionInfo = cluster.regions[regionIndex]; + if (shouldBeIsolated(regionInfo)) { + hasRegionToBeIsolated = true; + } else { + lastMovableRegion = regionIndex; + } + if (hasRegionToBeIsolated && lastMovableRegion != -1) { + fromServer = serverIndex; + break; + } + } + if (fromServer != -1) { + break; + } + } + if (fromServer == -1) { + return BalanceAction.NULL_ACTION; + } + + int toServer = pickOtherRandomServer(cluster, fromServer); + + if (!isWeighing) { + LOG.debug("Should move region {} off of server {} to server {}", + cluster.regions[lastMovableRegion].getEncodedName(), + cluster.servers[fromServer].getServerName(), cluster.servers[toServer].getServerName()); + } + + return getAction(fromServer, lastMovableRegion, toServer, -1); + } +} diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java new file mode 100644 index 000000000000..0f58b2d19096 --- /dev/null +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestBalancerConditionals.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.List; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestBalancerConditionals extends BalancerTestBase { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestBalancerConditionals.class); + + private static final ServerName SERVER_1 = ServerName.valueOf("server1", 12345, 1); + private static final ServerName SERVER_2 = ServerName.valueOf("server2", 12345, 1); + + private BalancerConditionals balancerConditionals; + private BalancerClusterState mockCluster; + + @Before + public void setUp() { + balancerConditionals = BalancerConditionals.INSTANCE; + mockCluster = mockCluster(new int[] { 0, 1, 2 }); + } + + @Test + public void testDefaultConfiguration() { + Configuration conf = new Configuration(); + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertEquals("No conditionals should be loaded by default", 0, + balancerConditionals.getConditionalClasses().size()); + } + + @Test + public void testSystemTableIsolationConditionalEnabled() { + Configuration conf = new Configuration(); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("SystemTableIsolationConditional should be active", + balancerConditionals.shouldSkipSloppyServerEvaluation()); + } + + @Test + public void testMetaTableIsolationConditionalEnabled() { + Configuration conf = new Configuration(); + conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("MetaTableIsolationConditional should be active", + balancerConditionals.shouldSkipSloppyServerEvaluation()); + } + + @Test + public void testCustomConditionalsViaConfiguration() { + Configuration conf = new Configuration(); + conf.set(BalancerConditionals.ADDITIONAL_CONDITIONALS_KEY, + MetaTableIsolationConditional.class.getName()); + + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("Custom conditionals should be loaded", + balancerConditionals.shouldSkipSloppyServerEvaluation()); + } + + @Test + public void testInvalidCustomConditionalClass() { + Configuration conf = new Configuration(); + conf.set(BalancerConditionals.ADDITIONAL_CONDITIONALS_KEY, "java.lang.String"); + + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertEquals("Invalid classes should not be loaded as conditionals", 0, + balancerConditionals.getConditionalClasses().size()); + } + + @Test + public void testNoViolationsWithoutConditionals() { + Configuration conf = new Configuration(); + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + RegionInfo regionInfo = RegionInfoBuilder.newBuilder(TableName.valueOf("test")).build(); + RegionPlan regionPlan = new RegionPlan(regionInfo, SERVER_1, SERVER_2); + + int violations = balancerConditionals.getConditionalViolationChange(List.of(regionPlan)); + + assertEquals("No conditionals should result in zero violations", 0, violations); + } + + @Test + public void testShouldSkipSloppyServerEvaluationWithMixedConditionals() { + Configuration conf = new Configuration(); + conf.setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + conf.setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + + balancerConditionals.loadConf(conf); + balancerConditionals.loadClusterState(mockCluster); + + assertTrue("Sloppy server evaluation should be skipped with relevant conditionals", + balancerConditionals.shouldSkipSloppyServerEvaluation()); + } +} diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDistributeReplicasConditional.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDistributeReplicasConditional.java new file mode 100644 index 000000000000..092a2d6e5e1c --- /dev/null +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestDistributeReplicasConditional.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestDistributeReplicasConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestDistributeReplicasConditional.class); + + @Test + public void testInevitableViolationWithSingleLocation() { + RegionInfo primaryRegion = createRegionInfo("table1", 0, 100, 0); + RegionInfo replicaRegion = createRegionInfo("table1", 0, 100, 1); + + int[][] serversPerLocation = { { 0, 1 } }; + int[] serverToLocationIndex = { 0, 0 }; // Both servers belong to the same location + int[][] regionsPerServer = { { 0 }, { 1 } }; + int primaryRegionIndex = 0; + int destinationServerIndex = 1; + RegionInfo[] regions = { primaryRegion, replicaRegion }; + + assertTrue("No violation when only one location exists", + DistributeReplicasConditional.checkViolation(regions, replicaRegion, destinationServerIndex, + serversPerLocation, serverToLocationIndex, regionsPerServer, primaryRegionIndex, "Host", + false)); + } + + @Test + public void testReplicaPlacedInSameHostAsPrimary() { + RegionInfo primaryRegion = createRegionInfo("table1", 0, 100, 0); + RegionInfo replicaRegion = createRegionInfo("table1", 0, 100, 1); + + // Simulate two servers (Server 0 and Server 1) on the same host (Host 0) + int[][] serversPerLocation = { { 0, 1 } }; + int[] serverToLocationIndex = { 0, 0 }; // Both servers belong to the same host (index 0) + int[][] regionsPerServer = { { 0 }, { 1 } }; // Server 0 hosts primary, Server 1 hosts replica + int primaryRegionIndex = 0; // Index of primary region + int destinationServerIndex = 1; // Replica is being moved to Server 1 + RegionInfo[] regions = { primaryRegion, replicaRegion }; + + assertTrue("Host violation detected when replica is placed on the same host as its primary", + DistributeReplicasConditional.checkViolation(regions, replicaRegion, destinationServerIndex, + serversPerLocation, serverToLocationIndex, regionsPerServer, primaryRegionIndex, "Host", + false)); + } + + @Test + public void testPrimaryPlacedInSameHostAsReplica() { + RegionInfo primaryRegion = createRegionInfo("table1", 0, 100, 0); + RegionInfo replicaRegion = createRegionInfo("table1", 0, 100, 1); + + int[][] serversPerLocation = { { 0, 1 } }; + int[] serverToLocationIndex = { 0, 0 }; // Both servers belong to the same host + int[][] regionsPerServer = { { 0 }, { 1 } }; + int primaryRegionIndex = 0; + int destinationServerIndex = 0; + RegionInfo[] regions = { primaryRegion, replicaRegion }; + + assertTrue("Violation detected when the primary is placed on the same host as its replica", + DistributeReplicasConditional.checkViolation(regions, primaryRegion, destinationServerIndex, + serversPerLocation, serverToLocationIndex, regionsPerServer, primaryRegionIndex, "Host", + false)); + } + + @Test + public void testReplicaPlacedInSameRackAsPrimary() { + RegionInfo primaryRegion = createRegionInfo("table1", 0, 100, 0); + RegionInfo replicaRegion = createRegionInfo("table1", 0, 100, 1); + + int[][] serversPerLocation = { { 0, 1 }, { 2 } }; + int[] serverToLocationIndex = { 0, 0, 1 }; + int[][] regionsPerServer = { { 0 }, { 1 }, {} }; + int primaryRegionIndex = 0; + int destinationServerIndex = 1; + RegionInfo[] regions = { primaryRegion, replicaRegion }; + + assertTrue("Rack violation detected when replica is placed on the same rack as its primary", + DistributeReplicasConditional.checkViolation(regions, replicaRegion, destinationServerIndex, + serversPerLocation, serverToLocationIndex, regionsPerServer, primaryRegionIndex, "Rack", + false)); + } + + @Test + public void testNoViolationOnDifferentHostAndRack() { + RegionInfo primaryRegion = createRegionInfo("table1", 0, 100, 0); + RegionInfo replicaRegion = createRegionInfo("table1", 0, 100, 1); + + int[][] serversPerLocation = { { 0 }, { 1 } }; + int[] serverToLocationIndex = { 0, 1 }; + int[][] regionsPerServer = { { 0 }, { 1 } }; + int primaryRegionIndex = 0; + int destinationServerIndex = 1; + RegionInfo[] regions = { primaryRegion, replicaRegion }; + + assertFalse("No violation when replica is placed on a different host and rack", + DistributeReplicasConditional.checkViolation(regions, replicaRegion, destinationServerIndex, + serversPerLocation, serverToLocationIndex, regionsPerServer, primaryRegionIndex, "Host", + false)); + } + + private static RegionInfo createRegionInfo(String tableName, int startKey, int stopKey, + int replicaId) { + return RegionInfoBuilder.newBuilder(TableName.valueOf(tableName)) + .setStartKey(new byte[] { (byte) startKey }).setEndKey(new byte[] { (byte) stopKey }) + .setReplicaId(replicaId).build(); + } +} diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationConditional.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationConditional.java new file mode 100644 index 000000000000..e30b2ea111bb --- /dev/null +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestMetaTableIsolationConditional.java @@ -0,0 +1,157 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestMetaTableIsolationConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestMetaTableIsolationConditional.class); + + private static final ServerName SERVER_1 = ServerName.valueOf("server1", 12345, 1); + private static final ServerName SERVER_2 = ServerName.valueOf("server2", 12345, 1); + private static final ServerName SERVER_3 = ServerName.valueOf("server3", 12345, 1); + + private static final RegionInfo META_REGION = + RegionInfoBuilder.newBuilder(TableName.META_TABLE_NAME).build(); + private static final RegionInfo USER_REGION = + RegionInfoBuilder.newBuilder(TableName.valueOf("userTable")).build(); + + /** + * Test that no violation is detected when `hbase:meta` is placed on an empty server. + */ + @Test + public void testNoViolationWhenMetaOnEmptyServer() { + Set serversHostingMeta = Collections.emptySet(); + Set emptyServers = new HashSet<>(Set.of(SERVER_1)); + + RegionPlan regionPlan = new RegionPlan(META_REGION, null, SERVER_1); + + assertFalse("No violation when `hbase:meta` is placed on an empty server", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that no violation is detected when `hbase:meta` is moved to a server already hosting + * `hbase:meta`. + */ + @Test + public void testNoViolationWhenMetaOnMetaServer() { + Set serversHostingMeta = new HashSet<>(Set.of(SERVER_1)); + Set emptyServers = Collections.emptySet(); + + RegionPlan regionPlan = new RegionPlan(META_REGION, null, SERVER_1); + + assertFalse("No violation when `hbase:meta` is placed on a server already hosting `hbase:meta`", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that a violation is detected when `hbase:meta` is placed on a server hosting other + * regions. + */ + @Test + public void testViolationWhenMetaOnNonEmptyServer() { + Set serversHostingMeta = Collections.emptySet(); + Set emptyServers = new HashSet<>(Set.of(SERVER_2)); + + RegionPlan regionPlan = new RegionPlan(META_REGION, null, SERVER_3); + + assertTrue("Violation detected when `hbase:meta` is placed on a server hosting other regions", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that a violation is detected when a non-meta region is placed on a server hosting + * `hbase:meta`. + */ + @Test + public void testViolationWhenNonMetaOnMetaServer() { + Set serversHostingMeta = new HashSet<>(Set.of(SERVER_1)); + Set emptyServers = Collections.emptySet(); + + RegionPlan regionPlan = new RegionPlan(USER_REGION, null, SERVER_1); + + assertTrue( + "Violation detected when a non-meta region is placed on a server hosting `hbase:meta`", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that no violation is detected when a non-meta region is placed on a server not hosting + * `hbase:meta`. + */ + @Test + public void testNoViolationWhenNonMetaOnNonMetaServer() { + Set serversHostingMeta = new HashSet<>(Set.of(SERVER_1)); + Set emptyServers = Collections.emptySet(); + + RegionPlan regionPlan = new RegionPlan(USER_REGION, null, SERVER_2); + + assertFalse( + "No violation when a non-meta region is placed on a server not hosting `hbase:meta`", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that no violation is detected when `hbase:meta` is placed on the only available empty + * server. + */ + @Test + public void testNoViolationWhenMetaOnLastEmptyServer() { + Set serversHostingMeta = Collections.emptySet(); + Set emptyServers = new HashSet<>(Set.of(SERVER_2)); + + RegionPlan regionPlan = new RegionPlan(META_REGION, null, SERVER_2); + + assertFalse("No violation when `hbase:meta` is placed on the only available empty server", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } + + /** + * Test that a violation is detected when `hbase:meta` is moved to a non-empty server. + */ + @Test + public void testViolationWhenMetaMovedToNonEmptyServer() { + Set serversHostingMeta = new HashSet<>(Set.of(SERVER_1)); + Set emptyServers = new HashSet<>(Set.of(SERVER_2)); + + RegionPlan regionPlan = new RegionPlan(META_REGION, SERVER_1, SERVER_3); + + assertTrue("Violation detected when `hbase:meta` is moved to a non-empty server", + MetaTableIsolationConditional.checkViolation(regionPlan, serversHostingMeta, emptyServers)); + } +} diff --git a/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestSystemTableIsolationConditional.java b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestSystemTableIsolationConditional.java new file mode 100644 index 000000000000..7cd05645adb7 --- /dev/null +++ b/hbase-balancer/src/test/java/org/apache/hadoop/hbase/master/balancer/TestSystemTableIsolationConditional.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.master.balancer; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.RegionInfoBuilder; +import org.apache.hadoop.hbase.master.RegionPlan; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +@Category(SmallTests.class) +public class TestSystemTableIsolationConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSystemTableIsolationConditional.class); + + private static final ServerName SERVER_1 = ServerName.valueOf("server1", 123, 1); + private static final ServerName SERVER_2 = ServerName.valueOf("server2", 123, 1); + + @Test + public void testSystemTableMovedToSystemTableServer() { + Set serversHostingSystemTables = new HashSet<>(); + serversHostingSystemTables.add(SERVER_1); + + RegionInfo systemTableRegion = + RegionInfoBuilder.newBuilder(TableName.valueOf("hbase:meta")).build(); + RegionPlan regionPlan = new RegionPlan(systemTableRegion, null, SERVER_1); + + assertFalse("No violation when system table is moved to a server hosting only system tables", + SystemTableIsolationConditional.checkViolation(regionPlan, serversHostingSystemTables, + Collections.emptySet())); + } + + @Test + public void testSystemTableMovedToNonSystemTableServer() { + Set serversHostingSystemTables = new HashSet<>(); + serversHostingSystemTables.add(SERVER_1); + + RegionInfo systemTableRegion = + RegionInfoBuilder.newBuilder(TableName.valueOf("hbase:meta")).build(); + RegionPlan regionPlan = new RegionPlan(systemTableRegion, null, SERVER_2); + + assertTrue( + "Violation detected when system table is moved to a server not hosting system tables", + SystemTableIsolationConditional.checkViolation(regionPlan, serversHostingSystemTables, + Collections.emptySet())); + } + + @Test + public void testNonSystemTableMovedToNonSystemTableServer() { + Set serversHostingSystemTables = new HashSet<>(); + serversHostingSystemTables.add(SERVER_1); + + RegionInfo nonSystemTableRegion = + RegionInfoBuilder.newBuilder(TableName.valueOf("testTable")).build(); + RegionPlan regionPlan = new RegionPlan(nonSystemTableRegion, null, SERVER_2); + + assertFalse("No violation when non-system table is moved to a server not hosting system tables", + SystemTableIsolationConditional.checkViolation(regionPlan, serversHostingSystemTables, + Collections.emptySet())); + } + + @Test + public void testNonSystemTableMovedToSystemTableServer() { + Set serversHostingSystemTables = new HashSet<>(); + serversHostingSystemTables.add(SERVER_1); + + RegionInfo nonSystemTableRegion = + RegionInfoBuilder.newBuilder(TableName.valueOf("testTable")).build(); + RegionPlan regionPlan = new RegionPlan(nonSystemTableRegion, null, SERVER_1); + + assertTrue( + "Violation detected when non-system table is moved to a server hosting system tables", + SystemTableIsolationConditional.checkViolation(regionPlan, serversHostingSystemTables, + Collections.emptySet())); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/BalancerConditionalsTestUtil.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/BalancerConditionalsTestUtil.java new file mode 100644 index 000000000000..f8ef822ea4bf --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/BalancerConditionalsTestUtil.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.balancer; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotEquals; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HRegionLocation; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.util.Bytes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +public class BalancerConditionalsTestUtil { + + private static final Logger LOG = LoggerFactory.getLogger(BalancerConditionalsTestUtil.class); + + private BalancerConditionalsTestUtil() { + } + + static byte[][] generateSplits(int numRegions) { + byte[][] splitKeys = new byte[numRegions - 1][]; + for (int i = 0; i < numRegions - 1; i++) { + splitKeys[i] = + Bytes.toBytes(String.format("%09d", (i + 1) * (Integer.MAX_VALUE / numRegions))); + } + return splitKeys; + } + + static void printRegionLocations(Connection connection) throws IOException { + Admin admin = connection.getAdmin(); + + // Get all table names in the cluster + Set tableNames = admin.listTableDescriptors(true).stream() + .map(TableDescriptor::getTableName).collect(Collectors.toSet()); + + // Group regions by server + Map>> serverToRegions = + admin.getClusterMetrics().getLiveServerMetrics().keySet().stream() + .collect(Collectors.toMap(server -> server, server -> { + try { + return listRegionsByTable(connection, server, tableNames); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + + // Pretty print region locations + StringBuilder regionLocationOutput = new StringBuilder(); + regionLocationOutput.append("Pretty printing region locations...\n"); + serverToRegions.forEach((server, tableRegions) -> { + regionLocationOutput.append("Server: " + server.getServerName() + "\n"); + tableRegions.forEach((table, regions) -> { + if (regions.isEmpty()) { + return; + } + regionLocationOutput.append(" Table: " + table.getNameAsString() + "\n"); + regions.forEach(region -> regionLocationOutput + .append(String.format(" Region: %s, start: %s, end: %s, replica: %s\n", + region.getEncodedName(), Bytes.toString(region.getStartKey()), + Bytes.toString(region.getEndKey()), region.getReplicaId()))); + }); + }); + LOG.info(regionLocationOutput.toString()); + } + + private static Map> listRegionsByTable(Connection connection, + ServerName server, Set tableNames) throws IOException { + Admin admin = connection.getAdmin(); + + // Find regions for each table + return tableNames.stream().collect(Collectors.toMap(tableName -> tableName, tableName -> { + List allRegions = null; + try { + allRegions = admin.getRegions(server); + } catch (IOException e) { + throw new RuntimeException(e); + } + return allRegions.stream().filter(region -> region.getTable().equals(tableName)) + .collect(Collectors.toList()); + })); + } + + static void validateReplicaDistribution(Connection connection, TableName tableName, + boolean shouldBeDistributed) { + Map> serverToRegions = null; + try { + serverToRegions = connection.getRegionLocator(tableName).getAllRegionLocations().stream() + .collect(Collectors.groupingBy(location -> location.getServerName(), + Collectors.mapping(location -> location.getRegion(), Collectors.toList()))); + } catch (IOException e) { + throw new RuntimeException(e); + } + + if (shouldBeDistributed) { + // Ensure no server hosts more than one replica of any region + for (Map.Entry> serverAndRegions : serverToRegions.entrySet()) { + List regionInfos = serverAndRegions.getValue(); + Set startKeys = new HashSet<>(); + for (RegionInfo regionInfo : regionInfos) { + // each region should have a distinct start key + assertFalse( + "Each region should have its own start key, " + + "demonstrating it is not a replica of any others on this host", + startKeys.contains(regionInfo.getStartKey())); + startKeys.add(regionInfo.getStartKey()); + } + } + } else { + // Ensure all replicas are on the same server + assertEquals("All regions should share one server", 1, serverToRegions.size()); + } + } + + static void validateRegionLocations(Map> tableToServers, + TableName productTableName, boolean shouldBeBalanced) { + ServerName metaServer = + tableToServers.get(TableName.META_TABLE_NAME).stream().findFirst().orElseThrow(); + ServerName quotaServer = + tableToServers.get(QuotaUtil.QUOTA_TABLE_NAME).stream().findFirst().orElseThrow(); + Set productServers = tableToServers.get(productTableName); + + if (shouldBeBalanced) { + for (ServerName server : productServers) { + assertNotEquals("Meta table and product table should not share servers", server, + metaServer); + assertNotEquals("Quota table and product table should not share servers", server, + quotaServer); + } + assertNotEquals("The meta server and quotas server should be different", metaServer, + quotaServer); + } else { + for (ServerName server : productServers) { + assertEquals("Meta table and product table must share servers", server, metaServer); + assertEquals("Quota table and product table must share servers", server, quotaServer); + } + assertEquals("The meta server and quotas server must be the same", metaServer, quotaServer); + } + } + + static Map> getTableToServers(Connection connection, + Set tableNames) { + return tableNames.stream().collect(Collectors.toMap(t -> t, t -> { + try { + return connection.getRegionLocator(t).getAllRegionLocations().stream() + .map(HRegionLocation::getServerName).collect(Collectors.toSet()); + } catch (IOException e) { + throw new RuntimeException(e); + } + })); + } + + @FunctionalInterface + interface AssertionRunnable { + void run() throws AssertionError; + } + + static void validateAssertionsWithRetries(HBaseTestingUtil testUtil, boolean runBalancerOnFailure, + AssertionRunnable assertion) { + validateAssertionsWithRetries(testUtil, runBalancerOnFailure, ImmutableSet.of(assertion)); + } + + static void validateAssertionsWithRetries(HBaseTestingUtil testUtil, boolean runBalancerOnFailure, + Set assertions) { + int maxAttempts = 10; + for (int i = 0; i < maxAttempts; i++) { + try { + for (AssertionRunnable assertion : assertions) { + assertion.run(); + } + } catch (AssertionError e) { + if (i == maxAttempts - 1) { + throw e; + } + try { + LOG.warn("Failed to validate region locations. Will retry", e); + Thread.sleep(1000); + BalancerConditionalsTestUtil.printRegionLocations(testUtil.getConnection()); + if (runBalancerOnFailure) { + testUtil.getAdmin().balance(); + } + Thread.sleep(1000); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + } + } + +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestLargerClusterBalancerConditionals.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestLargerClusterBalancerConditionals.java new file mode 100644 index 000000000000..be1f079aef18 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestLargerClusterBalancerConditionals.java @@ -0,0 +1,149 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.balancer; + +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.getTableToServers; +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateAssertionsWithRetries; +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateRegionLocations; +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateReplicaDistribution; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.balancer.BalancerConditionals; +import org.apache.hadoop.hbase.master.balancer.DistributeReplicasConditional; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category(LargeTests.class) +public class TestLargerClusterBalancerConditionals { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestLargerClusterBalancerConditionals.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestSystemTableIsolationBalancerConditionals.class); + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + + private static final int NUM_SERVERS = 18; + private static final int PRODUCT_TABLE_REGIONS_PER_SERVER = 10; + private static final int REPLICAS = 3; + + @Before + public void setUp() throws Exception { + TEST_UTIL.getConfiguration() + .setBoolean(BalancerConditionals.DISTRIBUTE_REPLICAS_CONDITIONALS_KEY, true); + TEST_UTIL.getConfiguration() + .setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(DistributeReplicasConditional.TEST_MODE_ENABLED_KEY, + true); + TEST_UTIL.getConfiguration().setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_BALANCER_PERIOD, 1000L); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + + // turn off replica cost functions + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaRackCostKey", 0); + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaHostCostKey", 0); + + TEST_UTIL.startMiniCluster(NUM_SERVERS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testTableIsolation() throws Exception { + Connection connection = TEST_UTIL.getConnection(); + Admin admin = connection.getAdmin(); + + // Create "product" table with 3 regions + TableName productTableName = TableName.valueOf("product"); + TableDescriptor productTableDescriptor = TableDescriptorBuilder.newBuilder(productTableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build()) + .setRegionReplication(REPLICAS).build(); + admin.createTable(productTableDescriptor, + BalancerConditionalsTestUtil.generateSplits(PRODUCT_TABLE_REGIONS_PER_SERVER * NUM_SERVERS)); + + Set tablesToBeSeparated = ImmutableSet. builder() + .add(TableName.META_TABLE_NAME).add(QuotaUtil.QUOTA_TABLE_NAME).add(productTableName).build(); + + // Pause the balancer + admin.balancerSwitch(false, true); + + // Move all regions (product, meta, and quotas) to one RegionServer + List allRegions = tablesToBeSeparated.stream().map(t -> { + try { + return admin.getRegions(t); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).flatMap(Collection::stream).toList(); + String targetServer = + TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName().getServerName(); + for (RegionInfo region : allRegions) { + admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(targetServer)); + } + + validateAssertionsWithRetries(TEST_UTIL, false, + ImmutableSet.of( + () -> validateRegionLocations(getTableToServers(connection, tablesToBeSeparated), + productTableName, false), + () -> validateReplicaDistribution(connection, productTableName, false))); + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + + // Unpause the balancer and run it + admin.balancerSwitch(true, true); + admin.balance(); + + validateAssertionsWithRetries(TEST_UTIL, true, + ImmutableSet.of( + () -> validateRegionLocations(getTableToServers(connection, tablesToBeSeparated), + productTableName, true), + () -> validateReplicaDistribution(connection, productTableName, true))); + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestReplicaDistributionBalancerConditional.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestReplicaDistributionBalancerConditional.java new file mode 100644 index 000000000000..874d1f684ddf --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestReplicaDistributionBalancerConditional.java @@ -0,0 +1,123 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.balancer; + +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateAssertionsWithRetries; + +import java.util.List; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.balancer.BalancerConditionals; +import org.apache.hadoop.hbase.master.balancer.DistributeReplicasConditional; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.ServerRegionReplicaUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@Category(LargeTests.class) +public class TestReplicaDistributionBalancerConditional { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestReplicaDistributionBalancerConditional.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestReplicaDistributionBalancerConditional.class); + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + private static final int REPLICAS = 3; + private static final int NUM_SERVERS = REPLICAS; + private static final int REGIONS_PER_SERVER = 5; + + @Before + public void setUp() throws Exception { + TEST_UTIL.getConfiguration() + .setBoolean(BalancerConditionals.DISTRIBUTE_REPLICAS_CONDITIONALS_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(DistributeReplicasConditional.TEST_MODE_ENABLED_KEY, + true); + TEST_UTIL.getConfiguration() + .setBoolean(ServerRegionReplicaUtil.REGION_REPLICA_REPLICATION_CONF_KEY, true); + TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_BALANCER_PERIOD, 1000L); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + + // turn off replica cost functions + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaRackCostKey", 0); + TEST_UTIL.getConfiguration() + .setLong("hbase.master.balancer.stochastic.regionReplicaHostCostKey", 0); + + TEST_UTIL.startMiniCluster(NUM_SERVERS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testReplicaDistribution() throws Exception { + Connection connection = TEST_UTIL.getConnection(); + Admin admin = connection.getAdmin(); + + // Create a "replicated_table" with region replicas + TableName replicatedTableName = TableName.valueOf("replicated_table"); + TableDescriptor replicatedTableDescriptor = + TableDescriptorBuilder.newBuilder(replicatedTableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build()) + .setRegionReplication(REPLICAS).build(); + admin.createTable(replicatedTableDescriptor, + BalancerConditionalsTestUtil.generateSplits(REGIONS_PER_SERVER * NUM_SERVERS)); + + // Pause the balancer + admin.balancerSwitch(false, true); + + // Collect all region replicas and place them on one RegionServer + List allRegions = admin.getRegions(replicatedTableName); + String targetServer = + TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName().getServerName(); + + for (RegionInfo region : allRegions) { + admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(targetServer)); + } + + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + validateAssertionsWithRetries(TEST_UTIL, false, () -> BalancerConditionalsTestUtil + .validateReplicaDistribution(connection, replicatedTableName, false)); + + // Unpause the balancer and trigger balancing + admin.balancerSwitch(true, true); + admin.balance(); + + validateAssertionsWithRetries(TEST_UTIL, true, () -> BalancerConditionalsTestUtil + .validateReplicaDistribution(connection, replicatedTableName, true)); + BalancerConditionalsTestUtil.printRegionLocations(TEST_UTIL.getConnection()); + } +} diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestSystemTableIsolationBalancerConditionals.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestSystemTableIsolationBalancerConditionals.java new file mode 100644 index 000000000000..01fb998235b7 --- /dev/null +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/balancer/TestSystemTableIsolationBalancerConditionals.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.balancer; + +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.getTableToServers; +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateAssertionsWithRetries; +import static org.apache.hadoop.hbase.balancer.BalancerConditionalsTestUtil.validateRegionLocations; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.Set; +import org.apache.hadoop.hbase.HBaseClassTestRule; +import org.apache.hadoop.hbase.HBaseTestingUtil; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.TableDescriptor; +import org.apache.hadoop.hbase.client.TableDescriptorBuilder; +import org.apache.hadoop.hbase.master.balancer.BalancerConditionals; +import org.apache.hadoop.hbase.quotas.QuotaUtil; +import org.apache.hadoop.hbase.testclassification.LargeTests; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableSet; + +@Category(LargeTests.class) +public class TestSystemTableIsolationBalancerConditionals { + + @ClassRule + public static final HBaseClassTestRule CLASS_RULE = + HBaseClassTestRule.forClass(TestSystemTableIsolationBalancerConditionals.class); + + private static final Logger LOG = + LoggerFactory.getLogger(TestSystemTableIsolationBalancerConditionals.class); + private static final HBaseTestingUtil TEST_UTIL = new HBaseTestingUtil(); + + // One for product table, one for meta, one for other system tables, and one extra + private static final int NUM_SERVERS = 3; + private static final int PRODUCT_TABLE_REGIONS_PER_SERVER = 5; + + @Before + public void setUp() throws Exception { + TEST_UTIL.getConfiguration().setBoolean(BalancerConditionals.ISOLATE_SYSTEM_TABLES_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(BalancerConditionals.ISOLATE_META_TABLE_KEY, true); + TEST_UTIL.getConfiguration().setBoolean(QuotaUtil.QUOTA_CONF_KEY, true); + TEST_UTIL.getConfiguration().setLong(HConstants.HBASE_BALANCER_PERIOD, 1000L); + TEST_UTIL.getConfiguration().setBoolean("hbase.master.balancer.stochastic.runMaxSteps", true); + + TEST_UTIL.startMiniCluster(NUM_SERVERS); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Test + public void testTableIsolation() throws Exception { + Connection connection = TEST_UTIL.getConnection(); + Admin admin = connection.getAdmin(); + + // Create "product" table with 3 regions + TableName productTableName = TableName.valueOf("product"); + TableDescriptor productTableDescriptor = TableDescriptorBuilder.newBuilder(productTableName) + .setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes("0")).build()) + .build(); + admin.createTable(productTableDescriptor, + BalancerConditionalsTestUtil.generateSplits(PRODUCT_TABLE_REGIONS_PER_SERVER * NUM_SERVERS)); + + Set tablesToBeSeparated = ImmutableSet. builder() + .add(TableName.META_TABLE_NAME).add(QuotaUtil.QUOTA_TABLE_NAME).add(productTableName).build(); + + // Pause the balancer + admin.balancerSwitch(false, true); + + // Move all regions (product, meta, and quotas) to one RegionServer + List allRegions = tablesToBeSeparated.stream().map(t -> { + try { + return admin.getRegions(t); + } catch (IOException e) { + throw new RuntimeException(e); + } + }).flatMap(Collection::stream).toList(); + String targetServer = + TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName().getServerName(); + for (RegionInfo region : allRegions) { + admin.move(region.getEncodedNameAsBytes(), Bytes.toBytes(targetServer)); + } + + validateAssertionsWithRetries(TEST_UTIL, false, + () -> validateRegionLocations(getTableToServers(connection, tablesToBeSeparated), + productTableName, false)); + + // Unpause the balancer and run it + admin.balancerSwitch(true, true); + admin.balance(); + + validateAssertionsWithRetries(TEST_UTIL, true, + () -> validateRegionLocations(getTableToServers(connection, tablesToBeSeparated), + productTableName, true)); + } +}