forked from apache/pulsar
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[feature][broker]Provide new load balance placement strategy implemen…
…tation based on the least resource usage with weight (apache#16281) email discussion thread: https://lists.apache.org/thread/36vyyhndr4og175k2bz3mfdf5ctd2xky ### Motivation See PIP-182 apache#16274 ### Modifications See apache#16274 The main idea of the new strategy is to unify the requirement of load shedding strategy and bundle placement strategy, which consider the resource usage with weight, including historical observations. How to calculate a score for a broker ? - use its historical load and short-term load data with weight, which can solve the case of load jitter in a broker. How to select a broker for assignning bundle ? - select a broker based on which one has the least average resource usage with weight. - the random selection algorithm is better than the `minScore` among low load brokers, and use `loadBalancerAverageResourceUsageDifferenceThresholdShedderPercentage ` to adjust the size of the `randomization pool`
- Loading branch information
Showing
6 changed files
with
271 additions
and
7 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
165 changes: 165 additions & 0 deletions
165
...src/main/java/org/apache/pulsar/broker/loadbalance/impl/LeastResourceUsageWithWeight.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,165 @@ | ||
| /** | ||
| * Licensed to the Apache Software Foundation (ASF) under one | ||
| * or more contributor license agreements. See the NOTICE file | ||
| * distributed with this work for additional information | ||
| * regarding copyright ownership. The ASF licenses this file | ||
| * to you under the Apache License, Version 2.0 (the | ||
| * "License"); you may not use this file except in compliance | ||
| * with the License. You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, | ||
| * software distributed under the License is distributed on an | ||
| * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | ||
| * KIND, either express or implied. See the License for the | ||
| * specific language governing permissions and limitations | ||
| * under the License. | ||
| */ | ||
| package org.apache.pulsar.broker.loadbalance.impl; | ||
|
|
||
| import java.util.ArrayList; | ||
| import java.util.HashMap; | ||
| import java.util.Map; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
| import java.util.concurrent.ThreadLocalRandom; | ||
| import lombok.extern.slf4j.Slf4j; | ||
| import org.apache.pulsar.broker.ServiceConfiguration; | ||
| import org.apache.pulsar.broker.loadbalance.LoadData; | ||
| import org.apache.pulsar.broker.loadbalance.ModularLoadManagerStrategy; | ||
| import org.apache.pulsar.policies.data.loadbalancer.BrokerData; | ||
| import org.apache.pulsar.policies.data.loadbalancer.BundleData; | ||
| import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData; | ||
|
|
||
| /** | ||
| * Placement strategy which selects a broker based on which one has the least resource usage with weight. | ||
| * This strategy takes into account the historical load percentage and short-term load percentage, and thus will not | ||
| * cause cluster fluctuations due to short-term load jitter. | ||
| */ | ||
| @Slf4j | ||
| public class LeastResourceUsageWithWeight implements ModularLoadManagerStrategy { | ||
| // Maintain this list to reduce object creation. | ||
| private final ArrayList<String> bestBrokers; | ||
| private final Map<String, Double> brokerAvgResourceUsageWithWeight; | ||
|
|
||
| public LeastResourceUsageWithWeight() { | ||
| this.bestBrokers = new ArrayList<>(); | ||
| this.brokerAvgResourceUsageWithWeight = new HashMap<>(); | ||
| } | ||
|
|
||
| // A broker's max resource usage with weight using its historical load and short-term load data with weight. | ||
| private double getMaxResourceUsageWithWeight(final String broker, final BrokerData brokerData, | ||
| final ServiceConfiguration conf) { | ||
| final double overloadThreshold = conf.getLoadBalancerBrokerOverloadedThresholdPercentage() / 100.0; | ||
| final double maxUsageWithWeight = | ||
| updateAndGetMaxResourceUsageWithWeight(broker, brokerData, conf); | ||
|
|
||
| if (maxUsageWithWeight > overloadThreshold) { | ||
| final LocalBrokerData localData = brokerData.getLocalData(); | ||
| log.warn( | ||
| "Broker {} is overloaded, max resource usage with weight percentage: {}%, " | ||
| + "CPU: {}%, MEMORY: {}%, DIRECT MEMORY: {}%, BANDWIDTH IN: {}%, " | ||
| + "BANDWIDTH OUT: {}%, CPU weight: {}, MEMORY weight: {}, DIRECT MEMORY weight: {}, " | ||
| + "BANDWIDTH IN weight: {}, BANDWIDTH OUT weight: {}", | ||
| broker, maxUsageWithWeight * 100, | ||
| localData.getCpu().percentUsage(), localData.getMemory().percentUsage(), | ||
| localData.getDirectMemory().percentUsage(), localData.getBandwidthIn().percentUsage(), | ||
| localData.getBandwidthOut().percentUsage(), conf.getLoadBalancerCPUResourceWeight(), | ||
| conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), | ||
| conf.getLoadBalancerBandwithInResourceWeight(), | ||
| conf.getLoadBalancerBandwithOutResourceWeight()); | ||
| } | ||
|
|
||
| if (log.isDebugEnabled()) { | ||
| log.debug("Broker {} has max resource usage with weight percentage: {}%", | ||
| brokerData.getLocalData().getWebServiceUrl(), maxUsageWithWeight * 100); | ||
| } | ||
| return maxUsageWithWeight; | ||
| } | ||
|
|
||
| /** | ||
| * Update and get the max resource usage with weight of broker according to the service configuration. | ||
| * | ||
| * @param broker the broker name. | ||
| * @param brokerData The broker load data. | ||
| * @param conf The service configuration. | ||
| * @return the max resource usage with weight of broker | ||
| */ | ||
| private double updateAndGetMaxResourceUsageWithWeight(String broker, BrokerData brokerData, | ||
| ServiceConfiguration conf) { | ||
| final double historyPercentage = conf.getLoadBalancerHistoryResourcePercentage(); | ||
| Double historyUsage = brokerAvgResourceUsageWithWeight.get(broker); | ||
| double resourceUsage = brokerData.getLocalData().getMaxResourceUsageWithWeight( | ||
| conf.getLoadBalancerCPUResourceWeight(), | ||
| conf.getLoadBalancerMemoryResourceWeight(), | ||
| conf.getLoadBalancerDirectMemoryResourceWeight(), | ||
| conf.getLoadBalancerBandwithInResourceWeight(), | ||
| conf.getLoadBalancerBandwithOutResourceWeight()); | ||
| historyUsage = historyUsage == null | ||
| ? resourceUsage : historyUsage * historyPercentage + (1 - historyPercentage) * resourceUsage; | ||
| if (log.isDebugEnabled()) { | ||
| log.debug( | ||
| "Broker {} get max resource usage with weight: {}, history resource percentage: {}%, CPU weight: " | ||
| + "{}, MEMORY weight: {}, DIRECT MEMORY weight: {}, BANDWIDTH IN weight: {}, BANDWIDTH " | ||
| + "OUT weight: {} ", | ||
| broker, historyUsage, historyPercentage, conf.getLoadBalancerCPUResourceWeight(), | ||
| conf.getLoadBalancerMemoryResourceWeight(), conf.getLoadBalancerDirectMemoryResourceWeight(), | ||
| conf.getLoadBalancerBandwithInResourceWeight(), | ||
| conf.getLoadBalancerBandwithOutResourceWeight()); | ||
| } | ||
| brokerAvgResourceUsageWithWeight.put(broker, historyUsage); | ||
| return historyUsage; | ||
| } | ||
|
|
||
| /** | ||
| * Find a suitable broker to assign the given bundle to. | ||
| * This method is not thread safety. | ||
| * | ||
| * @param candidates The candidates for which the bundle may be assigned. | ||
| * @param bundleToAssign The data for the bundle to assign. | ||
| * @param loadData The load data from the leader broker. | ||
| * @param conf The service configuration. | ||
| * @return The name of the selected broker as it appears on ZooKeeper. | ||
| */ | ||
| @Override | ||
| public Optional<String> selectBroker(Set<String> candidates, BundleData bundleToAssign, LoadData loadData, | ||
| ServiceConfiguration conf) { | ||
| bestBrokers.clear(); | ||
| // Maintain of list of all the best scoring brokers and then randomly | ||
| // select one of them at the end. | ||
| double totalUsage = 0.0d; | ||
| for (String broker : candidates) { | ||
| BrokerData brokerData = loadData.getBrokerData().get(broker); | ||
| double usageWithWeight = getMaxResourceUsageWithWeight(broker, brokerData, conf); | ||
| totalUsage += usageWithWeight; | ||
| } | ||
|
|
||
| final double avgUsage = totalUsage / candidates.size(); | ||
| final double diffThreshold = | ||
| conf.getLoadBalancerAverageResourceUsageDifferenceThresholdPercentage() / 100.0; | ||
| brokerAvgResourceUsageWithWeight.forEach((broker, avgResUsage) -> { | ||
| if (avgResUsage + diffThreshold <= avgUsage) { | ||
| bestBrokers.add(broker); | ||
| } | ||
| }); | ||
|
|
||
| if (bestBrokers.isEmpty()) { | ||
| // Assign randomly as all brokers are overloaded. | ||
| log.warn("Assign randomly as all {} brokers are overloaded.", candidates.size()); | ||
| bestBrokers.addAll(candidates); | ||
| } | ||
|
|
||
| if (bestBrokers.isEmpty()) { | ||
| // If still, it means there are no available brokers at this point. | ||
| log.error("There are no available brokers as candidates at this point for bundle: {}", bundleToAssign); | ||
| return Optional.empty(); | ||
| } | ||
|
|
||
| if (log.isDebugEnabled()) { | ||
| log.debug("Selected {} best brokers: {} from candidate brokers: {}", bestBrokers.size(), bestBrokers, | ||
| candidates); | ||
| } | ||
| return Optional.of(bestBrokers.get(ThreadLocalRandom.current().nextInt(bestBrokers.size()))); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters