Enable load balancing policy extensions

api/envoy/config/cluster/v3/cluster.proto

@@ -110,7 +110,7 @@ message Cluster {
     // this option or not.
     CLUSTER_PROVIDED = 6;
 
-    // [#not-implemented-hide:] Use the new :ref:`load_balancing_policy
+    // Use the new :ref:`load_balancing_policy
     // <envoy_v3_api_field_config.cluster.v3.Cluster.load_balancing_policy>` field to determine the LB policy.
     // [#next-major-version: In the v3 API, we should consider deprecating the lb_policy field
     // and instead using the new load_balancing_policy field as the one and only mechanism for
@@ -720,8 +720,7 @@ message Cluster {
 
   // The :ref:`load balancer type <arch_overview_load_balancing_types>` to use
   // when picking a host in the cluster.
-  // [#comment:TODO: Remove enum constraint :ref:`LOAD_BALANCING_POLICY_CONFIG<envoy_v3_api_enum_value_config.cluster.v3.Cluster.LbPolicy.LOAD_BALANCING_POLICY_CONFIG>` when implemented.]
-  LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true not_in: 7}];
+  LbPolicy lb_policy = 6 [(validate.rules).enum = {defined_only: true}];
 
   // Setting this is required for specifying members of
   // :ref:`STATIC<envoy_v3_api_enum_value_config.cluster.v3.Cluster.DiscoveryType.STATIC>`,
@@ -1008,7 +1007,7 @@ message Cluster {
   // servers of this cluster.
   repeated Filter filters = 40;
 
-  // [#not-implemented-hide:] New mechanism for LB policy configuration. Used only if the
+  // New mechanism for LB policy configuration. Used only if the
   // :ref:`lb_policy<envoy_v3_api_field_config.cluster.v3.Cluster.lb_policy>` field has the value
   // :ref:`LOAD_BALANCING_POLICY_CONFIG<envoy_v3_api_enum_value_config.cluster.v3.Cluster.LbPolicy.LOAD_BALANCING_POLICY_CONFIG>`.
   LoadBalancingPolicy load_balancing_policy = 41;
@@ -1073,7 +1072,7 @@ message Cluster {
   bool connection_pool_per_downstream_connection = 51;
 }
 
-// [#not-implemented-hide:] Extensible load balancing policy configuration.
+// Extensible load balancing policy configuration.
 //
 // Every LB policy defined via this mechanism will be identified via a unique name using reverse
 // DNS notation. If the policy needs configuration parameters, it must define a message for its
@@ -1104,9 +1103,12 @@ message LoadBalancingPolicy {
     reserved "config";
 
     // Required. The name of the LB policy.
-    string name = 1;
+    string name = 1 [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
 
-    google.protobuf.Any typed_config = 3;
+    google.protobuf.Any typed_config = 3
+        [deprecated = true, (envoy.annotations.deprecated_at_minor_version) = "3.0"];
+
+    core.v3.TypedExtensionConfig typed_extension_config = 4;
   }
 
   // Each client will iterate over the list in order and stop at the first policy that it

envoy/upstream/load_balancer.h

@@ -173,5 +173,26 @@ class ThreadAwareLoadBalancer {
 
 using ThreadAwareLoadBalancerPtr = std::unique_ptr<ThreadAwareLoadBalancer>;
 
+/**
+ * Factory for (thread-aware) load balancers. To support a load balancing policy of
+ * LOAD_BALANCING_POLICY_CONFIG, at least one load balancer factory corresponding to a policy in
+ * load_balancing_policy must be registered with Envoy. Envoy will use the first policy for which
+ * it has a registered factory.
+ */
+class TypedLoadBalancerFactory : public Config::UntypedFactory {
+public:
+  ~TypedLoadBalancerFactory() override = default;
+
+  /**
+   * @return ThreadAwareLoadBalancerPtr a new thread-aware load balancer.
+   */
+  virtual ThreadAwareLoadBalancerPtr
+  create(const PrioritySet& priority_set, ClusterStats& stats, Stats::Scope& stats_scope,
+         Runtime::Loader& runtime, Random::RandomGenerator& random,
+         const ::envoy::config::cluster::v3::LoadBalancingPolicy_Policy& lb_policy) PURE;
+
+  std::string category() const override { return "envoy.load_balancers"; }
+};
+
 } // namespace Upstream
 } // namespace Envoy

envoy/upstream/load_balancer_type.h

@@ -22,7 +22,8 @@ enum class LoadBalancerType {
   RingHash,
   OriginalDst,
   Maglev,
-  ClusterProvided
+  ClusterProvided,
+  LoadBalancingPolicyConfig
 };
 
 /**

envoy/upstream/upstream.h

@@ -692,6 +692,8 @@ using ProtocolOptionsConfigConstSharedPtr = std::shared_ptr<const ProtocolOption
  */
 class ClusterTypedMetadataFactory : public Envoy::Config::TypedMetadataFactory {};
 
+class TypedLoadBalancerFactory;
+
 /**
  * Information about a given upstream cluster.
  */
@@ -786,6 +788,19 @@ class ClusterInfo {
     return std::dynamic_pointer_cast<const Derived>(extensionProtocolOptions(name));
   }
 
+  /**
+   * @return const envoy::config::cluster::v3::LoadBalancingPolicy_Policy& the load balancing policy
+   * to use for this cluster.
+   */
+  virtual const envoy::config::cluster::v3::LoadBalancingPolicy_Policy&
+  loadBalancingPolicy() const PURE;
+
+  /**
+   * @return the load balancer factory for this cluster if the load balancing type is
+   * LOAD_BALANCING_POLICY_CONFIG.
+   */
+  virtual TypedLoadBalancerFactory* loadBalancerFactory() const PURE;
+
   /**
    * @return const envoy::config::cluster::v3::Cluster::CommonLbConfig& the common configuration for
    * all load balancers for this cluster.

source/common/upstream/BUILD

@@ -210,6 +210,14 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "load_balancer_factory_base_lib",
+    hdrs = ["load_balancer_factory_base.h"],
+    deps = [
+        ":load_balancer_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "load_stats_reporter_lib",
     srcs = ["load_stats_reporter.cc"],

source/common/upstream/cluster_manager_impl.cc

@@ -831,6 +831,13 @@ ClusterManagerImpl::loadCluster(const envoy::config::cluster::v3::Cluster& clust
     }
   } else if (cluster_reference.info()->lbType() == LoadBalancerType::ClusterProvided) {
     cluster_entry_it->second->thread_aware_lb_ = std::move(new_cluster_pair.second);
+  } else if (cluster_reference.info()->lbType() == LoadBalancerType::LoadBalancingPolicyConfig) {
+    const auto& policy = cluster_reference.info()->loadBalancingPolicy();
+    TypedLoadBalancerFactory* typed_lb_factory = cluster_reference.info()->loadBalancerFactory();
+    RELEASE_ASSERT(typed_lb_factory != nullptr, "ClusterInfo should contain a valid factory");
+    cluster_entry_it->second->thread_aware_lb_ =
+        typed_lb_factory->create(cluster_reference.prioritySet(), cluster_reference.info()->stats(),
+                                 cluster_reference.info()->statsScope(), runtime_, random_, policy);
   }
 
   updateClusterCounts();
@@ -1377,6 +1384,7 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::ClusterEntry(
       break;
     }
     case LoadBalancerType::ClusterProvided:
+    case LoadBalancerType::LoadBalancingPolicyConfig:
     case LoadBalancerType::RingHash:
     case LoadBalancerType::Maglev:
     case LoadBalancerType::OriginalDst: {

source/common/upstream/load_balancer_factory_base.h

@@ -0,0 +1,29 @@
+#pragma once
+
+#include "envoy/upstream/load_balancer.h"
+
+namespace Envoy {
+namespace Upstream {
+
+/**
+ * Base class for cluster provided load balancers and load balancers specified by load balancing
+ * policy config. This class should be extended directly if the load balancing policy specifies a
+ * thread-aware load balancer.
+ *
+ * TODO: provide a ThreadLocalLoadBalancer construct to abstract away thread-awareness from load
+ * balancing extensions that don't require it.
+ */
+class TypedLoadBalancerFactoryBase : public TypedLoadBalancerFactory {
+public:
+  // Upstream::TypedLoadBalancerFactory
+  std::string name() const override { return name_; }
+
+protected:
+  TypedLoadBalancerFactoryBase(const std::string& name) : name_(name) {}
+
+private:
+  const std::string name_;
+};
+
+} // namespace Upstream
+} // namespace Envoy

source/common/upstream/subset_lb.cc

@@ -781,8 +781,8 @@ SubsetLoadBalancer::PrioritySubsetImpl::PrioritySubsetImpl(const SubsetLoadBalan
 
   case LoadBalancerType::OriginalDst:
   case LoadBalancerType::ClusterProvided:
-    // LoadBalancerType::OriginalDst is blocked in the factory. LoadBalancerType::ClusterProvided
-    // is impossible because the subset LB returns a null load balancer from its factory.
+  case LoadBalancerType::LoadBalancingPolicyConfig:
+    // These load balancer types can only be created when there is no subset configuration.
     NOT_REACHED_GCOVR_EXCL_LINE;
   }