From a96b7ad5ee4c2593c7b214433449c4e696148edf Mon Sep 17 00:00:00 2001 From: code Date: Tue, 7 May 2024 21:09:54 +0800 Subject: [PATCH] upstream lb: move all subset related code to extensions/ (#33964) * upstream lb: move all subset related code to extensions/ Signed-off-by: wbpcode * address comments Signed-off-by: wbpcode --------- Signed-off-by: wbpcode Co-authored-by: wbpcode --- envoy/upstream/BUILD | 11 - envoy/upstream/load_balancer_type.h | 126 ------ envoy/upstream/upstream.h | 1 - mobile/test/performance/files_em_does_not_use | 1 + source/common/upstream/BUILD | 15 - source/common/upstream/cluster_manager_impl.h | 5 +- source/common/upstream/load_balancer_impl.h | 1 - source/common/upstream/subset_lb_config.cc | 53 --- .../load_balancing_policies/subset/BUILD | 20 +- .../load_balancing_policies/subset/config.cc | 43 +- .../load_balancing_policies/subset/config.h | 2 +- .../subset/subset_lb.cc | 160 +------ .../subset/subset_lb.h | 126 +----- .../subset/subset_lb_config.cc | 105 +++++ .../subset}/subset_lb_config.h | 147 +++++-- .../upstream/load_balancer_impl_test.cc | 137 ------ .../load_balancing_policies/subset/BUILD | 9 + .../subset/subset_benchmark.cc | 39 +- .../subset/subset_test.cc | 402 +++++++++++------- test/mocks/upstream/cluster_info.cc | 10 - test/mocks/upstream/cluster_info.h | 22 - 21 files changed, 560 insertions(+), 875 deletions(-) delete mode 100644 envoy/upstream/load_balancer_type.h delete mode 100644 source/common/upstream/subset_lb_config.cc create mode 100644 source/extensions/load_balancing_policies/subset/subset_lb_config.cc rename source/{common/upstream => extensions/load_balancing_policies/subset}/subset_lb_config.h (50%) diff --git a/envoy/upstream/BUILD b/envoy/upstream/BUILD index 8178b9d494e8..b2253b11bbfe 100644 --- a/envoy/upstream/BUILD +++ b/envoy/upstream/BUILD @@ -89,15 +89,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "load_balancer_type_interface", - hdrs = ["load_balancer_type.h"], - deps = [ - "//source/common/protobuf", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "locality_lib", hdrs = ["locality.h"], @@ -158,7 +149,6 @@ envoy_cc_library( external_deps = ["abseil_optional"], deps = [ ":health_check_host_monitor_interface", - ":load_balancer_type_interface", ":locality_lib", ":resource_manager_interface", "//envoy/common:callback", @@ -184,7 +174,6 @@ envoy_cc_library( deps = [ ":cluster_manager_interface", ":health_check_host_monitor_interface", - ":load_balancer_type_interface", ":locality_lib", ":resource_manager_interface", ":upstream_interface", diff --git a/envoy/upstream/load_balancer_type.h b/envoy/upstream/load_balancer_type.h deleted file mode 100644 index 167edd496a31..000000000000 --- a/envoy/upstream/load_balancer_type.h +++ /dev/null @@ -1,126 +0,0 @@ -#pragma once - -#include -#include -#include - -#include "envoy/common/pure.h" -#include "envoy/config/cluster/v3/cluster.pb.h" - -#include "source/common/protobuf/protobuf.h" - -namespace Envoy { -namespace Upstream { - -/** - * Type of load balancing to perform. - */ -enum class LoadBalancerType : uint8_t { - RoundRobin, - LeastRequest, - Random, - RingHash, - OriginalDst, - Maglev, - ClusterProvided, - LoadBalancingPolicyConfig -}; - -/** - * Subset selector configuration - */ -class SubsetSelector { -public: - virtual ~SubsetSelector() = default; - - /** - * @return keys defined for this selector - */ - virtual const std::set& selectorKeys() const PURE; - - /** - * @return fallback policy defined for this selector, or NOT_DEFINED - */ - virtual envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy - fallbackPolicy() const PURE; - - /** - * @return fallback keys subset defined for this selector, or empty set - */ - virtual const std::set& fallbackKeysSubset() const PURE; - - virtual bool singleHostPerSubset() const PURE; -}; - -using SubsetSelectorPtr = std::shared_ptr; - -/** - * Load Balancer subset configuration. - */ -class LoadBalancerSubsetInfo { -public: - virtual ~LoadBalancerSubsetInfo() = default; - - /** - * @return bool true if load balancer subsets are configured. - */ - virtual bool isEnabled() const PURE; - - /** - * @return LbSubsetFallbackPolicy the fallback policy used when - * route metadata does not match any subset. - */ - virtual envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy - fallbackPolicy() const PURE; - - /** - * @return LbSubsetMetadataFallbackPolicy the fallback policy used to try different route metadata - * until a host is found - */ - virtual envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy - metadataFallbackPolicy() const PURE; - - /** - * @return ProtobufWkt::Struct the struct describing the metadata for a - * host to be included in the default subset. - */ - virtual const ProtobufWkt::Struct& defaultSubset() const PURE; - - /* - * @return const std:vector>& a vector of - * sorted keys used to define load balancer subsets. - */ - virtual const std::vector& subsetSelectors() const PURE; - - /* - * @return bool whether routing to subsets should take locality weights into account. - */ - virtual bool localityWeightAware() const PURE; - - /* - * @return bool whether the locality weights should be scaled to compensate for the - * fraction of hosts removed from the original host set. - */ - virtual bool scaleLocalityWeight() const PURE; - - /* - * @return bool whether to attempt to select a host from the entire cluster if host - * selection from the fallback subset fails. - */ - virtual bool panicModeAny() const PURE; - - /* - * @return bool whether matching metadata should attempt to match against any of the - * elements in a list value defined in endpoint metadata. - */ - virtual bool listAsAny() const PURE; - - /* - * @return bool whether redundant key/value pairs is allowed in the request metadata. - */ - virtual bool allowRedundantKeys() const PURE; -}; - -} // namespace Upstream -} // namespace Envoy diff --git a/envoy/upstream/upstream.h b/envoy/upstream/upstream.h index caf170c3b6df..08d9b342ba9f 100644 --- a/envoy/upstream/upstream.h +++ b/envoy/upstream/upstream.h @@ -23,7 +23,6 @@ #include "envoy/stats/scope.h" #include "envoy/stats/stats.h" #include "envoy/upstream/health_check_host_monitor.h" -#include "envoy/upstream/load_balancer_type.h" #include "envoy/upstream/locality.h" #include "envoy/upstream/outlier_detection.h" #include "envoy/upstream/resource_manager.h" diff --git a/mobile/test/performance/files_em_does_not_use b/mobile/test/performance/files_em_does_not_use index 577362f5779c..e4c823572497 100644 --- a/mobile/test/performance/files_em_does_not_use +++ b/mobile/test/performance/files_em_does_not_use @@ -9,3 +9,4 @@ source/server/watchdog_impl.h source/server/options_impl.cc source/extensions/access_loggers/common/file_access_log_impl.h source/common/router/scoped_rds.h +source/extensions/load_balancing_policies/subset/subset_lb.h diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index efe386867b8b..28a3ef0a9e6a 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -257,27 +257,12 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "subset_lb_config_lib", - srcs = ["subset_lb_config.cc"], - hdrs = ["subset_lb_config.h"], - deps = [ - "//envoy/upstream:load_balancer_interface", - "//envoy/upstream:upstream_interface", - "//source/common/config:utility_lib", - "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto", - "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", - ], -) - envoy_cc_library( name = "load_balancer_lib", srcs = ["load_balancer_impl.cc"], hdrs = ["load_balancer_impl.h"], deps = [ ":scheduler_lib", - ":subset_lb_config_lib", "//envoy/common:random_generator_interface", "//envoy/runtime:runtime_interface", "//envoy/stats:stats_interface", diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 163ecdbee7d6..1a78fcf72fed 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -629,9 +629,8 @@ class ClusterManagerImpl : public ClusterManager, // Don't change the order of cluster_info_ and lb_factory_/lb_ as the the lb_factory_/lb_ // may keep a reference to the cluster_info_. ClusterInfoConstSharedPtr cluster_info_; - // LB factory if applicable. Not all load balancer types have a factory. LB types that have - // a factory will create a new LB on every membership update. LB types that don't have a - // factory will create an LB on construction and use it forever. + + // Factory to create active LB. LoadBalancerFactorySharedPtr lb_factory_; // Current active LB. LoadBalancerPtr lb_; diff --git a/source/common/upstream/load_balancer_impl.h b/source/common/upstream/load_balancer_impl.h index ef23f188722c..a7c31f5d12c0 100644 --- a/source/common/upstream/load_balancer_impl.h +++ b/source/common/upstream/load_balancer_impl.h @@ -22,7 +22,6 @@ #include "source/common/protobuf/utility.h" #include "source/common/runtime/runtime_protos.h" #include "source/common/upstream/edf_scheduler.h" -#include "source/common/upstream/subset_lb_config.h" namespace Envoy { namespace Upstream { diff --git a/source/common/upstream/subset_lb_config.cc b/source/common/upstream/subset_lb_config.cc deleted file mode 100644 index ef3cf228ebd7..000000000000 --- a/source/common/upstream/subset_lb_config.cc +++ /dev/null @@ -1,53 +0,0 @@ -#include "source/common/upstream/subset_lb_config.h" - -#include "source/common/config/utility.h" - -namespace Envoy { -namespace Upstream { - -SubsetSelectorImpl::SubsetSelectorImpl( - const Protobuf::RepeatedPtrField& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy, - const Protobuf::RepeatedPtrField& fallback_keys_subset, - bool single_host_per_subset) - : selector_keys_(selector_keys.begin(), selector_keys.end()), - fallback_keys_subset_(fallback_keys_subset.begin(), fallback_keys_subset.end()), - fallback_policy_(fallback_policy), single_host_per_subset_(single_host_per_subset) { - - if (fallback_policy_ != - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET) { - // defining fallback_keys_subset_ for a fallback policy other than KEYS_SUBSET doesn't have - // any effect and it is probably a user mistake. We should let the user know about it. - if (!fallback_keys_subset_.empty()) { - throwEnvoyExceptionOrPanic( - "fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); - } - return; - } - - // if KEYS_SUBSET fallback policy is selected, fallback_keys_subset must not be empty, because - // it would be the same as not defining fallback policy at all (global fallback policy would be - // used) - if (fallback_keys_subset_.empty()) { - throwEnvoyExceptionOrPanic("fallback_keys_subset cannot be empty"); - } - - // We allow only for a fallback to a subset of the selector keys because this is probably the - // only use case that makes sense (fallback from more specific selector to less specific - // selector). Potentially we can relax this constraint in the future if there will be a use case - // for this. - if (!std::includes(selector_keys_.begin(), selector_keys_.end(), fallback_keys_subset_.begin(), - fallback_keys_subset_.end())) { - throwEnvoyExceptionOrPanic("fallback_keys_subset must be a subset of selector keys"); - } - - // Enforce that the fallback_keys_subset_ set is smaller than the selector_keys_ set. Otherwise - // we could end up with a infinite recursion of SubsetLoadBalancer::chooseHost(). - if (selector_keys_.size() == fallback_keys_subset_.size()) { - throwEnvoyExceptionOrPanic("fallback_keys_subset cannot be equal to keys"); - } -} - -} // namespace Upstream -} // namespace Envoy diff --git a/source/extensions/load_balancing_policies/subset/BUILD b/source/extensions/load_balancing_policies/subset/BUILD index 7e001486b3a8..f119886fc3a6 100644 --- a/source/extensions/load_balancing_policies/subset/BUILD +++ b/source/extensions/load_balancing_policies/subset/BUILD @@ -14,6 +14,7 @@ envoy_cc_library( srcs = ["subset_lb.cc"], hdrs = ["subset_lb.h"], deps = [ + ":subset_lb_config_lib", "//envoy/runtime:runtime_interface", "//envoy/upstream:load_balancer_interface", "//source/common/common:assert_lib", @@ -21,11 +22,8 @@ envoy_cc_library( "//source/common/config:metadata_lib", "//source/common/protobuf", "//source/common/protobuf:utility_lib", - "//source/common/upstream:load_balancer_lib", "//source/common/upstream:upstream_lib", "//source/extensions/load_balancing_policies/common:factory_base", - "//source/extensions/load_balancing_policies/maglev:maglev_lb_lib", - "//source/extensions/load_balancing_policies/ring_hash:ring_hash_lb_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", @@ -47,3 +45,19 @@ envoy_cc_extension( "//source/common/upstream:load_balancer_lib", ], ) + +envoy_cc_extension( + name = "subset_lb_config_lib", + srcs = ["subset_lb_config.cc"], + hdrs = ["subset_lb_config.h"], + deps = [ + "//envoy/upstream:load_balancer_interface", + "//envoy/upstream:upstream_interface", + "//source/common/config:utility_lib", + "//source/common/upstream:load_balancer_lib", + "//source/common/upstream:upstream_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/common/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", + ], +) diff --git a/source/extensions/load_balancing_policies/subset/config.cc b/source/extensions/load_balancing_policies/subset/config.cc index 5d3d8c4542e1..cc6873c1c1a2 100644 --- a/source/extensions/load_balancing_policies/subset/config.cc +++ b/source/extensions/load_balancing_policies/subset/config.cc @@ -12,26 +12,6 @@ namespace Subset { using SubsetLbProto = envoy::extensions::load_balancing_policies::subset::v3::Subset; using ClusterProto = envoy::config::cluster::v3::Cluster; -class ChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator { -public: - ChildLoadBalancerCreatorImpl(const Upstream::SubsetLoadBalancerConfig& subset_config, - const Upstream::ClusterInfo& cluster_info) - : subset_config_(subset_config), cluster_info_(cluster_info) {} - - std::pair - createLoadBalancer(const Upstream::PrioritySet& child_priority_set, const Upstream::PrioritySet*, - Upstream::ClusterLbStats&, Stats::Scope&, Runtime::Loader& runtime, - Random::RandomGenerator& random, TimeSource& time_source) override { - return {subset_config_.createLoadBalancer(cluster_info_, child_priority_set, runtime, random, - time_source), - nullptr}; - } - -private: - const Upstream::SubsetLoadBalancerConfig& subset_config_; - const Upstream::ClusterInfo& cluster_info_; -}; - class LbFactory : public Upstream::LoadBalancerFactory { public: LbFactory(const Upstream::SubsetLoadBalancerConfig& subset_config, @@ -41,13 +21,9 @@ class LbFactory : public Upstream::LoadBalancerFactory { random_(random), time_source_(time_source) {} Upstream::LoadBalancerPtr create(Upstream::LoadBalancerParams params) override { - auto child_lb_creator = - std::make_unique(subset_config_, cluster_info_); - return std::make_unique( - subset_config_.subsetInfo(), std::move(child_lb_creator), params.priority_set, - params.local_priority_set, cluster_info_.lbStats(), cluster_info_.statsScope(), runtime_, - random_, time_source_); + subset_config_, cluster_info_, params.priority_set, params.local_priority_set, + cluster_info_.lbStats(), cluster_info_.statsScope(), runtime_, random_, time_source_); } bool recreateOnHostChange() const override { return false; } @@ -97,7 +73,6 @@ SubsetLbFactory::create(OptRef lb_config, Upstream::LoadBalancerConfigPtr SubsetLbFactory::loadConfig(const Protobuf::Message& config, ProtobufMessage::ValidationVisitor& visitor) { - auto active_or_legacy = Common::ActiveOrLegacy::get(&config); ASSERT(active_or_legacy.hasLegacy() || active_or_legacy.hasActive()); @@ -109,18 +84,8 @@ SubsetLbFactory::loadConfig(const Protobuf::Message& config, envoy::config::cluster::v3::Cluster::LbPolicy_Name( active_or_legacy.legacy()->lb_policy()))); } - - auto sub_lb_pair = - Upstream::LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProtoWithoutSubset( - *active_or_legacy.legacy(), visitor); - - if (!sub_lb_pair.ok()) { - throw EnvoyException(std::string(sub_lb_pair.status().message())); - } - - return std::make_unique( - active_or_legacy.legacy()->lb_subset_config(), std::move(sub_lb_pair->config), - sub_lb_pair->factory); + return std::make_unique(*active_or_legacy.legacy(), + visitor); } // Load the subset load balancer configuration. This will contains child load balancer diff --git a/source/extensions/load_balancing_policies/subset/config.h b/source/extensions/load_balancing_policies/subset/config.h index 76383690da2b..0c49d8862bfb 100644 --- a/source/extensions/load_balancing_policies/subset/config.h +++ b/source/extensions/load_balancing_policies/subset/config.h @@ -13,7 +13,7 @@ namespace LoadBalancingPolices { namespace Subset { class SubsetLbFactory - : public Upstream::TypedLoadBalancerFactoryBase { + : public Upstream::TypedLoadBalancerFactoryBase { public: SubsetLbFactory() : TypedLoadBalancerFactoryBase("envoy.load_balancing_policies.subset") {} diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.cc b/source/extensions/load_balancing_policies/subset/subset_lb.cc index 9ba717d83e96..2404f5c371cf 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.cc +++ b/source/extensions/load_balancing_policies/subset/subset_lb.cc @@ -12,8 +12,6 @@ #include "source/common/config/well_known_names.h" #include "source/common/protobuf/utility.h" #include "source/common/upstream/load_balancer_impl.h" -#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" -#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include "absl/container/node_hash_set.h" @@ -47,134 +45,25 @@ filterCriteriaBySelectors(const std::vector& subset_selectors using HostPredicate = std::function; -LegacyChildLoadBalancerCreatorImpl::LegacyChildLoadBalancerCreatorImpl( - LoadBalancerType lb_type, - OptRef lb_ring_hash_config, - OptRef lb_maglev_config, - OptRef round_robin_config, - OptRef least_request_config, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config) - : lb_type_(lb_type), - lb_ring_hash_config_( - lb_ring_hash_config.has_value() - ? std::make_unique( - lb_ring_hash_config.ref()) - : nullptr), - lb_maglev_config_( - lb_maglev_config.has_value() - ? std::make_unique( - lb_maglev_config.ref()) - : nullptr), - round_robin_config_( - round_robin_config.has_value() - ? std::make_unique( - round_robin_config.ref()) - : nullptr), - least_request_config_( - least_request_config.has_value() - ? std::make_unique( - least_request_config.ref()) - : nullptr), - common_config_(common_config) {} - -std::pair -LegacyChildLoadBalancerCreatorImpl::createLoadBalancer( - const Upstream::PrioritySet& child_priority_set, - const Upstream::PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope, - Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) { - switch (lb_type_) { - case Upstream::LoadBalancerType::LeastRequest: { - Upstream::LoadBalancerPtr lb = std::make_unique( - child_priority_set, local_priority_set, stats, runtime, random, common_config_, - lbLeastRequestConfig(), time_source); - return {nullptr, std::move(lb)}; - } - case Upstream::LoadBalancerType::Random: { - Upstream::LoadBalancerPtr lb = std::make_unique( - child_priority_set, local_priority_set, stats, runtime, random, common_config_); - return {nullptr, std::move(lb)}; - } - case Upstream::LoadBalancerType::RoundRobin: { - Upstream::LoadBalancerPtr lb = std::make_unique( - child_priority_set, local_priority_set, stats, runtime, random, common_config_, - lbRoundRobinConfig(), time_source); - return {nullptr, std::move(lb)}; - } - case Upstream::LoadBalancerType::RingHash: { - // TODO(mattklein123): The ring hash LB is thread aware, but currently the subset LB is not. - // We should make the subset LB thread aware since the calculations are costly, and then we - // can also use a thread aware sub-LB properly. The following works fine but is not optimal. - Upstream::ThreadAwareLoadBalancerPtr lb = std::make_unique( - child_priority_set, stats, scope, runtime, random, lbRingHashConfig(), common_config_); - return {std::move(lb), nullptr}; - } - case Upstream::LoadBalancerType::Maglev: { - // TODO(mattklein123): The Maglev LB is thread aware, but currently the subset LB is not. - // We should make the subset LB thread aware since the calculations are costly, and then we - // can also use a thread aware sub-LB properly. The following works fine but is not optimal. - - Upstream::ThreadAwareLoadBalancerPtr lb = std::make_unique( - child_priority_set, stats, scope, runtime, random, lbMaglevConfig(), common_config_); - return {std::move(lb), nullptr}; - } - case Upstream::LoadBalancerType::OriginalDst: - case Upstream::LoadBalancerType::ClusterProvided: - case Upstream::LoadBalancerType::LoadBalancingPolicyConfig: - // These load balancer types can only be created when there is no subset configuration. - PANIC("not implemented"); - } - return {nullptr, nullptr}; -} - -SubsetLoadBalancerConfig::SubsetLoadBalancerConfig( - const SubsetLoadbalancingPolicyProto& subset_config, - ProtobufMessage::ValidationVisitor& visitor) - : subset_info_(subset_config) { - - absl::InlinedVector missing_policies; - - for (const auto& policy : subset_config.subset_lb_policy().policies()) { - auto* factory = Config::Utility::getAndCheckFactory( - policy.typed_extension_config(), /*is_optional=*/true); - - if (factory != nullptr) { - // Load and validate the configuration. - auto sub_lb_proto_message = factory->createEmptyConfigProto(); - Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), - visitor, *sub_lb_proto_message); - - sub_load_balancer_config_ = factory->loadConfig(*sub_lb_proto_message, visitor); - sub_load_balancer_factory_ = factory; - break; - } - - missing_policies.push_back(policy.typed_extension_config().name()); - } - - if (sub_load_balancer_factory_ == nullptr) { - throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory " - "implementation for subset lb with names from [{}]", - absl::StrJoin(missing_policies, ", "))); - } -} - -SubsetLoadBalancer::SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets, - ChildLoadBalancerCreatorPtr child_lb, +SubsetLoadBalancer::SubsetLoadBalancer(const SubsetLoadBalancerConfig& lb_config, + const Upstream::ClusterInfo& cluster_info, const PrioritySet& priority_set, const PrioritySet* local_priority_set, ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, Random::RandomGenerator& random, TimeSource& time_source) - : stats_(stats), scope_(scope), runtime_(runtime), random_(random), time_source_(time_source), - fallback_policy_(subsets.fallbackPolicy()), - metadata_fallback_policy_(subsets.metadataFallbackPolicy()), - default_subset_metadata_(subsets.defaultSubset().fields().begin(), - subsets.defaultSubset().fields().end()), - subset_selectors_(subsets.subsetSelectors()), original_priority_set_(priority_set), - original_local_priority_set_(local_priority_set), child_lb_creator_(std::move(child_lb)), - locality_weight_aware_(subsets.localityWeightAware()), - scale_locality_weight_(subsets.scaleLocalityWeight()), list_as_any_(subsets.listAsAny()), - allow_redundant_keys_(subsets.allowRedundantKeys()) { - ASSERT(subsets.isEnabled()); + : lb_config_(lb_config), cluster_info_(cluster_info), stats_(stats), scope_(scope), + runtime_(runtime), random_(random), time_source_(time_source), + fallback_policy_(lb_config_.subsetInfo().fallbackPolicy()), + metadata_fallback_policy_(lb_config_.subsetInfo().metadataFallbackPolicy()), + default_subset_metadata_(lb_config_.subsetInfo().defaultSubset().fields().begin(), + lb_config_.subsetInfo().defaultSubset().fields().end()), + subset_selectors_(lb_config_.subsetInfo().subsetSelectors()), + original_priority_set_(priority_set), original_local_priority_set_(local_priority_set), + locality_weight_aware_(lb_config_.subsetInfo().localityWeightAware()), + scale_locality_weight_(lb_config_.subsetInfo().scaleLocalityWeight()), + list_as_any_(lb_config_.subsetInfo().listAsAny()), + allow_redundant_keys_(lb_config_.subsetInfo().allowRedundantKeys()) { + ASSERT(lb_config_.subsetInfo().isEnabled()); if (fallback_policy_ != envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK) { if (fallback_policy_ == envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT) { @@ -189,7 +78,7 @@ SubsetLoadBalancer::SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets, } } - if (subsets.panicModeAny()) { + if (lb_config_.subsetInfo().panicModeAny()) { initSubsetAnyOnce(); panic_mode_subset_ = subset_any_; } @@ -826,17 +715,12 @@ SubsetLoadBalancer::PrioritySubsetImpl::PrioritySubsetImpl(const SubsetLoadBalan // Create at least one host set. getOrCreateHostSet(0); - auto lb_pair = subset_lb.child_lb_creator_->createLoadBalancer( - *this, original_local_priority_set_, subset_lb.stats_, subset_lb.scope_, subset_lb.runtime_, - subset_lb.random_, subset_lb.time_source_); - - if (lb_pair.first != nullptr) { - thread_aware_lb_ = std::move(lb_pair.first); - thread_aware_lb_->initialize(); - lb_ = thread_aware_lb_->factory()->create({*this, original_local_priority_set_}); - } else { - lb_ = std::move(lb_pair.second); - } + thread_aware_lb_ = + subset_lb.lb_config_.createLoadBalancer(subset_lb.cluster_info_, *this, subset_lb.runtime_, + subset_lb.random_, subset_lb.time_source_); + ASSERT(thread_aware_lb_ != nullptr); + thread_aware_lb_->initialize(); + lb_ = thread_aware_lb_->factory()->create({*this, original_local_priority_set_}); triggerCallbacks(); } diff --git a/source/extensions/load_balancing_policies/subset/subset_lb.h b/source/extensions/load_balancing_policies/subset/subset_lb.h index 7b38896a8dde..e2b432d46795 100644 --- a/source/extensions/load_balancing_policies/subset/subset_lb.h +++ b/source/extensions/load_balancing_policies/subset/subset_lb.h @@ -22,6 +22,7 @@ #include "source/common/protobuf/utility.h" #include "source/common/upstream/load_balancer_impl.h" #include "source/common/upstream/upstream_impl.h" +#include "source/extensions/load_balancing_policies/subset/subset_lb_config.h" #include "absl/container/node_hash_map.h" #include "absl/types/optional.h" @@ -29,117 +30,15 @@ namespace Envoy { namespace Upstream { -class ChildLoadBalancerCreator { -public: - virtual ~ChildLoadBalancerCreator() = default; - - virtual std::pair - createLoadBalancer(const PrioritySet& child_priority_set, const PrioritySet* local_priority_set, - ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, - Random::RandomGenerator& random, TimeSource& time_source) PURE; -}; -using ChildLoadBalancerCreatorPtr = std::unique_ptr; - -// TODO(wbpcode): to remove the LegacyChildLoadBalancerCreatorImpl because it is not used anymore -// except for tests. -class LegacyChildLoadBalancerCreatorImpl : public Upstream::ChildLoadBalancerCreator { -public: - LegacyChildLoadBalancerCreatorImpl( - LoadBalancerType lb_type, - OptRef lb_ring_hash_config, - OptRef lb_maglev_config, - OptRef round_robin_config, - OptRef least_request_config, - const envoy::config::cluster::v3::Cluster::CommonLbConfig& common_config); - - std::pair - createLoadBalancer(const Upstream::PrioritySet& child_priority_set, - const Upstream::PrioritySet* local_priority_set, ClusterLbStats& stats, - Stats::Scope& scope, Runtime::Loader& runtime, Random::RandomGenerator& random, - TimeSource& time_source) override; - - OptRef lbRoundRobinConfig() const { - if (round_robin_config_ != nullptr) { - return *round_robin_config_; - } - return absl::nullopt; - } - - LoadBalancerType lbType() const { return lb_type_; } - - OptRef - lbLeastRequestConfig() const { - if (least_request_config_ != nullptr) { - return *least_request_config_; - } - return absl::nullopt; - } - - OptRef lbMaglevConfig() const { - if (lb_maglev_config_ != nullptr) { - return *lb_maglev_config_; - } - return absl::nullopt; - } - - OptRef lbRingHashConfig() const { - if (lb_ring_hash_config_ != nullptr) { - return *lb_ring_hash_config_; - } - return absl::nullopt; - } - -private: - const LoadBalancerType lb_type_; - const std::unique_ptr - lb_ring_hash_config_; - const std::unique_ptr - lb_maglev_config_; - const std::unique_ptr - round_robin_config_; - const std::unique_ptr - least_request_config_; - const envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; -}; - using HostHashSet = absl::flat_hash_set; -class SubsetLoadBalancerConfig : public Upstream::LoadBalancerConfig { -public: - SubsetLoadBalancerConfig(const SubsetLoadbalancingPolicyProto& subset_config, - ProtobufMessage::ValidationVisitor& visitor); - - SubsetLoadBalancerConfig(const LegacySubsetLoadbalancingPolicyProto& subset_config, - Upstream::LoadBalancerConfigPtr sub_load_balancer_config, - Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory) - : subset_info_(subset_config), sub_load_balancer_config_(std::move(sub_load_balancer_config)), - sub_load_balancer_factory_(sub_load_balancer_factory) { - ASSERT(sub_load_balancer_factory_ != nullptr, "sub_load_balancer_factory_ must not be nullptr"); - } - - Upstream::ThreadAwareLoadBalancerPtr - createLoadBalancer(const Upstream::ClusterInfo& cluster_info, - const Upstream::PrioritySet& child_priority_set, Runtime::Loader& runtime, - Random::RandomGenerator& random, TimeSource& time_source) const { - return sub_load_balancer_factory_->create(*sub_load_balancer_config_, cluster_info, - child_priority_set, runtime, random, time_source); - } - - const Upstream::LoadBalancerSubsetInfo& subsetInfo() const { return subset_info_; } - -private: - LoadBalancerSubsetInfoImpl subset_info_; - - Upstream::LoadBalancerConfigPtr sub_load_balancer_config_; - Upstream::TypedLoadBalancerFactory* sub_load_balancer_factory_{}; -}; - class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable { public: - SubsetLoadBalancer(const LoadBalancerSubsetInfo& subsets, ChildLoadBalancerCreatorPtr child_lb, - const PrioritySet& priority_set, const PrioritySet* local_priority_set, - ClusterLbStats& stats, Stats::Scope& scope, Runtime::Loader& runtime, - Random::RandomGenerator& random, TimeSource& time_source); + SubsetLoadBalancer(const SubsetLoadBalancerConfig& lb_config, + const Upstream::ClusterInfo& cluster_info, const PrioritySet& priority_set, + const PrioritySet* local_priority_set, ClusterLbStats& stats, + Stats::Scope& scope, Runtime::Loader& runtime, Random::RandomGenerator& random, + TimeSource& time_source); ~SubsetLoadBalancer() override; // Upstream::LoadBalancer @@ -158,6 +57,10 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable>; + static std::string describeMetadata(const SubsetMetadata& kvs); + private: struct SubsetSelectorFallbackParams; @@ -238,8 +141,6 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable; using PrioritySubsetImplPtr = std::unique_ptr; - using SubsetMetadata = std::vector>; - class LbSubsetEntry; struct SubsetSelectorMap; @@ -459,12 +360,13 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable extractSubsetMetadata(const std::set& subset_keys, const Host& host); - std::string describeMetadata(const SubsetMetadata& kvs); HostConstSharedPtr chooseHostWithMetadataFallbacks(LoadBalancerContext* context, const MetadataFallbacks& metadata_fallbacks); const ProtobufWkt::Value* getMetadataFallbackList(LoadBalancerContext* context) const; LoadBalancerContextWrapper removeMetadataFallbackList(LoadBalancerContext* context); + const SubsetLoadBalancerConfig& lb_config_; + const Upstream::ClusterInfo& cluster_info_; ClusterLbStats& stats_; Stats::Scope& scope_; Runtime::Loader& runtime_; @@ -482,8 +384,6 @@ class SubsetLoadBalancer : public LoadBalancer, Logger::Loggable& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig:: + LbSubsetSelector::LbSubsetSelectorFallbackPolicy fallback_policy, + const Protobuf::RepeatedPtrField& fallback_keys_subset, + bool single_host_per_subset) + : selector_keys_(selector_keys.begin(), selector_keys.end()), + fallback_keys_subset_(fallback_keys_subset.begin(), fallback_keys_subset.end()), + fallback_policy_(fallback_policy), single_host_per_subset_(single_host_per_subset) { + + if (fallback_policy_ != + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET) { + // defining fallback_keys_subset_ for a fallback policy other than KEYS_SUBSET doesn't have + // any effect and it is probably a user mistake. We should let the user know about it. + if (!fallback_keys_subset_.empty()) { + throw EnvoyException("fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); + } + return; + } + + // if KEYS_SUBSET fallback policy is selected, fallback_keys_subset must not be empty, because + // it would be the same as not defining fallback policy at all (global fallback policy would be + // used) + if (fallback_keys_subset_.empty()) { + throw EnvoyException("fallback_keys_subset cannot be empty"); + } + + // We allow only for a fallback to a subset of the selector keys because this is probably the + // only use case that makes sense (fallback from more specific selector to less specific + // selector). Potentially we can relax this constraint in the future if there will be a use case + // for this. + if (!std::includes(selector_keys_.begin(), selector_keys_.end(), fallback_keys_subset_.begin(), + fallback_keys_subset_.end())) { + throw EnvoyException("fallback_keys_subset must be a subset of selector keys"); + } + + // Enforce that the fallback_keys_subset_ set is smaller than the selector_keys_ set. Otherwise + // we could end up with a infinite recursion of SubsetLoadBalancer::chooseHost(). + if (selector_keys_.size() == fallback_keys_subset_.size()) { + throw EnvoyException("fallback_keys_subset cannot be equal to keys"); + } +} + +SubsetLoadBalancerConfig::SubsetLoadBalancerConfig(const SubsetLbConfigProto& subset_config, + ProtobufMessage::ValidationVisitor& visitor) + : subset_info_(std::make_unique(subset_config)) { + + absl::InlinedVector missing_policies; + + for (const auto& policy : subset_config.subset_lb_policy().policies()) { + auto* factory = Config::Utility::getAndCheckFactory( + policy.typed_extension_config(), /*is_optional=*/true); + + if (factory != nullptr) { + // Load and validate the configuration. + auto sub_lb_proto_message = factory->createEmptyConfigProto(); + Config::Utility::translateOpaqueConfig(policy.typed_extension_config().typed_config(), + visitor, *sub_lb_proto_message); + + child_lb_config_ = factory->loadConfig(*sub_lb_proto_message, visitor); + child_lb_factory_ = factory; + break; + } + + missing_policies.push_back(policy.typed_extension_config().name()); + } + + if (child_lb_factory_ == nullptr) { + throw EnvoyException(fmt::format("cluster: didn't find a registered load balancer factory " + "implementation for subset lb with names from [{}]", + absl::StrJoin(missing_policies, ", "))); + } +} + +SubsetLoadBalancerConfig::SubsetLoadBalancerConfig(const ClusterProto& cluster, + ProtobufMessage::ValidationVisitor& visitor) + : subset_info_(std::make_unique(cluster.lb_subset_config())) { + ASSERT(subset_info_->isEnabled()); + + auto sub_lb_pair = + LegacyLbPolicyConfigHelper::getTypedLbConfigFromLegacyProtoWithoutSubset(cluster, visitor); + if (!sub_lb_pair.ok()) { + throw EnvoyException(std::string(sub_lb_pair.status().message())); + } + + child_lb_factory_ = sub_lb_pair->factory; + ASSERT(child_lb_factory_ != nullptr); + child_lb_config_ = std::move(sub_lb_pair->config); +} + +SubsetLoadBalancerConfig::SubsetLoadBalancerConfig( + std::unique_ptr subset_info, TypedLoadBalancerFactory* child_factory, + LoadBalancerConfigPtr child_config) + : subset_info_(std::move(subset_info)), child_lb_factory_(child_factory), + child_lb_config_(std::move(child_config)) {} + +} // namespace Upstream +} // namespace Envoy diff --git a/source/common/upstream/subset_lb_config.h b/source/extensions/load_balancing_policies/subset/subset_lb_config.h similarity index 50% rename from source/common/upstream/subset_lb_config.h rename to source/extensions/load_balancing_policies/subset/subset_lb_config.h index 914c86a62851..6199d31a6f1b 100644 --- a/source/common/upstream/subset_lb_config.h +++ b/source/extensions/load_balancing_policies/subset/subset_lb_config.h @@ -10,27 +10,30 @@ namespace Envoy { namespace Upstream { +using SubsetLbConfigProto = envoy::extensions::load_balancing_policies::subset::v3::Subset; +using ClusterProto = envoy::config::cluster::v3::Cluster; +using LegacySubsetLbConfigProto = envoy::config::cluster::v3::Cluster::LbSubsetConfig; + /** - * Implementation of SubsetSelector. This is part of subset load balancer config and is used to - * store config of single selector. + * This is part of subset load balancer config and is used to store config of single selector. */ -class SubsetSelectorImpl : public SubsetSelector { +class SubsetSelector { public: - SubsetSelectorImpl(const Protobuf::RepeatedPtrField& selector_keys, - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: - LbSubsetSelectorFallbackPolicy fallback_policy, - const Protobuf::RepeatedPtrField& fallback_keys_subset, - bool single_host_per_subset); + SubsetSelector(const Protobuf::RepeatedPtrField& selector_keys, + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: + LbSubsetSelectorFallbackPolicy fallback_policy, + const Protobuf::RepeatedPtrField& fallback_keys_subset, + bool single_host_per_subset); // SubsetSelector - const std::set& selectorKeys() const override { return selector_keys_; } + const std::set& selectorKeys() const { return selector_keys_; } envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector:: LbSubsetSelectorFallbackPolicy - fallbackPolicy() const override { + fallbackPolicy() const { return fallback_policy_; } - const std::set& fallbackKeysSubset() const override { return fallback_keys_subset_; } - bool singleHostPerSubset() const override { return single_host_per_subset_; } + const std::set& fallbackKeysSubset() const { return fallback_keys_subset_; } + bool singleHostPerSubset() const { return single_host_per_subset_; } private: const std::set selector_keys_; @@ -41,14 +44,79 @@ class SubsetSelectorImpl : public SubsetSelector { const bool single_host_per_subset_ : 1; }; -using SubsetLoadbalancingPolicyProto = - envoy::extensions::load_balancing_policies::subset::v3::Subset; -using LegacySubsetLoadbalancingPolicyProto = envoy::config::cluster::v3::Cluster::LbSubsetConfig; +using SubsetSelectorPtr = std::shared_ptr; + +/** + * Load Balancer subset configuration. + */ +class LoadBalancerSubsetInfo { +public: + virtual ~LoadBalancerSubsetInfo() = default; + + /** + * @return bool true if load balancer subsets are configured. + */ + virtual bool isEnabled() const PURE; + + /** + * @return LbSubsetFallbackPolicy the fallback policy used when + * route metadata does not match any subset. + */ + virtual envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy + fallbackPolicy() const PURE; + + /** + * @return LbSubsetMetadataFallbackPolicy the fallback policy used to try different route metadata + * until a host is found + */ + virtual envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy + metadataFallbackPolicy() const PURE; + + /** + * @return ProtobufWkt::Struct the struct describing the metadata for a + * host to be included in the default subset. + */ + virtual const ProtobufWkt::Struct& defaultSubset() const PURE; + + /* + * @return const std:vector>& a vector of + * sorted keys used to define load balancer subsets. + */ + virtual const std::vector& subsetSelectors() const PURE; + + /* + * @return bool whether routing to subsets should take locality weights into account. + */ + virtual bool localityWeightAware() const PURE; + + /* + * @return bool whether the locality weights should be scaled to compensate for the + * fraction of hosts removed from the original host set. + */ + virtual bool scaleLocalityWeight() const PURE; + + /* + * @return bool whether to attempt to select a host from the entire cluster if host + * selection from the fallback subset fails. + */ + virtual bool panicModeAny() const PURE; + + /* + * @return bool whether matching metadata should attempt to match against any of the + * elements in a list value defined in endpoint metadata. + */ + virtual bool listAsAny() const PURE; + + /* + * @return bool whether redundant key/value pairs is allowed in the request metadata. + */ + virtual bool allowRedundantKeys() const PURE; +}; + +using LoadBalancerSubsetInfoPtr = std::unique_ptr; /** - * Implementation of LoadBalancerSubsetInfo. Both the legacy and extension subset proto configs - * are converted to this class. - * TODO(wbpcode): move this implementation to extensions directory. + * Both the legacy and extension subset proto configs are converted to this class. */ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { public: @@ -59,7 +127,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { using SubsetFallbackPolicy = envoy::config::cluster::v3::Cluster::LbSubsetConfig:: LbSubsetSelector::LbSubsetSelectorFallbackPolicy; - LoadBalancerSubsetInfoImpl(const SubsetLoadbalancingPolicyProto& subset_config) + LoadBalancerSubsetInfoImpl(const SubsetLbConfigProto& subset_config) : default_subset_(subset_config.default_subset()), fallback_policy_(static_cast(subset_config.fallback_policy())), metadata_fallback_policy_( @@ -71,7 +139,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { allow_redundant_keys_(subset_config.allow_redundant_keys()) { for (const auto& subset : subset_config.subset_selectors()) { if (!subset.keys().empty()) { - subset_selectors_.emplace_back(std::make_shared( + subset_selectors_.emplace_back(std::make_shared( subset.keys(), static_cast(subset.fallback_policy()), subset.fallback_keys_subset(), subset.single_host_per_subset())); } @@ -87,7 +155,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { } } - LoadBalancerSubsetInfoImpl(const LegacySubsetLoadbalancingPolicyProto& subset_config) + LoadBalancerSubsetInfoImpl(const LegacySubsetLbConfigProto& subset_config) : default_subset_(subset_config.default_subset()), fallback_policy_(subset_config.fallback_policy()), metadata_fallback_policy_(subset_config.metadata_fallback_policy()), @@ -97,7 +165,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { panic_mode_any_(subset_config.panic_mode_any()), list_as_any_(subset_config.list_as_any()) { for (const auto& subset : subset_config.subset_selectors()) { if (!subset.keys().empty()) { - subset_selectors_.emplace_back(std::make_shared( + subset_selectors_.emplace_back(std::make_shared( subset.keys(), subset.fallback_policy(), subset.fallback_keys_subset(), subset.single_host_per_subset())); } @@ -114,7 +182,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { return metadata_fallback_policy_; } const ProtobufWkt::Struct& defaultSubset() const override { return default_subset_; } - const std::vector& subsetSelectors() const override { + const std::vector& subsetSelectors() const override { return subset_selectors_; } bool localityWeightAware() const override { return locality_weight_aware_; } @@ -125,7 +193,7 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { private: const ProtobufWkt::Struct default_subset_; - std::vector subset_selectors_; + std::vector subset_selectors_; // Keep small members (bools and enums) at the end of class, to reduce alignment overhead. const FallbackPolicy fallback_policy_; const MetadataFallbackPolicy metadata_fallback_policy_; @@ -136,7 +204,36 @@ class LoadBalancerSubsetInfoImpl : public LoadBalancerSubsetInfo { const bool list_as_any_ : 1; const bool allow_redundant_keys_{}; }; -using DefaultLoadBalancerSubsetInfoImpl = ConstSingleton; + +using DefaultLoadBalancerSubsetInfo = ConstSingleton; + +class SubsetLoadBalancerConfig : public Upstream::LoadBalancerConfig { +public: + SubsetLoadBalancerConfig(const SubsetLbConfigProto& subset_config, + ProtobufMessage::ValidationVisitor& visitor); + SubsetLoadBalancerConfig(const ClusterProto& cluster, + ProtobufMessage::ValidationVisitor& visitor); + SubsetLoadBalancerConfig(LoadBalancerSubsetInfoPtr subset_info, + TypedLoadBalancerFactory* child_factory, + LoadBalancerConfigPtr child_config); + + Upstream::ThreadAwareLoadBalancerPtr + createLoadBalancer(const Upstream::ClusterInfo& cluster_info, + const Upstream::PrioritySet& child_priority_set, Runtime::Loader& runtime, + Random::RandomGenerator& random, TimeSource& time_source) const { + return child_lb_factory_->create( + makeOptRefFromPtr(child_lb_config_.get()), cluster_info, + child_priority_set, runtime, random, time_source); + } + std::string childLoadBalancerName() const { return child_lb_factory_->name(); } + + const LoadBalancerSubsetInfo& subsetInfo() const { return *subset_info_; } + +private: + LoadBalancerSubsetInfoPtr subset_info_; + Upstream::TypedLoadBalancerFactory* child_lb_factory_{}; + Upstream::LoadBalancerConfigPtr child_lb_config_; +}; } // namespace Upstream } // namespace Envoy diff --git a/test/common/upstream/load_balancer_impl_test.cc b/test/common/upstream/load_balancer_impl_test.cc index 54831d77d75d..61523f537c4d 100644 --- a/test/common/upstream/load_balancer_impl_test.cc +++ b/test/common/upstream/load_balancer_impl_test.cc @@ -3426,143 +3426,6 @@ INSTANTIATE_TEST_SUITE_P(PrimaryOrFailoverAndLegacyOrNew, RandomLoadBalancerTest LoadBalancerTestParam{false, false}, LoadBalancerTestParam{false, true})); -TEST(LoadBalancerSubsetInfoImplTest, DefaultConfigIsDiabled) { - auto subset_info = LoadBalancerSubsetInfoImpl( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance()); - - EXPECT_FALSE(subset_info.isEnabled()); - EXPECT_TRUE(subset_info.fallbackPolicy() == - envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK); - EXPECT_EQ(subset_info.defaultSubset().fields_size(), 0); - EXPECT_EQ(subset_info.subsetSelectors().size(), 0); -} - -TEST(LoadBalancerSubsetInfoImplTest, SubsetConfig) { - auto subset_value = ProtobufWkt::Value(); - subset_value.set_string_value("the value"); - - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - subset_config.set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET); - subset_config.mutable_default_subset()->mutable_fields()->insert({"key", subset_value}); - auto subset_selector1 = subset_config.mutable_subset_selectors()->Add(); - subset_selector1->add_keys("selector_key1"); - auto subset_selector2 = subset_config.mutable_subset_selectors()->Add(); - subset_selector2->add_keys("selector_key2"); - subset_selector2->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); - - auto subset_info = LoadBalancerSubsetInfoImpl(subset_config); - - EXPECT_TRUE(subset_info.isEnabled()); - EXPECT_TRUE(subset_info.fallbackPolicy() == - envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET); - EXPECT_EQ(subset_info.defaultSubset().fields_size(), 1); - EXPECT_EQ(subset_info.defaultSubset().fields().at("key").string_value(), - std::string("the value")); - EXPECT_EQ(subset_info.subsetSelectors().size(), 2); - EXPECT_EQ(subset_info.subsetSelectors()[0]->selectorKeys(), - std::set({"selector_key1"})); - EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackPolicy(), - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED); - EXPECT_EQ(subset_info.subsetSelectors()[1]->selectorKeys(), - std::set({"selector_key2"})); - EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackPolicy(), - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); -} - -TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetFallbackValid) { - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - auto selector1 = subset_config.mutable_subset_selectors()->Add(); - selector1->add_keys("key1"); - selector1->add_keys("key2"); - selector1->add_keys("key3"); - selector1->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - selector1->add_fallback_keys_subset("key1"); - selector1->add_fallback_keys_subset("key3"); - - auto selector2 = subset_config.mutable_subset_selectors()->Add(); - selector2->add_keys("key1"); - selector2->add_keys("key3"); - selector2->add_keys("key4"); - selector2->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - selector2->add_fallback_keys_subset("key4"); - - auto subset_info = LoadBalancerSubsetInfoImpl(subset_config); - - EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackPolicy(), - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - EXPECT_EQ(subset_info.subsetSelectors()[0]->selectorKeys(), - std::set({"key1", "key2", "key3"})); - EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackKeysSubset(), - std::set({"key1", "key3"})); - - EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackPolicy(), - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - EXPECT_EQ(subset_info.subsetSelectors()[1]->selectorKeys(), - std::set({"key1", "key3", "key4"})); - EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackKeysSubset(), - std::set({"key4"})); -} - -TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetForOtherPolicyInvalid) { - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - auto selector = subset_config.mutable_subset_selectors()->Add(); - - selector->add_keys("key1"); - selector->add_keys("key2"); - selector->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); - selector->add_fallback_keys_subset("key1"); - - EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, - "fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); -} - -TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetNotASubsetInvalid) { - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - auto selector = subset_config.mutable_subset_selectors()->Add(); - - selector->add_keys("key1"); - selector->add_keys("key2"); - selector->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - selector->add_fallback_keys_subset("key3"); - - EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, - "fallback_keys_subset must be a subset of selector keys"); -} - -TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetEmptyInvalid) { - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - auto selector = subset_config.mutable_subset_selectors()->Add(); - - selector->add_keys("key1"); - selector->add_keys("key2"); - selector->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - - EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, - "fallback_keys_subset cannot be empty"); -} - -TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetEqualKeysInvalid) { - auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); - auto selector = subset_config.mutable_subset_selectors()->Add(); - - selector->add_keys("key1"); - selector->add_keys("key2"); - selector->set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); - selector->add_fallback_keys_subset("key2"); - selector->add_fallback_keys_subset("key1"); - - EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, - "fallback_keys_subset cannot be equal to keys"); -} - TEST(LoadBalancerContextBaseTest, LoadBalancerContextBaseTest) { { LoadBalancerContextBase context; diff --git a/test/extensions/load_balancing_policies/subset/BUILD b/test/extensions/load_balancing_policies/subset/BUILD index 22a1130b3faf..50a924fdc7c3 100644 --- a/test/extensions/load_balancing_policies/subset/BUILD +++ b/test/extensions/load_balancing_policies/subset/BUILD @@ -54,6 +54,11 @@ envoy_extension_cc_test( "//source/common/upstream:load_balancer_lib", "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", + "//source/extensions/load_balancing_policies/least_request:config", + "//source/extensions/load_balancing_policies/maglev:config", + "//source/extensions/load_balancing_policies/random:config", + "//source/extensions/load_balancing_policies/ring_hash:config", + "//source/extensions/load_balancing_policies/round_robin:config", "//source/extensions/load_balancing_policies/subset:config", "//test/common/upstream:utility_lib", "//test/mocks:common_lib", @@ -69,6 +74,7 @@ envoy_extension_cc_test( "//test/test_common:simulated_time_system_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/round_robin/v3:pkg_cc_proto", ], ) @@ -80,9 +86,12 @@ envoy_extension_cc_benchmark_binary( "benchmark", ], deps = [ + "//source/extensions/load_balancing_policies/random:config", "//source/extensions/load_balancing_policies/subset:config", "//test/extensions/load_balancing_policies/common:benchmark_base_tester_lib", "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/random/v3:pkg_cc_proto", + "@envoy_api//envoy/extensions/load_balancing_policies/subset/v3:pkg_cc_proto", ], ) diff --git a/test/extensions/load_balancing_policies/subset/subset_benchmark.cc b/test/extensions/load_balancing_policies/subset/subset_benchmark.cc index e09337efca9c..6f3b454ce9d9 100644 --- a/test/extensions/load_balancing_policies/subset/subset_benchmark.cc +++ b/test/extensions/load_balancing_policies/subset/subset_benchmark.cc @@ -3,12 +3,13 @@ #include #include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/extensions/load_balancing_policies/random/v3/random.pb.h" +#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.h" +#include "envoy/extensions/load_balancing_policies/subset/v3/subset.pb.validate.h" #include "source/common/common/random_generator.h" #include "source/common/memory/stats.h" #include "source/common/upstream/upstream_impl.h" -#include "source/extensions/load_balancing_policies/maglev/maglev_lb.h" -#include "source/extensions/load_balancing_policies/ring_hash/ring_hash_lb.h" #include "source/extensions/load_balancing_policies/subset/subset_lb.h" #include "test/benchmark/main.h" @@ -30,20 +31,24 @@ class SubsetLbTester : public LoadBalancingPolices::Common::BaseTester { public: SubsetLbTester(uint64_t num_hosts, bool single_host_per_subset) : BaseTester(num_hosts, 0, 0, true /* attach metadata */) { - envoy::config::cluster::v3::Cluster::LbSubsetConfig subset_config; - subset_config.set_fallback_policy( - envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT); - auto* selector = subset_config.mutable_subset_selectors()->Add(); - selector->set_single_host_per_subset(single_host_per_subset); - *selector->mutable_keys()->Add() = std::string(metadata_key); - - subset_info_ = std::make_unique(subset_config); - auto child_lb_creator = std::make_unique( - Upstream::LoadBalancerType::Random, absl::nullopt, absl::nullopt, absl::nullopt, - absl::nullopt, common_config_); - lb_ = std::make_unique( - *subset_info_, std::move(child_lb_creator), priority_set_, &local_priority_set_, stats_, - stats_scope_, runtime_, random_, simTime()); + envoy::extensions::load_balancing_policies::subset::v3::Subset subset_config_proto{}; + subset_config_proto.set_fallback_policy( + envoy::extensions::load_balancing_policies::subset::v3::Subset::ANY_ENDPOINT); + auto* selector_proto = subset_config_proto.mutable_subset_selectors()->Add(); + selector_proto->set_single_host_per_subset(single_host_per_subset); + *selector_proto->mutable_keys()->Add() = std::string(metadata_key); + + auto* child_lb = subset_config_proto.mutable_subset_lb_policy()->mutable_policies()->Add(); + child_lb->mutable_typed_extension_config()->set_name("envoy.load_balancing_policies.random"); + envoy::extensions::load_balancing_policies::random::v3::Random random_lb_config; + child_lb->mutable_typed_extension_config()->mutable_typed_config()->PackFrom(random_lb_config); + + subset_config_ = std::make_unique( + subset_config_proto, ProtobufMessage::getStrictValidationVisitor()); + + lb_ = std::make_unique(*subset_config_, *info_, priority_set_, + &local_priority_set_, stats_, stats_scope_, + runtime_, random_, simTime()); const Upstream::HostVector& hosts = priority_set_.getOrCreateHostSet(0).hosts(); ASSERT(hosts.size() == num_hosts); @@ -64,7 +69,7 @@ class SubsetLbTester : public LoadBalancingPolices::Common::BaseTester { host_moved_, {}, random_.random(), absl::nullopt); } - std::unique_ptr subset_info_; + std::unique_ptr subset_config_; std::unique_ptr lb_; Upstream::HostVectorConstSharedPtr orig_hosts_; Upstream::HostVectorConstSharedPtr smaller_hosts_; diff --git a/test/extensions/load_balancing_policies/subset/subset_test.cc b/test/extensions/load_balancing_policies/subset/subset_test.cc index d9c241a30f3b..adf9b6d01b98 100644 --- a/test/extensions/load_balancing_policies/subset/subset_test.cc +++ b/test/extensions/load_balancing_policies/subset/subset_test.cc @@ -7,6 +7,8 @@ #include "envoy/config/cluster/v3/cluster.pb.h" #include "envoy/config/core/v3/base.pb.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.h" +#include "envoy/extensions/load_balancing_policies/round_robin/v3/round_robin.pb.validate.h" #include "source/common/common/logger.h" #include "source/common/config/metadata.h" @@ -37,41 +39,39 @@ using testing::ReturnRef; namespace Envoy { namespace Upstream { +namespace { -class SubsetLoadBalancerInternalStateTester { +class MockLoadBalancerSubsetInfo : public LoadBalancerSubsetInfo { public: - SubsetLoadBalancerInternalStateTester(std::shared_ptr lb) : lb_(lb) {} - - using MetadataVector = std::vector>; - - void testDescribeMetadata(std::string expected, const MetadataVector& metadata) { - const SubsetLoadBalancer::SubsetMetadata& subset_metadata(metadata); - EXPECT_EQ(expected, lb_.get()->describeMetadata(subset_metadata)); - } - - void validateLbTypeConfigs() const { - const auto* legacy_child_lb_creator = - dynamic_cast(lb_->child_lb_creator_.get()); - - if (legacy_child_lb_creator == nullptr) { - return; - } + MockLoadBalancerSubsetInfo(); + ~MockLoadBalancerSubsetInfo() override; + + // Upstream::LoadBalancerSubsetInfo + MOCK_METHOD(bool, isEnabled, (), (const)); + MOCK_METHOD(envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy, + fallbackPolicy, (), (const)); + MOCK_METHOD(envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy, + metadataFallbackPolicy, (), (const)); + MOCK_METHOD(const ProtobufWkt::Struct&, defaultSubset, (), (const)); + MOCK_METHOD(const std::vector&, subsetSelectors, (), (const)); + MOCK_METHOD(bool, localityWeightAware, (), (const)); + MOCK_METHOD(bool, scaleLocalityWeight, (), (const)); + MOCK_METHOD(bool, panicModeAny, (), (const)); + MOCK_METHOD(bool, listAsAny, (), (const)); + MOCK_METHOD(bool, allowRedundantKeys, (), (const)); + + std::vector subset_selectors_; +}; - // Each of these expects that an lb_type is set to that type iff the - // returned config for that type exists. - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RingHash, - legacy_child_lb_creator->lbRingHashConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::Maglev, - legacy_child_lb_creator->lbMaglevConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::RoundRobin, - legacy_child_lb_creator->lbRoundRobinConfig() != absl::nullopt); - EXPECT_EQ(legacy_child_lb_creator->lbType() == LoadBalancerType::LeastRequest, - legacy_child_lb_creator->lbLeastRequestConfig() != absl::nullopt); - } +MockLoadBalancerSubsetInfo::MockLoadBalancerSubsetInfo() { + ON_CALL(*this, isEnabled()).WillByDefault(Return(true)); + ON_CALL(*this, fallbackPolicy()) + .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); + ON_CALL(*this, defaultSubset()).WillByDefault(ReturnRef(ProtobufWkt::Struct::default_instance())); + ON_CALL(*this, subsetSelectors()).WillByDefault(ReturnRef(subset_selectors_)); +} -private: - std::shared_ptr lb_; -}; +MockLoadBalancerSubsetInfo::~MockLoadBalancerSubsetInfo() = default; class TestMetadataMatchCriterion : public Router::MetadataMatchCriterion { public: @@ -160,7 +160,142 @@ class TestMetadataMatchCriteria : public Router::MetadataMatchCriteria { std::vector matches_; }; -namespace SubsetLoadBalancerTest { +TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetFallbackValid) { + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + auto selector1 = subset_config.mutable_subset_selectors()->Add(); + selector1->add_keys("key1"); + selector1->add_keys("key2"); + selector1->add_keys("key3"); + selector1->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + selector1->add_fallback_keys_subset("key1"); + selector1->add_fallback_keys_subset("key3"); + + auto selector2 = subset_config.mutable_subset_selectors()->Add(); + selector2->add_keys("key1"); + selector2->add_keys("key3"); + selector2->add_keys("key4"); + selector2->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + selector2->add_fallback_keys_subset("key4"); + + auto subset_info = LoadBalancerSubsetInfoImpl(subset_config); + + EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackPolicy(), + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + EXPECT_EQ(subset_info.subsetSelectors()[0]->selectorKeys(), + std::set({"key1", "key2", "key3"})); + EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackKeysSubset(), + std::set({"key1", "key3"})); + + EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackPolicy(), + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + EXPECT_EQ(subset_info.subsetSelectors()[1]->selectorKeys(), + std::set({"key1", "key3", "key4"})); + EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackKeysSubset(), + std::set({"key4"})); +} + +TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetForOtherPolicyInvalid) { + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + auto selector = subset_config.mutable_subset_selectors()->Add(); + + selector->add_keys("key1"); + selector->add_keys("key2"); + selector->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); + selector->add_fallback_keys_subset("key1"); + + EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, + "fallback_keys_subset can be set only for KEYS_SUBSET fallback_policy"); +} + +TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetNotASubsetInvalid) { + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + auto selector = subset_config.mutable_subset_selectors()->Add(); + + selector->add_keys("key1"); + selector->add_keys("key2"); + selector->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + selector->add_fallback_keys_subset("key3"); + + EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, + "fallback_keys_subset must be a subset of selector keys"); +} + +TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetEmptyInvalid) { + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + auto selector = subset_config.mutable_subset_selectors()->Add(); + + selector->add_keys("key1"); + selector->add_keys("key2"); + selector->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + + EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, + "fallback_keys_subset cannot be empty"); +} + +TEST(LoadBalancerSubsetInfoImplTest, KeysSubsetEqualKeysInvalid) { + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + auto selector = subset_config.mutable_subset_selectors()->Add(); + + selector->add_keys("key1"); + selector->add_keys("key2"); + selector->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::KEYS_SUBSET); + selector->add_fallback_keys_subset("key2"); + selector->add_fallback_keys_subset("key1"); + + EXPECT_THROW_WITH_MESSAGE(LoadBalancerSubsetInfoImpl{subset_config}, EnvoyException, + "fallback_keys_subset cannot be equal to keys"); +} + +TEST(LoadBalancerSubsetInfoImplTest, DefaultConfigIsDiabled) { + auto subset_info = LoadBalancerSubsetInfoImpl( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance()); + + EXPECT_FALSE(subset_info.isEnabled()); + EXPECT_TRUE(subset_info.fallbackPolicy() == + envoy::config::cluster::v3::Cluster::LbSubsetConfig::NO_FALLBACK); + EXPECT_EQ(subset_info.defaultSubset().fields_size(), 0); + EXPECT_EQ(subset_info.subsetSelectors().size(), 0); +} + +TEST(LoadBalancerSubsetInfoImplTest, SubsetConfig) { + auto subset_value = ProtobufWkt::Value(); + subset_value.set_string_value("the value"); + + auto subset_config = envoy::config::cluster::v3::Cluster::LbSubsetConfig::default_instance(); + subset_config.set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET); + subset_config.mutable_default_subset()->mutable_fields()->insert({"key", subset_value}); + auto subset_selector1 = subset_config.mutable_subset_selectors()->Add(); + subset_selector1->add_keys("selector_key1"); + auto subset_selector2 = subset_config.mutable_subset_selectors()->Add(); + subset_selector2->add_keys("selector_key2"); + subset_selector2->set_fallback_policy( + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); + + auto subset_info = LoadBalancerSubsetInfoImpl(subset_config); + + EXPECT_TRUE(subset_info.isEnabled()); + EXPECT_TRUE(subset_info.fallbackPolicy() == + envoy::config::cluster::v3::Cluster::LbSubsetConfig::DEFAULT_SUBSET); + EXPECT_EQ(subset_info.defaultSubset().fields_size(), 1); + EXPECT_EQ(subset_info.defaultSubset().fields().at("key").string_value(), + std::string("the value")); + EXPECT_EQ(subset_info.subsetSelectors().size(), 2); + EXPECT_EQ(subset_info.subsetSelectors()[0]->selectorKeys(), + std::set({"selector_key1"})); + EXPECT_EQ(subset_info.subsetSelectors()[0]->fallbackPolicy(), + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::NOT_DEFINED); + EXPECT_EQ(subset_info.subsetSelectors()[1]->selectorKeys(), + std::set({"selector_key2"})); + EXPECT_EQ(subset_info.subsetSelectors()[1]->fallbackPolicy(), + envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetSelector::ANY_ENDPOINT); +} class TestLoadBalancerContext : public LoadBalancerContextBase { public: @@ -190,14 +325,27 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, public: SubsetLoadBalancerTest() : scope_(stats_store_.createScope("testprefix")), stat_names_(stats_store_.symbolTable()), - stats_(stat_names_, *stats_store_.rootScope()) { - least_request_lb_config_.mutable_choice_count()->set_value(2); - } + stats_(stat_names_, *stats_store_.rootScope()) {} using HostMetadata = std::map; using HostListMetadata = std::map>; using HostURLMetadataMap = std::map; + void initLbConfigAndLB(LoadBalancerSubsetInfoPtr subset_info = nullptr, bool zone_aware = false) { + lb_config_ = std::make_unique( + [&]() -> LoadBalancerSubsetInfoPtr { + if (subset_info == nullptr) { + return std::move(mock_subset_info_ptr_); + } + return std::move(subset_info); + }(), + Config::Utility::getFactoryByName(child_lb_name_), + std::move(child_lb_config_)); + lb_ = std::make_shared(*lb_config_, *info_, priority_set_, + zone_aware ? &local_priority_set_ : nullptr, stats_, + *scope_, runtime_, random_, simTime()); + } + void init() { init({ {"tcp://127.0.0.1:80", {{"version", "1.0"}}}, @@ -255,42 +403,14 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, } void init(const HostURLMetadataMap& host_metadata, - const HostURLMetadataMap& failover_host_metadata, bool use_actual_subset_info = false) { - - if (!use_actual_subset_info) { - EXPECT_CALL(subset_info_, isEnabled()).WillRepeatedly(Return(true)); - } - + const HostURLMetadataMap& failover_host_metadata, + LoadBalancerSubsetInfoPtr subset_info = nullptr) { configureHostSet(host_metadata, host_set_); if (!failover_host_metadata.empty()) { configureHostSet(failover_host_metadata, *priority_set_.getMockHostSet(1)); } - auto child_lb_creator = std::make_unique( - lb_type_, - lb_type_ == LoadBalancerType::RingHash - ? makeOptRef( - ring_hash_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::Maglev - ? makeOptRef( - maglev_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::RoundRobin - ? makeOptRef( - round_robin_lb_config_) - : absl::nullopt, - lb_type_ == LoadBalancerType::LeastRequest - ? makeOptRef( - least_request_lb_config_) - : absl::nullopt, - common_config_); - - lb_ = std::make_shared( - use_actual_subset_info ? static_cast(*actual_subset_info_) - : static_cast(subset_info_), - std::move(child_lb_creator), priority_set_, nullptr, stats_, *scope_, runtime_, random_, - simTime()); + initLbConfigAndLB(std::move(subset_info)); } void zoneAwareInit(const std::vector& host_metadata_per_locality, @@ -348,13 +468,7 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, std::make_shared(), HostsPerLocalityImpl::empty()), {}, {}, {}, 0, absl::nullopt); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - - lb_ = std::make_shared(subset_info_, std::move(child_lb_creator), - priority_set_, &local_priority_set_, stats_, *scope_, - runtime_, random_, simTime()); + initLbConfigAndLB(nullptr, true); } HostSharedPtr makeHost(const std::string& url, const HostMetadata& metadata) { @@ -420,8 +534,8 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, fallback_keys_subset_mapped.Add(std::string(it)); } - return std::make_shared( - selector_keys_mapped, fallback_policy, fallback_keys_subset_mapped, single_host_per_subset); + return std::make_shared(selector_keys_mapped, fallback_policy, + fallback_keys_subset_mapped, single_host_per_subset); } SubsetSelectorPtr makeSelector( @@ -530,11 +644,12 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, } } - void doLbTypeTest(LoadBalancerType type) { + void doChildLbNameTest(const std::string& child_lb_name) { EXPECT_CALL(subset_info_, fallbackPolicy()) .WillRepeatedly(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - lb_type_ = type; + child_lb_name_ = child_lb_name; + init({{"tcp://127.0.0.1:80", {{"version", "1.0"}}}}); EXPECT_EQ(host_set_.hosts_[0], lb_->chooseHost(nullptr)); @@ -543,6 +658,7 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, modifyHosts({added_host}, {host_set_.hosts_.back()}); EXPECT_EQ(added_host, lb_->chooseHost(nullptr)); + EXPECT_EQ(lb_->childLoadBalancerName(), child_lb_name); } MetadataConstSharedPtr buildMetadata(const std::string& version, bool is_default = false) const { @@ -588,19 +704,17 @@ class SubsetLoadBalancerTest : public Event::TestUsingSimulatedTime, return v; } - LoadBalancerType lb_type_{LoadBalancerType::RoundRobin}; + std::string child_lb_name_{"envoy.load_balancing_policies.round_robin"}; + LoadBalancerConfigPtr child_lb_config_; NiceMock priority_set_; MockHostSet& host_set_ = *priority_set_.getMockHostSet(0); // Mock subset info is used for testing most logic. - NiceMock subset_info_; - // Actual subset info is used for testing actual subset config parsing and behavior. - std::unique_ptr actual_subset_info_; + std::unique_ptr> mock_subset_info_ptr_{ + std::make_unique>()}; + NiceMock& subset_info_{*mock_subset_info_ptr_}; std::shared_ptr info_{new NiceMock()}; - envoy::config::cluster::v3::Cluster::RingHashLbConfig ring_hash_lb_config_; - envoy::config::cluster::v3::Cluster::MaglevLbConfig maglev_lb_config_; - envoy::config::cluster::v3::Cluster::LeastRequestLbConfig least_request_lb_config_; - envoy::config::cluster::v3::Cluster::RoundRobinLbConfig round_robin_lb_config_; - envoy::config::cluster::v3::Cluster::CommonLbConfig common_config_; + std::unique_ptr lb_config_; + NiceMock runtime_; NiceMock random_; Stats::IsolatedStoreImpl stats_store_; @@ -1567,13 +1681,7 @@ TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) { host_set_.healthy_hosts_ = host_set_.hosts_; host_set_.healthy_hosts_per_locality_ = host_set_.hosts_per_locality_; - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); - + initLbConfigAndLB(); TestLoadBalancerContext context_version({{"version", "1.0"}}); EXPECT_EQ(host_set_.hosts_[1], lb_->chooseHost(&context_version)); @@ -1584,33 +1692,23 @@ TEST_F(SubsetLoadBalancerTest, IgnoresHostsWithoutMetadata) { // Optimally these would also be some type of TEST_P, but that is a little bit complicated as // modifyHosts() also needs params. Clean this up. TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRoundRobin) { - doLbTypeTest(LoadBalancerType::RoundRobin); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); + doChildLbNameTest("envoy.load_balancing_policies.round_robin"); } TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesLeastRequest) { - doLbTypeTest(LoadBalancerType::LeastRequest); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); + doChildLbNameTest("envoy.load_balancing_policies.least_request"); } TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRandom) { - doLbTypeTest(LoadBalancerType::Random); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); + doChildLbNameTest("envoy.load_balancing_policies.random"); } TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesRingHash) { - doLbTypeTest(LoadBalancerType::RingHash); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); + doChildLbNameTest("envoy.load_balancing_policies.ring_hash"); } TEST_P(SubsetLoadBalancerTest, LoadBalancerTypesMaglev) { - doLbTypeTest(LoadBalancerType::Maglev); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.validateLbTypeConfigs(); + doChildLbNameTest("envoy.load_balancing_policies.maglev"); } TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) { @@ -1622,7 +1720,7 @@ TEST_F(SubsetLoadBalancerTest, ZoneAwareFallback) { EXPECT_CALL(subset_info_, subsetSelectors()).WillRepeatedly(ReturnRef(subset_selectors)); - common_config_.mutable_healthy_panic_threshold()->set_value(40); + info_->lb_config_.mutable_healthy_panic_threshold()->set_value(40); EXPECT_CALL(runtime_.snapshot_, getInteger("upstream.healthy_panic_threshold", 40)) .WillRepeatedly(Return(50)); EXPECT_CALL(runtime_.snapshot_, featureEnabled("upstream.zone_routing.enabled", 100)) @@ -2162,12 +2260,13 @@ TEST_F(SubsetLoadBalancerTest, DescribeMetadata) { ProtobufWkt::Value num_value; num_value.set_number_value(100); - auto tester = SubsetLoadBalancerInternalStateTester(lb_); - tester.testDescribeMetadata("version=\"abc\"", {{"version", str_value}}); - tester.testDescribeMetadata("number=100", {{"number", num_value}}); - tester.testDescribeMetadata("x=\"abc\", y=100", {{"x", str_value}, {"y", num_value}}); - tester.testDescribeMetadata("y=100, x=\"abc\"", {{"y", num_value}, {"x", str_value}}); - tester.testDescribeMetadata("", {}); + EXPECT_EQ("version=\"abc\"", SubsetLoadBalancer::describeMetadata({{"version", str_value}})); + EXPECT_EQ("number=100", SubsetLoadBalancer::describeMetadata({{"number", num_value}})); + EXPECT_EQ("x=\"abc\", y=100", + SubsetLoadBalancer::describeMetadata({{"x", str_value}, {"y", num_value}})); + EXPECT_EQ("y=100, x=\"abc\"", + SubsetLoadBalancer::describeMetadata({{"y", num_value}, {"x", str_value}})); + EXPECT_EQ("", SubsetLoadBalancer::describeMetadata({})); } TEST_F(SubsetLoadBalancerTest, DisabledLocalityWeightAwareness) { @@ -2187,12 +2286,7 @@ TEST_F(SubsetLoadBalancerTest, DisabledLocalityWeightAwareness) { }, host_set_, {1, 100}); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + initLbConfigAndLB(); TestLoadBalancerContext context({{"version", "1.1"}}); @@ -2213,12 +2307,7 @@ TEST_F(SubsetLoadBalancerTest, DoesNotCheckHostHealth) { EXPECT_CALL(*mock_host, weight()).WillRepeatedly(Return(1)); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + initLbConfigAndLB(); } TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) { @@ -2239,13 +2328,13 @@ TEST_F(SubsetLoadBalancerTest, EnabledLocalityWeightAwareness) { }, host_set_, {1, 100}); - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + auto* child_factory = + Config::Utility::getFactoryByName(child_lb_name_); + envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin rr_config; + rr_config.mutable_locality_lb_config()->mutable_locality_weighted_lb_config(); + child_lb_config_ = + child_factory->loadConfig(rr_config, ProtobufMessage::getStrictValidationVisitor()); + initLbConfigAndLB(); TestLoadBalancerContext context({{"version", "1.1"}}); @@ -2278,13 +2367,14 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeights) { }, host_set_, {50, 50}); - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + auto* child_factory = + Config::Utility::getFactoryByName(child_lb_name_); + envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin rr_config; + rr_config.mutable_locality_lb_config()->mutable_locality_weighted_lb_config(); + child_lb_config_ = + child_factory->loadConfig(rr_config, ProtobufMessage::getStrictValidationVisitor()); + initLbConfigAndLB(); + TestLoadBalancerContext context({{"version", "1.1"}}); // Since we scale the locality weights by number of hosts removed, we expect to see the second @@ -2327,13 +2417,14 @@ TEST_F(SubsetLoadBalancerTest, EnabledScaleLocalityWeightsRounding) { }, host_set_, {2, 2}); - common_config_.mutable_locality_weighted_lb_config(); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + auto* child_factory = + Config::Utility::getFactoryByName(child_lb_name_); + envoy::extensions::load_balancing_policies::round_robin::v3::RoundRobin rr_config; + rr_config.mutable_locality_lb_config()->mutable_locality_weighted_lb_config(); + child_lb_config_ = + child_factory->loadConfig(rr_config, ProtobufMessage::getStrictValidationVisitor()); + initLbConfigAndLB(); + TestLoadBalancerContext context({{"version", "1.0"}}); // We expect to see a 33/66 split because 2 * 1 / 2 = 1 and 2 * 3 / 4 = 1.5 -> 2 @@ -2363,12 +2454,7 @@ TEST_F(SubsetLoadBalancerTest, ScaleLocalityWeightsWithNoLocalityWeights) { }, host_set_); - auto child_lb_creator = std::make_unique( - lb_type_, ring_hash_lb_config_, maglev_lb_config_, round_robin_lb_config_, - least_request_lb_config_, common_config_); - lb_ = std::make_shared( - subset_info_, std::move(child_lb_creator), priority_set_, nullptr, stats_, - *stats_store_.rootScope(), runtime_, random_, simTime()); + initLbConfigAndLB(); } TEST_P(SubsetLoadBalancerTest, GaugesUpdatedOnDestroy) { @@ -2519,9 +2605,7 @@ TEST_F(SubsetLoadBalancerTest, AllowRedundantKeysForSubset) { envoy::extensions::load_balancing_policies::subset::v3::Subset subset_proto_config; TestUtility::loadFromYaml(yaml, subset_proto_config); - actual_subset_info_ = std::make_unique(subset_proto_config); - // Always be true for the LoadBalancerSubsetInfoImpl. - EXPECT_TRUE(actual_subset_info_->isEnabled()); + auto actual_subset_info = std::make_unique(subset_proto_config); // Add hosts initial hosts. init({{"tcp://127.0.0.1:80", {{"A", "A-V-0"}, {"B", "B-V-0"}, {"C", "C-V-0"}, {"D", "D-V-0"}}}, @@ -2531,7 +2615,7 @@ TEST_F(SubsetLoadBalancerTest, AllowRedundantKeysForSubset) { {"tcp://127.0.0.1:84", {{"A", "A-V-4"}, {"B", "B-V-4"}, {"C", "C-V-4"}, {"D", "D-V-4"}}}, {"tcp://127.0.0.1:85", {{"version", "1.0"}, {"stage", "dev"}}}, {"tcp://127.0.0.1:86", {{"version", "1.0"}, {"stage", "canary"}}}}, - {}, true); + {}, std::move(actual_subset_info)); TestLoadBalancerContext context_empty( std::initializer_list::value_type>{}); @@ -3232,6 +3316,6 @@ TEST(LoadBalancerContextWrapperTest, LoadBalancingContextWrapperTest) { wrapper.overrideHostToSelect(); } -} // namespace SubsetLoadBalancerTest +} // namespace } // namespace Upstream } // namespace Envoy diff --git a/test/mocks/upstream/cluster_info.cc b/test/mocks/upstream/cluster_info.cc index e8b32a821974..4038e57bd8c1 100644 --- a/test/mocks/upstream/cluster_info.cc +++ b/test/mocks/upstream/cluster_info.cc @@ -22,16 +22,6 @@ using testing::ReturnRef; namespace Envoy { namespace Upstream { -MockLoadBalancerSubsetInfo::MockLoadBalancerSubsetInfo() { - ON_CALL(*this, isEnabled()).WillByDefault(Return(false)); - ON_CALL(*this, fallbackPolicy()) - .WillByDefault(Return(envoy::config::cluster::v3::Cluster::LbSubsetConfig::ANY_ENDPOINT)); - ON_CALL(*this, defaultSubset()).WillByDefault(ReturnRef(ProtobufWkt::Struct::default_instance())); - ON_CALL(*this, subsetSelectors()).WillByDefault(ReturnRef(subset_selectors_)); -} - -MockLoadBalancerSubsetInfo::~MockLoadBalancerSubsetInfo() = default; - MockIdleTimeEnabledClusterInfo::MockIdleTimeEnabledClusterInfo() { ON_CALL(*this, idleTimeout()).WillByDefault(Return(std::chrono::milliseconds(1000))); } diff --git a/test/mocks/upstream/cluster_info.h b/test/mocks/upstream/cluster_info.h index a6df25bac0a4..b6d195ff0288 100644 --- a/test/mocks/upstream/cluster_info.h +++ b/test/mocks/upstream/cluster_info.h @@ -35,28 +35,6 @@ class FilterChainManager; namespace Upstream { -class MockLoadBalancerSubsetInfo : public LoadBalancerSubsetInfo { -public: - MockLoadBalancerSubsetInfo(); - ~MockLoadBalancerSubsetInfo() override; - - // Upstream::LoadBalancerSubsetInfo - MOCK_METHOD(bool, isEnabled, (), (const)); - MOCK_METHOD(envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetFallbackPolicy, - fallbackPolicy, (), (const)); - MOCK_METHOD(envoy::config::cluster::v3::Cluster::LbSubsetConfig::LbSubsetMetadataFallbackPolicy, - metadataFallbackPolicy, (), (const)); - MOCK_METHOD(const ProtobufWkt::Struct&, defaultSubset, (), (const)); - MOCK_METHOD(const std::vector&, subsetSelectors, (), (const)); - MOCK_METHOD(bool, localityWeightAware, (), (const)); - MOCK_METHOD(bool, scaleLocalityWeight, (), (const)); - MOCK_METHOD(bool, panicModeAny, (), (const)); - MOCK_METHOD(bool, listAsAny, (), (const)); - MOCK_METHOD(bool, allowRedundantKeys, (), (const)); - - std::vector subset_selectors_; -}; - // While this mock class doesn't have any direct use in public Envoy tests, it's // useful for constructing tests of downstream private filters that use // ClusterTypedMetadata.