diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 58d6b578bb..c4d86e758d 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -7,22 +7,40 @@ using namespace icinga; -void Checkable::AddDependency(const Dependency::Ptr& dep) +void Checkable::AddDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) { std::unique_lock lock(m_DependencyMutex); - m_Dependencies.insert(dep); + m_DependencyGroups.insert(dependencyGroup); } -void Checkable::RemoveDependency(const Dependency::Ptr& dep) +void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) { std::unique_lock lock(m_DependencyMutex); - m_Dependencies.erase(dep); + m_DependencyGroups.erase(dependencyGroup); +} + +std::vector Checkable::GetDependencyGroups() const +{ + std::unique_lock lock(m_DependencyMutex); + return {m_DependencyGroups.begin(), m_DependencyGroups.end()}; } std::vector Checkable::GetDependencies() const { std::unique_lock lock(m_DependencyMutex); - return std::vector(m_Dependencies.begin(), m_Dependencies.end()); + std::vector dependencies; + for (const auto& dependencyGroup : m_DependencyGroups) { + auto members(dependencyGroup->GetMembers(this)); + dependencies.insert(dependencies.end(), members.begin(), members.end()); + } + + return dependencies; +} + +bool Checkable::HasAnyDependencies() const +{ + std::unique_lock lock(m_DependencyMutex); + return !m_DependencyGroups.empty() || !m_ReverseDependencies.empty(); } void Checkable::AddReverseDependency(const Dependency::Ptr& dep) @@ -43,7 +61,7 @@ std::vector Checkable::GetReverseDependencies() const return std::vector(m_ReverseDependencies.begin(), m_ReverseDependencies.end()); } -bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency, int rstack) const +bool Checkable::IsReachable(DependencyType dt, int rstack) const { /* Anything greater than 256 causes recursion bus errors. */ int limit = 256; @@ -55,66 +73,53 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency return false; } - for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, failedDependency, rstack + 1)) - return false; - } - /* implicit dependency on host if this is a service */ const auto *service = dynamic_cast(this); if (service && (dt == DependencyState || dt == DependencyNotification)) { Host::Ptr host = service->GetHost(); if (host && host->GetState() != HostUp && host->GetStateType() == StateTypeHard) { - if (failedDependency) - *failedDependency = nullptr; - return false; } } - auto deps = GetDependencies(); - - std::unordered_map violated; // key: redundancy group, value: nullptr if satisfied, violating dependency otherwise - - for (const Dependency::Ptr& dep : deps) { - std::string redundancy_group = dep->GetRedundancyGroup(); - - if (!dep->IsAvailable(dt)) { - if (redundancy_group.empty()) { + for (auto& dependencyGroup : GetDependencyGroups()) { + if (!(dependencyGroup->GetState(dt, rstack + 1) & DependencyGroup::State::ReachableOK)) { + if (dependencyGroup->IsRedundancyGroup()) { // For non-redundant groups, this should already be logged. Log(LogDebug, "Checkable") - << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; - - if (failedDependency) - *failedDependency = dep; - - return false; + << "All dependencies in redundancy group '" << dependencyGroup->GetName() << "' have failed for checkable '" + << GetName() << "': Marking as unreachable."; } - - // tentatively mark this dependency group as failed unless it is already marked; - // so it either passed before (don't overwrite) or already failed (so don't care) - // note that std::unordered_map::insert() will not overwrite an existing entry - violated.insert(std::make_pair(redundancy_group, dep)); - } else if (!redundancy_group.empty()) { - violated[redundancy_group] = nullptr; + return false; } } - auto violator = std::find_if(violated.begin(), violated.end(), [](auto& v) { return v.second != nullptr; }); - if (violator != violated.end()) { - Log(LogDebug, "Checkable") - << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - - if (failedDependency) - *failedDependency = violator->second; + return true; +} +/** + * Checks whether the last check result of this Checkable affects its child dependencies. + * + * @return bool - Returns true if the Checkable affects its child dependencies, otherwise false. + */ +bool Checkable::AffectsChildren() const +{ + auto cr(GetLastCheckResult()); + if (!cr || IsStateOK(cr->GetState()) || !IsReachable()) { + // If there is no check result, the state is OK, or the Checkable is not reachable, we can't + // safely determine whether the Checkable affects its child dependencies. return false; } - if (failedDependency) - *failedDependency = nullptr; + for (auto& dep: GetReverseDependencies()) { + if (!dep->IsAvailable(DependencyState)) { + // If one of the child dependency is not available, then it's definitely due to the + // current Checkable state, so we don't need to verify the remaining ones. + return true; + } + } - return true; + return false; } std::set Checkable::GetParents() const @@ -145,6 +150,21 @@ std::set Checkable::GetChildren() const return parents; } +/** + * Retrieve the total number of all the children of the current Checkable. + * + * Note, due to the max recursion limit of 256, the returned number may not reflect + * the actual total number of children involved in the dependency chain. + * + * @return int - Returns the total number of all the children of the current Checkable. + */ +size_t Checkable::GetAllChildrenCount() const +{ + std::set children(GetChildren()); + GetAllChildrenInternal(children, 0); + return children.size(); +} + std::set Checkable::GetAllChildren() const { std::set children = GetChildren(); @@ -154,22 +174,36 @@ std::set Checkable::GetAllChildren() const return children; } +/** + * Retrieve all direct and indirect children of the current Checkable. + * + * Note, this function performs a recursive call chain traversing all the children of the current Checkable + * up to a certain limit (256). When that limit is reached, it will log a warning message and abort the operation. + * + * @param children - The set of children to be filled with all the children of the current Checkable. + * @param level - The current level of recursion. + */ void Checkable::GetAllChildrenInternal(std::set& children, int level) const { - if (level > 32) - return; + // The previous limit (32) doesn't seem to make sense, and appears to be some random number. + // So, this limit is set to 256 to match the limit in IsReachable(). + if (level > 256) { + Log(LogWarning, "Checkable") + << "Too many nested dependencies (>" << 256 << ") for checkable '" << GetName() << "': aborting traversal."; + return ; + } std::set localChildren; for (const Checkable::Ptr& checkable : children) { - std::set cChildren = checkable->GetChildren(); - - if (!cChildren.empty()) { + if (auto cChildren(checkable->GetChildren()); !cChildren.empty()) { GetAllChildrenInternal(cChildren, level + 1); localChildren.insert(cChildren.begin(), cChildren.end()); } - localChildren.insert(checkable); + if (level != 0) { // Recursion level 0 is the initiator, so checkable is already in the set. + localChildren.insert(checkable); + } } children.insert(localChildren.begin(), localChildren.end()); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index fcfbca9b28..6b8c7cfe0f 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -57,6 +57,7 @@ enum FlappingStateFilter class CheckCommand; class EventCommand; class Dependency; +class DependencyGroup; /** * An Icinga service. @@ -77,10 +78,12 @@ class Checkable : public ObjectImpl std::set GetParents() const; std::set GetChildren() const; std::set GetAllChildren() const; + size_t GetAllChildrenCount() const; void AddGroup(const String& name); - bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool IsReachable(DependencyType dt = DependencyState, int rstack = 0) const; + bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); @@ -182,9 +185,11 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ - void AddDependency(const intrusive_ptr& dep); - void RemoveDependency(const intrusive_ptr& dep); + void AddDependencyGroup(const intrusive_ptr& dependencyGroup); + void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); + std::vector> GetDependencyGroups() const; std::vector > GetDependencies() const; + bool HasAnyDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); void RemoveReverseDependency(const intrusive_ptr& dep); @@ -244,7 +249,7 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; - std::set > m_Dependencies; + std::set> m_DependencyGroups; std::set > m_ReverseDependencies; void GetAllChildrenInternal(std::set& children, int level = 0) const; diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 2843b906cb..0ad38d37d2 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -7,9 +7,12 @@ #include "base/initialize.hpp" #include "base/logger.hpp" #include "base/exception.hpp" +#include "base/utility.hpp" +#include "base/object-packer.hpp" #include #include #include +#include using namespace icinga; @@ -188,28 +191,91 @@ void Dependency::OnAllConfigLoaded() if (!m_Parent) BOOST_THROW_EXCEPTION(ScriptError("Dependency '" + GetName() + "' references a parent host/service which doesn't exist.", GetDebugInfo())); - m_Child->AddDependency(this); - m_Parent->AddReverseDependency(this); + if (!m_AssertNoCyclesForIndividualDeps) { + // Yes, this is rather a hack and introduces an inconsistent behaviour between file-based and + // runtime-created dependencies. However, I don't see any other possibility for handling this + // in a satisfiable manner. The reason for this change is simple, while Icinga DB doesn't need + // to know about every un/registered dependencies on startup, we still need to track and notify + // it about each and every dependency change at runtime to keep the database consistent. + DependencyGroup::Register(this); + m_Parent->AddReverseDependency(this); + } +} + +void Dependency::Start(bool runtimeCreated) +{ + if (runtimeCreated) { + std::vector previousGroups; + if (!GetRedundancyGroup().IsEmpty()) { + previousGroups = m_Child->GetDependencyGroups(); + } - if (m_AssertNoCyclesForIndividualDeps) { - DependencyCycleGraph graph; + DependencyGroup::Register(this); + m_Parent->AddReverseDependency(this); try { + DependencyCycleGraph graph; AssertNoDependencyCycle(m_Parent, graph); } catch (...) { - m_Child->RemoveDependency(this); + DependencyGroup::Unregister(this); m_Parent->RemoveReverseDependency(this); throw; } + + // Dependencies get activated before their parent Checkable objects, so if the child Checkable + // isn't active yet, it means that the Checkable is also created at runtime, and we don't need + // to notify Icinga DB, as the Checkable's OnVersionChanged signal will do that for us. + if (m_Child->IsActive()) { + std::vector outdatedGroups; + if (!previousGroups.empty()) { + auto newGroups(m_Child->GetDependencyGroups()); + std::set_difference( + previousGroups.begin(), + previousGroups.end(), + newGroups.begin(), + newGroups.end(), + std::back_inserter(outdatedGroups) + ); + } + DependencyGroup::OnMembersChanged(this, outdatedGroups); + } } + + ObjectImpl::Start(runtimeCreated); } void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this); + std::vector previousGroups; + std::set outdatedGroups; + if (runtimeRemoved && !GetRedundancyGroup().IsEmpty()) { + previousGroups = m_Child->GetDependencyGroups(); + for (auto& dependencyGroup: previousGroups) { + if (dependencyGroup->HasMember(this)) { + outdatedGroups.emplace(dependencyGroup); + break; + } + } + } + + DependencyGroup::Unregister(this); GetParent()->RemoveReverseDependency(this); + + if (runtimeRemoved) { + if (!previousGroups.empty()) { + auto newGroups(m_Child->GetDependencyGroups()); + std::set_difference( + previousGroups.begin(), + previousGroups.end(), + newGroups.begin(), + newGroups.end(), + std::inserter(outdatedGroups, outdatedGroups.end()) + ); + } + DependencyGroup::OnMembersChanged(this, {outdatedGroups.begin(), outdatedGroups.end()}); + } } bool Dependency::IsAvailable(DependencyType dt) const @@ -323,3 +389,474 @@ void Dependency::SetChild(intrusive_ptr child) { m_Child = child; } + +// Is the default (dummy) dependency group name used to group non-redundant dependencies. +static String l_DefaultDependencyGroup(Utility::NewUniqueID()); + +boost::signals2::signal&)> DependencyGroup::OnMembersChanged; + +std::mutex DependencyGroup::m_RegistryMutex; +DependencyGroup::RegistryType DependencyGroup::m_Registry; + +/** + * Register a dependency group in the global groups registry. + * + * At first, it tries to naively insert the dependency group into the registry. In case there is already an + * identical group in the registry, the insertion will just fail. In this case, it will move all + * members of the provided dependency group into the existing one and re-register the existing one recursively + * till there is no group with identical members left and the insertion finally succeeds. + * + * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex + * before calling this function. + * + * @param dependencyGroup The dependency group to register. + */ +void DependencyGroup::RegisterRedundancyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + if (auto it(m_Registry.insert(dependencyGroup.get())); !it.second) { + DependencyGroup::Ptr existingGroup(*it.first); + if (!existingGroup->IsRedundancyGroup()) { + // It's the dummy group, so just move the new members into the existing one, and it should be it. + dependencyGroup->MoveMembersTo(existingGroup); + } else { + // Erase it before we move the members into that group and change the hash. + m_Registry.erase(it.first); + dependencyGroup->MoveMembersTo(existingGroup); + RegisterRedundancyGroup(existingGroup); + } + } +} + +/** + * Refresh the registry of dependency groups. + * + * This function is used to refresh the global registry of dependency groups. + * It will first try to find an existing dependency group based on the provided dependency object's redundancy group. + * When it finds an identical group, it will move all its members (dependencies of the same child Checkable) into the + * new dependency group and add it to the registry. A nullptr as a new dependency group indicates that the provided + * dependency object should be unregistered from the registry. + * + * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex + * before calling this function. + * + * @param dependency The dependency object to refresh the registry for. + * @param newGroup The new dependency group to move the remaining Checkable dependencies into. + */ +void DependencyGroup::RefreshRegistry(const Dependency::Ptr& dependency, const DependencyGroup::Ptr& newGroup) +{ + auto [rangeBegin, rangeEnd] = m_Registry.get<1>().equal_range(dependency->GetRedundancyGroup()); + for (auto groupIt(rangeBegin); groupIt != rangeEnd; ++groupIt) { + DependencyGroup::Ptr existingGroup(*groupIt); + auto members(existingGroup->GetMembers(dependency->GetChild().get())); + if (members.empty()) { + continue; + } + + // Erase the existing dependency group from the registry, before we move the members + // out of it and change its identity, i.e. the hash value used by the registry. + m_Registry.erase(existingGroup.get()); + + DependencyGroup::Ptr replacementGroup(newGroup); + for (auto& member : members) { + // A nullptr newGroup means we want to unregister the provided dependency object. Otherwise, + // it should already be registered to the new dependency group, and we just need to move the + // remaining dependencies of the Checkable into that new group. + if (newGroup || member != dependency) { + if (!replacementGroup) { + replacementGroup = new DependencyGroup(existingGroup->GetName(), member); + } else { + // If there are any dependencies registered for the same child Checkable under the existing + // dependency group, we must move them into the new one. Meaning, the dependency group for + // that Checkable has changed, meanwhile for the other Checkables it's still the same. + replacementGroup->AddMember(member); + } + } + existingGroup->RemoveMember(member); + } + + if (existingGroup->HasMembers()) { + // Detach the existing dependency group from the child Checkable of the dependency + // object, as it's not a member of it anymore. Instead, we must... + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + + if (replacementGroup) { + // ...attach the new dependency group to the child Checkable. + dependency->GetChild()->AddDependencyGroup(replacementGroup); + RegisterRedundancyGroup(replacementGroup); + } + + // The existing dependency group still has some members left, so we must re-register + // it and enforce the rehashing of the members due to the removed ones above. + RegisterRedundancyGroup(existingGroup); + } else if (replacementGroup) { + // We were the last member of the existing dependency group, so instead of replacing it with the + // replacement group, we can just move back the members to it and re-add it to the registry. + replacementGroup->MoveMembersTo(existingGroup); + RegisterRedundancyGroup(existingGroup); + } else { + // The existing dependency group has no members left, so we must detach it from the child Checkable. + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + } + return; + } + + if (newGroup) { + // If we haven't found any existing dependency group to register the dependency to, we must + // attach the new dependency group to the child Checkable and register the new group to the registry. + dependency->GetChild()->AddDependencyGroup(newGroup); + RegisterRedundancyGroup(newGroup); + } +} + +/** + * Register the provided dependency to the global dependency group registry. + * + * @param dependency The dependency to register. + */ +void DependencyGroup::Register(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + auto groupName(dependency->GetRedundancyGroup()); + DependencyGroup::Ptr newGroup(new DependencyGroup(groupName.IsEmpty() ? l_DefaultDependencyGroup : groupName, dependency)); + if (m_Registry.empty() || groupName.IsEmpty()) { + dependency->GetChild()->AddDependencyGroup(newGroup); + RegisterRedundancyGroup(newGroup); + return; + } + + RefreshRegistry(dependency, newGroup); +} + +/** + * Unregister the provided dependency from the dependency group it was member of. + * + * @param dependency The dependency to unregister. + */ +void DependencyGroup::Unregister(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + if (dependency->GetRedundancyGroup().IsEmpty()) { + // Default dependency group of a given Checkable always produce the very same hash, so we can + // safely use any instance of it to find the existing one in the registry. + DependencyGroup::Ptr defaultGroup(new DependencyGroup(l_DefaultDependencyGroup, dependency)); + if (auto it(m_Registry.find(defaultGroup)); it != m_Registry.end()) { + DependencyGroup::Ptr existingGroup(*it); + existingGroup->RemoveMember(dependency); + if (!existingGroup->HasMembers()) { + // There are no members left in the default dependency group, so we must detach + // it from the child Checkable of the dependency object. + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + m_Registry.erase(it); + } + } + return; + } + + RefreshRegistry(dependency, Ptr()); +} + +/** + * Retrieve the size of the global dependency group registry. + * + * @return size_t - Returns the size of the global dependency groups registry. + */ +size_t DependencyGroup::GetRegistrySize() +{ + std::lock_guard lock(m_RegistryMutex); + return m_Registry.size(); +} + +DependencyGroup::DependencyGroup(String name, const Dependency::Ptr& dependency): m_Name(std::move(name)) +{ + AddMember(dependency); +} + +/** + * Create a composite key for the provided dependency. + * + * @param dependency The dependency object to create a composite key for. + * + * @return - Returns the composite key for the provided dependency. + */ +DependencyGroup::MemberTuple DependencyGroup::MakeCompositeKeyFor(const Dependency::Ptr& dependency) +{ + if (dependency->GetRedundancyGroup().IsEmpty()) { + // Just to make sure we don't have any duplicates in the default group. + return std::make_tuple(l_DefaultDependencyGroup, dependency->GetChild()->GetName(), 0, false); + } + + return std::make_tuple( + dependency->GetParent()->GetName(), + dependency->GetPeriodRaw(), + dependency->GetStateFilter(), + dependency->GetIgnoreSoftStates() + ); +} + +/** + * Check if the current dependency group is an explicitly configured redundancy group. + * + * @return bool - Returns true if the current dependency group isn't the dummy dependency group. + */ +bool DependencyGroup::IsRedundancyGroup() const +{ + return m_Name != l_DefaultDependencyGroup; +} + +/** + * Check if the current dependency group has any members. + * + * @return bool - Returns true if the current dependency group has any members. + */ +bool DependencyGroup::HasMembers() const +{ + std::lock_guard lock(m_Mutex); + return !m_Members.empty(); +} + +/** + * Check if the current dependency group has the provided dependency as a member. + * + * @param dependency The dependency to look for. + * + * @return bool - Returns true if the provided dependency is member of the current dependency group. + */ +bool DependencyGroup::HasMember(const Dependency::Ptr& dependency) const +{ + std::lock_guard lock(m_Mutex); + return m_Members.find(MakeCompositeKeyFor(dependency)) != m_Members.end(); +} + +/** + * Retrieve all members of the current dependency group the provided child Checkable depend on. + * + * Note, in order to ease duplicated dependencies exhaustion, the returned members are sorted by the parent Checkable. + * This way, all identical dependencies are placed next to each other, and you can easily consume them via a simple loop. + * + * @param child The child Checkable to look for. + * + * @return - Returns all members of the current dependency group the provided child depend on. + */ +std::vector DependencyGroup::GetMembers(const Checkable* child) const +{ + std::lock_guard lock(m_Mutex); + std::vector members; + for (auto& [_, dependencies] : m_Members) { + auto range(dependencies.equal_range(child)); + std::transform(range.first,range.second, std::back_inserter(members), [](const auto& member) { + return member.second; + }); + } + + std::sort(members.begin(), members.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + return members; +} + +/** + * Retrieve the number of members in the current dependency group. + * + * This function mainly exists for optimization purposes, i.e. instead of getting a copy of the members and + * counting them, we can directly query the number of members. + * + * @return - Returns the number of members in the current dependency group. + */ +size_t DependencyGroup::GetMemberCount() const +{ + std::lock_guard lock(m_Mutex); + return std::accumulate(m_Members.begin(), m_Members.end(), static_cast(0), [](int sum, const auto& pair) { + return sum + pair.second.size(); + }); +} + +/** + * Add a member to the current dependency group. + * + * @param member The dependency to add to the dependency group. + */ +void DependencyGroup::AddMember(const Dependency::Ptr& member) +{ + std::lock_guard lock(m_Mutex); + MemberTuple compositeKey(MakeCompositeKeyFor(member)); + if (auto it(m_Members.find(compositeKey)); it != m_Members.end()) { + it->second.emplace(member->GetChild().get(), member.get()); + } else { + m_Members.emplace(compositeKey, MemberValueType{{member->GetChild().get(), member.get()}}); + } +} + +/** + * Remove a member from the current dependency group. + * + * @param member The dependency to remove from the dependency group. + */ +void DependencyGroup::RemoveMember(const Dependency::Ptr& member) +{ + std::lock_guard lock(m_Mutex); + if (auto it(m_Members.find(MakeCompositeKeyFor(member))); it != m_Members.end()) { + auto [rangeBegin, rangeEnd] = it->second.equal_range(member->GetChild().get()); + for (auto memberIt(rangeBegin); memberIt != rangeEnd; ++memberIt) { + if (memberIt->second == member) { + // This will also remove the child Checkable from the multimap container + // entirely if this was the last member of it. + it->second.erase(memberIt); + // If the composite key has no more members left, we can remove it entirely as well. + if (it->second.empty()) { + m_Members.erase(it); + } + return; + } + } + } +} + +/** + * Move the members of the provided dependency group to the provided destination dependency group. + * + * @param dest The dependency group to move the members to. + */ +void DependencyGroup::MoveMembersTo(const DependencyGroup::Ptr& dest) +{ + VERIFY(this != dest); // Prevent from doing something stupid, i.e. deadlocking ourselves. + + std::lock_guard lock(m_Mutex); + DependencyGroup::Ptr thisPtr(this); // Just in case the Checkable below was our last reference. + for (auto& [_, members] : m_Members) { + Checkable::Ptr previousChild; + for (auto& [checkable, dependency] : members) { + dest->AddMember(dependency); + if (!previousChild || previousChild != checkable) { + previousChild = dependency->GetChild(); + previousChild->RemoveDependencyGroup(thisPtr); + previousChild->AddDependencyGroup(dest); + } + } + } +} + +/** + * Set the Icinga DB identifier for the current dependency group. + * + * The only usage of this function is the Icinga DB feature used to cache the unique hash of this dependency groups. + * + * @param identifier The Icinga DB identifier to set. + */ +void DependencyGroup::SetIcingaDBIdentifier(const String& identifier) +{ + std::lock_guard lock(m_Mutex); + m_IcingaDBIdentifier = identifier; +} + +/** + * Retrieve the Icinga DB identifier for the current dependency group. + * + * When the identifier is not already set by Icinga DB via the SetIcingaDBIdentifier method, + * this will just return an empty string. + * + * @return - Returns the Icinga DB identifier for the current dependency group. + */ +String DependencyGroup::GetIcingaDBIdentifier() const +{ + std::lock_guard lock(m_Mutex); + return m_IcingaDBIdentifier; +} + +/** + * Retrieve the (non-unique) name of the current dependency group. + * + * For explicitly configured redundancy groups, the name of the dependency group is the same as the one + * located in the configuration files. For non-redundant dependencies, on the other hand, the name is a randomly + * generated unique UUID (l_DefaultDependencyGroup). + * + * @return - Returns the name of the current dependency group. + */ +const String& DependencyGroup::GetName() const +{ + // We don't need to lock the mutex here, as the name is set once during + // the object construction and never changed afterwards. + return m_Name; +} + +/** + * Retrieve the unique composite key of the current dependency group. + * + * The composite key consists of some unique data of the group members, and should be used to generate a unique + * deterministic hash for the dependency group. Each key is a tuple of the parent name, the time period name + * (empty if not configured), the state filter, and the ignore soft states flag of the member. + * + * Additionally, to all the above mentioned keys, the non-unique dependency group name is also included. + * + * @return - Returns the composite key of the current dependency group. + */ +String DependencyGroup::GetCompositeKey() const +{ + std::lock_guard lock(m_Mutex); + if (!IsRedundancyGroup()) { + String childCheckableName; + if (!m_Members.empty()) { + childCheckableName = std::get<1>(m_Members.begin()->first); // See MemberTuple definition for the index. + } + return PackObject(new Array{GetName(), childCheckableName, 0, false}); + } + + Array::Ptr data(new Array{GetName()}); + for (auto& [compositeKey, _] : m_Members) { + auto [parentName, tpName, stateFilter, ignoreSoftStates] = compositeKey; + data->Add(std::move(parentName)); + data->Add(std::move(tpName)); + data->Add(stateFilter); + data->Add(ignoreSoftStates); + } + return PackObject(data); +} + +/** + * Retrieve the state of the current dependency group. + * + * The state of the dependency group is determined based on the state of the members of the group. + * This method returns a DependencyGroup::State::Unknown immediately when the group has no members. + * Otherwise, a dependency group is considered unreachable when none of the members is reachable. + * A reachable dependency group is failed when the edges connected to it are not available. + * + * @return - Returns the state of the current dependency group. + */ +DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) const +{ + MembersMap members; + { + // We don't want to hold the mutex lock for the entire evaluation, thus we just need to operate on a copy. + std::lock_guard lock(m_Mutex); + members = m_Members; + } + + bool isRedundant(IsRedundancyGroup()); + State state(members.empty() ? State::Unknown : State::Failed); + for (auto it(members.begin()); it != members.end() && (!isRedundant || state != State::ReachableOK); ++it) { + // Those are first batch of group members, i.e. they all share kind of the same Dependency config. + // So, for redundancy groups, we only need to check a single parent Checkable and dependency + // object of each such a group (see the preconditions of the below loop). + for (auto depIt(it->second.begin()); depIt != it->second.end() && (!isRedundant || depIt == it->second.begin()); ++depIt) { + auto [checkable, dependency] = *depIt; + if (!dependency->GetParent()->IsReachable(dt, rstack)) { + if (!isRedundant) { + // If any of the members is unreachable, the whole dependency group is unreachable, too. + return State::Unreachable; + } + state = State::UnreachableFailed; + } else if (!dependency->IsAvailable(dt)) { + if (!isRedundant) { + Log(LogDebug, "Checkable") + << "Non-redundant dependency '" << dependency->GetName() << "' failed for checkable '" + << checkable->GetName() << "': Marking as unreachable."; + + return State::Failed; + } + } else { + state = State::ReachableOK; + } + } + } + + return state; +} diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 6cebfaab1d..2e3ad77663 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -3,8 +3,13 @@ #ifndef DEPENDENCY_H #define DEPENDENCY_H +#include "base/shared-object.hpp" #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" +#include +#include +#include +#include namespace icinga { @@ -42,9 +47,27 @@ class Dependency final : public ObjectImpl void SetParent(intrusive_ptr parent); void SetChild(intrusive_ptr child); + /** + * A functor to compare the parent Checkable of a dependency with an explicitly provided parent Checkable. + * + * This is used to group dependencies by their parent Checkable (c++ 14 heterogeneous lookups). + */ + struct ParentComparator { + bool operator()(const Dependency::Ptr& dependency, const Checkable::Ptr& parent) const + { + return dependency->GetParent() < parent; + } + + bool operator()(const Checkable::Ptr& parent, const Dependency::Ptr& rhs) const + { + return parent < rhs->GetParent(); + } + }; + protected: void OnConfigLoaded() override; void OnAllConfigLoaded() override; + void Start(bool runtimeCreated) override; void Stop(bool runtimeRemoved) override; private: @@ -57,6 +80,132 @@ class Dependency final : public ObjectImpl static bool EvaluateApplyRule(const Checkable::Ptr& checkable, const ApplyRule& rule, bool skipFilter = false); }; +/** +* @ingroup icinga +*/ +class DependencyGroup final : public SharedObject +{ +public: + DECLARE_PTR_TYPEDEFS(DependencyGroup); + + /** + * Defines the key type of each dependency group members. + * + * For dependency groups **with** an explicitly configured redundancy group, that tuple consists of the dependency + * parent name, the dependency time period name (empty if not configured), the state filter, and the + * ignore soft states flag. + * + * For the non-redundant group (just a bunch of dependencies without a redundancy group) of a given Checkable, + * the tuple consists of the dependency group name (which is a randomly generated unique UUID), the child + * Checkable name, the state filter (is always 0), and the ignore soft states flag (is always false). + */ + using MemberTuple = std::tuple; + using MemberValueType = std::unordered_multimap; + using MembersMap = std::map; + + DependencyGroup(String name, const Dependency::Ptr& member); + + static void Register(const intrusive_ptr& dependency); + static void Unregister(const intrusive_ptr& dependency); + static size_t GetRegistrySize(); + + static MemberTuple MakeCompositeKeyFor(const Dependency::Ptr& dependency); + + bool IsRedundancyGroup() const; + bool HasMembers() const; + bool HasMember(const Dependency::Ptr& dependency) const; + std::vector GetMembers(const Checkable* child) const; + size_t GetMemberCount() const; + + void SetIcingaDBIdentifier(const String& identifier); + String GetIcingaDBIdentifier() const; + + const String& GetName() const; + String GetCompositeKey() const; + + enum State + { + Unknown = 1ull << 0, + Failed = 1ull << 1, + Unreachable = 1ull << 2, + ReachableOK = 1ull << 3, + UnreachableFailed = Unreachable | Failed, + }; + + State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + + static boost::signals2::signal&)> OnMembersChanged; + +protected: + void AddMember(const Dependency::Ptr& member); + void RemoveMember(const Dependency::Ptr& member); + void MoveMembersTo(const DependencyGroup::Ptr& other); + + static void RegisterRedundancyGroup(const DependencyGroup::Ptr& dependencyGroup); + static void RefreshRegistry(const intrusive_ptr& dependency, const DependencyGroup::Ptr& newGroup = nullptr); + +private: + mutable std::mutex m_Mutex; + String m_IcingaDBIdentifier; + String m_Name; + MembersMap m_Members; + + struct Hash { + /** + * Calculates the hash value of a dependency group used by DependencyGroup::RegistryType. + * + * @param dependencyGroup The dependency group to calculate the hash value for. + * + * @return Returns the hash value of the dependency group. + */ + size_t operator()(const DependencyGroup::Ptr& dependencyGroup) const + { + return std::hash{}(dependencyGroup->GetCompositeKey()); + } + }; + + struct Equal { + /** + * Checks two dependency groups for equality. + * + * The equality of two dependency groups is determined by the equality of their composite keys. + * That composite key consists of a tuple of the parent name, the time period name (empty if not configured), + * state filter, and the ignore soft states flag of the member. + * + * @param lhs The first dependency group to compare. + * @param rhs The second dependency group to compare. + * + * @return Returns true if the composite keys of the two dependency groups are equal. + */ + bool operator()(const DependencyGroup::Ptr& lhs, const DependencyGroup::Ptr& rhs) const + { + return lhs->GetCompositeKey() == rhs->GetCompositeKey(); + } + }; + + using RegistryType = boost::multi_index_container< + DependencyGroup*, // The type of the elements stored in the container. + boost::multi_index::indexed_by< + // The first index is a unique index based on the identity of the dependency group. + // The identity of the dependency group is determined by the provided Hash and Equal functors. + boost::multi_index::hashed_unique, Hash, Equal>, + // This non-unique index allows to search for dependency groups by their name, and reduces the overall + // runtime complexity. Without this index, we would have to iterate over all elements to find the on with + // the desired members and since std::unordered_set doesn't allow erasing elements while iterating, we would + // have to copy each of them to a temporary container, and then erase and reinsert them back t the original + // container. This produces way too much overhead, and slows down the startup time of Icinga 2 significantly. + boost::multi_index::hashed_non_unique< + boost::multi_index::const_mem_fun, + std::hash + > + > + >; + + // The global registry of dependency groups. + static std::mutex m_RegistryMutex; + static RegistryType m_Registry; +}; + } #endif /* DEPENDENCY_H */ diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 41de7ba23c..b588771041 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -20,6 +20,8 @@ public: class Dependency : CustomVarObject < DependencyNameComposer { + activation_priority -10; + load_after Host; load_after Service; @@ -77,7 +79,7 @@ class Dependency : CustomVarObject < DependencyNameComposer }}} }; - [config] String redundancy_group; + [config, no_user_modify] String redundancy_group; [config, navigation] name(TimePeriod) period (PeriodRaw) { navigate {{{ diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 920251969f..d53f8cebb3 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -19,6 +19,7 @@ #include "icinga/command.hpp" #include "icinga/compatutility.hpp" #include "icinga/customvarobject.hpp" +#include "icinga/dependency.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" @@ -132,6 +133,8 @@ void IcingaDB::ConfigStaticInitialize() IcingaDB::NextCheckUpdatedHandler(checkable); }); + DependencyGroup::OnMembersChanged.connect(&IcingaDB::DependencyGroupsChangedHandler); + Service::OnHostProblemChanged.connect([](const Service::Ptr& service, const CheckResult::Ptr&, const MessageOrigin::Ptr&) { IcingaDB::HostProblemChangedHandler(service); }); @@ -174,7 +177,7 @@ void IcingaDB::ConfigStaticInitialize() void IcingaDB::UpdateAllConfigObjects() { m_Rcon->Sync(); - m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "5"}, Prio::Heartbeat); + m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "6"}, Prio::Heartbeat); Log(LogInformation, "IcingaDB") << "Starting initial config/status dump"; double startTime = Utility::GetTime(); @@ -203,10 +206,20 @@ void IcingaDB::UpdateAllConfigObjects() m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "MAXLEN", "1", "*", "key", "*", "state", "wip"}, Prio::Config); const std::vector globalKeys = { - m_PrefixConfigObject + "customvar", - m_PrefixConfigObject + "action:url", - m_PrefixConfigObject + "notes:url", - m_PrefixConfigObject + "icon:image", + m_PrefixConfigObject + "customvar", + m_PrefixConfigObject + "action:url", + m_PrefixConfigObject + "notes:url", + m_PrefixConfigObject + "icon:image", + + // These keys aren't tied to a specific Checkable object but apply to all of them, and as such + // we've to make sure to clear them before we actually start dumping the actual objects. + // This also allows us to wait on all the Checkables to be dumped before we send a config + // dump done signal for those keys. + m_PrefixConfigObject + "dependency:node", + m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "dependency:edge:state", + m_PrefixConfigObject + "redundancygroup", + m_PrefixConfigObject + "redundancygroup:state", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -217,6 +230,7 @@ void IcingaDB::UpdateAllConfigObjects() m_DumpedGlobals.ActionUrl.Reset(); m_DumpedGlobals.NotesUrl.Reset(); m_DumpedGlobals.IconImage.Reset(); + m_DumpedGlobals.RedundancyGroup.Reset(); }); upq.ParallelFor(types, false, [this](const Type::Ptr& type) { @@ -788,6 +802,8 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } + InsertCheckableDependencies(checkable, hMSets, runtimeUpdate ? &runtimeUpdates : nullptr); + return; } @@ -1121,6 +1137,141 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } +void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates) +{ + // Only generate a dependency node event if the Checkable is actually part of some dependency graph. + // That's, it either depends on other Checkables or others depend on it, and in both cases, we have + // to at least generate a dependency node entry for it. + if (!checkable->HasAnyDependencies()) { + return; + } + + auto& hmsetDependencyNodes(hMSets[m_PrefixConfigObject + "dependency:node"]); + auto& hmsetDependencyEdges(hMSets[m_PrefixConfigObject + "dependency:edge"]); + auto& hmsetRedundancyGroups(hMSets[m_PrefixConfigObject + "redundancygroup"]); + + // If this isn't a runtime update, we need to send initial state updates for the dependencies as well. + auto& hmsetDependenciesStates(hMSets[m_PrefixConfigObject + "dependency:edge:state"]); + auto& hmsetRedundancyGroupsStates(hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + + auto [host, service] = GetHostService(checkable); + auto checkableId(GetObjectIdentifier(checkable)); + { + Dictionary::Ptr data(new Dictionary{{"environment_id", m_EnvironmentId}, {"host_id", GetObjectIdentifier(host)}}); + if (service) { + data->Set("service_id", checkableId); + } + + hmsetDependencyNodes.emplace_back(checkableId); + hmsetDependencyNodes.emplace_back(JsonEncode(data)); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, checkableId, m_PrefixConfigObject + "dependency:node", data); + } + } + + for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + String redundancyGroupId(dependencyGroup->GetIcingaDBIdentifier()); + bool syncSharedEdgeState(false); + if (dependencyGroup->IsRedundancyGroup()) { + redundancyGroupId = HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()}); + dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); + + // Sync redundancy group information only once unless it's a runtime update. + if (runtimeUpdates || m_DumpedGlobals.RedundancyGroup.IsNew(redundancyGroupId)) { + syncSharedEdgeState = true; + Dictionary::Ptr groupData(SerializeRedundancyGroup(dependencyGroup)); + hmsetRedundancyGroups.emplace_back(redundancyGroupId); + hmsetRedundancyGroups.emplace_back(JsonEncode(groupData)); + + Dictionary::Ptr nodeData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + }); + + hmsetDependencyNodes.emplace_back(redundancyGroupId); + hmsetDependencyNodes.emplace_back(JsonEncode(nodeData)); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } else { + auto stateAttrs(SerializeRedundancyGroup(dependencyGroup, true)); + hmsetRedundancyGroupsStates.emplace_back(redundancyGroupId); + hmsetRedundancyGroupsStates.emplace_back(JsonEncode(stateAttrs)); + + hmsetDependenciesStates.emplace_back(redundancyGroupId); + hmsetDependenciesStates.emplace_back(JsonEncode(Dictionary::Ptr(new Dictionary{ + {"id", redundancyGroupId}, + {"environment_id", m_EnvironmentId}, + {"failed", stateAttrs->Get("failed")}, + }))); + } + } + + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", checkableId}, + {"to_node_id", redundancyGroupId}, + // All redundancy group members share the same state, thus use the group ID as a reference. + {"dependency_edge_state_id", redundancyGroupId}, + {"display_name", dependencyGroup->GetName()}, + }); + + auto edgeId(HashValue(new Array{checkableId, redundancyGroupId})); + hmsetDependencyEdges.emplace_back(edgeId); + hmsetDependencyEdges.emplace_back(JsonEncode(data)); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } + + auto members(dependencyGroup->GetMembers(checkable.get())); + for (auto it(members.begin()); it != members.end(); /* no increment */) { + auto dependency(*it); + auto parent(dependency->GetParent()); + auto displayName(dependency->GetShortName()); + + Dictionary::Ptr memberStateAttrs; + if (!runtimeUpdates && (!dependencyGroup->IsRedundancyGroup() || syncSharedEdgeState)) { + memberStateAttrs = SerializeDependencyEdgeState(dependencyGroup, dependency); + } + + // Find all members that share the same parent starting from the iterator position (it position inclusively). + auto [begin, end] = std::equal_range(it, members.end(), parent, Dependency::ParentComparator{}); + // We might not need to process the element pointed to by (it) as well, so we need to + // exclude it from the returned range (notice the ++begin below). + std::for_each(++begin, end, [&displayName, &memberStateAttrs, &dependencyGroup](const Dependency::Ptr& member) { + displayName += ", " + member->GetShortName(); + if (memberStateAttrs && memberStateAttrs->Get("failed") == false) { + memberStateAttrs = SerializeDependencyEdgeState(dependencyGroup, member); + } + }); + it = end; // Advance the iterator to either end of the container or beginning of the next members block. + + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyGroup->IsRedundancyGroup() ? redundancyGroupId : checkableId}, + {"to_node_id", GetObjectIdentifier(parent)}, + {"display_name", displayName}, + }); + String edgeId(HashValue(new Array{data->Get("from_node_id"), data->Get("to_node_id")})); + data->Set("dependency_edge_state_id", edgeId); + + hmsetDependencyEdges.emplace_back(edgeId); + hmsetDependencyEdges.emplace_back(JsonEncode(data)); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } else if (memberStateAttrs) { + hmsetDependenciesStates.emplace_back(edgeId); + hmsetDependenciesStates.emplace_back(JsonEncode(memberStateAttrs)); + } + } + } +} + /** * Update the state information of a checkable in Redis. * @@ -1175,6 +1326,134 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) m_Rcon->FireAndForgetQuery(std::move(streamadd), Prio::RuntimeStateStream, {0, 1}); } + + UpdateDependenciesState(checkable, mode); +} + +/** + * Send dependencies state information of the given Checkable to Redis. + * + * For explicitly configured redundancy groups, the state information is always sent to Redis, regardless of the + * mode parameter. The mode parameter loosely controls how and which states are sent to Redis for non-redundant + * dependencies. The value of that parameter can be one of the following: + * - Volatile: Check each dependency object of the specified Checkable for its configuration before sending + * the state update. For instance, if the dependency is configured to ignore soft state changes, its state + * should not be different from the previous ones and will therefore not be sent to Redis. + * - RuntimeOnly: Performs an additional check to the one above and ignores dependency objects that are not within + * their time period. Once the dependencies pass all the necessary checks, these two mods only send the state + * information to the icinga:runtime:state Redis stream via XADD. + * - Full: Perform a full state update ignoring all the above mentioned checks. Additionally, set the same + * state information that is streamed into the pipeline to the Redis keys icinga:{dependency:edge,redundancygroup}:state + * as well. + * + * @param checkable The Checkable you want to send the dependencies state update for + * @param mode The type of operation you want to perform (Volatile, RuntimeOnly, or Full) + */ +void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, StateUpdate mode) const +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto dependencyGroups(checkable->GetDependencyGroups()); + if (dependencyGroups.empty()) { + return; + } + + RedisConnection::Queries streamStates, hmsets{{/* dependency states */}, {/* redundancy group states */}}; + auto addDependencyStateToStream([this, &streamStates](const String& redisKey, const Dictionary::Ptr& stateAttrs) { + RedisConnection::Query xAdd{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", + "runtime_type", "upsert", "redis_key", redisKey + }; + ObjectLock olock(stateAttrs); + for (auto& [key, value] : stateAttrs) { + xAdd.emplace_back(key); + xAdd.emplace_back(IcingaToStreamValue(value)); + } + streamStates.emplace_back(std::move(xAdd)); + }); + + auto now(Utility::GetTime()); + for (auto& dependencyGroup : dependencyGroups) { + bool isRedundancyGroup(dependencyGroup->IsRedundancyGroup()); + if (isRedundancyGroup && dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // Way too soon! The Icinga DB hash will be set during the initial config dump, but this state + // update seems to occur way too early. So, we've to skip it for now and wait for the next one. + // The m_ConfigDumpInProgress flag is probably still set to true at this point! + continue; + } + + auto members(dependencyGroup->GetMembers(checkable.get())); + for (auto it(members.begin()); it != members.end(); /* no increment */) { + auto dep(*it); + if (auto tp(dep->GetPeriod()); !isRedundancyGroup && mode != StateUpdate::Full && tp && !tp->IsInside(now)) { + // When the dependency is outside its time period, and we're not performing a full + // state update, we don't need to send any updates as it should always be reachable. + continue; + } + + Dictionary::Ptr stateAttrs(SerializeDependencyEdgeState(dependencyGroup, dep)); + // Find all members that share the same parent starting from the iterator position (it position inclusively). + auto [begin, end] = std::equal_range(it, members.end(), dep->GetParent(), Dependency::ParentComparator{}); + // We might not need to process the element pointed to by (it) as well, so we need to + // exclude it from the returned range (notice the ++begin below). + std::for_each(++begin, end, [&stateAttrs, &dependencyGroup](const Dependency::Ptr& member) { + if (stateAttrs->Get("failed") == false) { + stateAttrs = SerializeDependencyEdgeState(dependencyGroup, member); + } + }); + it = end; // Advance the iterator to either end of the container or beginning of the next members block. + + if (mode == StateUpdate::Full) { + // Actually we wouldn't need to send state updates via HMSET for dependency states here, + // as we're going to stream them via the runtime:state pipeline anyway. However, if we + // don't do this, Icinga DB is going to remove them from the DB and reinsert them again + // with each config dump. So, we've to refresh the non-stream keys regularly! + hmsets[0].emplace_back(stateAttrs->Get("id")); + hmsets[0].emplace_back(JsonEncode(stateAttrs)); + } + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", stateAttrs); + } + + if (isRedundancyGroup) { + Dictionary::Ptr stateAttrs(SerializeRedundancyGroup(dependencyGroup, true)); + Dictionary::Ptr groupSharedState(stateAttrs->ShallowClone()); + groupSharedState->Remove("redundancy_group_id"); + groupSharedState->Remove("is_reachable"); + groupSharedState->Remove("last_state_change"); + + if (mode == StateUpdate::Full) { + // Same as above, we've to refresh the group state information regularly to prevent + // Icinga DB from removing and reinserting them with each config dump. + hmsets[0].emplace_back(dependencyGroup->GetIcingaDBIdentifier()); + hmsets[0].emplace_back(JsonEncode(groupSharedState)); + hmsets[1].emplace_back(dependencyGroup->GetIcingaDBIdentifier()); + hmsets[1].emplace_back(JsonEncode(stateAttrs)); + } + addDependencyStateToStream(m_PrefixConfigObject + "redundancygroup:state", stateAttrs); + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", groupSharedState); + } + } + + if (!hmsets[0].empty() || !hmsets[1].empty()) { + if (!hmsets.front().empty()) { + hmsets[0].insert(hmsets[0].begin(), {"DEL", m_PrefixConfigObject + "dependency:edge:state"}); + } + if (!hmsets[1].empty()) { + hmsets[1].insert(hmsets[1].begin(), {"HMSET", m_PrefixConfigObject + "redundancygroup:state"}); + } + + hmsets.erase( + std::remove_if(hmsets.begin(), hmsets.end(), std::mem_fn(&RedisConnection::Query::empty)), + hmsets.end() + ); + m_Rcon->FireAndForgetQueries(std::move(hmsets), Prio::RuntimeStateSync); + } + + if (!streamStates.empty()) { + m_Rcon->FireAndForgetQueries(std::move(streamStates), Prio::RuntimeStateStream, {0, 1}); + } } // Used to update a single object, used for runtime updates @@ -1194,34 +1473,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile); } - std::vector > transaction = {{"MULTI"}}; - - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - for (auto& objectAttributes : runtimeUpdates) { - std::vector xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); - ObjectLock olock(objectAttributes); - - for (const Dictionary::Pair& kv : objectAttributes) { - String value = IcingaToStreamValue(kv.second); - if (!value.IsEmpty()) { - xAdd.emplace_back(kv.first); - xAdd.emplace_back(value); - } - } - - transaction.emplace_back(std::move(xAdd)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); - } + ExecuteRedisTransaction(hMSets, runtimeUpdates); if (checkable) { SendNextUpdate(checkable); @@ -1308,6 +1560,11 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("notes", checkable->GetNotes()); attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); + if (size_t affectedChildren (checkable->GetAllChildrenCount()); affectedChildren > 0) { + // Only set the Redis key if the Checkable has actually some child dependencies. + attributes->Set("affected_children", affectedChildren); + } + attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); @@ -2580,6 +2837,83 @@ void IcingaDB::SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dict } } +void IcingaDB::SendDependencyGroupsChanged(const Dependency::Ptr& dep, const std::vector& outdatedGroups) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto parent(dep->GetParent()); + auto child(dep->GetChild()); + if (!child->HasAnyDependencies()) { + // If the child Checkable has no parent and reverse dependencies, we can safely remove the dependency node. + DeleteRelationship(GetObjectIdentifier(child), "dependency:node"); + } + + RedisConnection::Queries hdels, xAdds; + auto deleteState([this, &hdels, &xAdds](const String& redisKey, const String& id) { + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigObject + redisKey, id}); + xAdds.emplace_back(RedisConnection::Query{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", "runtime_type", "delete", + "redis_key", m_PrefixConfigObject + redisKey, "id", id + }); + }); + + if (dep->GetRedundancyGroup().IsEmpty() && !dep->IsActive() && dep->GetExtension("ConfigObjectDeleted")) { + auto identifier(HashValue(new Array{GetObjectIdentifier(child), GetObjectIdentifier(parent)})); + // Remove the edge between the parent and child Checkable linked through the removed dependency. + DeleteRelationship(identifier, "dependency:edge"); + // The dependency object is going to be deleted, so we need to remove its state from Redis as well. + deleteState("dependency:edge:state", identifier); + } + + auto newGroups(child->GetDependencyGroups()); + for (auto& group : outdatedGroups) { + String redundancyGroupId(group->GetIcingaDBIdentifier()); + bool sameCompositeKey(redundancyGroupId == HashValue(new Array{m_EnvironmentId, group->GetCompositeKey()})); + if (!sameCompositeKey || !group->HasMembers() || std::find(newGroups.begin(), newGroups.end(), group) == newGroups.end()) { + // Remove the connection from the child Checkable to the redundancy group. + DeleteRelationship(HashValue(new Array{GetObjectIdentifier(child), redundancyGroupId}), "dependency:edge"); + + if (!sameCompositeKey || !group->HasMembers()) { + auto members(group->GetMembers(child.get())); + members.emplace_back(dep); + for (auto& member : members) { + // Remove the connection from the redundancy group to the parent Checkable of the removed + // dependency, if there's no other member in the redundancy group that references that very + // same parent Checkable or the dependency Group's Icinga DB identifier has changed. + auto groupParentId(HashValue(new Array{redundancyGroupId, GetObjectIdentifier(member->GetParent())})); + DeleteRelationship(groupParentId, "dependency:edge"); + deleteState("dependency:edge:state", groupParentId); + } + + // Remove the group entirely as it either has no members left or it's Icinga DB identifier has changed. + deleteState("dependency:edge:state", redundancyGroupId); + deleteState("redundancygroup:state", redundancyGroupId); + DeleteRelationship(redundancyGroupId, "dependency:node"); + DeleteRelationship(redundancyGroupId, "redundancygroup"); + } + } + } + + if (!hdels.empty()) { + m_Rcon->FireAndForgetQueries(std::move(hdels), Prio::RuntimeStateSync); + m_Rcon->FireAndForgetQueries(std::move(xAdds), Prio::RuntimeStateStream, {0, 1}); + + if (!newGroups.empty()) { + std::map hMSets; + std::vector runtimeUpdates; + InsertCheckableDependencies(child, hMSets, &runtimeUpdates); + + ExecuteRedisTransaction(hMSets, runtimeUpdates); + UpdateState(child, StateUpdate::Full); + } + } + + // The affect{ed,s}_children might now have different outcome, so we need to update the parent Checkable as well. + SendConfigUpdate(parent, true); +} + Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -2623,6 +2957,7 @@ Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); + attrs->Set("affects_children", checkable->AffectsChildren()); CheckResult::Ptr cr = checkable->GetLastCheckResult(); @@ -2760,6 +3095,7 @@ void IcingaDB::ReachabilityChangeHandler(const std::set& childre for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { for (auto& checkable : children) { rw->UpdateState(checkable, StateUpdate::Full); + IcingaDB::ReachabilityChangeHandler(checkable->GetChildren()); } } } @@ -2856,6 +3192,13 @@ void IcingaDB::NextCheckUpdatedHandler(const Checkable::Ptr& checkable) } } +void IcingaDB::DependencyGroupsChangedHandler(const Dependency::Ptr& dep, const std::vector& outdatedGroups) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupsChanged(dep, outdatedGroups); + } +} + void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) { for (auto& rw : ConfigType::GetObjectsByType()) { /* Host state changes affect is_handled and severity of services. */ @@ -2973,3 +3316,33 @@ void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithou m_Rcon->FireAndForgetQueries(queries, Prio::Config); } + +void IcingaDB::ExecuteRedisTransaction(std::map& hMSets, const std::vector& runtimeUpdates) const +{ + RedisConnection::Queries transaction{{"MULTI"}}; + for (auto& [redisKey, query] : hMSets) { + if (!query.empty()) { + query.insert(query.begin(), {"HMSET", redisKey}); + transaction.emplace_back(std::move(query)); + } + } + + for (auto& attrs : runtimeUpdates) { + RedisConnection::Query xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); + + ObjectLock olock(attrs); + for (auto& [key, value]: attrs) { + if (auto streamVal (IcingaToStreamValue(value)); !streamVal.IsEmpty()) { + xAdd.emplace_back(key); + xAdd.emplace_back(std::move(streamVal)); + } + } + + transaction.emplace_back(std::move(xAdd)); + } + + if (transaction.size() > 1) { + transaction.emplace_back(RedisConnection::Query{"EXEC"}); + m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); + } +} diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 35f503ab53..f8b96ef198 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -159,6 +159,54 @@ Dictionary::Ptr IcingaDB::SerializeVars(const Dictionary::Ptr& vars) return res; } +/** + * Serialize a dependency edge state for Icinga DB + * + * @param dependencyGroup The state of the group the dependency is part of. + * @param dep The dependency object to serialize. + * + * @return A dictionary with the serialized state. + */ +Dictionary::Ptr IcingaDB::SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep) +{ + auto child(dep->GetChild()); + Array::Ptr data(new Array{ + dependencyGroup->IsRedundancyGroup() ? dependencyGroup->GetIcingaDBIdentifier() : GetObjectIdentifier(child), + GetObjectIdentifier(dep->GetParent()) + }); + + return new Dictionary{ + {"id", HashValue(data)}, + {"environment_id", m_EnvironmentId}, + {"failed", !dep->IsAvailable(DependencyState) || !dep->GetParent()->IsReachable()} + }; +} + +/** + * Serialize the provided redundancy group or its state attributes. + * + * @param redundancyGroup The redundancy group object to serialize. + * @param serializeState Whether to serialize the state of the redundancy group. + * + * @return A dictionary with the serialized redundancy group. + */ +Dictionary::Ptr IcingaDB::SerializeRedundancyGroup(const DependencyGroup::Ptr& redundancyGroup, bool serializeState) +{ + if (serializeState) { + auto state(redundancyGroup->GetState()); + return new Dictionary{ + {"id", redundancyGroup->GetIcingaDBIdentifier()}, + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroup->GetIcingaDBIdentifier()}, + {"failed", !(state & DependencyGroup::State::ReachableOK)}, + {"is_reachable", !(state & DependencyGroup::State::Unreachable)}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}, + }; + } + + return new Dictionary{{"environment_id", m_EnvironmentId}, {"display_name", redundancyGroup->GetName()}}; +} + const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type) { switch (type) { diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 6652d9c1f4..e2db2eea8a 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -101,8 +101,11 @@ class IcingaDB : public ObjectImpl void DeleteKeys(const RedisConnection::Ptr& conn, const std::vector& keys, RedisConnection::QueryPriority priority); std::vector GetTypeOverwriteKeys(const String& type); std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); + void InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependenciesState(const Checkable::Ptr& checkable, StateUpdate mode) const; void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -112,6 +115,7 @@ class IcingaDB : public ObjectImpl void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false); + void ExecuteRedisTransaction(std::map& hMSets, const std::vector& runtimeUpdates) const; void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, @@ -136,6 +140,7 @@ class IcingaDB : public ObjectImpl void SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + void SendDependencyGroupsChanged(const Dependency::Ptr& dep, const std::vector& outdatedGroups); void ForwardHistoryEntries(); @@ -157,6 +162,8 @@ class IcingaDB : public ObjectImpl static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0)); static const char* GetNotificationTypeByEnum(NotificationType type); static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars); + static Dictionary::Ptr SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep); + static Dictionary::Ptr SerializeRedundancyGroup(const DependencyGroup::Ptr& redundancyGroup, bool serializeState = false); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); @@ -180,6 +187,7 @@ class IcingaDB : public ObjectImpl static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime); static void NewCheckResultHandler(const Checkable::Ptr& checkable); static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable); + static void DependencyGroupsChangedHandler(const Dependency::Ptr& dep, const std::vector& outdatedGroups); static void HostProblemChangedHandler(const Service::Ptr& service); static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry); static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime); @@ -225,7 +233,7 @@ class IcingaDB : public ObjectImpl std::atomic_size_t m_PendingRcons; struct { - DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; + DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage, RedundancyGroup; } m_DumpedGlobals; // m_EnvironmentId is shared across all IcingaDB objects (typically there is at most one, but it is perfectly fine diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a255178da2..eb7df6ce4b 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -230,6 +230,9 @@ add_boost_test(base icinga_checkresult/service_flapping_notification icinga_checkresult/suppressed_notification icinga_dependencies/multi_parent + icinga_dependencies/default_redundancy_group_registration_unregistration + icinga_dependencies/simple_redundancy_group_registration_unregistration + icinga_dependencies/mixed_redundancy_group_registration_unregsitration icinga_notification/strings icinga_notification/state_filter icinga_notification/type_filter diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 929b6ca0de..c542e254bf 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -9,6 +9,52 @@ using namespace icinga; BOOST_AUTO_TEST_SUITE(icinga_dependencies) +static Host::Ptr CreateHost(const std::string& name) +{ + Host::Ptr host = new Host(); + host->SetName(name); + return host; +} + +static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const std::string& name) +{ + Dependency::Ptr dep = new Dependency(); + dep->SetParent(parent); + dep->SetChild(child); + dep->SetName(name + "!" + child->GetName()); + return dep; +} + +static void RegisterDependency(Dependency::Ptr dep, const std::string& redundancyGroup) +{ + dep->SetRedundancyGroup(redundancyGroup); + DependencyGroup::Register(dep); + dep->GetParent()->AddReverseDependency(dep); +} + +static void AssertCheckableRedundancyGroup(Checkable::Ptr checkable, int dependencyCount, int redundancyGroupCount, int memberCount) +{ + BOOST_CHECK_MESSAGE( + dependencyCount == checkable->GetDependencies().size(), + "Dependency count mismatch for '" << checkable->GetName() << "' - expected=" << dependencyCount << "; got=" + << checkable->GetDependencies().size() + ); + auto redundancyGroups(checkable->GetDependencyGroups()); + BOOST_CHECK_MESSAGE( + redundancyGroupCount == redundancyGroups.size(), + "Redundancy group count mismatch for '" << checkable->GetName() << "'" << " - expected=" << redundancyGroupCount + << "; got=" << redundancyGroups.size() + ); + if (redundancyGroupCount > 0) { + BOOST_REQUIRE_MESSAGE(1 <= redundancyGroups.size(), "Checkable '" << checkable->GetName() << "' should have at least one redundancy group."); + BOOST_CHECK_MESSAGE( + memberCount == redundancyGroups.begin()->get()->GetMemberCount(), + "Member count mismatch for '" << checkable->GetName() << "'" << " - expected=" << memberCount + << "; got=" << redundancyGroups.begin()->get()->GetMemberCount() + ); + } +} + BOOST_AUTO_TEST_CASE(multi_parent) { /* One child host, two parent hosts. Simulate multi-parent dependencies. */ @@ -20,53 +66,28 @@ BOOST_AUTO_TEST_CASE(multi_parent) * - Parent objects need a CheckResult object * - Dependencies need a StateFilter */ - Host::Ptr parentHost1 = new Host(); - parentHost1->SetActive(true); - parentHost1->SetMaxCheckAttempts(1); - parentHost1->Activate(); - parentHost1->SetAuthority(true); + Host::Ptr parentHost1 = CreateHost("parentHost1"); parentHost1->SetStateRaw(ServiceCritical); parentHost1->SetStateType(StateTypeHard); parentHost1->SetLastCheckResult(new CheckResult()); - Host::Ptr parentHost2 = new Host(); - parentHost2->SetActive(true); - parentHost2->SetMaxCheckAttempts(1); - parentHost2->Activate(); - parentHost2->SetAuthority(true); + Host::Ptr parentHost2 = CreateHost("parentHost2"); parentHost2->SetStateRaw(ServiceOK); parentHost2->SetStateType(StateTypeHard); parentHost2->SetLastCheckResult(new CheckResult()); - Host::Ptr childHost = new Host(); - childHost->SetActive(true); - childHost->SetMaxCheckAttempts(1); - childHost->Activate(); - childHost->SetAuthority(true); + Host::Ptr childHost = CreateHost("childHost"); childHost->SetStateRaw(ServiceOK); childHost->SetStateType(StateTypeHard); /* Build the dependency tree. */ - Dependency::Ptr dep1 = new Dependency(); - - dep1->SetParent(parentHost1); - dep1->SetChild(childHost); + Dependency::Ptr dep1 (CreateDependency(parentHost1, childHost, "dep1")); dep1->SetStateFilter(StateFilterUp); + RegisterDependency(dep1, ""); - // Reverse dependencies - childHost->AddDependency(dep1); - parentHost1->AddReverseDependency(dep1); - - Dependency::Ptr dep2 = new Dependency(); - - dep2->SetParent(parentHost2); - dep2->SetChild(childHost); + Dependency::Ptr dep2 (CreateDependency(parentHost2, childHost, "dep2")); dep2->SetStateFilter(StateFilterUp); - - // Reverse dependencies - childHost->AddDependency(dep2); - parentHost2->AddReverseDependency(dep2); - + RegisterDependency(dep2, ""); /* Test the reachability from this point. * parentHost1 is DOWN, parentHost2 is UP. @@ -80,15 +101,29 @@ BOOST_AUTO_TEST_CASE(multi_parent) /* The only DNS server is DOWN. * Expected result: childHost is unreachable. */ - dep1->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep1); // Remove the dep and re-add it with a configured redundancy group. + RegisterDependency(dep1, "DNS"); BOOST_CHECK(childHost->IsReachable() == false); /* 1/2 DNS servers is DOWN. * Expected result: childHost is reachable. */ - dep2->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep2); + RegisterDependency(dep2, "DNS"); BOOST_CHECK(childHost->IsReachable() == true); + auto grandParentHost(CreateHost("GrandParentHost")); + grandParentHost->SetLastCheckResult(new CheckResult()); + grandParentHost->SetStateRaw(ServiceCritical); + grandParentHost->SetStateType(StateTypeHard); + + Dependency::Ptr dep3 (CreateDependency(grandParentHost, parentHost1, "dep3")); + dep3->SetStateFilter(StateFilterUp); + RegisterDependency(dep3, ""); + // The grandparent is DOWN but the DNS redundancy group has to be still reachable. + BOOST_CHECK_EQUAL(true, childHost->IsReachable()); + DependencyGroup::Unregister(dep3); + /* Both DNS servers are DOWN. * Expected result: childHost is unreachable. */ @@ -98,4 +133,161 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); } +BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("C")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, ""); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, ""); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(simple_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA (CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + // Still 1 redundancy group, but there should be 4 members now, i.e. 2 for each child Checkable. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + // After unregistering depCA, childHostC should have a new redundancy group with only depCB as member, and... + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + // ...childHostD should still have the same redundancy group as before but also with only two members. + AssertCheckableRedundancyGroup(childHostD, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depDA); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group member, i.e. + // both child hosts should have the same redundancy group with only depCB as member. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depDA); + DependencyGroup::Unregister(depDB); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group member. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostE(CreateHost("childE")); + Dependency::Ptr depEA(CreateDependency(depCA->GetParent(), childHostE, "depEA")); + RegisterDependency(depEA, "redundant"); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEB(CreateDependency(depCB->GetParent(), childHostE, "depEB")); + RegisterDependency(depEB, "redundant"); + // All 3 hosts share the same group, and each host has 2 members, thus 6 members in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEZ(CreateDependency(CreateHost("Z"), childHostE, "depEZ")); + RegisterDependency(depEZ, "redundant"); + // Child host E should have a new redundancy group with 3 members and the other two should still share the same group. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 3, 1, 3); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depEA); // Re-register depEA and instead... + DependencyGroup::Unregister(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. + // All 3 hosts share the same group again, and each host has 2 members, thus 6 members in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + DependencyGroup::Unregister(depDB); + DependencyGroup::Unregister(depEB); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 2); + // Child host C has now a separate group with only depCB as member, and child hosts D and E share the same group. + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostE, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + BOOST_AUTO_TEST_SUITE_END()