From 12c4711183113ab2815c6e732475059772cc3161 Mon Sep 17 00:00:00 2001 From: Alvar Penning Date: Mon, 21 Oct 2024 16:00:39 +0200 Subject: [PATCH 01/20] IcingaDB: Start keeping track of Host/Service to Dependency relationship This does not work in this state! Trying to refresh Dependency if a Host or Service being member of this Dependency has a state change. --- lib/icingadb/icingadb-objects.cpp | 200 ++++++++++++++++++++++++++++++ lib/icingadb/icingadb.cpp | 2 + lib/icingadb/icingadb.hpp | 3 + 3 files changed, 205 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 920251969f2..e46012ba1e9 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -19,6 +19,7 @@ #include "icinga/command.hpp" #include "icinga/compatutility.hpp" #include "icinga/customvarobject.hpp" +#include "icinga/dependency.hpp" #include "icinga/host.hpp" #include "icinga/service.hpp" #include "icinga/hostgroup.hpp" @@ -61,6 +62,7 @@ std::vector IcingaDB::GetTypes() // Then sync them for similar reasons. Downtime::TypeInstance, Comment::TypeInstance, + Dependency::TypeInstance, HostGroup::TypeInstance, ServiceGroup::TypeInstance, @@ -791,6 +793,137 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S return; } + if (type == Dependency::TypeInstance) { + auto& dependencyNodes (hMSets[m_PrefixConfigObject + "dependency:node"]); + auto& dependencyEdges (hMSets[m_PrefixConfigObject + "dependency:edge"]); + auto& redundancyGroups (hMSets[m_PrefixConfigObject + "redundancygroup"]); + + Dependency::Ptr dependency = static_pointer_cast(object); + + Host::Ptr parentHost, childHost; + Service::Ptr parentService, childService; + tie(parentHost, parentService) = GetHostService(dependency->GetParent()); + tie(childHost, childService) = GetHostService(dependency->GetChild()); + String redundancyGroup = dependency->GetRedundancyGroup(); + + String redundancyGroupId, dependencyNodeParentId, dependencyNodeChildId, dependencyNodeReduId; + + Dictionary::Ptr parentNodeData, childNodeData; + + if (parentService) { + dependencyNodeParentId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(parentHost), + GetObjectIdentifier(parentService)})); + parentNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(parentHost)}, + {"service_id", GetObjectIdentifier(parentService)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(parentService), dependency); + } else { + dependencyNodeParentId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(parentHost)})); + parentNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(parentHost)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(parentHost), dependency); + } + + if (childService) { + dependencyNodeChildId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(childHost), + GetObjectIdentifier(childService)})); + childNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(childHost)}, + {"service_id", GetObjectIdentifier(childService)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(childService), dependency); + } else { + dependencyNodeChildId = HashValue(new Array({ + m_EnvironmentId, + GetObjectIdentifier(childHost)})); + childNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"host_id", GetObjectIdentifier(childHost)}}); + + m_CheckablesToDependencies->Set(GetObjectIdentifier(childHost), dependency); + } + + dependencyNodes.emplace_back(dependencyNodeParentId); + dependencyNodes.emplace_back(JsonEncode(parentNodeData)); + dependencyNodes.emplace_back(dependencyNodeChildId); + dependencyNodes.emplace_back(JsonEncode(childNodeData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeParentId, m_PrefixConfigObject + "dependency:node", parentNodeData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeChildId, m_PrefixConfigObject + "dependency:node", childNodeData); + } + + if (!redundancyGroup.IsEmpty()) { + /* TODO: name should be suffixed with names of all children. + * however, at this point I don't have this information, + * only the direct neighbors. + */ + redundancyGroupId = HashValue(new Array({m_EnvironmentId, redundancyGroup, dependencyNodeChildId})); + dependencyNodeReduId = redundancyGroupId; + + redundancyGroups.emplace_back(redundancyGroupId); + Dictionary::Ptr groupData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"name", redundancyGroupId}, + {"display_name", redundancyGroup}}); + redundancyGroups.emplace_back(JsonEncode(groupData)); + + dependencyNodes.emplace_back(dependencyNodeReduId); + Dictionary::Ptr reduNodeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}}); + dependencyNodes.emplace_back(JsonEncode(reduNodeData)); + + String edgeInId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeReduId})); + dependencyEdges.emplace_back(edgeInId); + Dictionary::Ptr edgeInData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeChildId}, + {"to_node_id", dependencyNodeReduId}}); + dependencyEdges.emplace_back(JsonEncode(edgeInData)); + + String edgeOutId = HashValue(new Array({m_EnvironmentId, dependencyNodeReduId, dependencyNodeParentId})); + dependencyEdges.emplace_back(edgeOutId); + Dictionary::Ptr edgeOutData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeReduId}, + {"to_node_id", dependencyNodeParentId}, + {"dependency_id", GetObjectIdentifier(dependency)}}); + dependencyEdges.emplace_back(JsonEncode(edgeOutData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeReduId, m_PrefixConfigObject + "dependency:node", reduNodeData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeInId, m_PrefixConfigObject + "dependency:edge", edgeInData); + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeOutId, m_PrefixConfigObject + "dependency:edge", edgeOutData); + } + } else { + String edgeId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeParentId})); + dependencyEdges.emplace_back(edgeId); + Dictionary::Ptr edgeData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"from_node_id", dependencyNodeChildId}, + {"to_node_id", dependencyNodeParentId}, + {"dependency_id", GetObjectIdentifier(dependency)}}); + dependencyEdges.emplace_back(JsonEncode(edgeData)); + + if (runtimeUpdate) { + AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", edgeData); + } + } + } + if (type == TimePeriod::TypeInstance) { TimePeriod::Ptr timeperiod = static_pointer_cast(object); @@ -1121,6 +1254,47 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } +void IcingaDB::UpdateDependencyState(const Dependency::Ptr& dependency) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto& redundancyGroupStates (hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + + String redundancyGroup = dependency->GetRedundancyGroup(); + + if (!redundancyGroup.IsEmpty()) { + Host::Ptr childHost; + Service::Ptr childService; + tie(childHost, childService) = GetHostService(dependency->GetChild()); + + String dependencyNodeChildId = HashValue( + (childService) + ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) + : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); + String redundancyGroupId = HashValue(new Array({ + m_EnvironmentId, + redundancyGroup, + dependencyNodeChildId})); + + redundancyGroupStates.emplace_back(redundancyGroupId); + Dictionary::Ptr groupStateData = new Dictionary({ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + {"failed", !((childService) ? childService->IsReachable() : childHost->IsReachable())}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}}); + redundancyGroupStates.emplace_back(JsonEncode(groupStateData)); + + // TODO + // AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup:state", groupStateData); + // dataClone->Set("id", objectKey); // redundancyGroupId + // dataClone->Set("redis_key", redisKey); // m_PrefixConfigObject + "redundancygroup:state" + // dataClone->Set("runtime_type", "upsert"); + // runtimeUpdates.emplace_back(dataClone); + } +} + /** * Update the state information of a checkable in Redis. * @@ -1450,6 +1624,32 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a return true; } + if (type == Dependency::TypeInstance) { + Dependency::Ptr dependency = static_pointer_cast(object); + String redundancyGroup = dependency->GetRedundancyGroup(); + + attributes->Set("name", GetObjectIdentifier(dependency)); + + if (!redundancyGroup.IsEmpty()) { + Host::Ptr childHost; + Service::Ptr childService; + tie(childHost, childService) = GetHostService(dependency->GetChild()); + + String dependencyNodeChildId = HashValue( + (childService) + ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) + : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); + String redundancyGroupId = HashValue(new Array({ + m_EnvironmentId, + redundancyGroup, + dependencyNodeChildId})); + + attributes->Set("redundancy_group_id", redundancyGroupId); + } + + return true; + } + if (type == Downtime::TypeInstance) { Downtime::Ptr downtime = static_pointer_cast(object); diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 8d3b9099bd7..3c623259b58 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -38,6 +38,8 @@ IcingaDB::IcingaDB() m_PrefixConfigObject = "icinga:"; m_PrefixConfigCheckSum = "icinga:checksum:"; + + m_CheckablesToDependencies = new Dictionary(); } void IcingaDB::Validate(int types, const ValidationUtils& utils) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 6652d9c1f4d..8eb5c946ddd 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -103,6 +103,7 @@ class IcingaDB : public ObjectImpl std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependencyState(const Dependency::Ptr& dependency); void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -224,6 +225,8 @@ class IcingaDB : public ObjectImpl std::unordered_map m_Rcons; std::atomic_size_t m_PendingRcons; + Dictionary::Ptr m_CheckablesToDependencies; + struct { DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; } m_DumpedGlobals; From dae5e117e2787254d47ffaea1db98cbb10aa0b31 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:15:00 +0100 Subject: [PATCH 02/20] Checkable: Introduce `GetAllChildrenCount()` method --- lib/icinga/checkable-dependency.cpp | 41 ++++++++++++++++++++++++----- lib/icinga/checkable.hpp | 1 + 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 58d6b578bb8..addb4863462 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -145,6 +145,21 @@ std::set Checkable::GetChildren() const return parents; } +/** + * Retrieve the total number of all the children of the current Checkable. + * + * Note, due to the max recursion limit of 256, the returned number may not reflect + * the actual total number of children involved in the dependency chain. + * + * @return int - Returns the total number of all the children of the current Checkable. + */ +size_t Checkable::GetAllChildrenCount() const +{ + std::set children(GetChildren()); + GetAllChildrenInternal(children, 0); + return children.size(); +} + std::set Checkable::GetAllChildren() const { std::set children = GetChildren(); @@ -154,22 +169,36 @@ std::set Checkable::GetAllChildren() const return children; } +/** + * Retrieve all direct and indirect children of the current Checkable. + * + * Note, this function performs a recursive call chain traversing all the children of the current Checkable + * up to a certain limit (256). When that limit is reached, it will log a warning message and abort the operation. + * + * @param children - The set of children to be filled with all the children of the current Checkable. + * @param level - The current level of recursion. + */ void Checkable::GetAllChildrenInternal(std::set& children, int level) const { - if (level > 32) - return; + // The previous limit (32) doesn't seem to make sense, and appears to be some random number. + // So, this limit is set to 256 to match the limit in IsReachable(). + if (level > 256) { + Log(LogWarning, "Checkable") + << "Too many nested dependencies (>" << 256 << ") for checkable '" << GetName() << "': aborting traversal."; + return ; + } std::set localChildren; for (const Checkable::Ptr& checkable : children) { - std::set cChildren = checkable->GetChildren(); - - if (!cChildren.empty()) { + if (auto cChildren(checkable->GetChildren()); !cChildren.empty()) { GetAllChildrenInternal(cChildren, level + 1); localChildren.insert(cChildren.begin(), cChildren.end()); } - localChildren.insert(checkable); + if (level != 0) { // Recursion level 0 is the initiator, so checkable is already in the set. + localChildren.insert(checkable); + } } children.insert(localChildren.begin(), localChildren.end()); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index fcfbca9b281..c9e54b0f5a4 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -77,6 +77,7 @@ class Checkable : public ObjectImpl std::set GetParents() const; std::set GetChildren() const; std::set GetAllChildren() const; + size_t GetAllChildrenCount() const; void AddGroup(const String& name); From 9418634c6a16d8542b0b91c0a675cb0c7532ae47 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:19:55 +0100 Subject: [PATCH 03/20] IcingaDB: Add `affected_children` to `Host/Service` Redis updates --- lib/icingadb/icingadb-objects.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e46012ba1e9..6753813ca69 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1482,6 +1482,11 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a attributes->Set("notes", checkable->GetNotes()); attributes->Set("icon_image_alt", checkable->GetIconImageAlt()); + if (size_t affectedChildren (checkable->GetAllChildrenCount()); affectedChildren > 0) { + // Only set the Redis key if the Checkable has actually some child dependencies. + attributes->Set("affected_children", affectedChildren); + } + attributes->Set("checkcommand_id", GetObjectIdentifier(checkable->GetCheckCommand())); Endpoint::Ptr commandEndpoint = checkable->GetCommandEndpoint(); From d4699d1fce77ceaf7c07afeea3f7e059517945ff Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 2 Dec 2024 14:37:09 +0100 Subject: [PATCH 04/20] IcingaDB: Sync `affects_children` as part of runtime state updates --- lib/icinga/checkable-dependency.cpp | 25 +++++++++++++++++++++++++ lib/icinga/checkable.hpp | 1 + lib/icingadb/icingadb-objects.cpp | 1 + 3 files changed, 27 insertions(+) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index addb4863462..90a78debc47 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -117,6 +117,31 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency return true; } +/** + * Checks whether the last check result of this Checkable affects its child dependencies. + * + * @return bool - Returns true if the Checkable affects its child dependencies, otherwise false. + */ +bool Checkable::AffectsChildren() const +{ + auto cr(GetLastCheckResult()); + if (!cr || IsStateOK(cr->GetState()) || !IsReachable()) { + // If there is no check result, the state is OK, or the Checkable is not reachable, we can't + // safely determine whether the Checkable affects its child dependencies. + return false; + } + + for (auto& dep: GetReverseDependencies()) { + if (!dep->IsAvailable(DependencyState)) { + // If one of the child dependency is not available, then it's definitely due to the + // current Checkable state, so we don't need to verify the remaining ones. + return true; + } + } + + return false; +} + std::set Checkable::GetParents() const { std::set parents; diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index c9e54b0f5a4..c6413fa1b09 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -82,6 +82,7 @@ class Checkable : public ObjectImpl void AddGroup(const String& name); bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 6753813ca69..e51e2d215cc 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -2828,6 +2828,7 @@ Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) attrs->Set("check_attempt", checkable->GetCheckAttempt()); attrs->Set("is_active", checkable->IsActive()); + attrs->Set("affects_children", checkable->AffectsChildren()); CheckResult::Ptr cr = checkable->GetLastCheckResult(); From dff9be01ebbb013fe0f88456f5bc3571f20a5407 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 09:09:31 +0100 Subject: [PATCH 05/20] Dependency: Don't allow to change `redundancy_group` at runtime Otherwise, it would require too much code changes to properly handle redundancy group runtime modification in Icinga DB for no real benefit. --- lib/icinga/dependency.ti | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 41de7ba23cf..38f5859aeaa 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -77,7 +77,7 @@ class Dependency : CustomVarObject < DependencyNameComposer }}} }; - [config] String redundancy_group; + [config, no_user_modify] String redundancy_group; [config, navigation] name(TimePeriod) period (PeriodRaw) { navigate {{{ From 04815ab5d602f4a0c070dd2935414ed76437ef9a Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 10:42:21 +0100 Subject: [PATCH 06/20] Introduce `DependencyGroup` helper class --- lib/icinga/checkable-dependency.cpp | 1 - lib/icinga/dependency.cpp | 227 ++++++++++++++++++++++++++++ lib/icinga/dependency.hpp | 52 +++++++ 3 files changed, 279 insertions(+), 1 deletion(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 90a78debc47..873358d8b19 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -6,7 +6,6 @@ #include using namespace icinga; - void Checkable::AddDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 2843b906cb8..caf1d58473a 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -7,9 +7,12 @@ #include "base/initialize.hpp" #include "base/logger.hpp" #include "base/exception.hpp" +#include "base/utility.hpp" +#include "base/object-packer.hpp" #include #include #include +#include using namespace icinga; @@ -323,3 +326,227 @@ void Dependency::SetChild(intrusive_ptr child) { m_Child = child; } + +// Is the default (dummy) dependency group name used to group non-redundant dependencies. +static String l_DefaultDependencyGroup(Utility::NewUniqueID()); + +DependencyGroup::DependencyGroup(String name, const Dependency::Ptr& dependency): m_Name(std::move(name)) +{ + AddMember(dependency); +} + +/** + * Create a composite key for the provided dependency. + * + * @param dependency The dependency object to create a composite key for. + * + * @return - Returns the composite key for the provided dependency. + */ +DependencyGroup::MemberTuple DependencyGroup::MakeCompositeKeyFor(const Dependency::Ptr& dependency) +{ + if (dependency->GetRedundancyGroup().IsEmpty()) { + // Just to make sure we don't have any duplicates in the default group. + return std::make_tuple(l_DefaultDependencyGroup, dependency->GetChild()->GetName(), 0, false); + } + + return std::make_tuple( + dependency->GetParent()->GetName(), + dependency->GetPeriodRaw(), + dependency->GetStateFilter(), + dependency->GetIgnoreSoftStates() + ); +} + +/** + * Check if the current dependency group is an explicitly configured redundancy group. + * + * @return bool - Returns true if the current dependency group isn't the dummy dependency group. + */ +bool DependencyGroup::IsRedundancyGroup() const +{ + return m_Name != l_DefaultDependencyGroup; +} + +/** + * Check if the current dependency group has any members. + * + * @return bool - Returns true if the current dependency group has any members. + */ +bool DependencyGroup::HasMembers() const +{ + std::lock_guard lock(m_Mutex); + return !m_Members.empty(); +} + +/** + * Check if the current dependency group has the provided dependency as a member. + * + * @param dependency The dependency to look for. + * + * @return bool - Returns true if the provided dependency is member of the current dependency group. + */ +bool DependencyGroup::HasMember(const Dependency::Ptr& dependency) const +{ + std::lock_guard lock(m_Mutex); + return m_Members.find(MakeCompositeKeyFor(dependency)) != m_Members.end(); +} + +/** + * Retrieve all members of the current dependency group the provided child Checkable depend on. + * + * Note, in order to ease duplicated dependencies exhaustion, the returned members are sorted by the parent Checkable. + * This way, all identical dependencies are placed next to each other, and you can easily consume them via a simple loop. + * + * @param child The child Checkable to look for. + * + * @return - Returns all members of the current dependency group the provided child depend on. + */ +std::vector DependencyGroup::GetMembers(const Checkable* child) const +{ + std::lock_guard lock(m_Mutex); + std::vector members; + for (auto& [_, dependencies] : m_Members) { + auto range(dependencies.equal_range(child)); + std::transform(range.first,range.second, std::back_inserter(members), [](const auto& member) { + return member.second; + }); + } + + std::sort(members.begin(), members.end(), [](const Dependency::Ptr& lhs, const Dependency::Ptr& rhs) { + return lhs->GetParent() < rhs->GetParent(); + }); + return members; +} + +/** + * Retrieve the number of members in the current dependency group. + * + * This function mainly exists for optimization purposes, i.e. instead of getting a copy of the members and + * counting them, we can directly query the number of members. + * + * @return - Returns the number of members in the current dependency group. + */ +size_t DependencyGroup::GetMemberCount() const +{ + std::lock_guard lock(m_Mutex); + return std::accumulate(m_Members.begin(), m_Members.end(), static_cast(0), [](int sum, const auto& pair) { + return sum + pair.second.size(); + }); +} + +/** + * Add a member to the current dependency group. + * + * @param member The dependency to add to the dependency group. + */ +void DependencyGroup::AddMember(const Dependency::Ptr& member) +{ + std::lock_guard lock(m_Mutex); + MemberTuple compositeKey(MakeCompositeKeyFor(member)); + if (auto it(m_Members.find(compositeKey)); it != m_Members.end()) { + it->second.emplace(member->GetChild().get(), member.get()); + } else { + m_Members.emplace(compositeKey, MemberValueType{{member->GetChild().get(), member.get()}}); + } +} + +/** + * Remove a member from the current dependency group. + * + * @param member The dependency to remove from the dependency group. + */ +void DependencyGroup::RemoveMember(const Dependency::Ptr& member) +{ + std::lock_guard lock(m_Mutex); + if (auto it(m_Members.find(MakeCompositeKeyFor(member))); it != m_Members.end()) { + auto [rangeBegin, rangeEnd] = it->second.equal_range(member->GetChild().get()); + for (auto memberIt(rangeBegin); memberIt != rangeEnd; ++memberIt) { + if (memberIt->second == member) { + // This will also remove the child Checkable from the multimap container + // entirely if this was the last member of it. + it->second.erase(memberIt); + // If the composite key has no more members left, we can remove it entirely as well. + if (it->second.empty()) { + m_Members.erase(it); + } + return; + } + } + } +} + +/** + * Set the Icinga DB identifier for the current dependency group. + * + * The only usage of this function is the Icinga DB feature used to cache the unique hash of this dependency groups. + * + * @param identifier The Icinga DB identifier to set. + */ +void DependencyGroup::SetIcingaDBIdentifier(const String& identifier) +{ + std::lock_guard lock(m_Mutex); + m_IcingaDBIdentifier = identifier; +} + +/** + * Retrieve the Icinga DB identifier for the current dependency group. + * + * When the identifier is not already set by Icinga DB via the SetIcingaDBIdentifier method, + * this will just return an empty string. + * + * @return - Returns the Icinga DB identifier for the current dependency group. + */ +String DependencyGroup::GetIcingaDBIdentifier() const +{ + std::lock_guard lock(m_Mutex); + return m_IcingaDBIdentifier; +} + +/** + * Retrieve the (non-unique) name of the current dependency group. + * + * For explicitly configured redundancy groups, the name of the dependency group is the same as the one + * located in the configuration files. For non-redundant dependencies, on the other hand, the name is a randomly + * generated unique UUID (l_DefaultDependencyGroup). + * + * @return - Returns the name of the current dependency group. + */ +const String& DependencyGroup::GetName() const +{ + // We don't need to lock the mutex here, as the name is set once during + // the object construction and never changed afterwards. + return m_Name; +} + +/** + * Retrieve the unique composite key of the current dependency group. + * + * The composite key consists of some unique data of the group members, and should be used to generate a unique + * deterministic hash for the dependency group. Each key is a tuple of the parent name, the time period name + * (empty if not configured), the state filter, and the ignore soft states flag of the member. + * + * Additionally, to all the above mentioned keys, the non-unique dependency group name is also included. + * + * @return - Returns the composite key of the current dependency group. + */ +String DependencyGroup::GetCompositeKey() const +{ + std::lock_guard lock(m_Mutex); + if (!IsRedundancyGroup()) { + String childCheckableName; + if (!m_Members.empty()) { + childCheckableName = std::get<1>(m_Members.begin()->first); // See MemberTuple definition for the index. + } + return PackObject(new Array{GetName(), childCheckableName, 0, false}); + } + + Array::Ptr data(new Array{GetName()}); + for (auto& [compositeKey, _] : m_Members) { + auto [parentName, tpName, stateFilter, ignoreSoftStates] = compositeKey; + data->Add(std::move(parentName)); + data->Add(std::move(tpName)); + data->Add(stateFilter); + data->Add(ignoreSoftStates); + } + return PackObject(data); +} diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 6cebfaab1d7..253d1887550 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -3,8 +3,10 @@ #ifndef DEPENDENCY_H #define DEPENDENCY_H +#include "base/shared-object.hpp" #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" +#include namespace icinga { @@ -57,6 +59,56 @@ class Dependency final : public ObjectImpl static bool EvaluateApplyRule(const Checkable::Ptr& checkable, const ApplyRule& rule, bool skipFilter = false); }; +/** +* @ingroup icinga +*/ +class DependencyGroup final : public SharedObject +{ +public: + DECLARE_PTR_TYPEDEFS(DependencyGroup); + + /** + * Defines the key type of each dependency group members. + * + * For dependency groups **with** an explicitly configured redundancy group, that tuple consists of the dependency + * parent name, the dependency time period name (empty if not configured), the state filter, and the + * ignore soft states flag. + * + * For the non-redundant group (just a bunch of dependencies without a redundancy group) of a given Checkable, + * the tuple consists of the dependency group name (which is a randomly generated unique UUID), the child + * Checkable name, the state filter (is always 0), and the ignore soft states flag (is always false). + */ + using MemberTuple = std::tuple; + using MemberValueType = std::unordered_multimap; + using MembersMap = std::map; + + DependencyGroup(String name, const Dependency::Ptr& member); + + static MemberTuple MakeCompositeKeyFor(const Dependency::Ptr& dependency); + + bool IsRedundancyGroup() const; + bool HasMembers() const; + bool HasMember(const Dependency::Ptr& dependency) const; + std::vector GetMembers(const Checkable* child) const; + size_t GetMemberCount() const; + + void SetIcingaDBIdentifier(const String& identifier); + String GetIcingaDBIdentifier() const; + + const String& GetName() const; + String GetCompositeKey() const; + +protected: + void AddMember(const Dependency::Ptr& member); + void RemoveMember(const Dependency::Ptr& member); + +private: + mutable std::mutex m_Mutex; + String m_IcingaDBIdentifier; + String m_Name; + MembersMap m_Members; +}; + } #endif /* DEPENDENCY_H */ From 132effd65bcb37ba93c50d040c7c1e57cc802574 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 12 Dec 2024 13:34:30 +0100 Subject: [PATCH 07/20] DependencyGroup: Add a global registry & deduplication logic --- lib/icinga/checkable-dependency.cpp | 13 ++ lib/icinga/checkable.hpp | 4 + lib/icinga/dependency.cpp | 195 ++++++++++++++++++++++++++++ lib/icinga/dependency.hpp | 66 ++++++++++ 4 files changed, 278 insertions(+) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 873358d8b19..13206dcbed2 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -6,6 +6,19 @@ #include using namespace icinga; + +void Checkable::AddDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + std::unique_lock lock(m_DependencyMutex); + m_DependencyGroups.insert(dependencyGroup); +} + +void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + std::unique_lock lock(m_DependencyMutex); + m_DependencyGroups.erase(dependencyGroup); +} + void Checkable::AddDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index c6413fa1b09..9038bf9a1c1 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -57,6 +57,7 @@ enum FlappingStateFilter class CheckCommand; class EventCommand; class Dependency; +class DependencyGroup; /** * An Icinga service. @@ -184,6 +185,8 @@ class Checkable : public ObjectImpl bool IsFlapping() const; /* Dependencies */ + void AddDependencyGroup(const intrusive_ptr& dependencyGroup); + void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); void AddDependency(const intrusive_ptr& dep); void RemoveDependency(const intrusive_ptr& dep); std::vector > GetDependencies() const; @@ -246,6 +249,7 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; + std::set> m_DependencyGroups; std::set > m_Dependencies; std::set > m_ReverseDependencies; diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index caf1d58473a..1a1bcb8b995 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -330,6 +330,177 @@ void Dependency::SetChild(intrusive_ptr child) // Is the default (dummy) dependency group name used to group non-redundant dependencies. static String l_DefaultDependencyGroup(Utility::NewUniqueID()); +std::mutex DependencyGroup::m_RegistryMutex; +DependencyGroup::RegistryType DependencyGroup::m_Registry; + +/** + * Register a dependency group in the global groups registry. + * + * At first, it tries to naively insert the dependency group into the registry. In case there is already an + * identical group in the registry, the insertion will just fail. In this case, it will move all + * members of the provided dependency group into the existing one and re-register the existing one recursively + * till there is no group with identical members left and the insertion finally succeeds. + * + * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex + * before calling this function. + * + * @param dependencyGroup The dependency group to register. + */ +void DependencyGroup::RegisterRedundancyGroup(const DependencyGroup::Ptr& dependencyGroup) +{ + if (auto it(m_Registry.insert(dependencyGroup.get())); !it.second) { + DependencyGroup::Ptr existingGroup(*it.first); + if (!existingGroup->IsRedundancyGroup()) { + // It's the dummy group, so just move the new members into the existing one, and it should be it. + dependencyGroup->MoveMembersTo(existingGroup); + } else { + // Erase it before we move the members into that group and change the hash. + m_Registry.erase(it.first); + dependencyGroup->MoveMembersTo(existingGroup); + RegisterRedundancyGroup(existingGroup); + } + } +} + +/** + * Refresh the registry of dependency groups. + * + * This function is used to refresh the global registry of dependency groups. + * It will first try to find an existing dependency group based on the provided dependency object's redundancy group. + * When it finds an identical group, it will move all its members (dependencies of the same child Checkable) into the + * new dependency group and add it to the registry. A nullptr as a new dependency group indicates that the provided + * dependency object should be unregistered from the registry. + * + * Note: This is a helper function intended for internal use only, and you should acquire the global registry mutex + * before calling this function. + * + * @param dependency The dependency object to refresh the registry for. + * @param newGroup The new dependency group to move the remaining Checkable dependencies into. + */ +void DependencyGroup::RefreshRegistry(const Dependency::Ptr& dependency, const DependencyGroup::Ptr& newGroup) +{ + auto [rangeBegin, rangeEnd] = m_Registry.get<1>().equal_range(dependency->GetRedundancyGroup()); + for (auto groupIt(rangeBegin); groupIt != rangeEnd; ++groupIt) { + DependencyGroup::Ptr existingGroup(*groupIt); + auto members(existingGroup->GetMembers(dependency->GetChild().get())); + if (members.empty()) { + continue; + } + + // Erase the existing dependency group from the registry, before we move the members + // out of it and change its identity, i.e. the hash value used by the registry. + m_Registry.erase(existingGroup.get()); + + DependencyGroup::Ptr replacementGroup(newGroup); + for (auto& member : members) { + // A nullptr newGroup means we want to unregister the provided dependency object. Otherwise, + // it should already be registered to the new dependency group, and we just need to move the + // remaining dependencies of the Checkable into that new group. + if (newGroup || member != dependency) { + if (!replacementGroup) { + replacementGroup = new DependencyGroup(existingGroup->GetName(), member); + } else { + // If there are any dependencies registered for the same child Checkable under the existing + // dependency group, we must move them into the new one. Meaning, the dependency group for + // that Checkable has changed, meanwhile for the other Checkables it's still the same. + replacementGroup->AddMember(member); + } + } + existingGroup->RemoveMember(member); + } + + if (existingGroup->HasMembers()) { + // Detach the existing dependency group from the child Checkable of the dependency + // object, as it's not a member of it anymore. Instead, we must... + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + + if (replacementGroup) { + // ...attach the new dependency group to the child Checkable. + dependency->GetChild()->AddDependencyGroup(replacementGroup); + RegisterRedundancyGroup(replacementGroup); + } + + // The existing dependency group still has some members left, so we must re-register + // it and enforce the rehashing of the members due to the removed ones above. + RegisterRedundancyGroup(existingGroup); + } else if (replacementGroup) { + // We were the last member of the existing dependency group, so instead of replacing it with the + // replacement group, we can just move back the members to it and re-add it to the registry. + replacementGroup->MoveMembersTo(existingGroup); + RegisterRedundancyGroup(existingGroup); + } else { + // The existing dependency group has no members left, so we must detach it from the child Checkable. + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + } + return; + } + + if (newGroup) { + // If we haven't found any existing dependency group to register the dependency to, we must + // attach the new dependency group to the child Checkable and register the new group to the registry. + dependency->GetChild()->AddDependencyGroup(newGroup); + RegisterRedundancyGroup(newGroup); + } +} + +/** + * Register the provided dependency to the global dependency group registry. + * + * @param dependency The dependency to register. + */ +void DependencyGroup::Register(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + auto groupName(dependency->GetRedundancyGroup()); + DependencyGroup::Ptr newGroup(new DependencyGroup(groupName.IsEmpty() ? l_DefaultDependencyGroup : groupName, dependency)); + if (m_Registry.empty() || groupName.IsEmpty()) { + dependency->GetChild()->AddDependencyGroup(newGroup); + RegisterRedundancyGroup(newGroup); + return; + } + + RefreshRegistry(dependency, newGroup); +} + +/** + * Unregister the provided dependency from the dependency group it was member of. + * + * @param dependency The dependency to unregister. + */ +void DependencyGroup::Unregister(const Dependency::Ptr& dependency) +{ + std::lock_guard lock(m_RegistryMutex); + if (dependency->GetRedundancyGroup().IsEmpty()) { + // Default dependency group of a given Checkable always produce the very same hash, so we can + // safely use any instance of it to find the existing one in the registry. + DependencyGroup::Ptr defaultGroup(new DependencyGroup(l_DefaultDependencyGroup, dependency)); + if (auto it(m_Registry.find(defaultGroup)); it != m_Registry.end()) { + DependencyGroup::Ptr existingGroup(*it); + existingGroup->RemoveMember(dependency); + if (!existingGroup->HasMembers()) { + // There are no members left in the default dependency group, so we must detach + // it from the child Checkable of the dependency object. + dependency->GetChild()->RemoveDependencyGroup(existingGroup); + m_Registry.erase(it); + } + } + return; + } + + RefreshRegistry(dependency, Ptr()); +} + +/** + * Retrieve the size of the global dependency group registry. + * + * @return size_t - Returns the size of the global dependency groups registry. + */ +size_t DependencyGroup::GetRegistrySize() +{ + std::lock_guard lock(m_RegistryMutex); + return m_Registry.size(); +} + DependencyGroup::DependencyGroup(String name, const Dependency::Ptr& dependency): m_Name(std::move(name)) { AddMember(dependency); @@ -475,6 +646,30 @@ void DependencyGroup::RemoveMember(const Dependency::Ptr& member) } } +/** + * Move the members of the provided dependency group to the provided destination dependency group. + * + * @param dest The dependency group to move the members to. + */ +void DependencyGroup::MoveMembersTo(const DependencyGroup::Ptr& dest) +{ + VERIFY(this != dest); // Prevent from doing something stupid, i.e. deadlocking ourselves. + + std::lock_guard lock(m_Mutex); + DependencyGroup::Ptr thisPtr(this); // Just in case the Checkable below was our last reference. + for (auto& [_, members] : m_Members) { + Checkable::Ptr previousChild; + for (auto& [checkable, dependency] : members) { + dest->AddMember(dependency); + if (!previousChild || previousChild != checkable) { + previousChild = dependency->GetChild(); + previousChild->RemoveDependencyGroup(thisPtr); + previousChild->AddDependencyGroup(dest); + } + } + } +} + /** * Set the Icinga DB identifier for the current dependency group. * diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 253d1887550..d054c61985e 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -6,6 +6,9 @@ #include "base/shared-object.hpp" #include "icinga/i2-icinga.hpp" #include "icinga/dependency-ti.hpp" +#include +#include +#include #include namespace icinga @@ -84,6 +87,10 @@ class DependencyGroup final : public SharedObject DependencyGroup(String name, const Dependency::Ptr& member); + static void Register(const intrusive_ptr& dependency); + static void Unregister(const intrusive_ptr& dependency); + static size_t GetRegistrySize(); + static MemberTuple MakeCompositeKeyFor(const Dependency::Ptr& dependency); bool IsRedundancyGroup() const; @@ -101,12 +108,71 @@ class DependencyGroup final : public SharedObject protected: void AddMember(const Dependency::Ptr& member); void RemoveMember(const Dependency::Ptr& member); + void MoveMembersTo(const DependencyGroup::Ptr& other); + + static void RegisterRedundancyGroup(const DependencyGroup::Ptr& dependencyGroup); + static void RefreshRegistry(const intrusive_ptr& dependency, const DependencyGroup::Ptr& newGroup = nullptr); private: mutable std::mutex m_Mutex; String m_IcingaDBIdentifier; String m_Name; MembersMap m_Members; + + struct Hash { + /** + * Calculates the hash value of a dependency group used by DependencyGroup::RegistryType. + * + * @param dependencyGroup The dependency group to calculate the hash value for. + * + * @return Returns the hash value of the dependency group. + */ + size_t operator()(const DependencyGroup::Ptr& dependencyGroup) const + { + return std::hash{}(dependencyGroup->GetCompositeKey()); + } + }; + + struct Equal { + /** + * Checks two dependency groups for equality. + * + * The equality of two dependency groups is determined by the equality of their composite keys. + * That composite key consists of a tuple of the parent name, the time period name (empty if not configured), + * state filter, and the ignore soft states flag of the member. + * + * @param lhs The first dependency group to compare. + * @param rhs The second dependency group to compare. + * + * @return Returns true if the composite keys of the two dependency groups are equal. + */ + bool operator()(const DependencyGroup::Ptr& lhs, const DependencyGroup::Ptr& rhs) const + { + return lhs->GetCompositeKey() == rhs->GetCompositeKey(); + } + }; + + using RegistryType = boost::multi_index_container< + DependencyGroup*, // The type of the elements stored in the container. + boost::multi_index::indexed_by< + // The first index is a unique index based on the identity of the dependency group. + // The identity of the dependency group is determined by the provided Hash and Equal functors. + boost::multi_index::hashed_unique, Hash, Equal>, + // This non-unique index allows to search for dependency groups by their name, and reduces the overall + // runtime complexity. Without this index, we would have to iterate over all elements to find the on with + // the desired members and since std::unordered_set doesn't allow erasing elements while iterating, we would + // have to copy each of them to a temporary container, and then erase and reinsert them back t the original + // container. This produces way too much overhead, and slows down the startup time of Icinga 2 significantly. + boost::multi_index::hashed_non_unique< + boost::multi_index::const_mem_fun, + std::hash + > + > + >; + + // The global registry of dependency groups. + static std::mutex m_RegistryMutex; + static RegistryType m_Registry; }; } From db5cc76417ee897c441a6e61e4ed149e609617c3 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 11:21:01 +0100 Subject: [PATCH 08/20] Add `DependencyGroup::GetState()` helper method --- lib/icinga/dependency.cpp | 50 +++++++++++++++++++++++++++++++++++++++ lib/icinga/dependency.hpp | 11 +++++++++ 2 files changed, 61 insertions(+) diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 1a1bcb8b995..0dedf508906 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -745,3 +745,53 @@ String DependencyGroup::GetCompositeKey() const } return PackObject(data); } + +/** + * Retrieve the state of the current dependency group. + * + * The state of the dependency group is determined based on the state of the members of the group. + * This method returns a DependencyGroup::State::Unknown immediately when the group has no members. + * Otherwise, a dependency group is considered unreachable when none of the members is reachable. + * A reachable dependency group is failed when the edges connected to it are not available. + * + * @return - Returns the state of the current dependency group. + */ +DependencyGroup::State DependencyGroup::GetState(DependencyType dt, int rstack) const +{ + MembersMap members; + { + // We don't want to hold the mutex lock for the entire evaluation, thus we just need to operate on a copy. + std::lock_guard lock(m_Mutex); + members = m_Members; + } + + bool isRedundant(IsRedundancyGroup()); + State state(members.empty() ? State::Unknown : State::Failed); + for (auto it(members.begin()); it != members.end() && (!isRedundant || state != State::ReachableOK); ++it) { + // Those are first batch of group members, i.e. they all share kind of the same Dependency config. + // So, for redundancy groups, we only need to check a single parent Checkable and dependency + // object of each such a group (see the preconditions of the below loop). + for (auto depIt(it->second.begin()); depIt != it->second.end() && (!isRedundant || depIt == it->second.begin()); ++depIt) { + auto [checkable, dependency] = *depIt; + if (!dependency->GetParent()->IsReachable(dt, rstack)) { + if (!isRedundant) { + // If any of the members is unreachable, the whole dependency group is unreachable, too. + return State::Unreachable; + } + state = State::UnreachableFailed; + } else if (!dependency->IsAvailable(dt)) { + if (!isRedundant) { + Log(LogDebug, "Checkable") + << "Non-redundant dependency '" << dependency->GetName() << "' failed for checkable '" + << checkable->GetName() << "': Marking as unreachable."; + + return State::Failed; + } + } else { + state = State::ReachableOK; + } + } + } + + return state; +} diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index d054c61985e..fbb92470d73 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -105,6 +105,17 @@ class DependencyGroup final : public SharedObject const String& GetName() const; String GetCompositeKey() const; + enum State + { + Unknown = 1ull << 0, + Failed = 1ull << 1, + Unreachable = 1ull << 2, + ReachableOK = 1ull << 3, + UnreachableFailed = Unreachable | Failed, + }; + + State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + protected: void AddMember(const Dependency::Ptr& member); void RemoveMember(const Dependency::Ptr& member); From cab3a1a729004552fa88bfae1aa7fc870cf862aa Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 10:55:16 +0100 Subject: [PATCH 09/20] Checkable: Store dependencies grouped by their redundancy group --- lib/icinga/checkable-dependency.cpp | 20 ++++++++++---------- lib/icinga/checkable.hpp | 4 +--- lib/icinga/dependency.cpp | 6 +++--- test/icinga-dependencies.cpp | 4 ++-- 4 files changed, 16 insertions(+), 18 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 13206dcbed2..b73a3a2d21e 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -19,22 +19,22 @@ void Checkable::RemoveDependencyGroup(const DependencyGroup::Ptr& dependencyGrou m_DependencyGroups.erase(dependencyGroup); } -void Checkable::AddDependency(const Dependency::Ptr& dep) +std::vector Checkable::GetDependencyGroups() const { - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.insert(dep); -} - -void Checkable::RemoveDependency(const Dependency::Ptr& dep) -{ - std::unique_lock lock(m_DependencyMutex); - m_Dependencies.erase(dep); + std::unique_lock lock(m_DependencyMutex); + return {m_DependencyGroups.begin(), m_DependencyGroups.end()}; } std::vector Checkable::GetDependencies() const { std::unique_lock lock(m_DependencyMutex); - return std::vector(m_Dependencies.begin(), m_Dependencies.end()); + std::vector dependencies; + for (const auto& dependencyGroup : m_DependencyGroups) { + auto members(dependencyGroup->GetMembers(this)); + dependencies.insert(dependencies.end(), members.begin(), members.end()); + } + + return dependencies; } void Checkable::AddReverseDependency(const Dependency::Ptr& dep) diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 9038bf9a1c1..bdf19075f03 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -187,8 +187,7 @@ class Checkable : public ObjectImpl /* Dependencies */ void AddDependencyGroup(const intrusive_ptr& dependencyGroup); void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); - void AddDependency(const intrusive_ptr& dep); - void RemoveDependency(const intrusive_ptr& dep); + std::vector> GetDependencyGroups() const; std::vector > GetDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); @@ -250,7 +249,6 @@ class Checkable : public ObjectImpl /* Dependencies */ mutable std::mutex m_DependencyMutex; std::set> m_DependencyGroups; - std::set > m_Dependencies; std::set > m_ReverseDependencies; void GetAllChildrenInternal(std::set& children, int level = 0) const; diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index 0dedf508906..eebeb014cd3 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -191,7 +191,7 @@ void Dependency::OnAllConfigLoaded() if (!m_Parent) BOOST_THROW_EXCEPTION(ScriptError("Dependency '" + GetName() + "' references a parent host/service which doesn't exist.", GetDebugInfo())); - m_Child->AddDependency(this); + DependencyGroup::Register(this); m_Parent->AddReverseDependency(this); if (m_AssertNoCyclesForIndividualDeps) { @@ -200,7 +200,7 @@ void Dependency::OnAllConfigLoaded() try { AssertNoDependencyCycle(m_Parent, graph); } catch (...) { - m_Child->RemoveDependency(this); + DependencyGroup::Unregister(this); m_Parent->RemoveReverseDependency(this); throw; } @@ -211,7 +211,7 @@ void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); - GetChild()->RemoveDependency(this); + DependencyGroup::Unregister(this); GetParent()->RemoveReverseDependency(this); } diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 929b6ca0de5..86735cdb90d 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(multi_parent) dep1->SetStateFilter(StateFilterUp); // Reverse dependencies - childHost->AddDependency(dep1); + DependencyGroup::Register(dep1); parentHost1->AddReverseDependency(dep1); Dependency::Ptr dep2 = new Dependency(); @@ -64,7 +64,7 @@ BOOST_AUTO_TEST_CASE(multi_parent) dep2->SetStateFilter(StateFilterUp); // Reverse dependencies - childHost->AddDependency(dep2); + DependencyGroup::Register(dep2); parentHost2->AddReverseDependency(dep2); From d63647134389ea0ed9bc4c08e0fd38a825661a01 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 11:00:26 +0100 Subject: [PATCH 10/20] IcingaDB: Dump checkables dependencies config to redis correctly --- lib/icinga/checkable-dependency.cpp | 6 + lib/icinga/checkable.hpp | 1 + lib/icinga/dependency.hpp | 17 ++ lib/icingadb/icingadb-objects.cpp | 299 ++++++++++------------------ lib/icingadb/icingadb.cpp | 2 - lib/icingadb/icingadb.hpp | 7 +- 6 files changed, 136 insertions(+), 196 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index b73a3a2d21e..480df7bf450 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -37,6 +37,12 @@ std::vector Checkable::GetDependencies() const return dependencies; } +bool Checkable::HasAnyDependencies() const +{ + std::unique_lock lock(m_DependencyMutex); + return !m_DependencyGroups.empty() || !m_ReverseDependencies.empty(); +} + void Checkable::AddReverseDependency(const Dependency::Ptr& dep) { std::unique_lock lock(m_DependencyMutex); diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index bdf19075f03..6fcfc1385d0 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -189,6 +189,7 @@ class Checkable : public ObjectImpl void RemoveDependencyGroup(const intrusive_ptr& dependencyGroup); std::vector> GetDependencyGroups() const; std::vector > GetDependencies() const; + bool HasAnyDependencies() const; void AddReverseDependency(const intrusive_ptr& dep); void RemoveReverseDependency(const intrusive_ptr& dep); diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index fbb92470d73..6511791c154 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -47,6 +47,23 @@ class Dependency final : public ObjectImpl void SetParent(intrusive_ptr parent); void SetChild(intrusive_ptr child); + /** + * A functor to compare the parent Checkable of a dependency with an explicitly provided parent Checkable. + * + * This is used to group dependencies by their parent Checkable (c++ 14 heterogeneous lookups). + */ + struct ParentComparator { + bool operator()(const Dependency::Ptr& dependency, const Checkable::Ptr& parent) const + { + return dependency->GetParent() < parent; + } + + bool operator()(const Checkable::Ptr& parent, const Dependency::Ptr& rhs) const + { + return parent < rhs->GetParent(); + } + }; + protected: void OnConfigLoaded() override; void OnAllConfigLoaded() override; diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index e51e2d215cc..13162cb2aed 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -62,7 +62,6 @@ std::vector IcingaDB::GetTypes() // Then sync them for similar reasons. Downtime::TypeInstance, Comment::TypeInstance, - Dependency::TypeInstance, HostGroup::TypeInstance, ServiceGroup::TypeInstance, @@ -205,10 +204,18 @@ void IcingaDB::UpdateAllConfigObjects() m_Rcon->FireAndForgetQuery({"XADD", "icinga:dump", "MAXLEN", "1", "*", "key", "*", "state", "wip"}, Prio::Config); const std::vector globalKeys = { - m_PrefixConfigObject + "customvar", - m_PrefixConfigObject + "action:url", - m_PrefixConfigObject + "notes:url", - m_PrefixConfigObject + "icon:image", + m_PrefixConfigObject + "customvar", + m_PrefixConfigObject + "action:url", + m_PrefixConfigObject + "notes:url", + m_PrefixConfigObject + "icon:image", + + // These keys aren't tied to a specific Checkable object but apply to all of them, and as such + // we've to make sure to clear them before we actually start dumping the actual objects. + // This also allows us to wait on all the Checkables to be dumped before we send a config + // dump done signal for those keys. + m_PrefixConfigObject + "dependency:node", + m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "redundancygroup", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -219,6 +226,7 @@ void IcingaDB::UpdateAllConfigObjects() m_DumpedGlobals.ActionUrl.Reset(); m_DumpedGlobals.NotesUrl.Reset(); m_DumpedGlobals.IconImage.Reset(); + m_DumpedGlobals.RedundancyGroup.Reset(); }); upq.ParallelFor(types, false, [this](const Type::Ptr& type) { @@ -790,138 +798,9 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } - return; - } - - if (type == Dependency::TypeInstance) { - auto& dependencyNodes (hMSets[m_PrefixConfigObject + "dependency:node"]); - auto& dependencyEdges (hMSets[m_PrefixConfigObject + "dependency:edge"]); - auto& redundancyGroups (hMSets[m_PrefixConfigObject + "redundancygroup"]); - - Dependency::Ptr dependency = static_pointer_cast(object); - - Host::Ptr parentHost, childHost; - Service::Ptr parentService, childService; - tie(parentHost, parentService) = GetHostService(dependency->GetParent()); - tie(childHost, childService) = GetHostService(dependency->GetChild()); - String redundancyGroup = dependency->GetRedundancyGroup(); - - String redundancyGroupId, dependencyNodeParentId, dependencyNodeChildId, dependencyNodeReduId; - - Dictionary::Ptr parentNodeData, childNodeData; - - if (parentService) { - dependencyNodeParentId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(parentHost), - GetObjectIdentifier(parentService)})); - parentNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(parentHost)}, - {"service_id", GetObjectIdentifier(parentService)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(parentService), dependency); - } else { - dependencyNodeParentId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(parentHost)})); - parentNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(parentHost)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(parentHost), dependency); - } - - if (childService) { - dependencyNodeChildId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(childHost), - GetObjectIdentifier(childService)})); - childNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(childHost)}, - {"service_id", GetObjectIdentifier(childService)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(childService), dependency); - } else { - dependencyNodeChildId = HashValue(new Array({ - m_EnvironmentId, - GetObjectIdentifier(childHost)})); - childNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"host_id", GetObjectIdentifier(childHost)}}); - - m_CheckablesToDependencies->Set(GetObjectIdentifier(childHost), dependency); - } - - dependencyNodes.emplace_back(dependencyNodeParentId); - dependencyNodes.emplace_back(JsonEncode(parentNodeData)); - dependencyNodes.emplace_back(dependencyNodeChildId); - dependencyNodes.emplace_back(JsonEncode(childNodeData)); - - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeParentId, m_PrefixConfigObject + "dependency:node", parentNodeData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeChildId, m_PrefixConfigObject + "dependency:node", childNodeData); - } - - if (!redundancyGroup.IsEmpty()) { - /* TODO: name should be suffixed with names of all children. - * however, at this point I don't have this information, - * only the direct neighbors. - */ - redundancyGroupId = HashValue(new Array({m_EnvironmentId, redundancyGroup, dependencyNodeChildId})); - dependencyNodeReduId = redundancyGroupId; - - redundancyGroups.emplace_back(redundancyGroupId); - Dictionary::Ptr groupData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"name", redundancyGroupId}, - {"display_name", redundancyGroup}}); - redundancyGroups.emplace_back(JsonEncode(groupData)); - - dependencyNodes.emplace_back(dependencyNodeReduId); - Dictionary::Ptr reduNodeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"redundancy_group_id", redundancyGroupId}}); - dependencyNodes.emplace_back(JsonEncode(reduNodeData)); - - String edgeInId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeReduId})); - dependencyEdges.emplace_back(edgeInId); - Dictionary::Ptr edgeInData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeChildId}, - {"to_node_id", dependencyNodeReduId}}); - dependencyEdges.emplace_back(JsonEncode(edgeInData)); - - String edgeOutId = HashValue(new Array({m_EnvironmentId, dependencyNodeReduId, dependencyNodeParentId})); - dependencyEdges.emplace_back(edgeOutId); - Dictionary::Ptr edgeOutData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeReduId}, - {"to_node_id", dependencyNodeParentId}, - {"dependency_id", GetObjectIdentifier(dependency)}}); - dependencyEdges.emplace_back(JsonEncode(edgeOutData)); - - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, dependencyNodeReduId, m_PrefixConfigObject + "dependency:node", reduNodeData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeInId, m_PrefixConfigObject + "dependency:edge", edgeInData); - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeOutId, m_PrefixConfigObject + "dependency:edge", edgeOutData); - } - } else { - String edgeId = HashValue(new Array({m_EnvironmentId, dependencyNodeChildId, dependencyNodeParentId})); - dependencyEdges.emplace_back(edgeId); - Dictionary::Ptr edgeData = new Dictionary({ - {"environment_id", m_EnvironmentId}, - {"from_node_id", dependencyNodeChildId}, - {"to_node_id", dependencyNodeParentId}, - {"dependency_id", GetObjectIdentifier(dependency)}}); - dependencyEdges.emplace_back(JsonEncode(edgeData)); + InsertCheckableDependencies(checkable, hMSets, runtimeUpdate ? &runtimeUpdates : nullptr); - if (runtimeUpdate) { - AddObjectDataToRuntimeUpdates(runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", edgeData); - } - } + return; } if (type == TimePeriod::TypeInstance) { @@ -1254,44 +1133,110 @@ void IcingaDB::InsertObjectDependencies(const ConfigObject::Ptr& object, const S } } -void IcingaDB::UpdateDependencyState(const Dependency::Ptr& dependency) +void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates) { - if (!m_Rcon || !m_Rcon->IsConnected()) { + // Only generate a dependency node event if the Checkable is actually part of some dependency graph. + // That's, it either depends on other Checkables or others depend on it, and in both cases, we have + // to at least generate a dependency node entry for it. + if (!checkable->HasAnyDependencies()) { return; } - auto& redundancyGroupStates (hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + auto& hmsetDependencyNodes(hMSets[m_PrefixConfigObject + "dependency:node"]); + auto& hmsetDependencyEdges(hMSets[m_PrefixConfigObject + "dependency:edge"]); + auto& hmsetRedundancyGroups(hMSets[m_PrefixConfigObject + "redundancygroup"]); + + auto [host, service] = GetHostService(checkable); + auto checkableId(GetObjectIdentifier(checkable)); + { + Dictionary::Ptr data(new Dictionary{{"environment_id", m_EnvironmentId}, {"host_id", GetObjectIdentifier(host)}}); + if (service) { + data->Set("service_id", checkableId); + } + + hmsetDependencyNodes.emplace_back(checkableId); + hmsetDependencyNodes.emplace_back(JsonEncode(data)); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, checkableId, m_PrefixConfigObject + "dependency:node", data); + } + } + + for (auto& dependencyGroup : checkable->GetDependencyGroups()) { + String redundancyGroupId(dependencyGroup->GetIcingaDBIdentifier()); + if (dependencyGroup->IsRedundancyGroup()) { + redundancyGroupId = HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()}); + dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); + + // Sync redundancy group information only once unless it's a runtime update. + if (runtimeUpdates || m_DumpedGlobals.RedundancyGroup.IsNew(redundancyGroupId)) { + Dictionary::Ptr groupData(new Dictionary{{"environment_id", m_EnvironmentId}, {"name", dependencyGroup->GetName()}}); + hmsetRedundancyGroups.emplace_back(redundancyGroupId); + hmsetRedundancyGroups.emplace_back(JsonEncode(groupData)); + + Dictionary::Ptr nodeData(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroupId}, + }); + + hmsetDependencyNodes.emplace_back(redundancyGroupId); + hmsetDependencyNodes.emplace_back(JsonEncode(nodeData)); + + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); + AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } + } + + Dictionary::Ptr data(new Dictionary{ + {"environment_id", m_EnvironmentId}, + {"from_node_id", checkableId}, + {"to_node_id", redundancyGroupId}, + // All redundancy group members share the same state, thus use the group ID as a reference. + {"dependency_edge_state_id", redundancyGroupId}, + {"display_name", dependencyGroup->GetName()}, + }); - String redundancyGroup = dependency->GetRedundancyGroup(); + auto edgeId(HashValue(new Array{checkableId, redundancyGroupId})); + hmsetDependencyEdges.emplace_back(edgeId); + hmsetDependencyEdges.emplace_back(JsonEncode(data)); - if (!redundancyGroup.IsEmpty()) { - Host::Ptr childHost; - Service::Ptr childService; - tie(childHost, childService) = GetHostService(dependency->GetChild()); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } - String dependencyNodeChildId = HashValue( - (childService) - ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) - : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); - String redundancyGroupId = HashValue(new Array({ - m_EnvironmentId, - redundancyGroup, - dependencyNodeChildId})); + auto members(dependencyGroup->GetMembers(checkable.get())); + for (auto it(members.begin()); it != members.end(); /* no increment */) { + auto dependency(*it); + auto parent(dependency->GetParent()); + auto displayName(dependency->GetShortName()); + + // Find all members that share the same parent starting from the iterator position (it position inclusively). + auto [begin, end] = std::equal_range(it, members.end(), parent, Dependency::ParentComparator{}); + // We might not need to process the element pointed to by (it) as well, so we need to + // exclude it from the returned range (notice the ++begin below). + std::for_each(++begin, end, [&displayName](const Dependency::Ptr& member) { + displayName += ", " + member->GetShortName(); + }); + it = end; // Advance the iterator to either end of the container or beginning of the next members block. - redundancyGroupStates.emplace_back(redundancyGroupId); - Dictionary::Ptr groupStateData = new Dictionary({ + Dictionary::Ptr data(new Dictionary{ {"environment_id", m_EnvironmentId}, - {"redundancy_group_id", redundancyGroupId}, - {"failed", !((childService) ? childService->IsReachable() : childHost->IsReachable())}, - {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}}); - redundancyGroupStates.emplace_back(JsonEncode(groupStateData)); + {"from_node_id", dependencyGroup->IsRedundancyGroup() ? redundancyGroupId : checkableId}, + {"to_node_id", GetObjectIdentifier(parent)}, + {"display_name", displayName}, + }); + String edgeId(HashValue(new Array{data->Get("from_node_id"), data->Get("to_node_id")})); + data->Set("dependency_edge_state_id", edgeId); + + hmsetDependencyEdges.emplace_back(edgeId); + hmsetDependencyEdges.emplace_back(JsonEncode(data)); - // TODO - // AddObjectDataToRuntimeUpdates(runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup:state", groupStateData); - // dataClone->Set("id", objectKey); // redundancyGroupId - // dataClone->Set("redis_key", redisKey); // m_PrefixConfigObject + "redundancygroup:state" - // dataClone->Set("runtime_type", "upsert"); - // runtimeUpdates.emplace_back(dataClone); + if (runtimeUpdates) { + AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } + } } } @@ -1629,32 +1574,6 @@ bool IcingaDB::PrepareObject(const ConfigObject::Ptr& object, Dictionary::Ptr& a return true; } - if (type == Dependency::TypeInstance) { - Dependency::Ptr dependency = static_pointer_cast(object); - String redundancyGroup = dependency->GetRedundancyGroup(); - - attributes->Set("name", GetObjectIdentifier(dependency)); - - if (!redundancyGroup.IsEmpty()) { - Host::Ptr childHost; - Service::Ptr childService; - tie(childHost, childService) = GetHostService(dependency->GetChild()); - - String dependencyNodeChildId = HashValue( - (childService) - ? new Array({ m_EnvironmentId, GetObjectIdentifier(childHost), GetObjectIdentifier(childService) }) - : new Array({ m_EnvironmentId, GetObjectIdentifier(childHost) })); - String redundancyGroupId = HashValue(new Array({ - m_EnvironmentId, - redundancyGroup, - dependencyNodeChildId})); - - attributes->Set("redundancy_group_id", redundancyGroupId); - } - - return true; - } - if (type == Downtime::TypeInstance) { Downtime::Ptr downtime = static_pointer_cast(object); diff --git a/lib/icingadb/icingadb.cpp b/lib/icingadb/icingadb.cpp index 3c623259b58..8d3b9099bd7 100644 --- a/lib/icingadb/icingadb.cpp +++ b/lib/icingadb/icingadb.cpp @@ -38,8 +38,6 @@ IcingaDB::IcingaDB() m_PrefixConfigObject = "icinga:"; m_PrefixConfigCheckSum = "icinga:checksum:"; - - m_CheckablesToDependencies = new Dictionary(); } void IcingaDB::Validate(int types, const ValidationUtils& utils) diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 8eb5c946ddd..2ddafcf59d5 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -101,9 +101,10 @@ class IcingaDB : public ObjectImpl void DeleteKeys(const RedisConnection::Ptr& conn, const std::vector& keys, RedisConnection::QueryPriority priority); std::vector GetTypeOverwriteKeys(const String& type); std::vector GetTypeDumpSignalKeys(const Type::Ptr& type); + void InsertCheckableDependencies(const Checkable::Ptr& checkable, std::map& hMSets, + std::vector* runtimeUpdates); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); - void UpdateDependencyState(const Dependency::Ptr& dependency); void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -225,10 +226,8 @@ class IcingaDB : public ObjectImpl std::unordered_map m_Rcons; std::atomic_size_t m_PendingRcons; - Dictionary::Ptr m_CheckablesToDependencies; - struct { - DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage; + DumpedGlobals CustomVar, ActionUrl, NotesUrl, IconImage, RedundancyGroup; } m_DumpedGlobals; // m_EnvironmentId is shared across all IcingaDB objects (typically there is at most one, but it is perfectly fine From 88d818c8e73835d88cdc1c8fb6ff1bdf5b2c46d9 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 4 Dec 2024 16:47:32 +0100 Subject: [PATCH 11/20] IcingaDB: Sync dependencies states to Redis --- lib/icingadb/icingadb-objects.cpp | 132 +++++++++++++++++++++++++++++- lib/icingadb/icingadb-utility.cpp | 48 +++++++++++ lib/icingadb/icingadb.hpp | 3 + 3 files changed, 182 insertions(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 13162cb2aed..1617c40a52d 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -215,7 +215,9 @@ void IcingaDB::UpdateAllConfigObjects() // dump done signal for those keys. m_PrefixConfigObject + "dependency:node", m_PrefixConfigObject + "dependency:edge", + m_PrefixConfigObject + "dependency:edge:state", m_PrefixConfigObject + "redundancygroup", + m_PrefixConfigObject + "redundancygroup:state", }; DeleteKeys(m_Rcon, globalKeys, Prio::Config); DeleteKeys(m_Rcon, {"icinga:nextupdate:host", "icinga:nextupdate:service"}, Prio::Config); @@ -1170,7 +1172,7 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: // Sync redundancy group information only once unless it's a runtime update. if (runtimeUpdates || m_DumpedGlobals.RedundancyGroup.IsNew(redundancyGroupId)) { - Dictionary::Ptr groupData(new Dictionary{{"environment_id", m_EnvironmentId}, {"name", dependencyGroup->GetName()}}); + Dictionary::Ptr groupData(SerializeRedundancyGroup(dependencyGroup)); hmsetRedundancyGroups.emplace_back(redundancyGroupId); hmsetRedundancyGroups.emplace_back(JsonEncode(groupData)); @@ -1294,6 +1296,134 @@ void IcingaDB::UpdateState(const Checkable::Ptr& checkable, StateUpdate mode) m_Rcon->FireAndForgetQuery(std::move(streamadd), Prio::RuntimeStateStream, {0, 1}); } + + UpdateDependenciesState(checkable, mode); +} + +/** + * Send dependencies state information of the given Checkable to Redis. + * + * For explicitly configured redundancy groups, the state information is always sent to Redis, regardless of the + * mode parameter. The mode parameter loosely controls how and which states are sent to Redis for non-redundant + * dependencies. The value of that parameter can be one of the following: + * - Volatile: Check each dependency object of the specified Checkable for its configuration before sending + * the state update. For instance, if the dependency is configured to ignore soft state changes, its state + * should not be different from the previous ones and will therefore not be sent to Redis. + * - RuntimeOnly: Performs an additional check to the one above and ignores dependency objects that are not within + * their time period. Once the dependencies pass all the necessary checks, these two mods only send the state + * information to the icinga:runtime:state Redis stream via XADD. + * - Full: Perform a full state update ignoring all the above mentioned checks. Additionally, set the same + * state information that is streamed into the pipeline to the Redis keys icinga:{dependency:edge,redundancygroup}:state + * as well. + * + * @param checkable The Checkable you want to send the dependencies state update for + * @param mode The type of operation you want to perform (Volatile, RuntimeOnly, or Full) + */ +void IcingaDB::UpdateDependenciesState(const Checkable::Ptr& checkable, StateUpdate mode) const +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto dependencyGroups(checkable->GetDependencyGroups()); + if (dependencyGroups.empty()) { + return; + } + + RedisConnection::Queries streamStates, hmsets{{/* dependency states */}, {/* redundancy group states */}}; + auto addDependencyStateToStream([this, &streamStates](const String& redisKey, const Dictionary::Ptr& stateAttrs) { + RedisConnection::Query xAdd{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", + "runtime_type", "upsert", "redis_key", redisKey + }; + ObjectLock olock(stateAttrs); + for (auto& [key, value] : stateAttrs) { + xAdd.emplace_back(key); + xAdd.emplace_back(IcingaToStreamValue(value)); + } + streamStates.emplace_back(std::move(xAdd)); + }); + + auto now(Utility::GetTime()); + for (auto& dependencyGroup : dependencyGroups) { + bool isRedundancyGroup(dependencyGroup->IsRedundancyGroup()); + if (isRedundancyGroup && dependencyGroup->GetIcingaDBIdentifier().IsEmpty()) { + // Way too soon! The Icinga DB hash will be set during the initial config dump, but this state + // update seems to occur way too early. So, we've to skip it for now and wait for the next one. + // The m_ConfigDumpInProgress flag is probably still set to true at this point! + continue; + } + + auto members(dependencyGroup->GetMembers(checkable.get())); + for (auto it(members.begin()); it != members.end(); /* no increment */) { + auto dep(*it); + if (auto tp(dep->GetPeriod()); !isRedundancyGroup && mode != StateUpdate::Full && tp && !tp->IsInside(now)) { + // When the dependency is outside its time period, and we're not performing a full + // state update, we don't need to send any updates as it should always be reachable. + continue; + } + + Dictionary::Ptr stateAttrs(SerializeDependencyEdgeState(dependencyGroup, dep)); + // Find all members that share the same parent starting from the iterator position (it position inclusively). + auto [begin, end] = std::equal_range(it, members.end(), dep->GetParent(), Dependency::ParentComparator{}); + // We might not need to process the element pointed to by (it) as well, so we need to + // exclude it from the returned range (notice the ++begin below). + std::for_each(++begin, end, [&stateAttrs, &dependencyGroup](const Dependency::Ptr& member) { + if (stateAttrs->Get("failed") == false) { + stateAttrs = SerializeDependencyEdgeState(dependencyGroup, member); + } + }); + it = end; // Advance the iterator to either end of the container or beginning of the next members block. + + if (mode == StateUpdate::Full) { + // Actually we wouldn't need to send state updates via HMSET for dependency states here, + // as we're going to stream them via the runtime:state pipeline anyway. However, if we + // don't do this, Icinga DB is going to remove them from the DB and reinsert them again + // with each config dump. So, we've to refresh the non-stream keys regularly! + hmsets[0].emplace_back(stateAttrs->Get("id")); + hmsets[0].emplace_back(JsonEncode(stateAttrs)); + } + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", stateAttrs); + } + + if (isRedundancyGroup) { + Dictionary::Ptr stateAttrs(SerializeRedundancyGroup(dependencyGroup, true)); + Dictionary::Ptr groupSharedState(stateAttrs->ShallowClone()); + groupSharedState->Remove("redundancy_group_id"); + groupSharedState->Remove("is_reachable"); + groupSharedState->Remove("last_state_change"); + + if (mode == StateUpdate::Full) { + // Same as above, we've to refresh the group state information regularly to prevent + // Icinga DB from removing and reinserting them with each config dump. + hmsets[0].emplace_back(dependencyGroup->GetIcingaDBIdentifier()); + hmsets[0].emplace_back(JsonEncode(groupSharedState)); + hmsets[1].emplace_back(dependencyGroup->GetIcingaDBIdentifier()); + hmsets[1].emplace_back(JsonEncode(stateAttrs)); + } + addDependencyStateToStream(m_PrefixConfigObject + "redundancygroup:state", stateAttrs); + addDependencyStateToStream(m_PrefixConfigObject + "dependency:edge:state", groupSharedState); + } + } + + if (!hmsets[0].empty() || !hmsets[1].empty()) { + if (!hmsets.front().empty()) { + hmsets[0].insert(hmsets[0].begin(), {"DEL", m_PrefixConfigObject + "dependency:edge:state"}); + } + if (!hmsets[1].empty()) { + hmsets[1].insert(hmsets[1].begin(), {"HMSET", m_PrefixConfigObject + "redundancygroup:state"}); + } + + hmsets.erase( + std::remove_if(hmsets.begin(), hmsets.end(), std::mem_fn(&RedisConnection::Query::empty)), + hmsets.end() + ); + m_Rcon->FireAndForgetQueries(std::move(hmsets), Prio::RuntimeStateSync); + } + + if (!streamStates.empty()) { + m_Rcon->FireAndForgetQueries(std::move(streamStates), Prio::RuntimeStateStream, {0, 1}); + } } // Used to update a single object, used for runtime updates diff --git a/lib/icingadb/icingadb-utility.cpp b/lib/icingadb/icingadb-utility.cpp index 35f503ab53d..f8b96ef1984 100644 --- a/lib/icingadb/icingadb-utility.cpp +++ b/lib/icingadb/icingadb-utility.cpp @@ -159,6 +159,54 @@ Dictionary::Ptr IcingaDB::SerializeVars(const Dictionary::Ptr& vars) return res; } +/** + * Serialize a dependency edge state for Icinga DB + * + * @param dependencyGroup The state of the group the dependency is part of. + * @param dep The dependency object to serialize. + * + * @return A dictionary with the serialized state. + */ +Dictionary::Ptr IcingaDB::SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep) +{ + auto child(dep->GetChild()); + Array::Ptr data(new Array{ + dependencyGroup->IsRedundancyGroup() ? dependencyGroup->GetIcingaDBIdentifier() : GetObjectIdentifier(child), + GetObjectIdentifier(dep->GetParent()) + }); + + return new Dictionary{ + {"id", HashValue(data)}, + {"environment_id", m_EnvironmentId}, + {"failed", !dep->IsAvailable(DependencyState) || !dep->GetParent()->IsReachable()} + }; +} + +/** + * Serialize the provided redundancy group or its state attributes. + * + * @param redundancyGroup The redundancy group object to serialize. + * @param serializeState Whether to serialize the state of the redundancy group. + * + * @return A dictionary with the serialized redundancy group. + */ +Dictionary::Ptr IcingaDB::SerializeRedundancyGroup(const DependencyGroup::Ptr& redundancyGroup, bool serializeState) +{ + if (serializeState) { + auto state(redundancyGroup->GetState()); + return new Dictionary{ + {"id", redundancyGroup->GetIcingaDBIdentifier()}, + {"environment_id", m_EnvironmentId}, + {"redundancy_group_id", redundancyGroup->GetIcingaDBIdentifier()}, + {"failed", !(state & DependencyGroup::State::ReachableOK)}, + {"is_reachable", !(state & DependencyGroup::State::Unreachable)}, + {"last_state_change", TimestampToMilliseconds(Utility::GetTime())}, + }; + } + + return new Dictionary{{"environment_id", m_EnvironmentId}, {"display_name", redundancyGroup->GetName()}}; +} + const char* IcingaDB::GetNotificationTypeByEnum(NotificationType type) { switch (type) { diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index 2ddafcf59d5..ff61b2c793d 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -105,6 +105,7 @@ class IcingaDB : public ObjectImpl std::vector* runtimeUpdates); void InsertObjectDependencies(const ConfigObject::Ptr& object, const String typeName, std::map>& hMSets, std::vector& runtimeUpdates, bool runtimeUpdate); + void UpdateDependenciesState(const Checkable::Ptr& checkable, StateUpdate mode) const; void UpdateState(const Checkable::Ptr& checkable, StateUpdate mode); void SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpdate); void CreateConfigUpdate(const ConfigObject::Ptr& object, const String type, std::map>& hMSets, @@ -159,6 +160,8 @@ class IcingaDB : public ObjectImpl static String CalcEventID(const char* eventType, const ConfigObject::Ptr& object, double eventTime = 0, NotificationType nt = NotificationType(0)); static const char* GetNotificationTypeByEnum(NotificationType type); static Dictionary::Ptr SerializeVars(const Dictionary::Ptr& vars); + static Dictionary::Ptr SerializeDependencyEdgeState(const DependencyGroup::Ptr& dependencyGroup, const Dependency::Ptr& dep); + static Dictionary::Ptr SerializeRedundancyGroup(const DependencyGroup::Ptr& redundancyGroup, bool serializeState = false); static String HashValue(const Value& value); static String HashValue(const Value& value, const std::set& propertiesBlacklist, bool propertiesWhitelist = false); From a830eba50bed61d470c38b9fc923838e5dba8108 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 5 Dec 2024 16:21:32 +0100 Subject: [PATCH 12/20] IcingaDB: Sync dependencies initial states on config dump --- lib/icingadb/icingadb-objects.cpp | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 1617c40a52d..a5087bea6af 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -1149,6 +1149,10 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: auto& hmsetDependencyEdges(hMSets[m_PrefixConfigObject + "dependency:edge"]); auto& hmsetRedundancyGroups(hMSets[m_PrefixConfigObject + "redundancygroup"]); + // If this isn't a runtime update, we need to send initial state updates for the dependencies as well. + auto& hmsetDependenciesStates(hMSets[m_PrefixConfigObject + "dependency:edge:state"]); + auto& hmsetRedundancyGroupsStates(hMSets[m_PrefixConfigObject + "redundancygroup:state"]); + auto [host, service] = GetHostService(checkable); auto checkableId(GetObjectIdentifier(checkable)); { @@ -1166,12 +1170,14 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: for (auto& dependencyGroup : checkable->GetDependencyGroups()) { String redundancyGroupId(dependencyGroup->GetIcingaDBIdentifier()); + bool syncSharedEdgeState(false); if (dependencyGroup->IsRedundancyGroup()) { redundancyGroupId = HashValue(new Array{m_EnvironmentId, dependencyGroup->GetCompositeKey()}); dependencyGroup->SetIcingaDBIdentifier(redundancyGroupId); // Sync redundancy group information only once unless it's a runtime update. if (runtimeUpdates || m_DumpedGlobals.RedundancyGroup.IsNew(redundancyGroupId)) { + syncSharedEdgeState = true; Dictionary::Ptr groupData(SerializeRedundancyGroup(dependencyGroup)); hmsetRedundancyGroups.emplace_back(redundancyGroupId); hmsetRedundancyGroups.emplace_back(JsonEncode(groupData)); @@ -1187,6 +1193,17 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: if (runtimeUpdates) { AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "redundancygroup", groupData); AddObjectDataToRuntimeUpdates(*runtimeUpdates, redundancyGroupId, m_PrefixConfigObject + "dependency:node", nodeData); + } else { + auto stateAttrs(SerializeRedundancyGroup(dependencyGroup, true)); + hmsetRedundancyGroupsStates.emplace_back(redundancyGroupId); + hmsetRedundancyGroupsStates.emplace_back(JsonEncode(stateAttrs)); + + hmsetDependenciesStates.emplace_back(redundancyGroupId); + hmsetDependenciesStates.emplace_back(JsonEncode(Dictionary::Ptr(new Dictionary{ + {"id", redundancyGroupId}, + {"environment_id", m_EnvironmentId}, + {"failed", stateAttrs->Get("failed")}, + }))); } } @@ -1214,12 +1231,20 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: auto parent(dependency->GetParent()); auto displayName(dependency->GetShortName()); + Dictionary::Ptr memberStateAttrs; + if (!runtimeUpdates && (!dependencyGroup->IsRedundancyGroup() || syncSharedEdgeState)) { + memberStateAttrs = SerializeDependencyEdgeState(dependencyGroup, dependency); + } + // Find all members that share the same parent starting from the iterator position (it position inclusively). auto [begin, end] = std::equal_range(it, members.end(), parent, Dependency::ParentComparator{}); // We might not need to process the element pointed to by (it) as well, so we need to // exclude it from the returned range (notice the ++begin below). - std::for_each(++begin, end, [&displayName](const Dependency::Ptr& member) { + std::for_each(++begin, end, [&displayName, &memberStateAttrs, &dependencyGroup](const Dependency::Ptr& member) { displayName += ", " + member->GetShortName(); + if (memberStateAttrs && memberStateAttrs->Get("failed") == false) { + memberStateAttrs = SerializeDependencyEdgeState(dependencyGroup, member); + } }); it = end; // Advance the iterator to either end of the container or beginning of the next members block. @@ -1237,6 +1262,9 @@ void IcingaDB::InsertCheckableDependencies(const Checkable::Ptr& checkable, std: if (runtimeUpdates) { AddObjectDataToRuntimeUpdates(*runtimeUpdates, edgeId, m_PrefixConfigObject + "dependency:edge", data); + } else if (memberStateAttrs) { + hmsetDependenciesStates.emplace_back(edgeId); + hmsetDependenciesStates.emplace_back(JsonEncode(memberStateAttrs)); } } } From dae5f6f0c11ae113c57b609f1378454980c186c8 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 9 Dec 2024 11:19:57 +0100 Subject: [PATCH 13/20] IcingaDB: Handle runtime removed dependencies correctly --- lib/icinga/dependency.cpp | 73 ++++++++++++++- lib/icinga/dependency.hpp | 3 + lib/icingadb/icingadb-objects.cpp | 145 ++++++++++++++++++++++++------ lib/icingadb/icingadb.hpp | 3 + 4 files changed, 192 insertions(+), 32 deletions(-) diff --git a/lib/icinga/dependency.cpp b/lib/icinga/dependency.cpp index eebeb014cd3..0ad38d37d2e 100644 --- a/lib/icinga/dependency.cpp +++ b/lib/icinga/dependency.cpp @@ -191,28 +191,91 @@ void Dependency::OnAllConfigLoaded() if (!m_Parent) BOOST_THROW_EXCEPTION(ScriptError("Dependency '" + GetName() + "' references a parent host/service which doesn't exist.", GetDebugInfo())); - DependencyGroup::Register(this); - m_Parent->AddReverseDependency(this); + if (!m_AssertNoCyclesForIndividualDeps) { + // Yes, this is rather a hack and introduces an inconsistent behaviour between file-based and + // runtime-created dependencies. However, I don't see any other possibility for handling this + // in a satisfiable manner. The reason for this change is simple, while Icinga DB doesn't need + // to know about every un/registered dependencies on startup, we still need to track and notify + // it about each and every dependency change at runtime to keep the database consistent. + DependencyGroup::Register(this); + m_Parent->AddReverseDependency(this); + } +} + +void Dependency::Start(bool runtimeCreated) +{ + if (runtimeCreated) { + std::vector previousGroups; + if (!GetRedundancyGroup().IsEmpty()) { + previousGroups = m_Child->GetDependencyGroups(); + } - if (m_AssertNoCyclesForIndividualDeps) { - DependencyCycleGraph graph; + DependencyGroup::Register(this); + m_Parent->AddReverseDependency(this); try { + DependencyCycleGraph graph; AssertNoDependencyCycle(m_Parent, graph); } catch (...) { DependencyGroup::Unregister(this); m_Parent->RemoveReverseDependency(this); throw; } + + // Dependencies get activated before their parent Checkable objects, so if the child Checkable + // isn't active yet, it means that the Checkable is also created at runtime, and we don't need + // to notify Icinga DB, as the Checkable's OnVersionChanged signal will do that for us. + if (m_Child->IsActive()) { + std::vector outdatedGroups; + if (!previousGroups.empty()) { + auto newGroups(m_Child->GetDependencyGroups()); + std::set_difference( + previousGroups.begin(), + previousGroups.end(), + newGroups.begin(), + newGroups.end(), + std::back_inserter(outdatedGroups) + ); + } + DependencyGroup::OnMembersChanged(this, outdatedGroups); + } } + + ObjectImpl::Start(runtimeCreated); } void Dependency::Stop(bool runtimeRemoved) { ObjectImpl::Stop(runtimeRemoved); + std::vector previousGroups; + std::set outdatedGroups; + if (runtimeRemoved && !GetRedundancyGroup().IsEmpty()) { + previousGroups = m_Child->GetDependencyGroups(); + for (auto& dependencyGroup: previousGroups) { + if (dependencyGroup->HasMember(this)) { + outdatedGroups.emplace(dependencyGroup); + break; + } + } + } + DependencyGroup::Unregister(this); GetParent()->RemoveReverseDependency(this); + + if (runtimeRemoved) { + if (!previousGroups.empty()) { + auto newGroups(m_Child->GetDependencyGroups()); + std::set_difference( + previousGroups.begin(), + previousGroups.end(), + newGroups.begin(), + newGroups.end(), + std::inserter(outdatedGroups, outdatedGroups.end()) + ); + } + DependencyGroup::OnMembersChanged(this, {outdatedGroups.begin(), outdatedGroups.end()}); + } } bool Dependency::IsAvailable(DependencyType dt) const @@ -330,6 +393,8 @@ void Dependency::SetChild(intrusive_ptr child) // Is the default (dummy) dependency group name used to group non-redundant dependencies. static String l_DefaultDependencyGroup(Utility::NewUniqueID()); +boost::signals2::signal&)> DependencyGroup::OnMembersChanged; + std::mutex DependencyGroup::m_RegistryMutex; DependencyGroup::RegistryType DependencyGroup::m_Registry; diff --git a/lib/icinga/dependency.hpp b/lib/icinga/dependency.hpp index 6511791c154..2e3ad776631 100644 --- a/lib/icinga/dependency.hpp +++ b/lib/icinga/dependency.hpp @@ -67,6 +67,7 @@ class Dependency final : public ObjectImpl protected: void OnConfigLoaded() override; void OnAllConfigLoaded() override; + void Start(bool runtimeCreated) override; void Stop(bool runtimeRemoved) override; private: @@ -133,6 +134,8 @@ class DependencyGroup final : public SharedObject State GetState(DependencyType dt = DependencyState, int rstack = 0) const; + static boost::signals2::signal&)> OnMembersChanged; + protected: void AddMember(const Dependency::Ptr& member); void RemoveMember(const Dependency::Ptr& member); diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index a5087bea6af..0f17ebf434f 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -133,6 +133,8 @@ void IcingaDB::ConfigStaticInitialize() IcingaDB::NextCheckUpdatedHandler(checkable); }); + DependencyGroup::OnMembersChanged.connect(&IcingaDB::DependencyGroupsChangedHandler); + Service::OnHostProblemChanged.connect([](const Service::Ptr& service, const CheckResult::Ptr&, const MessageOrigin::Ptr&) { IcingaDB::HostProblemChangedHandler(service); }); @@ -1471,34 +1473,7 @@ void IcingaDB::SendConfigUpdate(const ConfigObject::Ptr& object, bool runtimeUpd UpdateState(checkable, runtimeUpdate ? StateUpdate::Full : StateUpdate::Volatile); } - std::vector > transaction = {{"MULTI"}}; - - for (auto& kv : hMSets) { - if (!kv.second.empty()) { - kv.second.insert(kv.second.begin(), {"HMSET", kv.first}); - transaction.emplace_back(std::move(kv.second)); - } - } - - for (auto& objectAttributes : runtimeUpdates) { - std::vector xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); - ObjectLock olock(objectAttributes); - - for (const Dictionary::Pair& kv : objectAttributes) { - String value = IcingaToStreamValue(kv.second); - if (!value.IsEmpty()) { - xAdd.emplace_back(kv.first); - xAdd.emplace_back(value); - } - } - - transaction.emplace_back(std::move(xAdd)); - } - - if (transaction.size() > 1) { - transaction.push_back({"EXEC"}); - m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); - } + ExecuteRedisTransaction(hMSets, runtimeUpdates); if (checkable) { SendNextUpdate(checkable); @@ -2862,6 +2837,83 @@ void IcingaDB::SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dict } } +void IcingaDB::SendDependencyGroupsChanged(const Dependency::Ptr& dep, const std::vector& outdatedGroups) +{ + if (!m_Rcon || !m_Rcon->IsConnected()) { + return; + } + + auto parent(dep->GetParent()); + auto child(dep->GetChild()); + if (!child->HasAnyDependencies()) { + // If the child Checkable has no parent and reverse dependencies, we can safely remove the dependency node. + DeleteRelationship(GetObjectIdentifier(child), "dependency:node"); + } + + RedisConnection::Queries hdels, xAdds; + auto deleteState([this, &hdels, &xAdds](const String& redisKey, const String& id) { + hdels.emplace_back(RedisConnection::Query{"HDEL", m_PrefixConfigObject + redisKey, id}); + xAdds.emplace_back(RedisConnection::Query{ + "XADD", "icinga:runtime:state", "MAXLEN", "~", "1000000", "*", "runtime_type", "delete", + "redis_key", m_PrefixConfigObject + redisKey, "id", id + }); + }); + + if (dep->GetRedundancyGroup().IsEmpty() && !dep->IsActive() && dep->GetExtension("ConfigObjectDeleted")) { + auto identifier(HashValue(new Array{GetObjectIdentifier(child), GetObjectIdentifier(parent)})); + // Remove the edge between the parent and child Checkable linked through the removed dependency. + DeleteRelationship(identifier, "dependency:edge"); + // The dependency object is going to be deleted, so we need to remove its state from Redis as well. + deleteState("dependency:edge:state", identifier); + } + + auto newGroups(child->GetDependencyGroups()); + for (auto& group : outdatedGroups) { + String redundancyGroupId(group->GetIcingaDBIdentifier()); + bool sameCompositeKey(redundancyGroupId == HashValue(new Array{m_EnvironmentId, group->GetCompositeKey()})); + if (!sameCompositeKey || !group->HasMembers() || std::find(newGroups.begin(), newGroups.end(), group) == newGroups.end()) { + // Remove the connection from the child Checkable to the redundancy group. + DeleteRelationship(HashValue(new Array{GetObjectIdentifier(child), redundancyGroupId}), "dependency:edge"); + + if (!sameCompositeKey || !group->HasMembers()) { + auto members(group->GetMembers(child.get())); + members.emplace_back(dep); + for (auto& member : members) { + // Remove the connection from the redundancy group to the parent Checkable of the removed + // dependency, if there's no other member in the redundancy group that references that very + // same parent Checkable or the dependency Group's Icinga DB identifier has changed. + auto groupParentId(HashValue(new Array{redundancyGroupId, GetObjectIdentifier(member->GetParent())})); + DeleteRelationship(groupParentId, "dependency:edge"); + deleteState("dependency:edge:state", groupParentId); + } + + // Remove the group entirely as it either has no members left or it's Icinga DB identifier has changed. + deleteState("dependency:edge:state", redundancyGroupId); + deleteState("redundancygroup:state", redundancyGroupId); + DeleteRelationship(redundancyGroupId, "dependency:node"); + DeleteRelationship(redundancyGroupId, "redundancygroup"); + } + } + } + + if (!hdels.empty()) { + m_Rcon->FireAndForgetQueries(std::move(hdels), Prio::RuntimeStateSync); + m_Rcon->FireAndForgetQueries(std::move(xAdds), Prio::RuntimeStateStream, {0, 1}); + + if (!newGroups.empty()) { + std::map hMSets; + std::vector runtimeUpdates; + InsertCheckableDependencies(child, hMSets, &runtimeUpdates); + + ExecuteRedisTransaction(hMSets, runtimeUpdates); + UpdateState(child, StateUpdate::Full); + } + } + + // The affect{ed,s}_children might now have different outcome, so we need to update the parent Checkable as well. + SendConfigUpdate(parent, true); +} + Dictionary::Ptr IcingaDB::SerializeState(const Checkable::Ptr& checkable) { Dictionary::Ptr attrs = new Dictionary(); @@ -3139,6 +3191,13 @@ void IcingaDB::NextCheckUpdatedHandler(const Checkable::Ptr& checkable) } } +void IcingaDB::DependencyGroupsChangedHandler(const Dependency::Ptr& dep, const std::vector& outdatedGroups) +{ + for (auto& rw : ConfigType::GetObjectsByType()) { + rw->SendDependencyGroupsChanged(dep, outdatedGroups); + } +} + void IcingaDB::HostProblemChangedHandler(const Service::Ptr& service) { for (auto& rw : ConfigType::GetObjectsByType()) { /* Host state changes affect is_handled and severity of services. */ @@ -3256,3 +3315,33 @@ void IcingaDB::DeleteRelationship(const String& id, const String& redisKeyWithou m_Rcon->FireAndForgetQueries(queries, Prio::Config); } + +void IcingaDB::ExecuteRedisTransaction(std::map& hMSets, const std::vector& runtimeUpdates) const +{ + RedisConnection::Queries transaction{{"MULTI"}}; + for (auto& [redisKey, query] : hMSets) { + if (!query.empty()) { + query.insert(query.begin(), {"HMSET", redisKey}); + transaction.emplace_back(std::move(query)); + } + } + + for (auto& attrs : runtimeUpdates) { + RedisConnection::Query xAdd({"XADD", "icinga:runtime", "MAXLEN", "~", "1000000", "*"}); + + ObjectLock olock(attrs); + for (auto& [key, value]: attrs) { + if (auto streamVal (IcingaToStreamValue(value)); !streamVal.IsEmpty()) { + xAdd.emplace_back(key); + xAdd.emplace_back(std::move(streamVal)); + } + } + + transaction.emplace_back(std::move(xAdd)); + } + + if (transaction.size() > 1) { + transaction.emplace_back(RedisConnection::Query{"EXEC"}); + m_Rcon->FireAndForgetQueries(std::move(transaction), Prio::Config, {1}); + } +} diff --git a/lib/icingadb/icingadb.hpp b/lib/icingadb/icingadb.hpp index ff61b2c793d..e2db2eea8af 100644 --- a/lib/icingadb/icingadb.hpp +++ b/lib/icingadb/icingadb.hpp @@ -115,6 +115,7 @@ class IcingaDB : public ObjectImpl void AddObjectDataToRuntimeUpdates(std::vector& runtimeUpdates, const String& objectKey, const String& redisKey, const Dictionary::Ptr& data); void DeleteRelationship(const String& id, const String& redisKeyWithoutPrefix, bool hasChecksum = false); + void ExecuteRedisTransaction(std::map& hMSets, const std::vector& runtimeUpdates) const; void SendSentNotification( const Notification::Ptr& notification, const Checkable::Ptr& checkable, const std::set& users, @@ -139,6 +140,7 @@ class IcingaDB : public ObjectImpl void SendCommandEnvChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCommandArgumentsChanged(const ConfigObject::Ptr& command, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); void SendCustomVarsChanged(const ConfigObject::Ptr& object, const Dictionary::Ptr& oldValues, const Dictionary::Ptr& newValues); + void SendDependencyGroupsChanged(const Dependency::Ptr& dep, const std::vector& outdatedGroups); void ForwardHistoryEntries(); @@ -185,6 +187,7 @@ class IcingaDB : public ObjectImpl static void FlappingChangeHandler(const Checkable::Ptr& checkable, double changeTime); static void NewCheckResultHandler(const Checkable::Ptr& checkable); static void NextCheckUpdatedHandler(const Checkable::Ptr& checkable); + static void DependencyGroupsChangedHandler(const Dependency::Ptr& dep, const std::vector& outdatedGroups); static void HostProblemChangedHandler(const Service::Ptr& service); static void AcknowledgementSetHandler(const Checkable::Ptr& checkable, const String& author, const String& comment, AcknowledgementType type, bool persistent, double changeTime, double expiry); static void AcknowledgementClearedHandler(const Checkable::Ptr& checkable, const String& removedBy, double changeTime); From 6893325a9bc0927906ecb91234350f10b87f8e85 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Thu, 12 Dec 2024 18:31:05 +0100 Subject: [PATCH 14/20] Checkable: Drop unused `failedDependency` argument from `IsReachable()` --- lib/icinga/checkable-dependency.cpp | 16 ++-------------- lib/icinga/checkable.hpp | 2 +- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index 480df7bf450..ca38aac4734 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -61,7 +61,7 @@ std::vector Checkable::GetReverseDependencies() const return std::vector(m_ReverseDependencies.begin(), m_ReverseDependencies.end()); } -bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency, int rstack) const +bool Checkable::IsReachable(DependencyType dt, int rstack) const { /* Anything greater than 256 causes recursion bus errors. */ int limit = 256; @@ -74,7 +74,7 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency } for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, failedDependency, rstack + 1)) + if (!checkable->IsReachable(dt, rstack + 1)) return false; } @@ -84,9 +84,6 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Host::Ptr host = service->GetHost(); if (host && host->GetState() != HostUp && host->GetStateType() == StateTypeHard) { - if (failedDependency) - *failedDependency = nullptr; - return false; } } @@ -103,9 +100,6 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Log(LogDebug, "Checkable") << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; - if (failedDependency) - *failedDependency = dep; - return false; } @@ -123,15 +117,9 @@ bool Checkable::IsReachable(DependencyType dt, Dependency::Ptr *failedDependency Log(LogDebug, "Checkable") << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - if (failedDependency) - *failedDependency = violator->second; - return false; } - if (failedDependency) - *failedDependency = nullptr; - return true; } diff --git a/lib/icinga/checkable.hpp b/lib/icinga/checkable.hpp index 6fcfc1385d0..6b8c7cfe0fa 100644 --- a/lib/icinga/checkable.hpp +++ b/lib/icinga/checkable.hpp @@ -82,7 +82,7 @@ class Checkable : public ObjectImpl void AddGroup(const String& name); - bool IsReachable(DependencyType dt = DependencyState, intrusive_ptr *failedDependency = nullptr, int rstack = 0) const; + bool IsReachable(DependencyType dt = DependencyState, int rstack = 0) const; bool AffectsChildren() const; AcknowledgementType GetAcknowledgement(); From 46184646666f46738cf0147c4b2ae8f2274d77d5 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:22:21 +0100 Subject: [PATCH 15/20] Checkable: Use redundancy groups state in `IsReachable` --- lib/icinga/checkable-dependency.cpp | 38 +++++------------------------ 1 file changed, 6 insertions(+), 32 deletions(-) diff --git a/lib/icinga/checkable-dependency.cpp b/lib/icinga/checkable-dependency.cpp index ca38aac4734..c4d86e758d5 100644 --- a/lib/icinga/checkable-dependency.cpp +++ b/lib/icinga/checkable-dependency.cpp @@ -73,11 +73,6 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const return false; } - for (const Checkable::Ptr& checkable : GetParents()) { - if (!checkable->IsReachable(dt, rstack + 1)) - return false; - } - /* implicit dependency on host if this is a service */ const auto *service = dynamic_cast(this); if (service && (dt == DependencyState || dt == DependencyNotification)) { @@ -88,38 +83,17 @@ bool Checkable::IsReachable(DependencyType dt, int rstack) const } } - auto deps = GetDependencies(); - - std::unordered_map violated; // key: redundancy group, value: nullptr if satisfied, violating dependency otherwise - - for (const Dependency::Ptr& dep : deps) { - std::string redundancy_group = dep->GetRedundancyGroup(); - - if (!dep->IsAvailable(dt)) { - if (redundancy_group.empty()) { + for (auto& dependencyGroup : GetDependencyGroups()) { + if (!(dependencyGroup->GetState(dt, rstack + 1) & DependencyGroup::State::ReachableOK)) { + if (dependencyGroup->IsRedundancyGroup()) { // For non-redundant groups, this should already be logged. Log(LogDebug, "Checkable") - << "Non-redundant dependency '" << dep->GetName() << "' failed for checkable '" << GetName() << "': Marking as unreachable."; - - return false; + << "All dependencies in redundancy group '" << dependencyGroup->GetName() << "' have failed for checkable '" + << GetName() << "': Marking as unreachable."; } - - // tentatively mark this dependency group as failed unless it is already marked; - // so it either passed before (don't overwrite) or already failed (so don't care) - // note that std::unordered_map::insert() will not overwrite an existing entry - violated.insert(std::make_pair(redundancy_group, dep)); - } else if (!redundancy_group.empty()) { - violated[redundancy_group] = nullptr; + return false; } } - auto violator = std::find_if(violated.begin(), violated.end(), [](auto& v) { return v.second != nullptr; }); - if (violator != violated.end()) { - Log(LogDebug, "Checkable") - << "All dependencies in redundancy group '" << violator->first << "' have failed for checkable '" << GetName() << "': Marking as unreachable."; - - return false; - } - return true; } From e6327ccefaa82dfdee67e09eedfab52d7250fbbc Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:22:52 +0100 Subject: [PATCH 16/20] tests: Add unittests for the redundancy groups registry --- test/icinga-dependencies.cpp | 260 ++++++++++++++++++++++++++++++----- 1 file changed, 226 insertions(+), 34 deletions(-) diff --git a/test/icinga-dependencies.cpp b/test/icinga-dependencies.cpp index 86735cdb90d..c542e254bfb 100644 --- a/test/icinga-dependencies.cpp +++ b/test/icinga-dependencies.cpp @@ -9,6 +9,52 @@ using namespace icinga; BOOST_AUTO_TEST_SUITE(icinga_dependencies) +static Host::Ptr CreateHost(const std::string& name) +{ + Host::Ptr host = new Host(); + host->SetName(name); + return host; +} + +static Dependency::Ptr CreateDependency(Checkable::Ptr parent, Checkable::Ptr child, const std::string& name) +{ + Dependency::Ptr dep = new Dependency(); + dep->SetParent(parent); + dep->SetChild(child); + dep->SetName(name + "!" + child->GetName()); + return dep; +} + +static void RegisterDependency(Dependency::Ptr dep, const std::string& redundancyGroup) +{ + dep->SetRedundancyGroup(redundancyGroup); + DependencyGroup::Register(dep); + dep->GetParent()->AddReverseDependency(dep); +} + +static void AssertCheckableRedundancyGroup(Checkable::Ptr checkable, int dependencyCount, int redundancyGroupCount, int memberCount) +{ + BOOST_CHECK_MESSAGE( + dependencyCount == checkable->GetDependencies().size(), + "Dependency count mismatch for '" << checkable->GetName() << "' - expected=" << dependencyCount << "; got=" + << checkable->GetDependencies().size() + ); + auto redundancyGroups(checkable->GetDependencyGroups()); + BOOST_CHECK_MESSAGE( + redundancyGroupCount == redundancyGroups.size(), + "Redundancy group count mismatch for '" << checkable->GetName() << "'" << " - expected=" << redundancyGroupCount + << "; got=" << redundancyGroups.size() + ); + if (redundancyGroupCount > 0) { + BOOST_REQUIRE_MESSAGE(1 <= redundancyGroups.size(), "Checkable '" << checkable->GetName() << "' should have at least one redundancy group."); + BOOST_CHECK_MESSAGE( + memberCount == redundancyGroups.begin()->get()->GetMemberCount(), + "Member count mismatch for '" << checkable->GetName() << "'" << " - expected=" << memberCount + << "; got=" << redundancyGroups.begin()->get()->GetMemberCount() + ); + } +} + BOOST_AUTO_TEST_CASE(multi_parent) { /* One child host, two parent hosts. Simulate multi-parent dependencies. */ @@ -20,53 +66,28 @@ BOOST_AUTO_TEST_CASE(multi_parent) * - Parent objects need a CheckResult object * - Dependencies need a StateFilter */ - Host::Ptr parentHost1 = new Host(); - parentHost1->SetActive(true); - parentHost1->SetMaxCheckAttempts(1); - parentHost1->Activate(); - parentHost1->SetAuthority(true); + Host::Ptr parentHost1 = CreateHost("parentHost1"); parentHost1->SetStateRaw(ServiceCritical); parentHost1->SetStateType(StateTypeHard); parentHost1->SetLastCheckResult(new CheckResult()); - Host::Ptr parentHost2 = new Host(); - parentHost2->SetActive(true); - parentHost2->SetMaxCheckAttempts(1); - parentHost2->Activate(); - parentHost2->SetAuthority(true); + Host::Ptr parentHost2 = CreateHost("parentHost2"); parentHost2->SetStateRaw(ServiceOK); parentHost2->SetStateType(StateTypeHard); parentHost2->SetLastCheckResult(new CheckResult()); - Host::Ptr childHost = new Host(); - childHost->SetActive(true); - childHost->SetMaxCheckAttempts(1); - childHost->Activate(); - childHost->SetAuthority(true); + Host::Ptr childHost = CreateHost("childHost"); childHost->SetStateRaw(ServiceOK); childHost->SetStateType(StateTypeHard); /* Build the dependency tree. */ - Dependency::Ptr dep1 = new Dependency(); - - dep1->SetParent(parentHost1); - dep1->SetChild(childHost); + Dependency::Ptr dep1 (CreateDependency(parentHost1, childHost, "dep1")); dep1->SetStateFilter(StateFilterUp); + RegisterDependency(dep1, ""); - // Reverse dependencies - DependencyGroup::Register(dep1); - parentHost1->AddReverseDependency(dep1); - - Dependency::Ptr dep2 = new Dependency(); - - dep2->SetParent(parentHost2); - dep2->SetChild(childHost); + Dependency::Ptr dep2 (CreateDependency(parentHost2, childHost, "dep2")); dep2->SetStateFilter(StateFilterUp); - - // Reverse dependencies - DependencyGroup::Register(dep2); - parentHost2->AddReverseDependency(dep2); - + RegisterDependency(dep2, ""); /* Test the reachability from this point. * parentHost1 is DOWN, parentHost2 is UP. @@ -80,15 +101,29 @@ BOOST_AUTO_TEST_CASE(multi_parent) /* The only DNS server is DOWN. * Expected result: childHost is unreachable. */ - dep1->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep1); // Remove the dep and re-add it with a configured redundancy group. + RegisterDependency(dep1, "DNS"); BOOST_CHECK(childHost->IsReachable() == false); /* 1/2 DNS servers is DOWN. * Expected result: childHost is reachable. */ - dep2->SetRedundancyGroup("DNS"); + DependencyGroup::Unregister(dep2); + RegisterDependency(dep2, "DNS"); BOOST_CHECK(childHost->IsReachable() == true); + auto grandParentHost(CreateHost("GrandParentHost")); + grandParentHost->SetLastCheckResult(new CheckResult()); + grandParentHost->SetStateRaw(ServiceCritical); + grandParentHost->SetStateType(StateTypeHard); + + Dependency::Ptr dep3 (CreateDependency(grandParentHost, parentHost1, "dep3")); + dep3->SetStateFilter(StateFilterUp); + RegisterDependency(dep3, ""); + // The grandparent is DOWN but the DNS redundancy group has to be still reachable. + BOOST_CHECK_EQUAL(true, childHost->IsReachable()); + DependencyGroup::Unregister(dep3); + /* Both DNS servers are DOWN. * Expected result: childHost is unreachable. */ @@ -98,4 +133,161 @@ BOOST_AUTO_TEST_CASE(multi_parent) BOOST_CHECK(childHost->IsReachable() == false); } +BOOST_AUTO_TEST_CASE(default_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("C")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, ""); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, ""); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(simple_redundancy_group_registration_unregistration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA (CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + // Still 1 redundancy group, but there should be 4 members now, i.e. 2 for each child Checkable. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + // After unregistering depCA, childHostC should have a new redundancy group with only depCB as member, and... + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + // ...childHostD should still have the same redundancy group as before but also with only two members. + AssertCheckableRedundancyGroup(childHostD, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depDA); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group member, i.e. + // both child hosts should have the same redundancy group with only depCB as member. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depDA); + DependencyGroup::Unregister(depDB); + // Nothing should have changed for childHostC, but childHostD should now have a fewer group member. + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + +BOOST_AUTO_TEST_CASE(mixed_redundancy_group_registration_unregsitration) +{ + Checkable::Ptr childHostC(CreateHost("childC")); + Dependency::Ptr depCA(CreateDependency(CreateHost("A"), childHostC, "depCA")); + RegisterDependency(depCA, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostD(CreateHost("childD")); + Dependency::Ptr depDA(CreateDependency(depCA->GetParent(), childHostD, "depDA")); + RegisterDependency(depDA, "redundant"); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depCB(CreateDependency(CreateHost("B"), childHostC, "depCB")); + RegisterDependency(depCB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 2); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depDB(CreateDependency(depCB->GetParent(), childHostD, "depDB")); + RegisterDependency(depDB, "redundant"); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Checkable::Ptr childHostE(CreateHost("childE")); + Dependency::Ptr depEA(CreateDependency(depCA->GetParent(), childHostE, "depEA")); + RegisterDependency(depEA, "redundant"); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 1); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEB(CreateDependency(depCB->GetParent(), childHostE, "depEB")); + RegisterDependency(depEB, "redundant"); + // All 3 hosts share the same group, and each host has 2 members, thus 6 members in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + Dependency::Ptr depEZ(CreateDependency(CreateHost("Z"), childHostE, "depEZ")); + RegisterDependency(depEZ, "redundant"); + // Child host E should have a new redundancy group with 3 members and the other two should still share the same group. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 3, 1, 3); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 4); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 2); + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Register(depEA); // Re-register depEA and instead... + DependencyGroup::Unregister(depEZ); // ...unregister depEZ and check if all the hosts share the same group again. + // All 3 hosts share the same group again, and each host has 2 members, thus 6 members in total. + AssertCheckableRedundancyGroup(childHostC, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostD, 2, 1, 6); + AssertCheckableRedundancyGroup(childHostE, 2, 1, 6); + BOOST_CHECK_EQUAL(1, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCA); + DependencyGroup::Unregister(depDB); + DependencyGroup::Unregister(depEB); + AssertCheckableRedundancyGroup(childHostC, 1, 1, 1); + AssertCheckableRedundancyGroup(childHostD, 1, 1, 2); + AssertCheckableRedundancyGroup(childHostE, 1, 1, 2); + // Child host C has now a separate group with only depCB as member, and child hosts D and E share the same group. + BOOST_CHECK_EQUAL(2, DependencyGroup::GetRegistrySize()); + + DependencyGroup::Unregister(depCB); + DependencyGroup::Unregister(depDA); + DependencyGroup::Unregister(depEA); + AssertCheckableRedundancyGroup(childHostC, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostD, 0, 0, 0); + AssertCheckableRedundancyGroup(childHostE, 0, 0, 0); + BOOST_CHECK_EQUAL(0, DependencyGroup::GetRegistrySize()); +} + BOOST_AUTO_TEST_SUITE_END() From 941fa5743e739422f19b3fbf8814d6730b80a0b3 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 16 Dec 2024 09:23:47 +0100 Subject: [PATCH 17/20] IcingaDB: Bump expected redis version to `6` --- lib/icingadb/icingadb-objects.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 0f17ebf434f..9a89db7865e 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -177,7 +177,7 @@ void IcingaDB::ConfigStaticInitialize() void IcingaDB::UpdateAllConfigObjects() { m_Rcon->Sync(); - m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "5"}, Prio::Heartbeat); + m_Rcon->FireAndForgetQuery({"XADD", "icinga:schema", "MAXLEN", "1", "*", "version", "6"}, Prio::Heartbeat); Log(LogInformation, "IcingaDB") << "Starting initial config/status dump"; double startTime = Utility::GetTime(); From 8ae1fb398dc79926a61ad2d6cc466a44a5208d2d Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Fri, 10 Jan 2025 13:24:27 +0100 Subject: [PATCH 18/20] Activate `Dependency` objects before their parent objects --- lib/icinga/dependency.ti | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/icinga/dependency.ti b/lib/icinga/dependency.ti index 38f5859aeaa..b5887710418 100644 --- a/lib/icinga/dependency.ti +++ b/lib/icinga/dependency.ti @@ -20,6 +20,8 @@ public: class Dependency : CustomVarObject < DependencyNameComposer { + activation_priority -10; + load_after Host; load_after Service; From b8c0a72174a4b76b8b782ded432ac647049674ac Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 13 Jan 2025 09:57:42 +0100 Subject: [PATCH 19/20] tests: Add the new unittests to the `CMakefile.txt` --- test/CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index a255178da2c..eb7df6ce4be 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -230,6 +230,9 @@ add_boost_test(base icinga_checkresult/service_flapping_notification icinga_checkresult/suppressed_notification icinga_dependencies/multi_parent + icinga_dependencies/default_redundancy_group_registration_unregistration + icinga_dependencies/simple_redundancy_group_registration_unregistration + icinga_dependencies/mixed_redundancy_group_registration_unregsitration icinga_notification/strings icinga_notification/state_filter icinga_notification/type_filter From 17ba7c9594bde9b49fc69f6b9f793d16a17b2bd9 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 15 Jan 2025 17:27:28 +0100 Subject: [PATCH 20/20] IcingaDB: Send reachablity state updates for all children recursively --- lib/icingadb/icingadb-objects.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/icingadb/icingadb-objects.cpp b/lib/icingadb/icingadb-objects.cpp index 9a89db7865e..d53f8cebb3b 100644 --- a/lib/icingadb/icingadb-objects.cpp +++ b/lib/icingadb/icingadb-objects.cpp @@ -3095,6 +3095,7 @@ void IcingaDB::ReachabilityChangeHandler(const std::set& childre for (const IcingaDB::Ptr& rw : ConfigType::GetObjectsByType()) { for (auto& checkable : children) { rw->UpdateState(checkable, StateUpdate::Full); + IcingaDB::ReachabilityChangeHandler(checkable->GetChildren()); } } }