feat: [PAYMCLOUD-203] Refactor alert prefix variable and refine event…

… filters (#403) * Refactor alert prefix variable and refine event filters. Replaced `slack_message_prefix` with a more generic `alert_prefix` variable for greater consistency across alerts. Updated templates and conditions to apply the new variable and extended filtering rules for both Slack and OpsGenie to refine dropped and matched Kubernetes event reasons. * Simplify alert message formatting in Opsgenie templates Removed redundant square brackets around the severity level in Opsgenie alert messages for a cleaner and more consistent format. The change applies to both Sev3 and Sev1 alerts. * Fix indentation issues in default.tftpl template Corrected inconsistent indentation in the Opsgenie configuration section of the template. These changes improve template readability and ensure proper YAML structure for configuration parsing.
pagopa · Jan 17, 2025 · d5790b1 · d5790b1
1 parent cf14027
commit d5790b1
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 19 deletions.
diff --git a/kubernetes_event_exporter/main.tf b/kubernetes_event_exporter/main.tf
@@ -13,11 +13,11 @@ resource "helm_release" "helm_this" {
     var.custom_config == null ?
     templatefile("${path.module}/templates/default.tftpl",
       {
+        alert_prefix           = var.alert_prefix
         enable_slack           = var.enable_slack, # SLACK PARAM BELOW
         slack_receiver_name    = var.slack_receiver_name,
         slack_token            = var.slack_token,
         slack_channel          = var.slack_channel,
-        slack_message_prefix   = var.slack_message_prefix,
         slack_title            = var.slack_title,
         slack_author           = var.slack_author,
         enable_opsgenie        = var.enable_opsgenie, # OPSGENIE PARAM BELOW

diff --git a/kubernetes_event_exporter/templates/default.tftpl b/kubernetes_event_exporter/templates/default.tftpl
@@ -10,7 +10,7 @@ config:
       slack:
         token: "${slack_token}"
         channel: "${slack_channel}"
-        message: "${slack_message_prefix} {{ .Message }}"
+        message: "${alert_prefix} {{ .Message }}"
         title: "${slack_title}"
         author_name: "${slack_author}"
         fields:
@@ -24,7 +24,7 @@ config:
       opsgenie:
         apiKey: "${opsgenie_api_key}"
         priority: "P3"
-        message: "[INFRA-pagoPa][AKS Sev3] {{ .Reason }} for {{ .InvolvedObject.Namespace }}/{{ .InvolvedObject.Name }} on K8s cluster"
+        message: "${alert_prefix}[Sev3] {{ .Reason }} for {{ .InvolvedObject.Namespace }}/{{ .InvolvedObject.Name }} on K8s cluster"
         alias: "{{ .UID }}"
         description: "<pre>{{ toPrettyJson . }}</pre>"
         tags:
@@ -36,7 +36,7 @@ config:
       opsgenie:
         apiKey: "${opsgenie_api_key}"
         priority: "P1"
-        message: "[INFRA-pagoPa][AKS Sev1] {{ .Reason }} for {{ .InvolvedObject.Namespace }}/{{ .InvolvedObject.Name }} on K8s cluster"
+        message: "${alert_prefix}[Sev1] {{ .Reason }} for {{ .InvolvedObject.Namespace }}/{{ .InvolvedObject.Name }} on K8s cluster"
         alias: "{{ .UID }}"
         description: "<pre>{{ toPrettyJson . }}</pre>"
         tags:
@@ -52,11 +52,17 @@ config:
       %{ if enable_slack }
       - drop:
         - reason: "Unhealthy"
+        - kind: "HorizontalPodAutoscaler"
+        - kind: "ScaledObjectCheckFailed"
         - reason: "FailedToUpdateEndpoint"
         - reason: "FailedScheduling"
         - reason: "EgressBlocked"
-        - kind: "HorizontalPodAutoscaler"
-        - kind: "ScaledObjectCheckFailed"
+        - reason: "OOMKilling"
+        - reason: "RebootScheduled"
+        - reason: "RedeployScheduled"
+        - reason: "FreezeScheduled"
+        - reason: "TerminateScheduled"
+        - reason: "PreemptScheduled"
         match:
           - receiver: "slack"
             type: "Warning"
@@ -73,15 +79,37 @@ config:
         - reason: "FailedToUpdateEndpoint"
         - reason: "FailedScheduling"
         - reason: "EgressBlocked"
+        - reason: "OOMKilling"
+        - reason: "RebootScheduled"
+        - reason: "RedeployScheduled"
+        - reason: "FreezeScheduled"
+        - reason: "TerminateScheduled"
+        - reason: "PreemptScheduled"
         match:
-          - receiver: ${opsgenie_receiver_name}-warning
-            reason: "OOMKilling"
-          - receiver: ${opsgenie_receiver_name}-warning
-            reason: "*Scheduled"
           - receiver: "${opsgenie_receiver_name}-critical"
             type: "Warning"
           - receiver: "${opsgenie_receiver_name}-critical"
-            reason: "*Failed*"
-          - receiver: "${opsgenie_receiver_name}-critical"
+            reason: "Failed"
+      - drop:
+        - reason: "Unhealthy"
+        - kind: "HorizontalPodAutoscaler"
+        - kind: "ScaledObjectCheckFailed"
+        - reason: "FailedToUpdateEndpoint"
+        - reason: "FailedScheduling"
+        - reason: "EgressBlocked"
+        match:
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "OOMKilling"
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "RebootScheduled"
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "RedeployScheduled"
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "FreezeScheduled"
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "TerminateScheduled"
+          - receiver: ${opsgenie_receiver_name}-warning
+            reason: "PreemptScheduled"
+          - receiver: "${opsgenie_receiver_name}-warning"
             reason: "NotTriggerScaleUp"
-      %{ endif }
+        %{ endif }
diff --git a/kubernetes_event_exporter/variables.tf b/kubernetes_event_exporter/variables.tf
@@ -30,6 +30,15 @@ variable "custom_variables" {
   description = "(Optional) This maps contains the custom variable declare by the user on the custom_config"
 }
 
+#
+# ⚠️ General alert parameters
+#
+variable "alert_prefix" {
+  type        = string
+  description = "(Optional) Formatting the message prefix of your alert."
+  default     = "pagoPa"
+}
+
 #
 # 💬 SLACK Parameters
 #
@@ -56,12 +65,6 @@ variable "slack_channel" {
   description = "(Optional) Slack channel for receive messages from exporter."
 }
 
-variable "slack_message_prefix" {
-  type        = string
-  description = "(Optional) Formatting the message prefix of your slack alert."
-  default     = "Received a Kubernetes Event:"
-}
-
 variable "slack_title" {
   type        = string
   description = "(Optional) The name of message title of your app."