From f9a918ba24c31b799a2706f50c026aed57bdcaea Mon Sep 17 00:00:00 2001 From: Silviana Horga Date: Wed, 31 Jan 2024 11:05:56 +0000 Subject: [PATCH 1/4] Create hmpps domain services alarms pagerduty/slack connection --- terraform/pagerduty/aws.tf | 88 +++++++------- .../pagerduty/member-services-integrations.tf | 107 +++++++++++++++++- 2 files changed, 151 insertions(+), 44 deletions(-) diff --git a/terraform/pagerduty/aws.tf b/terraform/pagerduty/aws.tf index 77f14c0bd..cca4affa4 100644 --- a/terraform/pagerduty/aws.tf +++ b/terraform/pagerduty/aws.tf @@ -18,49 +18,51 @@ resource "aws_secretsmanager_secret" "pagerduty_integration_keys" { resource "aws_secretsmanager_secret_version" "pagerduty_integration_keys" { secret_id = aws_secretsmanager_secret.pagerduty_integration_keys.id secret_string = jsonencode({ - core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key, - ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key, - tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key, - networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key, - operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key, - security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key, - nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key, - nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key, - laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key, - laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key, - laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key, - laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key, - jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key, - jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key, - iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key, - iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key, - laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key, - laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key, - hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key, - hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key, - hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key, - oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key, - oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key - test_alarms = pagerduty_service_integration.test_alarms.integration_key, - laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key, - laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key - laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key, - laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key, - csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key, - dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key, - planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key, - ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key, - ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key, - wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key, - wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key, - pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key, - pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key, - tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key, - tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key, - dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key, - dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key, - laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key, - laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key + core_alerts_cloudwatch = pagerduty_service_integration.core_alerts_cloudwatch.integration_key, + ddos_cloudwatch = pagerduty_service_integration.ddos_cloudwatch.integration_key, + tgw_cloudwatch = pagerduty_service_integration.tgw_cloudwatch.integration_key, + networking_cloudwatch = pagerduty_service_integration.networking_cloudwatch.integration_key, + operations_cloudwatch = pagerduty_service_integration.operations_cloudwatch.integration_key, + security_cloudwatch = pagerduty_service_integration.security_cloudwatch.integration_key, + nomis_alarms = pagerduty_service_integration.nomis_cloudwatch.integration_key, + nomis_nonprod_alarms = pagerduty_service_integration.nomis_nonprod_cloudwatch.integration_key, + laa_mlra_nonprod_alarms = pagerduty_service_integration.laa_mlra_nonprod_cloudwatch.integration_key, + laa_mlra_prod_alarms = pagerduty_service_integration.laa_mlra_prod_cloudwatch.integration_key, + laa_oas_nonprod_alarms = pagerduty_service_integration.laa_oas_nonprod_cloudwatch.integration_key, + laa_oas_prod_alarms = pagerduty_service_integration.laa_oas_prod_cloudwatch.integration_key, + jitbit_nonprod_alarms = pagerduty_service_integration.jitbit_nonprod_cloudwatch.integration_key, + jitbit_prod_alarms = pagerduty_service_integration.jitbit_prod_cloudwatch.integration_key, + iaps_nonprod_alarms = pagerduty_service_integration.iaps_nonprod_cloudwatch.integration_key, + iaps_prod_alarms = pagerduty_service_integration.iaps_prod_cloudwatch.integration_key, + laa_mojfin_prod_alarms = pagerduty_service_integration.laa_mojfin_prod_cloudwatch.integration_key, + laa_mojfin_non_prod_alarms = pagerduty_service_integration.laa_mojfin_non_prod_cloudwatch.integration_key, + hmpps_shef_dba_high_priority = pagerduty_service_integration.hmpps_shef_dba_high_priority.integration_key, + hmpps_shef_dba_low_priority = pagerduty_service_integration.hmpps_shef_dba_low_priority.integration_key, + hmpps_shef_dba_non_prod = pagerduty_service_integration.hmpps_shef_dba_non_prod.integration_key, + oasys_alarms = pagerduty_service_integration.oasys_cloudwatch.integration_key, + oasys_nonprod_alarms = pagerduty_service_integration.oasys_nonprod_cloudwatch.integration_key + test_alarms = pagerduty_service_integration.test_alarms.integration_key, + laa_portal_nonprod_alarms = pagerduty_service_integration.laa_portal_nonprod_cloudwatch.integration_key, + laa_portal_prod_alarms = pagerduty_service_integration.laa_portal_prod_cloudwatch.integration_key + laa_maat_nonprod_alarms = pagerduty_service_integration.laa_maat_nonprod_cloudwatch.integration_key, + laa_maat_prod_alarm = pagerduty_service_integration.laa_maat_prod_cloudwatch.integration_key, + csr_alarms = pagerduty_service_integration.csr_cloudwatch.integration_key, + dpr_nonprod_alarms = pagerduty_service_integration.dpr_nonprod_cloudwatch.integration_key, + planetfm_alarms = pagerduty_service_integration.planetfm_cloudwatch.integration_key, + ncas_non_prod_alarms = pagerduty_service_integration.ncas_non_prod_cloudwatch.integration_key, + ncas_prod_alarms = pagerduty_service_integration.ncas_prod_cloudwatch.integration_key, + wardship_non_prod_alarms = pagerduty_service_integration.wardship_non_prod_cloudwatch.integration_key, + wardship_prod_alarms = pagerduty_service_integration.wardship_prod_cloudwatch.integration_key, + pra_non_prod_alarms = pagerduty_service_integration.pra_non_prod_cloudwatch.integration_key, + pra_prod_alarms = pagerduty_service_integration.pra_prod_cloudwatch.integration_key, + tipstaff_non_prod_alarms = pagerduty_service_integration.tipstaff_non_prod_cloudwatch.integration_key, + tipstaff_prod_alarms = pagerduty_service_integration.tipstaff_prod_cloudwatch.integration_key, + dacp_non_prod_alarms = pagerduty_service_integration.dacp_non_prod_cloudwatch.integration_key, + dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key, + laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key, + laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key, + hmpps_domain_services_alarms = pagerduty_service_integration.hmpps_domain_services_cloudwatch.integration_key, + hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key }) } diff --git a/terraform/pagerduty/member-services-integrations.tf b/terraform/pagerduty/member-services-integrations.tf index 478d7b2a1..583c645df 100644 --- a/terraform/pagerduty/member-services-integrations.tf +++ b/terraform/pagerduty/member-services-integrations.tf @@ -1680,4 +1680,109 @@ resource "pagerduty_slack_connection" "laa_maat_api_prod_connection" { priorities = ["*"] } } -# Slack channel: #laa-alerts-maat-prod \ No newline at end of file +# Slack channel: #laa-alerts-maat-prod + +# HMPPS Domain Services +resource "pagerduty_service" "hmpps_domain_services" { + name = "HMPPS Domain Services Alarms" + description = "HMPPS Domain Services Alarms" + auto_resolve_timeout = 345600 + acknowledgement_timeout = "null" + escalation_policy = pagerduty_escalation_policy.member_policy.id + alert_creation = "create_alerts_and_incidents" +} + +resource "pagerduty_service_integration" "hmpps_domain_services_cloudwatch" { + name = data.pagerduty_vendor.cloudwatch.name + service = pagerduty_service.hmpps_domain_services.id + vendor = data.pagerduty_vendor.cloudwatch.id +} + +resource "pagerduty_slack_connection" "hmpps_domain_services_connection" { + source_id = pagerduty_service.nomis.id + source_type = "service_reference" + workspace_id = local.slack_workspace_id + channel_id = "C04E4FM3KS7" + notification_type = "responder" + lifecycle { + ignore_changes = [ + config, + ] + } + config { + events = [ + "incident.triggered", + "incident.acknowledged", + "incident.escalated", + "incident.resolved", + "incident.reassigned", + "incident.annotated", + "incident.unacknowledged", + "incident.delegated", + "incident.priority_updated", + "incident.action_invocation.created", + "incident.action_invocation.terminated", + "incident.action_invocation.updated", + "incident.responder.added", + "incident.responder.replied", + "incident.status_update_published", + "incident.reopened" + ] + + priorities = ["*"] + } +} + +# Slack channel: #dso_alerts_modernisation_platform + +# HMPPS Domain Services Non Prod + +resource "pagerduty_service" "hmpps_domain_services_nonprod" { + name = "HMPPS Domain Services Alarms Non Prod" + description = "HMPPS Domain Services Alarms Non Prod" + auto_resolve_timeout = 345600 + acknowledgement_timeout = "null" + escalation_policy = pagerduty_escalation_policy.member_policy.id + alert_creation = "create_alerts_and_incidents" +} + +resource "pagerduty_service_integration" "hmpps_domain_services_nonprod_cloudwatch" { + name = data.pagerduty_vendor.cloudwatch.name + service = pagerduty_service.hmpps_domain_services_nonprod.id + vendor = data.pagerduty_vendor.cloudwatch.id +} + +resource "pagerduty_slack_connection" "hmpps_domain_services_nonprod_connection" { + source_id = pagerduty_service.hmpps_domain_services_nonprod.id + source_type = "service_reference" + workspace_id = local.slack_workspace_id + channel_id = "C04QGQML68P" + notification_type = "responder" + lifecycle { + ignore_changes = [ + config, + ] + } + config { + events = [ + "incident.triggered", + "incident.acknowledged", + "incident.escalated", + "incident.resolved", + "incident.reassigned", + "incident.annotated", + "incident.unacknowledged", + "incident.delegated", + "incident.action_invocation.created", + "incident.action_invocation.terminated", + "incident.action_invocation.updated", + "incident.priority_updated", + "incident.responder.added", + "incident.responder.replied", + "incident.status_update_published", + "incident.reopened" + ] + priorities = ["*"] + } +} +# Slack channel: #dso_alerts_devtest_modernisation_platform \ No newline at end of file From a33590c47eb96276727831b6eee0c0f4e7537537 Mon Sep 17 00:00:00 2001 From: Silviana Horga Date: Wed, 31 Jan 2024 11:32:44 +0000 Subject: [PATCH 2/4] Change naming for consistency --- terraform/pagerduty/aws.tf | 2 +- .../pagerduty/member-services-integrations.tf | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/terraform/pagerduty/aws.tf b/terraform/pagerduty/aws.tf index cca4affa4..cb5c8ff60 100644 --- a/terraform/pagerduty/aws.tf +++ b/terraform/pagerduty/aws.tf @@ -61,7 +61,7 @@ resource "aws_secretsmanager_secret_version" "pagerduty_integration_keys" { dacp_prod_alarms = pagerduty_service_integration.dacp_prod_cloudwatch.integration_key, laa_maat_api_nonprod_alarms = pagerduty_service_integration.laa_maat_api_nonprod_cloudwatch.integration_key, laa_maat_api_prod_alarms = pagerduty_service_integration.laa_maat_api_prod_cloudwatch.integration_key, - hmpps_domain_services_alarms = pagerduty_service_integration.hmpps_domain_services_cloudwatch.integration_key, + hmpps_domain_services_prod_alarms = pagerduty_service_integration.hmpps_domain_services_prod_cloudwatch.integration_key, hmpps_domain_services_nonprod_alarms = pagerduty_service_integration.hmpps_domain_services_nonprod_cloudwatch.integration_key }) } diff --git a/terraform/pagerduty/member-services-integrations.tf b/terraform/pagerduty/member-services-integrations.tf index 583c645df..fc6afc247 100644 --- a/terraform/pagerduty/member-services-integrations.tf +++ b/terraform/pagerduty/member-services-integrations.tf @@ -1683,23 +1683,23 @@ resource "pagerduty_slack_connection" "laa_maat_api_prod_connection" { # Slack channel: #laa-alerts-maat-prod # HMPPS Domain Services -resource "pagerduty_service" "hmpps_domain_services" { - name = "HMPPS Domain Services Alarms" - description = "HMPPS Domain Services Alarms" +resource "pagerduty_service" "hmpps_domain_services_prod" { + name = "HMPPS Domain Services Prod Alarms" + description = "HMPPS Domain Services Prod Alarms" auto_resolve_timeout = 345600 acknowledgement_timeout = "null" escalation_policy = pagerduty_escalation_policy.member_policy.id alert_creation = "create_alerts_and_incidents" } -resource "pagerduty_service_integration" "hmpps_domain_services_cloudwatch" { +resource "pagerduty_service_integration" "hmpps_domain_services_prod_cloudwatch" { name = data.pagerduty_vendor.cloudwatch.name - service = pagerduty_service.hmpps_domain_services.id + service = pagerduty_service.hmpps_domain_services_prod.id vendor = data.pagerduty_vendor.cloudwatch.id } -resource "pagerduty_slack_connection" "hmpps_domain_services_connection" { - source_id = pagerduty_service.nomis.id +resource "pagerduty_slack_connection" "hmpps_domain_services_prod_connection" { + source_id = pagerduty_service.hmpps_domain_services_prod.id source_type = "service_reference" workspace_id = local.slack_workspace_id channel_id = "C04E4FM3KS7" From b7a794d2cea2a2d1b1b633fedada589277bd8df2 Mon Sep 17 00:00:00 2001 From: Silviana Horga Date: Wed, 31 Jan 2024 11:39:23 +0000 Subject: [PATCH 3/4] Update comment --- terraform/pagerduty/member-services-integrations.tf | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/terraform/pagerduty/member-services-integrations.tf b/terraform/pagerduty/member-services-integrations.tf index fc6afc247..8589e9548 100644 --- a/terraform/pagerduty/member-services-integrations.tf +++ b/terraform/pagerduty/member-services-integrations.tf @@ -1682,7 +1682,7 @@ resource "pagerduty_slack_connection" "laa_maat_api_prod_connection" { } # Slack channel: #laa-alerts-maat-prod -# HMPPS Domain Services +# HMPPS Domain Services Prod resource "pagerduty_service" "hmpps_domain_services_prod" { name = "HMPPS Domain Services Prod Alarms" description = "HMPPS Domain Services Prod Alarms" @@ -1736,7 +1736,6 @@ resource "pagerduty_slack_connection" "hmpps_domain_services_prod_connection" { # Slack channel: #dso_alerts_modernisation_platform # HMPPS Domain Services Non Prod - resource "pagerduty_service" "hmpps_domain_services_nonprod" { name = "HMPPS Domain Services Alarms Non Prod" description = "HMPPS Domain Services Alarms Non Prod" From 6502c13ede16986168e74eebf7a75d3b738ce67f Mon Sep 17 00:00:00 2001 From: Silviana Horga Date: Wed, 31 Jan 2024 11:44:57 +0000 Subject: [PATCH 4/4] Update description for consistency --- terraform/pagerduty/member-services-integrations.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/pagerduty/member-services-integrations.tf b/terraform/pagerduty/member-services-integrations.tf index 8589e9548..9ed0c1782 100644 --- a/terraform/pagerduty/member-services-integrations.tf +++ b/terraform/pagerduty/member-services-integrations.tf @@ -1684,8 +1684,8 @@ resource "pagerduty_slack_connection" "laa_maat_api_prod_connection" { # HMPPS Domain Services Prod resource "pagerduty_service" "hmpps_domain_services_prod" { - name = "HMPPS Domain Services Prod Alarms" - description = "HMPPS Domain Services Prod Alarms" + name = "HMPPS Domain Services Alarms Prod" + description = "HMPPS Domain Services Alarms Prod" auto_resolve_timeout = 345600 acknowledgement_timeout = "null" escalation_policy = pagerduty_escalation_policy.member_policy.id