[IBCDPE-1007] Monitoring and security scanning (#14)

* Implementing monitoring and security scanning into the cluster

README.md

@@ -201,3 +201,4 @@ This document describes the abbreviated process below:
 }
 ```
 - Add a new `spacelift_aws_integration` resources to the `common-resources/aws-integrations` directory.
+

dev/main.tf

@@ -8,4 +8,5 @@ resource "spacelift_space" "development" {
 module "dpe-sandbox-spacelift" {
   source          = "./spacelift/dpe-sandbox"
   parent_space_id = spacelift_space.development.id
+  admin_stack_id  = var.admin_stack_id
 }

dev/spacelift/dpe-sandbox/main.tf

@@ -41,6 +41,16 @@ resource "spacelift_stack" "k8s-stack-deployments" {
   space_id                = spacelift_space.dpe-sandbox.id
 }
 
+# resource "spacelift_stack_dependency" "dependency-on-admin-stack" {
+#   for_each = {
+#     k8s-stack             = spacelift_stack.k8s-stack,
+#     k8s-stack-deployments = spacelift_stack.k8s-stack-deployments
+#   }
+
+#   stack_id            = each.value.id
+#   depends_on_stack_id = var.admin_stack_id
+# }
+
 resource "spacelift_context_attachment" "k8s-kubeconfig-hooks" {
   context_id = "kubernetes-deployments-kubeconfig"
   stack_id   = spacelift_stack.k8s-stack-deployments.id
@@ -118,15 +128,15 @@ resource "spacelift_stack_destructor" "k8s-stack-destructor" {
 
 resource "spacelift_aws_integration_attachment" "k8s-aws-integration-attachment" {
   # org-sagebase-dnt-dev-aws-integration
-  integration_id = "01J3DNYVM4AWWSDY3QEVRMQ076"
+  integration_id = "01J3R9GX6DC09QV7NV872DDYR3"
   stack_id       = spacelift_stack.k8s-stack.id
   read           = true
   write          = true
 }
 
 resource "spacelift_aws_integration_attachment" "k8s-deployments-aws-integration-attachment" {
   # org-sagebase-dnt-dev-aws-integration
-  integration_id = "01J3DNYVM4AWWSDY3QEVRMQ076"
+  integration_id = "01J3R9GX6DC09QV7NV872DDYR3"
   stack_id       = spacelift_stack.k8s-stack-deployments.id
   read           = true
   write          = true

dev/spacelift/dpe-sandbox/variables.tf

@@ -10,3 +10,8 @@ variable "tags" {
     "CostCenter" = "No Program / 000000"
   }
 }
+
+variable "admin_stack_id" {
+  description = "ID of the admin stack"
+  type        = string
+}

dev/stacks/dpe-sandbox-k8s-deployments/main.tf

@@ -1,10 +1,28 @@
 module "sage-aws-eks-autoscaler" {
   source  = "spacelift.io/sagebionetworks/sage-aws-eks-autoscaler/aws"
-  version = "0.3.2"
+  version = "0.4.2"
 
+  cluster_name           = var.cluster_name
   cluster_name           = var.cluster_name
   private_vpc_subnet_ids = var.private_subnet_ids
   vpc_id                 = var.vpc_id
   node_security_group_id = var.node_security_group_id
   spotinst_account       = var.spotinst_account
+  # desired_capacity       = 2
+}
+
+module "victoria-metrics" {
+  source  = "spacelift.io/sagebionetworks/victoria-metrics/aws"
+  version = "0.0.7"
+}
+
+module "trivy-operator" {
+  source  = "spacelift.io/sagebionetworks/trivy-operator/aws"
+  version = "0.0.12"
+}
+
+module "airflow" {
+  source       = "spacelift.io/sagebionetworks/airflow/aws"
+  version      = "0.0.1"
+  cluster_name = var.cluster_name
 }

dev/stacks/dpe-sandbox-k8s/main.tf

@@ -1,14 +1,14 @@
 module "sage-aws-vpc" {
   source             = "spacelift.io/sagebionetworks/sage-aws-vpc/aws"
-  version            = "0.3.3"
+  version            = "0.3.4"
   vpc_name           = "dpe-sandbox"
   capture_flow_logs  = true
   flow_log_retention = 1
 }
 
 module "sage-aws-eks" {
   source  = "spacelift.io/sagebionetworks/sage-aws-eks/aws"
-  version = "0.3.9"
+  version = "0.4.0"
 
   cluster_name                      = "dpe-k8-sandbox"
   private_vpc_subnet_ids            = module.sage-aws-vpc.private_subnet_ids
@@ -20,4 +20,14 @@ module "sage-aws-eks" {
   pod_security_group_enforcing_mode = "standard"
   aws_account_id                    = "631692904429"
   private_subnet_cidrs              = module.sage-aws-vpc.vpc_private_subnet_cidrs
+  cluster_name                      = "dpe-k8-sandbox"
+  private_vpc_subnet_ids            = module.sage-aws-vpc.private_subnet_ids
+  vpc_id                            = module.sage-aws-vpc.vpc_id
+  vpc_security_group_id             = module.sage-aws-vpc.vpc_security_group_id
+  enable_policy_event_logs          = true
+  capture_cloudwatch_logs           = true
+  cloudwatch_retention              = 1
+  pod_security_group_enforcing_mode = "standard"
+  aws_account_id                    = "631692904429"
+  private_subnet_cidrs              = module.sage-aws-vpc.vpc_private_subnet_cidrs
 }

dev/variables.tf

@@ -2,3 +2,8 @@ variable "parent_space_id" {
   description = "ID of the parent spacelift space"
   type        = string
 }
+
+variable "admin_stack_id" {
+  description = "ID of the admin stack"
+  type        = string
+}

main.tf

@@ -17,7 +17,7 @@ resource "spacelift_stack" "root_administrative_stack" {
 
   administrative          = true
   autodeploy              = true
-  branch                  = "main"
+  branch                  = "ibcdpe-1007-monitoring"
   description             = "Manages other spacelift resources"
   name                    = "Root Spacelift Administrative Stack"
   project_root            = ""
@@ -56,4 +56,5 @@ module "dev-resources" {
     module.terraform-registry,
   ]
   parent_space_id = spacelift_space.environment.id
+  admin_stack_id  = spacelift_stack.root_administrative_stack.id
 }