From 1629913a3d5e3997a11b1e2ec4b90a0ee29b5ef8 Mon Sep 17 00:00:00 2001
From: Juanadelacuesta <8647634+Juanadelacuesta@users.noreply.github.com>
Date: Thu, 16 Jan 2025 22:57:14 +0100
Subject: [PATCH] func: unify keys and make them cluster grouped

---
 e2e/terraform/Makefile                        |  8 ++++----
 e2e/terraform/README.md                       |  4 ++--
 e2e/terraform/provision-infra/compute.tf      | 12 +++++------
 .../provision-infra/consul-servers.tf         | 10 +++++-----
 .../provision-infra/keys/nomad_root_token     |  1 -
 e2e/terraform/provision-infra/main.tf         |  3 ++-
 e2e/terraform/provision-infra/nomad-acls.tf   | 11 +++++-----
 e2e/terraform/provision-infra/nomad.tf        |  9 ++++++---
 e2e/terraform/provision-infra/outputs.tf      | 20 +++++++++----------
 .../provision-infra/provision-nomad/main.tf   | 12 +++++------
 .../provision-infra/provision-nomad/tls.tf    |  4 ++--
 .../provision-nomad/variables.tf              |  6 ++++++
 .../scripts/bootstrap-nomad.sh                |  4 ++--
 e2e/terraform/provision-infra/tls_ca.tf       |  4 ++--
 e2e/terraform/provision-infra/tls_client.tf   |  8 ++++----
 15 files changed, 63 insertions(+), 53 deletions(-)
 delete mode 100644 e2e/terraform/provision-infra/keys/nomad_root_token

diff --git a/e2e/terraform/Makefile b/e2e/terraform/Makefile
index 47af11e006d..fa77c60f815 100644
--- a/e2e/terraform/Makefile
+++ b/e2e/terraform/Makefile
@@ -45,10 +45,10 @@ destroy_full:
 # don't run this by default in plan/apply because it prevents you from
 # updating a running cluster
 tidy:
-	rm -rf keys
-	mkdir keys
-	chmod 0700 keys
-	rm -rf uploads/*
+	rm -rf provision-infra/keys
+	mkdir -p provision-infra/keys
+	chmod 0700 provision-infra/keys
+	rm -rf provision-infra/uploads/*
 	git checkout uploads/README.md
 	rm -f terraform.tfstate.*.backup
 	rm custom.tfvars
diff --git a/e2e/terraform/README.md b/e2e/terraform/README.md
index aca0b7a85d4..80d510ff33e 100644
--- a/e2e/terraform/README.md
+++ b/e2e/terraform/README.md
@@ -140,14 +140,14 @@ about the cluster:
 You can use Terraform outputs above to access nodes via ssh:
 
 ```sh
-ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
+ssh -i keys/<cluster-name>/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
 ```
 
 The Windows client runs OpenSSH for convenience, but has a different
 user and will drop you into a Powershell shell instead of bash:
 
 ```sh
-ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
+ssh -i keys/<cluster-name>/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
 ```
 
 ## Teardown
diff --git a/e2e/terraform/provision-infra/compute.tf b/e2e/terraform/provision-infra/compute.tf
index 6290cb33d7e..ccf7f082b9c 100644
--- a/e2e/terraform/provision-infra/compute.tf
+++ b/e2e/terraform/provision-infra/compute.tf
@@ -102,10 +102,10 @@ data "aws_ami" "ubuntu_jammy_amd64" {
     values = ["Ubuntu"]
   }
 
-  filter {
+/*   filter {
     name   = "tag:BuilderSha"
     values = [data.external.packer_sha.result["sha"]]
-  }
+  } */
 }
 
 data "aws_ami" "ubuntu_jammy" {
@@ -122,10 +122,10 @@ data "aws_ami" "ubuntu_jammy" {
     values = ["Ubuntu"]
   }
 
-  filter {
+/*   filter {
     name   = "tag:BuilderSha"
     values = [data.external.packer_sha.result["sha"]]
-  }
+  } */
 }
 
 data "aws_ami" "windows_2016" {
@@ -144,8 +144,8 @@ data "aws_ami" "windows_2016" {
     values = ["Windows2016"]
   }
 
-  filter {
+ /*  filter {
     name   = "tag:BuilderSha"
     values = [data.external.packer_sha.result["sha"]]
-  }
+  } */
 }
diff --git a/e2e/terraform/provision-infra/consul-servers.tf b/e2e/terraform/provision-infra/consul-servers.tf
index 67740b2d96e..35d96534d80 100644
--- a/e2e/terraform/provision-infra/consul-servers.tf
+++ b/e2e/terraform/provision-infra/consul-servers.tf
@@ -10,7 +10,7 @@ resource "random_uuid" "consul_initial_management_token" {}
 
 resource "local_sensitive_file" "consul_initial_management_token" {
   content         = random_uuid.consul_initial_management_token.result
-  filename        = "${path.module}/keys/consul_initial_management_token"
+  filename        = "${local.keys_dir}/consul_initial_management_token"
   file_permission = "0600"
 }
 
@@ -91,13 +91,13 @@ resource "null_resource" "upload_consul_server_configs" {
     user            = "ubuntu"
     host            = aws_instance.consul_server.public_ip
     port            = 22
-    private_key     = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key     = file("${local.keys_dir}/${local.random_name}.pem")
     target_platform = "unix"
     timeout         = "15m"
   }
 
   provisioner "file" {
-    source      = "${path.module}/keys/tls_ca.crt"
+    source      = "${local.keys_dir}/tls_ca.crt"
     destination = "/tmp/consul_ca.pem"
   }
   provisioner "file" {
@@ -133,7 +133,7 @@ resource "null_resource" "install_consul_server_configs" {
     user            = "ubuntu"
     host            = aws_instance.consul_server.public_ip
     port            = 22
-    private_key     = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key     = file("${local.keys_dir}/${local.random_name}.pem")
     target_platform = "unix"
     timeout         = "15m"
   }
@@ -169,7 +169,7 @@ resource "null_resource" "bootstrap_consul_acls" {
     command = "${path.module}/scripts/bootstrap-consul.sh"
     environment = {
       CONSUL_HTTP_ADDR           = "https://${aws_instance.consul_server.public_ip}:8501"
-      CONSUL_CACERT              = "${path.module}/keys/tls_ca.crt"
+      CONSUL_CACERT              = "${local.keys_dir}/tls_ca.crt"
       CONSUL_HTTP_TOKEN          = "${random_uuid.consul_initial_management_token.result}"
       CONSUL_AGENT_TOKEN         = "${random_uuid.consul_agent_token.result}"
       NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}"
diff --git a/e2e/terraform/provision-infra/keys/nomad_root_token b/e2e/terraform/provision-infra/keys/nomad_root_token
deleted file mode 100644
index f82682b3dd1..00000000000
--- a/e2e/terraform/provision-infra/keys/nomad_root_token
+++ /dev/null
@@ -1 +0,0 @@
-92f98e6b-32ad-1b80-fefe-fb8f5f84096d
diff --git a/e2e/terraform/provision-infra/main.tf b/e2e/terraform/provision-infra/main.tf
index 27403ebb748..0e79652080e 100644
--- a/e2e/terraform/provision-infra/main.tf
+++ b/e2e/terraform/provision-infra/main.tf
@@ -16,12 +16,13 @@ resource "random_password" "windows_admin_password" {
 locals {
   random_name = "${var.name}-${random_pet.e2e.id}"
   uploads_dir = "${path.module}/provision-nomad/uploads/${random_pet.e2e.id}"
+  keys_dir    = "${path.module}/keys/${random_pet.e2e.id}"
 }
 
 # Generates keys to use for provisioning and access
 module "keys" {
   name    = local.random_name
-  path    = "${path.module}/../keys"
+  path    = "${local.keys_dir}"
   source  = "mitchellh/dynamic-keys/aws"
   version = "v2.0.0"
 }
diff --git a/e2e/terraform/provision-infra/nomad-acls.tf b/e2e/terraform/provision-infra/nomad-acls.tf
index 846bce89eab..fded7388027 100644
--- a/e2e/terraform/provision-infra/nomad-acls.tf
+++ b/e2e/terraform/provision-infra/nomad-acls.tf
@@ -14,16 +14,17 @@ resource "null_resource" "bootstrap_nomad_acls" {
     command = "${path.module}/scripts/bootstrap-nomad.sh"
     environment = {
       NOMAD_ADDR        = "https://${aws_instance.server.0.public_ip}:4646"
-      NOMAD_CACERT      = "${path.module}/keys/tls_ca.crt"
-      NOMAD_CLIENT_CERT = "${path.module}/keys/tls_api_client.crt"
-      NOMAD_CLIENT_KEY  = "${path.module}/keys/tls_api_client.key"
+      NOMAD_CACERT      = "${local.keys_dir}/tls_ca.crt"
+      NOMAD_CLIENT_CERT = "${local.keys_dir}/tls_api_client.crt"
+      NOMAD_CLIENT_KEY  = "${local.keys_dir}/tls_api_client.key"
+      NOMAD_TOKEN_PATH  = "${local.keys_dir}"
     }
   }
 }
 
 data "local_sensitive_file" "nomad_token" {
   depends_on = [null_resource.bootstrap_nomad_acls]
-  filename   = "${path.module}/keys/nomad_root_token"
+  filename   = "${local.keys_dir}/nomad_root_token"
 }
 
 # push the token out to the servers for humans to use.
@@ -53,7 +54,7 @@ resource "null_resource" "root_nomad_env_servers" {
     user        = "ubuntu"
     host        = aws_instance.server[count.index].public_ip
     port        = 22
-    private_key = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key = file("${local.keys_dir}/${local.random_name}.pem")
     timeout     = "5m"
   }
   provisioner "remote-exec" {
diff --git a/e2e/terraform/provision-infra/nomad.tf b/e2e/terraform/provision-infra/nomad.tf
index b753673bfde..cd79a0f6a53 100644
--- a/e2e/terraform/provision-infra/nomad.tf
+++ b/e2e/terraform/provision-infra/nomad.tf
@@ -23,12 +23,13 @@ module "nomad_server" {
   aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "ubuntu"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }
 
@@ -52,12 +53,13 @@ module "nomad_client_ubuntu_jammy" {
   tls_ca_cert = tls_self_signed_cert.ca.cert_pem
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "ubuntu"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }
 
@@ -83,11 +85,12 @@ module "nomad_client_windows_2016" {
   tls_ca_cert = tls_self_signed_cert.ca.cert_pem
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "Administrator"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }
diff --git a/e2e/terraform/provision-infra/outputs.tf b/e2e/terraform/provision-infra/outputs.tf
index bf82c9d046f..80574a7bcd8 100644
--- a/e2e/terraform/provision-infra/outputs.tf
+++ b/e2e/terraform/provision-infra/outputs.tf
@@ -26,16 +26,16 @@ Then you can run tests from the e2e directory with:
 ssh into servers with:
 
 %{for ip in aws_instance.server.*.public_ip~}
-   ssh -i keys/${local.random_name}.pem ubuntu@${ip}
+   ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
 %{endfor~}
 
 ssh into clients with:
 
 %{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem ubuntu@${ip}
+    ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
 %{endfor~}
 %{for ip in aws_instance.client_windows_2016.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem Administrator@${ip}
+    ssh -i keys/${local.random_name}/${local.random_name}.pem Administrator@${ip}
 %{endfor~}
 
 EOM
@@ -49,14 +49,14 @@ output "environment" {
   sensitive   = true
   value       = <<EOM
 export NOMAD_ADDR=https://${aws_instance.server[0].public_ip}:4646
-export NOMAD_CACERT=${abspath(path.module)}/keys/tls_ca.crt
-export NOMAD_CLIENT_CERT=${abspath(path.module)}/keys/tls_api_client.crt
-export NOMAD_CLIENT_KEY=${abspath(path.module)}/keys/tls_api_client.key
+export NOMAD_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
+export NOMAD_CLIENT_CERT=${abspath(local.keys_dir)}/tls_api_client.crt
+export NOMAD_CLIENT_KEY=${abspath(local.keys_dir)}/tls_api_client.key
 export NOMAD_TOKEN=${data.local_sensitive_file.nomad_token.content}
 export NOMAD_E2E=1
 export CONSUL_HTTP_ADDR=https://${aws_instance.consul_server.public_ip}:8501
 export CONSUL_HTTP_TOKEN=${local_sensitive_file.consul_initial_management_token.content}
-export CONSUL_CACERT=${abspath(path.module)}/keys/tls_ca.crt
+export CONSUL_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
 EOM
 }
 
@@ -65,15 +65,15 @@ output "nomad_addr" {
 }
 
 output "ca_file" {
-  value = "${abspath(path.module)}/keys/tls_ca.crt"
+  value = "${abspath(local.keys_dir)}/tls_ca.crt"
 }
 
 output "cert_file" {
-  value = "${abspath(path.module)}/keys/tls_api_client.crt"
+  value = "${abspath(local.keys_dir)}/tls_api_client.crt"
 }
 
 output "key_file" {
-  value = "${abspath(path.module)}/keys/tls_api_client.key"
+  value = "${abspath(local.keys_dir)}/tls_api_client.key"
 }
 
 output "nomad_token" {
diff --git a/e2e/terraform/provision-infra/provision-nomad/main.tf b/e2e/terraform/provision-infra/provision-nomad/main.tf
index c21adeb9a0c..c0b78158f02 100644
--- a/e2e/terraform/provision-infra/provision-nomad/main.tf
+++ b/e2e/terraform/provision-infra/provision-nomad/main.tf
@@ -67,7 +67,7 @@ resource "null_resource" "upload_consul_configs" {
     destination = "/tmp/consul_cert.pem"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/tls_ca.crt"
+    source      = "${var.keys_dir}/tls_ca.crt"
     destination = "/tmp/consul_ca.crt"
   }
   provisioner "file" {
@@ -136,23 +136,23 @@ resource "null_resource" "upload_nomad_configs" {
     destination = "/tmp/agent-${var.instance.public_ip}.crt"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/tls_api_client.key"
+    source      = "${var.keys_dir}/tls_api_client.key"
     destination = "/tmp/tls_proxy.key"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/tls_api_client.crt"
+    source      = "${var.keys_dir}/tls_api_client.crt"
     destination = "/tmp/tls_proxy.crt"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/tls_ca.crt"
+    source      = "${var.keys_dir}/tls_ca.crt"
     destination = "/tmp/ca.crt"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/self_signed.key"
+    source      = "${var.keys_dir}/self_signed.key"
     destination = "/tmp/self_signed.key"
   }
   provisioner "file" {
-    source      = "${path.module}/../keys/self_signed.crt"
+    source      = "${var.keys_dir}/self_signed.crt"
     destination = "/tmp/self_signed.crt"
   }
 }
diff --git a/e2e/terraform/provision-infra/provision-nomad/tls.tf b/e2e/terraform/provision-infra/provision-nomad/tls.tf
index 90798801ac4..b24179f87db 100644
--- a/e2e/terraform/provision-infra/provision-nomad/tls.tf
+++ b/e2e/terraform/provision-infra/provision-nomad/tls.tf
@@ -34,10 +34,10 @@ resource "tls_locally_signed_cert" "nomad" {
 
 resource "local_sensitive_file" "nomad_client_key" {
   content  = tls_private_key.nomad.private_key_pem
-  filename = "keys/agent-${var.instance.public_ip}.key"
+  filename = "${var.keys_dir}/agent-${var.instance.public_ip}.key"
 }
 
 resource "local_sensitive_file" "nomad_client_cert" {
   content  = tls_locally_signed_cert.nomad.cert_pem
-  filename = "keys/agent-${var.instance.public_ip}.crt"
+  filename = "${var.keys_dir}/agent-${var.instance.public_ip}.crt"
 }
diff --git a/e2e/terraform/provision-infra/provision-nomad/variables.tf b/e2e/terraform/provision-infra/provision-nomad/variables.tf
index 5ba745e059a..ef5c3e347fd 100644
--- a/e2e/terraform/provision-infra/provision-nomad/variables.tf
+++ b/e2e/terraform/provision-infra/provision-nomad/variables.tf
@@ -93,3 +93,9 @@ variable "uploads_dir" {
   description = "Directory where all the configuration files for nomad, consul and vault will be taken from to configure each nomad node"
   default     = ""
 }
+
+variable "keys_dir" {
+  type        = string
+  description = "Directory where all the configuration TLS and SSH keys and certificates will be stored fro provisioning"
+  default     = ""
+}
diff --git a/e2e/terraform/provision-infra/scripts/bootstrap-nomad.sh b/e2e/terraform/provision-infra/scripts/bootstrap-nomad.sh
index b231ce7672e..7c7a004eb06 100755
--- a/e2e/terraform/provision-infra/scripts/bootstrap-nomad.sh
+++ b/e2e/terraform/provision-infra/scripts/bootstrap-nomad.sh
@@ -19,8 +19,8 @@ set -e
 
 export NOMAD_TOKEN="$ROOT_TOKEN"
 
-mkdir -p ../keys
-echo $NOMAD_TOKEN > "${DIR}/../keys/nomad_root_token"
+mkdir -p "$NOMAD_TOKEN_PATH"
+echo $NOMAD_TOKEN > "${NOMAD_TOKEN_PATH}/nomad_root_token"
 echo NOMAD_TOKEN=$NOMAD_TOKEN
 
 # Our default policy after bootstrapping will be full-access. Without
diff --git a/e2e/terraform/provision-infra/tls_ca.tf b/e2e/terraform/provision-infra/tls_ca.tf
index d2aaa9a1b8c..94481341d9c 100644
--- a/e2e/terraform/provision-infra/tls_ca.tf
+++ b/e2e/terraform/provision-infra/tls_ca.tf
@@ -23,11 +23,11 @@ resource "tls_self_signed_cert" "ca" {
 }
 
 resource "local_sensitive_file" "ca_key" {
-  filename = "${path.module}/keys/tls_ca.key"
+  filename = "${local.keys_dir}/tls_ca.key"
   content  = tls_private_key.ca.private_key_pem
 }
 
 resource "local_sensitive_file" "ca_cert" {
-  filename = "${path.module}/keys/tls_ca.crt"
+  filename = "${local.keys_dir}/tls_ca.crt"
   content  = tls_self_signed_cert.ca.cert_pem
 }
diff --git a/e2e/terraform/provision-infra/tls_client.tf b/e2e/terraform/provision-infra/tls_client.tf
index 3b184b73454..74dce73f86e 100644
--- a/e2e/terraform/provision-infra/tls_client.tf
+++ b/e2e/terraform/provision-infra/tls_client.tf
@@ -34,12 +34,12 @@ resource "tls_locally_signed_cert" "api_client" {
 
 resource "local_sensitive_file" "api_client_key" {
   content  = tls_private_key.api_client.private_key_pem
-  filename = "${path.module}/keys/tls_api_client.key"
+  filename = "${local.keys_dir}/tls_api_client.key"
 }
 
 resource "local_sensitive_file" "api_client_cert" {
   content  = tls_locally_signed_cert.api_client.cert_pem
-  filename = "${path.module}/keys/tls_api_client.crt"
+  filename = "${local.keys_dir}/tls_api_client.crt"
 }
 
 # Self signed cert for reverse proxy
@@ -66,10 +66,10 @@ resource "tls_self_signed_cert" "self_signed" {
 
 resource "local_sensitive_file" "self_signed_key" {
   content  = tls_private_key.self_signed.private_key_pem
-  filename = "${path.module}/keys/self_signed.key"
+  filename = "${local.keys_dir}/self_signed.key"
 }
 
 resource "local_sensitive_file" "self_signed_cert" {
   content  = tls_self_signed_cert.self_signed.cert_pem
-  filename = "${path.module}/keys/self_signed.crt"
+  filename = "${local.keys_dir}/self_signed.crt"
 }