[F-net-11478] Make keys directory cluster grouped

e2e/terraform/Makefile

@@ -8,7 +8,7 @@ custom.tfvars:
 	echo 'nomad_local_binary = "$(PKG_PATH)"' > custom.tfvars
 	echo 'volumes = false' >> custom.tfvars
 	echo 'client_count_linux = 3' >> custom.tfvars
-	echo 'client_count_windows_2016_amd64 = 0' >> custom.tfvars
+	echo 'client_count_windows_2016 = 0' >> custom.tfvars
 	echo 'consul_license = "$(shell cat $(CONSUL_LICENSE_PATH))"' >> custom.tfvars
 	echo 'nomad_license = "$(shell cat $(NOMAD_LICENSE_PATH))"' >> custom.tfvars
 
@@ -45,10 +45,10 @@ destroy_full:
 # don't run this by default in plan/apply because it prevents you from
 # updating a running cluster
 tidy:
-	rm -rf keys
-	mkdir keys
-	chmod 0700 keys
-	rm -rf uploads/*
+	rm -rf provision-infra/keys
+	mkdir -p provision-infra/keys
+	chmod 0700 provision-infra/keys
+	rm -rf provision-infra/uploads/*
 	git checkout uploads/README.md
 	rm -f terraform.tfstate.*.backup
 	rm custom.tfvars

e2e/terraform/README.md

@@ -52,7 +52,7 @@ region                           = "us-east-1"
 instance_type                    = "t2.medium"
 server_count                     = "3"
 client_count_linux               = "4"
-client_count_windows_2016_amd64  = "1"
+client_count_windows_2016        = "1"
 ```
 
 You will also need a Consul Enterprise license file and a Nomad Enterprise license file.
@@ -140,14 +140,14 @@ about the cluster:
 You can use Terraform outputs above to access nodes via ssh:
 
 ```sh
-ssh -i keys/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
+ssh -i keys/{cluster-name}/nomad-e2e-*.pem ubuntu@${EC2_IP_ADDR}
 ```
 
 The Windows client runs OpenSSH for convenience, but has a different
 user and will drop you into a Powershell shell instead of bash:
 
 ```sh
-ssh -i keys/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
+ssh -i keys/{cluster-name}/nomad-e2e-*.pem Administrator@${EC2_IP_ADDR}
 ```
 
 ## Teardown

e2e/terraform/main.tf

@@ -8,13 +8,13 @@ provider "aws" {
 module "provision-infra" {
   source = "./provision-infra"
 
-  server_count                    = var.server_count
-  client_count_linux              = var.client_count_linux
-  client_count_windows_2016_amd64 = var.client_count_windows_2016_amd64
-  nomad_local_binary              = var.nomad_local_binary
-  nomad_license                   = var.nomad_license
-  consul_license                  = var.consul_license
-  nomad_region                    = var.nomad_region
-  instance_architecture           = var.instance_architecture
-  name                            = var.name
+  server_count              = var.server_count
+  client_count_linux        = var.client_count_linux
+  client_count_windows_2016 = var.client_count_windows_2016
+  nomad_local_binary        = var.nomad_local_binary
+  nomad_license             = var.nomad_license
+  consul_license            = var.consul_license
+  nomad_region              = var.nomad_region
+  instance_arch             = var.instance_arch
+  name                      = var.name
 }

e2e/terraform/provision-infra/compute.tf

@@ -2,12 +2,13 @@
 # SPDX-License-Identifier: BUSL-1.1
 
 locals {
-  ami_prefix        = "nomad-e2e-v3"
-  ubuntu_image_name = "ubuntu-jammy-${var.instance_architecture}"
+  ami_prefix         = "nomad-e2e-v3"
+  ubuntu_image_name  = "ubuntu-jammy-${var.instance_arch}"
+  windows_image_name = "windows-2016-${var.instance_arch}"
 }
 
 resource "aws_instance" "server" {
-  ami                    = data.aws_ami.ubuntu_jammy_amd64.image_id
+  ami                    = data.aws_ami.ubuntu_jammy.image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
   vpc_security_group_ids = [aws_security_group.servers.id] # see also the secondary ENI
@@ -42,12 +43,12 @@ resource "aws_instance" "client_ubuntu_jammy" {
 
 
 
-resource "aws_instance" "client_windows_2016_amd64" {
-  ami                    = data.aws_ami.windows_2016_amd64[0].image_id
+resource "aws_instance" "client_windows_2016" {
+  ami                    = data.aws_ami.windows_2016[0].image_id
   instance_type          = var.instance_type
   key_name               = module.keys.key_name
   vpc_security_group_ids = [aws_security_group.clients.id]
-  count                  = var.client_count_windows_2016_amd64
+  count                  = var.client_count_windows_2016
   iam_instance_profile   = data.aws_iam_instance_profile.nomad_e2e_cluster.name
   availability_zone      = var.availability_zone
 
@@ -127,15 +128,15 @@ data "aws_ami" "ubuntu_jammy" {
   }
 }
 
-data "aws_ami" "windows_2016_amd64" {
-  count = var.client_count_windows_2016_amd64 > 0 ? 1 : 0
+data "aws_ami" "windows_2016" {
+  count = var.client_count_windows_2016 > 0 ? 1 : 0
 
   most_recent = true
   owners      = ["self"]
 
   filter {
     name   = "name"
-    values = ["${local.ami_prefix}-windows-2016-amd64-*"]
+    values = ["${local.ami_prefix}-windows-2016-${local.windows_image_name}-*"]
   }
 
   filter {

e2e/terraform/provision-infra/consul-servers.tf

@@ -10,7 +10,7 @@ resource "random_uuid" "consul_initial_management_token" {}
 
 resource "local_sensitive_file" "consul_initial_management_token" {
   content         = random_uuid.consul_initial_management_token.result
-  filename        = "${path.module}/keys/consul_initial_management_token"
+  filename        = "${local.keys_dir}/consul_initial_management_token"
   file_permission = "0600"
 }
 
@@ -91,13 +91,13 @@ resource "null_resource" "upload_consul_server_configs" {
     user            = "ubuntu"
     host            = aws_instance.consul_server.public_ip
     port            = 22
-    private_key     = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key     = file("${local.keys_dir}/${local.random_name}.pem")
     target_platform = "unix"
     timeout         = "15m"
   }
 
   provisioner "file" {
-    source      = "${path.module}/keys/tls_ca.crt"
+    source      = "${local.keys_dir}/tls_ca.crt"
     destination = "/tmp/consul_ca.pem"
   }
   provisioner "file" {
@@ -133,7 +133,7 @@ resource "null_resource" "install_consul_server_configs" {
     user            = "ubuntu"
     host            = aws_instance.consul_server.public_ip
     port            = 22
-    private_key     = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key     = file("${local.keys_dir}/${local.random_name}.pem")
     target_platform = "unix"
     timeout         = "15m"
   }
@@ -169,7 +169,7 @@ resource "null_resource" "bootstrap_consul_acls" {
     command = "${path.module}/scripts/bootstrap-consul.sh"
     environment = {
       CONSUL_HTTP_ADDR           = "https://${aws_instance.consul_server.public_ip}:8501"
-      CONSUL_CACERT              = "${path.module}/keys/tls_ca.crt"
+      CONSUL_CACERT              = "${local.keys_dir}/tls_ca.crt"
       CONSUL_HTTP_TOKEN          = "${random_uuid.consul_initial_management_token.result}"
       CONSUL_AGENT_TOKEN         = "${random_uuid.consul_agent_token.result}"
       NOMAD_CLUSTER_CONSUL_TOKEN = "${random_uuid.consul_token_for_nomad.result}"

e2e/terraform/provision-infra/main.tf

@@ -16,12 +16,13 @@ resource "random_password" "windows_admin_password" {
 locals {
   random_name = "${var.name}-${random_pet.e2e.id}"
   uploads_dir = "${path.module}/provision-nomad/uploads/${random_pet.e2e.id}"
+  keys_dir    = "${path.module}/keys/${random_pet.e2e.id}"
 }
 
 # Generates keys to use for provisioning and access
 module "keys" {
   name    = local.random_name
-  path    = "${path.module}/../keys"
+  path    = "${local.keys_dir}"
   source  = "mitchellh/dynamic-keys/aws"
   version = "v2.0.0"
 }

e2e/terraform/provision-infra/nomad-acls.tf

@@ -14,16 +14,17 @@ resource "null_resource" "bootstrap_nomad_acls" {
     command = "${path.module}/scripts/bootstrap-nomad.sh"
     environment = {
       NOMAD_ADDR        = "https://${aws_instance.server.0.public_ip}:4646"
-      NOMAD_CACERT      = "${path.module}/keys/tls_ca.crt"
-      NOMAD_CLIENT_CERT = "${path.module}/keys/tls_api_client.crt"
-      NOMAD_CLIENT_KEY  = "${path.module}/keys/tls_api_client.key"
+      NOMAD_CACERT      = "${local.keys_dir}/tls_ca.crt"
+      NOMAD_CLIENT_CERT = "${local.keys_dir}/tls_api_client.crt"
+      NOMAD_CLIENT_KEY  = "${local.keys_dir}/tls_api_client.key"
+      NOMAD_TOKEN_PATH  = "${local.keys_dir}"
     }
   }
 }
 
 data "local_sensitive_file" "nomad_token" {
   depends_on = [null_resource.bootstrap_nomad_acls]
-  filename   = "${path.module}/keys/nomad_root_token"
+  filename   = "${local.keys_dir}/nomad_root_token"
 }
 
 # push the token out to the servers for humans to use.
@@ -53,7 +54,7 @@ resource "null_resource" "root_nomad_env_servers" {
     user        = "ubuntu"
     host        = aws_instance.server[count.index].public_ip
     port        = 22
-    private_key = file("${path.module}/../keys/${local.random_name}.pem")
+    private_key = file("${local.keys_dir}/${local.random_name}.pem")
     timeout     = "5m"
   }
   provisioner "remote-exec" {

e2e/terraform/provision-infra/nomad.tf

@@ -23,12 +23,13 @@ module "nomad_server" {
   aws_kms_key_id = data.aws_kms_alias.e2e.target_key_id
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "ubuntu"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }
 
@@ -40,54 +41,56 @@ module "nomad_client_ubuntu_jammy" {
   count      = var.client_count_linux
 
   platform           = "linux"
-  arch               = "linux_amd64"
+  arch               = "linux_${var.instance_arch}"
   role               = "client"
   index              = count.index
   instance           = aws_instance.client_ubuntu_jammy[count.index]
   nomad_license      = var.nomad_license
   nomad_region       = var.nomad_region
-  nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy_amd64) ? var.nomad_local_binary_client_ubuntu_jammy_amd64[count.index] : var.nomad_local_binary
+  nomad_local_binary = count.index < length(var.nomad_local_binary_client_ubuntu_jammy) ? var.nomad_local_binary_client_ubuntu_jammy[count.index] : var.nomad_local_binary
 
   tls_ca_key  = tls_private_key.ca.private_key_pem
   tls_ca_cert = tls_self_signed_cert.ca.cert_pem
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "ubuntu"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }
 
 
 # TODO: split out the different Windows targets (2016, 2019) when they're
 # available
-module "nomad_client_windows_2016_amd64" {
+module "nomad_client_windows_2016" {
   source     = "./provision-nomad"
-  depends_on = [aws_instance.client_windows_2016_amd64]
-  count      = var.client_count_windows_2016_amd64
+  depends_on = [aws_instance.client_windows_2016]
+  count      = var.client_count_windows_2016
 
   platform = "windows"
-  arch     = "windows_amd64"
+  arch     = "windows_${var.instance_arch}"
   role     = "client"
   index    = count.index
-  instance = aws_instance.client_windows_2016_amd64[count.index]
+  instance = aws_instance.client_windows_2016[count.index]
 
   nomad_region       = var.nomad_region
   nomad_license      = var.nomad_license
-  nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016_amd64) ? var.nomad_local_binary_client_windows_2016_amd64[count.index] : ""
+  nomad_local_binary = count.index < length(var.nomad_local_binary_client_windows_2016) ? var.nomad_local_binary_client_windows_2016[count.index] : ""
 
   tls_ca_key  = tls_private_key.ca.private_key_pem
   tls_ca_cert = tls_self_signed_cert.ca.cert_pem
 
   uploads_dir = local.uploads_dir
+  keys_dir    = local.keys_dir
 
   connection = {
     type        = "ssh"
     user        = "Administrator"
     port        = 22
-    private_key = "${path.module}/../keys/${local.random_name}.pem"
+    private_key = "${local.keys_dir}/${local.random_name}.pem"
   }
 }

e2e/terraform/provision-infra/outputs.tf

@@ -10,7 +10,7 @@ output "linux_clients" {
 }
 
 output "windows_clients" {
-  value = aws_instance.client_windows_2016_amd64.*.public_ip
+  value = aws_instance.client_windows_2016.*.public_ip
 }
 
 output "message" {
@@ -26,16 +26,16 @@ Then you can run tests from the e2e directory with:
 ssh into servers with:
 
 %{for ip in aws_instance.server.*.public_ip~}
-   ssh -i keys/${local.random_name}.pem ubuntu@${ip}
+   ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
 %{endfor~}
 
 ssh into clients with:
 
 %{for ip in aws_instance.client_ubuntu_jammy.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem ubuntu@${ip}
+    ssh -i keys/${local.random_name}/${local.random_name}.pem ubuntu@${ip}
 %{endfor~}
-%{for ip in aws_instance.client_windows_2016_amd64.*.public_ip~}
-    ssh -i keys/${local.random_name}.pem Administrator@${ip}
+%{for ip in aws_instance.client_windows_2016.*.public_ip~}
+    ssh -i keys/${local.random_name}/${local.random_name}.pem Administrator@${ip}
 %{endfor~}
 
 EOM
@@ -49,14 +49,14 @@ output "environment" {
   sensitive   = true
   value       = <<EOM
 export NOMAD_ADDR=https://${aws_instance.server[0].public_ip}:4646
-export NOMAD_CACERT=${abspath(path.module)}/keys/tls_ca.crt
-export NOMAD_CLIENT_CERT=${abspath(path.module)}/keys/tls_api_client.crt
-export NOMAD_CLIENT_KEY=${abspath(path.module)}/keys/tls_api_client.key
+export NOMAD_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
+export NOMAD_CLIENT_CERT=${abspath(local.keys_dir)}/tls_api_client.crt
+export NOMAD_CLIENT_KEY=${abspath(local.keys_dir)}/tls_api_client.key
 export NOMAD_TOKEN=${data.local_sensitive_file.nomad_token.content}
 export NOMAD_E2E=1
 export CONSUL_HTTP_ADDR=https://${aws_instance.consul_server.public_ip}:8501
 export CONSUL_HTTP_TOKEN=${local_sensitive_file.consul_initial_management_token.content}
-export CONSUL_CACERT=${abspath(path.module)}/keys/tls_ca.crt
+export CONSUL_CACERT=${abspath(local.keys_dir)}/tls_ca.crt
 EOM
 }
 
@@ -65,15 +65,15 @@ output "nomad_addr" {
 }
 
 output "ca_file" {
-  value = "${abspath(path.module)}/keys/tls_ca.crt"
+  value = "${abspath(local.keys_dir)}/tls_ca.crt"
 }
 
 output "cert_file" {
-  value = "${abspath(path.module)}/keys/tls_api_client.crt"
+  value = "${abspath(local.keys_dir)}/tls_api_client.crt"
 }
 
 output "key_file" {
-  value = "${abspath(path.module)}/keys/tls_api_client.key"
+  value = "${abspath(local.keys_dir)}/tls_api_client.key"
 }
 
 output "nomad_token" {