test: add E2E vaultcompat test for JWT auth flow (#18822)

Test the JWT auth flow using real Nomad and Vault agents.

command/job_plan_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/testutil"
 	"github.com/mitchellh/cli"
 	"github.com/shoenig/test/must"
@@ -188,7 +189,7 @@ func TestPlanCommand_From_Files(t *testing.T) {
 	s := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
 		c.Vault.Address = v.HTTPAddr
 		c.Vault.Enabled = true
-		c.Vault.AllowUnauthenticated = false
+		c.Vault.AllowUnauthenticated = pointer.Of(false)
 		c.Vault.Token = v.RootToken
 	})
 	defer s.Stop()

command/job_validate_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 
 	"github.com/hashicorp/nomad/ci"
+	"github.com/hashicorp/nomad/helper/pointer"
 	"github.com/hashicorp/nomad/testutil"
 	"github.com/mitchellh/cli"
 	"github.com/stretchr/testify/require"
@@ -29,7 +30,7 @@ func TestValidateCommand_Files(t *testing.T) {
 	s := testutil.NewTestServer(t, func(c *testutil.TestServerConfig) {
 		c.Vault.Address = v.HTTPAddr
 		c.Vault.Enabled = true
-		c.Vault.AllowUnauthenticated = false
+		c.Vault.AllowUnauthenticated = pointer.Of(false)
 		c.Vault.Token = v.RootToken
 	})
 	defer s.Stop()

e2e/vaultcompat/cluster_setup_test.go

@@ -0,0 +1,84 @@
+// Copyright (c) HashiCorp, Inc.
+// SPDX-License-Identifier: BUSL-1.1
+
+package vaultcompat
+
+import "fmt"
+
+const (
+	// jwtPath is where the JWT auth method is mounted in Vault.
+	// Use a non-default value for a more realistic scenario.
+	jwtPath = "nomad_jwt"
+)
+
+// roleLegacy is the legacy recommendation for nomad cluster role.
+var roleLegacy = map[string]interface{}{
+	"disallowed_policies": "nomad-server",
+	"explicit_max_ttl":    0, // use old name for vault compatibility
+	"name":                "nomad-cluster",
+	"orphan":              false,
+	"period":              259200, // use old name for vault compatibility
+	"renewable":           true,
+}
+
+// authConfigJWT is the configuration for the JWT auth method used by Nomad.
+func authConfigJWT(jwksURL string) map[string]any {
+	return map[string]any{
+		"jwks_url":           jwksURL,
+		"jwt_supported_algs": []string{"EdDSA"},
+		"default_role":       "nomad-workloads",
+	}
+}
+
+// roleWID is the recommended role for Nomad workloads when using JWT and
+// workload identity.
+func roleWID(policies []string) map[string]any {
+	return map[string]any{
+		"role_type":               "jwt",
+		"bound_audiences":         "vault.io",
+		"user_claim":              "/nomad_job_id",
+		"user_claim_json_pointer": true,
+		"claim_mappings": map[string]any{
+			"nomad_namespace": "nomad_namespace",
+			"nomad_job_id":    "nomad_job_id",
+		},
+		"token_ttl":      "30m",
+		"token_type":     "service",
+		"token_period":   "72h",
+		"token_policies": policies,
+	}
+}
+
+// policyWID is a templated Vault policy that grants tasks access to secret
+// paths prefixed by <namespace>/<job>.
+func policyWID(mountAccessor string) string {
+	return fmt.Sprintf(`
+path "secret/data/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/{{identity.entity.aliases.%[1]s.metadata.nomad_job_id}}/*" {
+  capabilities = ["read"]
+}
+
+path "secret/data/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/{{identity.entity.aliases.%[1]s.metadata.nomad_job_id}}" {
+  capabilities = ["read"]
+}
+
+path "secret/metadata/{{identity.entity.aliases.%[1]s.metadata.nomad_namespace}}/*" {
+  capabilities = ["list"]
+}
+
+path "secret/metadata/*" {
+  capabilities = ["list"]
+}
+`, mountAccessor)
+}
+
+// policyRestricted is Vault policy that only grants read access to a specific
+// path.
+const policyRestricted = `
+path "secret/data/restricted" {
+  capabilities = ["read"]
+}
+
+path "secret/metadata/restricted" {
+  capabilities = ["list"]
+}
+`

e2e/vaultcompat/input/cat_jwt.hcl

@@ -0,0 +1,111 @@
+# Copyright (c) HashiCorp, Inc.
+# SPDX-License-Identifier: BUSL-1.1
+
+job "cat_jwt" {
+  type = "batch"
+
+  // Tasks in this group are expected to succeed and run to completion.
+  group "success" {
+    vault {}
+
+    // Task default_identity uses the default workload identity injected by the
+    // server and the inherits the Vault configuration from the group.
+    task "default_identity" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/default/cat_jwt"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+    }
+
+    // Task custom_identity uses a custom workload identity configuration for
+    // Vault that exposes the JWT as a file and expand on the group Vault
+    // configuration.
+    task "custom_identity" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args = [
+          "${NOMAD_SECRETS_DIR}/secret.txt",
+          "${NOMAD_SECRETS_DIR}/nomad_vault_default.jwt",
+        ]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      vault {
+        role = "nomad-restricted"
+      }
+
+      identity {
+        name = "vault_default"
+        aud  = ["vault.io"]
+        ttl  = "10m"
+        file = true
+      }
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+  }
+
+  // Tasks in this group are expected to fail or never complete.
+  group "fail" {
+
+    // Task unauthorized fails to access secrets it doesn't have access to.
+    task "unauthorized" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/secret.txt"]
+      }
+
+      template {
+        data        = <<EOF
+{{with secret "secret/data/restricted"}}{{.Data.data.secret}}{{end}}
+EOF
+        destination = "${NOMAD_SECRETS_DIR}/secret.txt"
+      }
+
+      vault {}
+    }
+
+    // Task missing_vault fails to access the Vault token because it doesn't
+    // have a vault block, so Nomad doesn't derive a token.
+    task "missing_vault" {
+      driver = "raw_exec"
+
+      config {
+        command = "cat"
+        args    = ["${NOMAD_SECRETS_DIR}/vault_token"]
+      }
+    }
+
+    restart {
+      attempts = 0
+      mode     = "fail"
+    }
+
+    reschedule {
+      attempts  = 0
+      unlimited = false
+    }
+  }
+}