From 211cd5827a5697592fe127b8d2aefb521223d12d Mon Sep 17 00:00:00 2001 From: Erik Osterman Date: Sun, 13 Jan 2019 11:32:13 -0800 Subject: [PATCH] Support TF_CLI_* Arguments (#3) * Update makefile * Export mode * Update README.yaml Co-Authored-By: osterman --- Makefile | 12 ++--- README.md | 66 ++++++++++++++++++++++++++-- README.yaml | 65 +++++++++++++++++++++++++-- main.go | 123 +++++++++++++++++++++++++++++++++++++++++----------- 4 files changed, 225 insertions(+), 41 deletions(-) diff --git a/Makefile b/Makefile index 4406f12..3005441 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,8 @@ SHELL = /bin/bash PATH:=$(PATH):$(GOPATH)/bin -include $(shell curl --silent -o .build-harness "https://raw.githubusercontent.com/cloudposse/build-harness/master/templates/Makefile.build-harness"; echo .build-harness) +-include $(shell curl -sSL -o .build-harness "https://git.io/build-harness"; echo .build-harness) +build: go/build + @exit 0 -.PHONY : go-get -go-get: - go get - - -.PHONY : go-build -go-build: go-get - CGO_ENABLED=0 go build -v -o "./dist/bin/tfenv" *.go diff --git a/README.md b/README.md index 1801987..11ae8a5 100644 --- a/README.md +++ b/README.md @@ -6,9 +6,12 @@ # tfenv [![Build Status](https://travis-ci.org/cloudposse/tfenv.svg?branch=master)](https://travis-ci.org/cloudposse/tfenv) [![Latest Release](https://img.shields.io/github/release/cloudposse/tfenv.svg)](https://github.com/cloudposse/tfenv/releases/latest) [![Slack Community](https://slack.cloudposse.com/badge.svg)](https://slack.cloudposse.com) -Command line utility to transform environment variables for use with Terraform (e.g. `HOSTNAME` → `TF_VAR_hostname`) +Command line utility to transform environment variables for use with Terraform. +(e.g. `HOSTNAME` → `TF_VAR_hostname`) -__NOTE__: `tfenv` is **not** a [terraform version manager](https://github.com/tfutils/tfenv). +It can also intelligently map environment variables to terraform command line arguments (e.g. `TF_CLI_INIT_BACKEND_CONFIG_BUCKET=example` → `TF_CLI_ARGS_init=-backend-config=bucket=example`). + +__NOTE__: `tfenv` is **not** a [terraform version manager](https://github.com/tfutils/tfenv). It strictly manages environment variables. --- @@ -45,6 +48,7 @@ If you answer "yes" to any of these questions, then look no further! * Have you ever wished you could easily pass environment variables to terraform *without* adding the `TF_VAR_` prefix? * Do you use [`chamber`](https://github.com/segmentio/chamber) and get annoyed when it transforms environment variables to uppercase? * Would you like to use common environment variables names with terraform? (e.g. `USER` or `AWS_REGION`) +* Is there some argument to `terraform init` you want to specify with an environment variable? (e.g. a `-backend-config` property) **Yes?** Great! Then this utility is for you. @@ -57,6 +61,11 @@ The `tfenv` utility will perform the following transformations: __NOTE__: `tfenv` will preserve the existing environment and add the new environment variables with `TF_VAR_`. This is because some terraform providers expect non-`TF_VAR_*` prefixed environment variables. Additionally, when using the `local-exec` provisioner, it's convenient to use regular environment variables. See our [`terraform-null-smtp-mail`](https://github.com/cloudposse/terraform-null-smtp-mail) module for an example of using this pattern. + +**But wait, there's more!** + +With `tfenv` we can surgically assign a value to any terraform argument using per-argument environment variables. + ## Usage @@ -75,9 +84,58 @@ The basic usage looks like this. We're going to run some `command` and pass it ` So for example, we can pass our current environment to terraform by simply running: ```sh - tfenv terraform plan +tfenv terraform plan +``` + +### Direnv + +You can use `tfenv` with direnv very easily. Running `tfenv` without any arguments will emit `export` statements. + +Example `.envrc`: + +```sh +# Export terraform environment +tfenv +``` + +### Bash + +Load the terraform environment into your shell. + +Just add the following into your shell script: + +```sh +source <(tfenv) ``` +### Terraform Args + +With `tfenv` we can populate the [`TF_CLI_ARGS` and `TF_CLI_ARGS_*` environment variables](https://www.terraform.io/docs/configuration/environment-variables.html#tf_cli_args-and-tf_cli_args_name) automatically. This makes it easy to toggle settings. + +For example, if you want to pass `-backend-config=bucket=terraform-state-bucket` to `terraform init`, then you would do the following: + +```sh +export TF_CLI_INIT_BACKEND_CONFIG_BUCKET=terraform-state-bucket +``` + +Running `tfenv` will populate the `TF_CLI_ARGS_init=-backend-config=bucket=terraform-state-bucket` + +Multiple arguments can be specified and they will be properly concatenated. + +### Initializing Modules + +Terraform as the built-in capability to initialize "root modules" from a remote sources by passing the `-from-module` argument to `terraform init`. + +We can turn this into a 12-factor style invocation using `tfenv`. + +```sh +export TF_CLI_INIT_FROM_MODULE=git::git@github.com:ImpactHealthio/terraform-root-modules.git//aws/$(SERVICE)?ref=tags/0.5.7 +source <(tfenv) +terraform init +``` + +Learn more about `TF_CLI_ARGS` and `TF_CLI_ARGS_*` in the [official documentation](https://www.terraform.io/docs/configuration/environment-variables.html#tf_cli_args-and-tf_cli_args_name). + @@ -207,7 +265,7 @@ In general, PRs are welcome. We follow the typical "fork-and-pull" Git workflow. ## Copyright -Copyright © 2017-2018 [Cloud Posse, LLC](https://cpco.io/copyright) +Copyright © 2017-2019 [Cloud Posse, LLC](https://cpco.io/copyright) diff --git a/README.yaml b/README.yaml index fae3124..d60e7f3 100644 --- a/README.yaml +++ b/README.yaml @@ -30,9 +30,12 @@ badges: # Short description of this project description: |- - Command line utility to transform environment variables for use with Terraform (e.g. `HOSTNAME` → `TF_VAR_hostname`) + Command line utility to transform environment variables for use with Terraform. + (e.g. `HOSTNAME` → `TF_VAR_hostname`) + + It can also intelligently map environment variables to terraform command line arguments (e.g. `TF_CLI_INIT_BACKEND_CONFIG_BUCKET=example` → `TF_CLI_ARGS_init=-backend-config=bucket=example`). - __NOTE__: `tfenv` is **not** a [terraform version manager](https://github.com/tfutils/tfenv). + __NOTE__: `tfenv` is **not** a [terraform version manager](https://github.com/tfutils/tfenv). It strictly manages environment variables. introduction: |- @@ -41,6 +44,7 @@ introduction: |- * Have you ever wished you could easily pass environment variables to terraform *without* adding the `TF_VAR_` prefix? * Do you use [`chamber`](https://github.com/segmentio/chamber) and get annoyed when it transforms environment variables to uppercase? * Would you like to use common environment variables names with terraform? (e.g. `USER` or `AWS_REGION`) + * Is there some argument to `terraform init` you want to specify with an environment variable? (e.g. a `-backend-config` property) **Yes?** Great! Then this utility is for you. @@ -54,6 +58,11 @@ introduction: |- __NOTE__: `tfenv` will preserve the existing environment and add the new environment variables with `TF_VAR_`. This is because some terraform providers expect non-`TF_VAR_*` prefixed environment variables. Additionally, when using the `local-exec` provisioner, it's convenient to use regular environment variables. See our [`terraform-null-smtp-mail`](https://github.com/cloudposse/terraform-null-smtp-mail) module for an example of using this pattern. + **But wait, there's more!** + + With `tfenv` we can surgically assign a value to any terraform argument using per-argument environment variables. + + # How to use this project usage: |- @@ -72,9 +81,59 @@ usage: |- So for example, we can pass our current environment to terraform by simply running: ```sh - tfenv terraform plan + tfenv terraform plan + ``` + + ### Direnv + + You can use `tfenv` with direnv very easily. Running `tfenv` without any arguments will emit `export` statements. + + Example `.envrc`: + + ```sh + # Export terraform environment + tfenv ``` + ### Bash + + Load the terraform environment into your shell. + + Just add the following into your shell script: + + ```sh + source <(tfenv) + ``` + + ### Terraform Args + + With `tfenv` we can populate the [`TF_CLI_ARGS` and `TF_CLI_ARGS_*` environment variables](https://www.terraform.io/docs/configuration/environment-variables.html#tf_cli_args-and-tf_cli_args_name) automatically. This makes it easy to toggle settings. + + For example, if you want to pass `-backend-config=bucket=terraform-state-bucket` to `terraform init`, then you would do the following: + + ```sh + export TF_CLI_INIT_BACKEND_CONFIG_BUCKET=terraform-state-bucket + ``` + + Running `tfenv` will populate the `TF_CLI_ARGS_init=-backend-config=bucket=terraform-state-bucket` + + Multiple arguments can be specified and they will be properly concatenated. + + ### Initializing Modules + + Terraform has the built-in capability to initialize "root modules" from a remote sources by passing the `-from-module` argument to `terraform init`. + + We can turn this into a 12-factor style invocation using `tfenv`. + + ```sh + export TF_CLI_INIT_FROM_MODULE=git::git@github.com:ImpactHealthio/terraform-root-modules.git//aws/$(SERVICE)?ref=tags/0.5.7 + source <(tfenv) + terraform init + ``` + + Learn more about `TF_CLI_ARGS` and `TF_CLI_ARGS_*` in the [official documentation](https://www.terraform.io/docs/configuration/environment-variables.html#tf_cli_args-and-tf_cli_args_name). + + related: - name: "Packages" description: "Cloud Posse installer and distribution of native apps" diff --git a/main.go b/main.go index 1caeb80..3535a4a 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,8 @@ package main import ( - // "fmt" + "fmt" + "github.com/taskcluster/shell" "log" "os" "os/exec" @@ -44,8 +45,16 @@ func main() { // Blacklist of excluded environment variables. Processed *before* whitelist. var tfenvBlacklist = getEnv("TFENV_BLACKLIST", "^(AWS_ACCESS_KEY_ID|AWS_SECRET_ACCESS_KEY)$") - // The command that was executed - cmd := os.Args[0] + // Args that we pass to TF_CLI_ARGS_init + var tfCliArgsInit []string + var tfCliArgsPlan []string + var tfCliArgsApply []string + var tfCliArgsDestroy []string + var tfCliArgs []string + + reTfCliInitBackend := regexp.MustCompile("^TF_CLI_INIT_BACKEND_CONFIG_(.*)") + reTfCliCommand := regexp.MustCompile("^TF_CLI_(INIT|PLAN|APPLY|DESTROY)_(.*)") + reTfCliDefault := regexp.MustCompile("^TF_CLI_DEFAULT_(.*)") reTfVar := regexp.MustCompile("^" + tfenvPrefix) reTrim := regexp.MustCompile("(^_+|_+$)") @@ -58,10 +67,45 @@ func main() { env = append(env, e) // Begin normalization of environment variable - pair := strings.Split(e, "=") - - // Process the blacklist for exclusions, then the whitelist for inclusions - if !reBlacklist.MatchString(pair[0]) && reWhitelist.MatchString(pair[0]) { + pair := strings.SplitN(e, "=", 2) + + originalEnvName := pair[0] + + // `TF_CLI_ARGS_init`: Map `TF_CLI_INIT_BACKEND_CONFIG_FOO=value` to `-backend-config=foo=value` + if reTfCliInitBackend.MatchString(pair[0]) { + match := reTfCliInitBackend.FindStringSubmatch(pair[0]) + arg := reDedupe.ReplaceAllString(match[1], "-") + arg = strings.ToLower(arg) + arg = "-backend-config=" + arg + "=" + pair[1] + tfCliArgsInit = append(tfCliArgsInit, arg) + } else if reTfCliCommand.MatchString(pair[0]) { + // `TF_CLI_ARGS_plan`: Map `TF_CLI_PLAN_SOMETHING=value` to `-something=value` + match := reTfCliCommand.FindStringSubmatch(pair[0]) + cmd := reDedupe.ReplaceAllString(match[1], "-") + cmd = strings.ToLower(cmd) + + param := reDedupe.ReplaceAllString(match[2], "-") + param = strings.ToLower(param) + arg := "-" + param + "=" + pair[1] + switch cmd { + case "init": + tfCliArgsInit = append(tfCliArgsInit, arg) + case "plan": + tfCliArgsPlan = append(tfCliArgsPlan, arg) + case "apply": + tfCliArgsApply = append(tfCliArgsApply, arg) + case "destroy": + tfCliArgsDestroy = append(tfCliArgsDestroy, arg) + } + } else if reTfCliDefault.MatchString(pair[0]) { + // `TF_CLI_ARGS`: Map `TF_CLI_DEFAULT_SOMETHING=value` to `-something=value` + match := reTfCliDefault.FindStringSubmatch(pair[0]) + param := reDedupe.ReplaceAllString(match[1], "-") + param = strings.ToLower(param) + arg := "-" + param + "=" + pair[1] + tfCliArgs = append(tfCliArgs, arg) + } else if !reBlacklist.MatchString(pair[0]) && reWhitelist.MatchString(pair[0]) { + // Process the blacklist for exclusions, then the whitelist for inclusions // Strip off TF_VAR_ prefix so we can simplify normalization pair[0] = reTfVar.ReplaceAllString(pair[0], "") @@ -75,35 +119,64 @@ func main() { pair[0] = reDedupe.ReplaceAllString(pair[0], "_") // prepend TF_VAR_, if not there already - if len(pair[0]) != 0 { + if len(pair[0]) > 0 { pair[0] = tfenvPrefix + pair[0] - envvar := pair[0] + "=" + pair[1] - //fmt.Println(envvar) - env = append(env, envvar) + if strings.Compare(pair[0], originalEnvName) != 0 { + envvar := pair[0] + "=" + pair[1] + //fmt.Println(envvar) + env = append(env, envvar) + } } } } - sort.Strings(env) + if len(tfCliArgsInit) > 0 { + env = append(env, "TF_CLI_ARGS_init="+strings.Join(tfCliArgsInit, " ")) + } - if len(os.Args) < 2 { - log.Fatalf("error: %v command args...", cmd) + if len(tfCliArgsPlan) > 0 { + env = append(env, "TF_CLI_ARGS_plan="+strings.Join(tfCliArgsPlan, " ")) } - // The command that will be executed - exe := os.Args[1] + if len(tfCliArgsApply) > 0 { + env = append(env, "TF_CLI_ARGS_apply="+strings.Join(tfCliArgsApply, " ")) + } - // The command + any arguments - args = append(args, os.Args[1:]...) + if len(tfCliArgsDestroy) > 0 { + env = append(env, "TF_CLI_ARGS_destroy="+strings.Join(tfCliArgsDestroy, " ")) + } - // Lookup path for executable - binary, binaryPathErr := exec.LookPath(exe) - if binaryPathErr != nil { - log.Fatalf("error: failed to find executable `%v`: %v", exe, binaryPathErr) + if len(tfCliArgs) > 0 { + env = append(env, "TF_CLI_ARGS="+strings.Join(tfCliArgs, " ")) } - execErr := syscall.Exec(binary, args, env) - if execErr != nil { - log.Fatalf("error: exec failed: %v", execErr) + sort.Strings(env) + + // The command that was executed + cmd := os.Args[0] + + if len(os.Args) < 2 { + for _, envvar := range env { + // Begin normalization of environment variable + pair := strings.SplitN(envvar, "=", 2) + fmt.Printf("export %v=%v\n", pair[0], shell.Escape(pair[1])) + } + } else { + // The command that will be executed + exe := os.Args[1] + + // The command + any arguments + args = append(args, os.Args[1:]...) + + // Lookup path for executable + binary, binaryPathErr := exec.LookPath(exe) + if binaryPathErr != nil { + log.Fatalf("error: %v failed to find executable `%v`: %v", cmd, exe, binaryPathErr) + } + + execErr := syscall.Exec(binary, args, env) + if execErr != nil { + log.Fatalf("error: %v exec failed: %v", cmd, execErr) + } } }