Add Support for Testing Golden Snapshots (#939)

* backup

* backup

* backup

* support colorized diffs

* support colorized diffs

* golden snapshots

* validate test-cases schema

* validate test-cases schema

* Update schema path

* Update schema path

* try to use script for tty

* fix for workingn with pty on GHA

* remove script hack

* colorize unified diff

* Fix more snapshots

* add readme

* update snaphots and test cases

* disable snapshots on version check

* update match patterns

* change logs level

* set github token

* pass github token to step

* fix windows tests

* tweak git attributes and add empty dir testing

* add empty dir testing, update docs

* remove debug command

* fix regex for windows

* pass GITHUB_TOKEN on tests

* add skip conditions

* update docs, fix errors with workflows and tests

* reword

.editorconfig

@@ -2,6 +2,27 @@
 insert_final_newline = true
 end_of_line = lf
 
+# Binary files (override to ensure no text-related rules are applied)
+[*.{png,jpg,gif,svg,pdf,ai,eps,mp4}]
+insert_final_newline = false
+end_of_line = unset
+indent_style = unset
+indent_size = unset
+
+# Override for machine generated binary HTML files in Screengrabs directory
+[website/src/components/Screengrabs/**/*.html]
+insert_final_newline = false
+end_of_line = unset
+indent_style = unset
+indent_size = unset
+
+# Override for machine generated binary files in tests/snapshots directory for golden snapshots
+[test/snapshots/**/*.golden]
+insert_final_newline = false
+end_of_line = unset
+indent_style = unset
+indent_size = unset
+
 # Override for Makefile
 [{Makefile,makefile,GNUmakefile}]
 indent_style = tab

.gitattributes

@@ -6,6 +6,9 @@ docs linguist-documentation=true
 # Screengrabs are binary HTML files that are automatically generated
 website/src/components/Screengrabs/**/*.html linguist-generated=true binary
 
+# Golden snapshots should be treated a raw output to prevent line-ending conversions
+tests/snapshots/**/*.golden linguist-generated=true -text
+
 # Mark binary files to prevent normalization
 *.png binary
 *.svg binary
@@ -16,3 +19,8 @@ website/src/components/Screengrabs/**/*.html linguist-generated=true binary
 *.eps binary
 *.ansi binary
 *.mp4 binary
+
+# Reduce merge conflicts that can occur when go.mod and go.sum files are updated
+# Run `go mod tidy` to update the go.sum file
+go.sum linguist-generated=true merge=union
+go.mod linguist-generated=true merge=union

.github/workflows/test.yml

@@ -72,6 +72,25 @@ jobs:
           path: |
             ./build/
 
+  tidy:
+    name: Tidy Go Modules
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: "go.mod"
+        id: go
+
+      - uses: j0hnsmith/go-mod-check@v1
+        with:
+          working-directory: ${{ github.workspace }}
+          # optional, only if you're happy to have `replace ` lines in your go.mod file
+          skip-replace-check: true
+
   # run acceptance tests
   test:
     name: Acceptance Tests
@@ -140,8 +159,23 @@ jobs:
         run: |
           make deps
 
+      # Enable this after merging test-cases
+      # Only seems to work with remote schema files
+      #- name: Validate YAML Schema for Test Cases
+      #  uses: InoUno/[email protected]
+      #  with:
+      #    root: "tests/test-cases"
+      #    schemaMapping: |
+      #      {
+      #        "schema.json": [
+      #          "**/*.yaml"
+      #        ]
+      #       }
+
       - name: Acceptance tests
         timeout-minutes: 10
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: make testacc
 
   docker:
@@ -352,6 +386,8 @@ jobs:
       - name: Run tests in ${{ matrix.demo-folder }} for ${{ matrix.flavor.target }}
         working-directory: ${{ matrix.demo-folder }}
         if: matrix.flavor.target == 'linux' || matrix.flavor.target == 'macos'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           atmos test
 
@@ -369,6 +405,8 @@ jobs:
         working-directory: ${{ matrix.demo-folder }}
         if: matrix.flavor.target == 'windows'
         shell: pwsh
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           atmos test
 
@@ -446,7 +484,7 @@ jobs:
       - name: Check out code
         uses: actions/checkout@v4
 
-      - name: Validate YAML Schema
+      - name: Validate YAML Schema for Stacks
         uses: InoUno/[email protected]
         with:
           root: "examples/${{ matrix.demo-folder }}/stacks"

.vscode/settings.json

@@ -0,0 +1,6 @@
+{
+    "yaml.customTags": [
+        "!not scalar"
+    ],
+    "[mdx]": { "editor.defaultFormatter": null, "editor.formatOnSave": false },
+}

Makefile

@@ -1,6 +1,7 @@
 # This works because `go list ./...` excludes vendor directories by default in modern versions of Go (1.11+).
 # No need for grep or additional filtering.
 TEST ?= $$(go list ./...)
+TESTARGS ?=
 SHELL := /bin/bash
 #GOOS=darwin
 #GOOS=linux

cmd/docs.go

@@ -116,6 +116,12 @@ var docsCmd = &cobra.Command{
 
 		// Opens atmos.tools docs if no component argument is provided
 		var err error
+
+		if os.Getenv("GO_TEST") == "1" {
+			u.LogDebug(atmosConfig, "Skipping browser launch in test environment")
+			return // Skip launching the browser
+		}
+
 		switch runtime.GOOS {
 		case "linux":
 			err = exec.Command("xdg-open", atmosDocsURL).Start()

cmd/version.go

@@ -28,7 +28,9 @@ var versionCmd = &cobra.Command{
 			u.LogErrorAndExit(schema.AtmosConfiguration{}, err)
 		}
 
-		u.PrintMessage(fmt.Sprintf("\U0001F47D Atmos %s on %s/%s", version.Version, runtime.GOOS, runtime.GOARCH))
+		atmosIcon := "\U0001F47D"
+
+		u.PrintMessage(fmt.Sprintf("%s Atmos %s on %s/%s", atmosIcon, version.Version, runtime.GOOS, runtime.GOARCH))
 		fmt.Println()
 
 		if checkFlag {
@@ -45,7 +47,10 @@ var versionCmd = &cobra.Command{
 				}
 				latestRelease := strings.TrimPrefix(latestReleaseTag, "v")
 				currentRelease := strings.TrimPrefix(version.Version, "v")
-				if latestRelease != currentRelease {
+
+				if latestRelease == currentRelease {
+					u.PrintMessage(fmt.Sprintf("You are running the latest version of Atmos (%s)", latestRelease))
+				} else {
 					u.PrintMessageToUpgradeToAtmosLatestRelease(latestRelease)
 				}
 			}

internal/exec/vendor_component_utils.go

@@ -16,8 +16,8 @@ import (
 	"github.com/cloudposse/atmos/pkg/schema"
 	u "github.com/cloudposse/atmos/pkg/utils"
 	"github.com/hairyhenderson/gomplate/v3"
-	"github.com/mattn/go-isatty"
 	cp "github.com/otiai10/copy"
+	"golang.org/x/term"
 )
 
 // findComponentConfigFile identifies the component vendoring config file (`component.yaml` or `component.yml`)
@@ -359,5 +359,7 @@ func ExecuteComponentVendorInternal(
 
 // CheckTTYSupport checks if stdout supports TTY for displaying the progress UI.
 func CheckTTYSupport() bool {
-	return isatty.IsTerminal(os.Stdout.Fd())
+	fd := int(os.Stdout.Fd())
+	isTerminal := term.IsTerminal(fd)
+	return isTerminal
 }

tests/README.md

@@ -0,0 +1,109 @@
+# Tests
+
+We have automated tests in packages, as well as standalone tests in this directory.
+
+Smoke tests are implemented to verify the basic functionality and expected behavior of the compiled `atmos` binary, simulating real-world usage scenarios.
+
+```shell
+├── cli_test.go                                      # Responsible for smoke testing
+├── fixtures/
+│   ├── components/
+│   │   └── terraform/                               # Components that are conveniently reused for tests
+│   ├── scenarios/                                   # Test scenarios consisting of stack configurations (valid & invalid)
+│   │   ├── complete/                                # Advanced stack configuration with both broken and valid stacks
+│   │   ├── metadata/                                # Test configurations for `metadata.*`
+│   │   └── relative-paths/                          # Test configurations for relative imports
+│   └── schemas/                                     # Schemas used for JSON validation
+├── snapshots/                                       # Golden snapshots (what we expect output to look like)
+│   ├── TestCLICommands.stderr.golden
+│   └── TestCLICommands_which_atmos.stdout.golden
+└── test-cases/
+    ├── complete.yaml
+    ├── core.yaml
+    ├── demo-custom-command.yaml
+    ├── demo-stacks.yaml
+    ├── demo-vendoring.yaml
+    ├── metadata.yaml
+    ├── relative-paths.yaml
+    └── schema.json                                 # JSON schema for validation
+
+```
+
+## Test Cases
+
+Our convention is to implement a test-case configuration file per scenario. Then place all smoke tests related to that scenario in the file.
+
+###  Environment Variables
+
+The tests will automatically set some environment variables:
+
+- `GO_TEST=1` is always set, so commands in atmos can disable certain functionality during tests
+- `TERM` is set when `tty: true` to emulate a proper terminal
+
+### Flags
+
+To regenerate ALL snapshots pass the `-regenerate-snaphosts` flag.
+
+> ![WARNING]
+>
+> #### This will regenerate all the snapshots
+>
+> After regenerating, make sure to review the differences:
+>
+> ```shell
+> git diff tests/snapshots
+> ```
+
+To regenerate the snapshots for a specific test, just run:
+
+(replace `TestCLICommands/check_atmos_--help_in_empty-dir` with your test name)
+
+```shell
+go test ./tests -v -run 'TestCLICommands/check_atmos_--help_in_empty-dir' -timeout 2m -regenerate-snapshots
+```
+
+After generating new golden snapshots, don't forget to add them.
+
+```shell
+git add tests/snapshots/*
+```
+
+### Example Configuration
+
+We support an explicit type `!not` on the `expect.stdout` and `expect.stderr` sections (not on `expect.diff`)
+
+Snapshots are enabled by setting the `snapshots` flag, and using the `expect.diff` to ignore line-level differences. If no differences are expected, use an empty list. Note, things like paths will change between local development and CI, so some differences are often expected.
+
+We recommend testing incorrect usage with `expect.exit_code` of non-zero. For example, passing unsupported arguments.
+
+```yaml
+# yaml-language-server: $schema=schema.json
+
+tests:
+  - name: atmos circuit-breaker
+    description: >                            # Friendly description of what this test is verifying
+      Ensure atmos breaks the infinite loop when shell depth exceeds maximum (10).
+
+    enabled: true                             # Whether or not to enable this check
+
+    skip:                                     # Conditions when to skip
+      os: !not windows                        # Do not run on Windows (e.g. PTY not supported)
+                                              # Use "darwin" for macOS
+                                              # Use "linux" for Linux ;)
+
+    snapshot: true                            # Enable golden snapshot. Use together with `expect.diff`
+
+    workdir: "fixtures/scenarios/complete/"   # Location to execute command
+    env:
+      SOME_ENV: true                          # Set an environment variable called "SOME_ENV"
+    command: "atmos"                          # Command to run
+    args:                                     # Arguments or flags passed to command
+      - "help"
+
+    expect:                                   # Assertions
+      diff: []                                # List of expected differences
+      stdout:                                 # Expected output to stdout or TTY. All TTY output is directed to stdout
+      stderr:                                 # Expected output to stderr;
+        - "^$"                                # Expect no output
+      exit_code: 0                            # Expected exit code
+```