From cae070fb3385a182d293f592a24b1346bcecce96 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Mon, 18 Nov 2024 09:44:45 -0800 Subject: [PATCH 1/6] Create test-runner command for rerunning tests --- Makefile | 15 +- cmd/tools/test-runner/main.go | 9 + go.mod | 1 + go.sum | 3 + tests/callbacks_test.go | 1 + tools/testrunner/testdata/junit-attempt-1.xml | 30 ++ tools/testrunner/testdata/junit-attempt-2.xml | 26 ++ tools/testrunner/testrunner.go | 323 ++++++++++++++++++ tools/testrunner/testrunner_test.go | 134 ++++++++ 9 files changed, 533 insertions(+), 9 deletions(-) create mode 100644 cmd/tools/test-runner/main.go create mode 100644 tools/testrunner/testdata/junit-attempt-1.xml create mode 100644 tools/testrunner/testdata/junit-attempt-2.xml create mode 100644 tools/testrunner/testrunner.go create mode 100644 tools/testrunner/testrunner_test.go diff --git a/Makefile b/Makefile index 05522ce8d0d..a18afd0a629 100644 --- a/Makefile +++ b/Makefile @@ -52,10 +52,7 @@ TEST_TAG_FLAG := -tags $(ALL_TEST_TAGS) TEST_TIMEOUT ?= 20m # Number of retries for *-coverage targets. -# NOTE: This is incompatible with TEST_ARGS which specify the `-run` flag due to how gotestsum selects which tests to -# retry. -# *WARNING*: This is disabled for now; see https://github.com/gotestyourself/gotestsum/issues/423 -FAILED_TEST_RETRIES ?= 0 +FAILED_TEST_RETRIES ?= 2 # Whether or not to test with the race detector. All of (1 on y yes t true) are true values. TEST_RACE_FLAG ?= on @@ -405,13 +402,13 @@ prepare-coverage-test: $(GOTESTSUM) $(TEST_OUTPUT_ROOT) unit-test-coverage: prepare-coverage-test @printf $(COLOR) "Run unit tests with coverage..." - $(GOTESTSUM) --rerun-fails=$(FAILED_TEST_RETRIES) --rerun-fails-max-failures=10 --junitfile $(NEW_REPORT) --packages $(UNIT_TEST_DIRS) -- \ + go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(UNIT_TEST_DIRS) -- \ $(COMPILED_TEST_ARGS) \ -coverprofile=$(NEW_COVER_PROFILE) integration-test-coverage: prepare-coverage-test @printf $(COLOR) "Run integration tests with coverage..." - $(GOTESTSUM) --rerun-fails=$(FAILED_TEST_RETRIES) --rerun-fails-max-failures=10 --junitfile $(NEW_REPORT) --packages $(INTEGRATION_TEST_DIRS) -- \ + go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(INTEGRATION_TEST_DIRS) -- \ $(COMPILED_TEST_ARGS) \ -coverprofile=$(NEW_COVER_PROFILE) $(INTEGRATION_TEST_COVERPKG) @@ -421,21 +418,21 @@ pre-build-functional-test-coverage: prepare-coverage-test functional-test-coverage: prepare-coverage-test @printf $(COLOR) "Run functional tests with coverage with $(PERSISTENCE_DRIVER) driver..." - $(GOTESTSUM) --rerun-fails=$(FAILED_TEST_RETRIES) --rerun-fails-max-failures=10 --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_ROOT) -- \ + go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_ROOT) -- \ $(COMPILED_TEST_ARGS) \ -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) functional-test-xdc-coverage: prepare-coverage-test @printf $(COLOR) "Run functional test for cross DC with coverage with $(PERSISTENCE_DRIVER) driver..." - $(GOTESTSUM) --rerun-fails=$(FAILED_TEST_RETRIES) --rerun-fails-max-failures=10 --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_XDC_ROOT) -- \ + go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_XDC_ROOT) -- \ $(COMPILED_TEST_ARGS) \ -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) functional-test-ndc-coverage: prepare-coverage-test @printf $(COLOR) "Run functional test for NDC with coverage with $(PERSISTENCE_DRIVER) driver..." - $(GOTESTSUM) --rerun-fails=$(FAILED_TEST_RETRIES) --rerun-fails-max-failures=10 --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_NDC_ROOT) -- \ + go run ./cmd/tools/test-runner $(GOTESTSUM) -retries $(FAILED_TEST_RETRIES) --junitfile $(NEW_REPORT) --packages $(FUNCTIONAL_TEST_NDC_ROOT) -- \ $(COMPILED_TEST_ARGS) \ -coverprofile=$(NEW_COVER_PROFILE) $(FUNCTIONAL_TEST_COVERPKG) \ -args -persistenceType=$(PERSISTENCE_TYPE) -persistenceDriver=$(PERSISTENCE_DRIVER) diff --git a/cmd/tools/test-runner/main.go b/cmd/tools/test-runner/main.go new file mode 100644 index 00000000000..68e51529f09 --- /dev/null +++ b/cmd/tools/test-runner/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "go.temporal.io/server/tools/testrunner" +) + +func main() { + testrunner.Main() +} diff --git a/go.mod b/go.mod index 0a5ef644250..7326e911bb4 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,7 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/go.sum b/go.sum index 2ee3849ab8b..a731e808843 100644 --- a/go.sum +++ b/go.sum @@ -124,6 +124,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= @@ -170,6 +171,8 @@ github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jstemmer/go-junit-report/v2 v2.1.0 h1:X3+hPYlSczH9IMIpSC9CQSZA0L+BipYafciZUWHEmsc= +github.com/jstemmer/go-junit-report/v2 v2.1.0/go.mod h1:mgHVr7VUo5Tn8OLVr1cKnLuEy0M92wdRntM99h7RkgQ= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= diff --git a/tests/callbacks_test.go b/tests/callbacks_test.go index 1fa75829224..e06e9acbddb 100644 --- a/tests/callbacks_test.go +++ b/tests/callbacks_test.go @@ -96,6 +96,7 @@ func (s *CallbacksSuite) runNexusCompletionHTTPServer(h *completionHandler, list } func (s *CallbacksSuite) TestWorkflowCallbacks_InvalidArgument() { + s.Equal(1, 2) ctx := testcore.NewContext() taskQueue := testcore.RandomizeStr(s.T().Name()) workflowType := "test" diff --git a/tools/testrunner/testdata/junit-attempt-1.xml b/tools/testrunner/testdata/junit-attempt-1.xml new file mode 100644 index 00000000000..0d39195ee22 --- /dev/null +++ b/tools/testrunner/testdata/junit-attempt-1.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + diff --git a/tools/testrunner/testdata/junit-attempt-2.xml b/tools/testrunner/testdata/junit-attempt-2.xml new file mode 100644 index 00000000000..c7b4150a050 --- /dev/null +++ b/tools/testrunner/testdata/junit-attempt-2.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go new file mode 100644 index 00000000000..43ce3342fa2 --- /dev/null +++ b/tools/testrunner/testrunner.go @@ -0,0 +1,323 @@ +package testrunner + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "iter" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "slices" + "strconv" + "strings" + + "github.com/google/uuid" + "github.com/jstemmer/go-junit-report/v2/junit" +) + +type node struct { + children map[string]node +} + +func (n node) visitor(path ...string) func(yield func(string, node) bool) { + return func(yield func(string, node) bool) { + for name, child := range n.children { + path := append(path, name) + if !yield(strings.Join(path, "/"), child) { + return + } + child.visitor(path...)(yield) + } + } +} + +func (n node) walk() iter.Seq2[string, node] { + return n.visitor() +} + +type attempt struct { + runner *runner + number int + junitXmlPath string + exitErr *exec.ExitError + suites junit.Testsuites +} + +func (a *attempt) run(ctx context.Context, args []string) error { + args = append([]string{"--junitfile", a.junitXmlPath}, args...) + log.Printf("starting test attempt %d with args: %v", a.number, args) + // This must run from the repo root. + cmd := exec.CommandContext(ctx, a.runner.gotestsumExecutable, args...) + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + cmd.Stdin = os.Stdin + if err := cmd.Run(); err != nil { + return err + } + return nil +} + +func (a *attempt) recordResult(exitErr *exec.ExitError) error { + var suites junit.Testsuites + + f, err := os.Open(a.junitXmlPath) + if err != nil { + return err + } + defer f.Close() + + decoder := xml.NewDecoder(f) + if err := decoder.Decode(&suites); err != nil { + return err + } + a.exitErr = exitErr + a.suites = suites + return nil +} + +func (a attempt) failures() []string { + root := node{children: make(map[string]node)} + + for _, suite := range a.suites.Suites { + for _, tc := range suite.Testcases { + if tc.Failure != nil { + n := root + for _, part := range strings.Split(tc.Name, "/") { + child, ok := n.children[part] + if !ok { + child = node{children: make(map[string]node)} + n.children[part] = child + } + n = child + } + } + } + } + + leafFailures := make([]string, 0) + + for path, n := range root.walk() { + if len(n.children) > 0 { + continue + } + leafFailures = append(leafFailures, path) + } + + return leafFailures +} + +type runner struct { + gotestsumExecutable string + junitOutputPath string + attempts []*attempt + retries int +} + +func newRunner(gotestsumExecutable string) *runner { + return &runner{ + gotestsumExecutable: gotestsumExecutable, + attempts: make([]*attempt, 0), + } +} + +func (r *runner) sanitizeAndParseArgs(args []string) ([]string, error) { + var sanitizedArgs []string + type action struct { + f func(string) error + err string + } + var next *action + for i, arg := range args { + if next != nil { + if err := next.f(arg); err != nil { + return nil, err + } + next = nil + continue + } else if arg == "-retries" { + next = &action{ + f: func(arg string) error { + var err error + r.retries, err = strconv.Atoi(arg) + return err + }, + err: "got -retries flag with no value", + } + continue + } else if strings.HasPrefix(arg, "-retries=") { + var err error + r.retries, err = strconv.Atoi(arg[len("-retries="):]) + if err != nil { + return nil, err + } + continue + } else if arg == "--junitfile" { + // --junitfile is used by gotestsum + next = &action{ + f: func(arg string) error { + r.junitOutputPath = arg + return nil + }, + err: "got --junitfile flag with no value", + } + continue + } else if strings.HasPrefix(arg, "--junitfile=") { + // --junitfile is used by gotestsum + r.junitOutputPath = arg[len("--junitfile="):] + continue + } else if arg == "--" { + // Forward all arguments from -- on. + sanitizedArgs = append(sanitizedArgs, args[i:]...) + break + } + sanitizedArgs = append(sanitizedArgs, arg) + } + if next != nil { + return nil, fmt.Errorf("incomplete command line arguments: %s", next.err) + } + if r.junitOutputPath == "" { + return nil, fmt.Errorf("missing required argument --junitfile") + } + return sanitizedArgs, nil +} + +func (r *runner) newAttempt() *attempt { + f := filepath.Join(os.TempDir(), fmt.Sprintf("temporalio-temporal-%s-junit.xml", uuid.NewString())) + a := &attempt{ + runner: r, + number: len(r.attempts) + 1, + junitXmlPath: f, + } + r.attempts = append(r.attempts, a) + return a +} + +func (r *runner) combineAttempts() junit.Testsuites { + var combined junit.Testsuites + + for i, attempt := range r.attempts { + combined.Tests += attempt.suites.Tests + combined.Errors += attempt.suites.Errors + combined.Failures += attempt.suites.Failures + combined.Skipped += attempt.suites.Skipped + combined.Disabled += attempt.suites.Disabled + combined.Time += attempt.suites.Time + + if i == 0 { + combined.XMLName = attempt.suites.XMLName + combined.Name = attempt.suites.Name + combined.Suites = attempt.suites.Suites + continue + } + + for _, suite := range attempt.suites.Suites { + cpy := suite + cpy.Name += fmt.Sprintf(" (retry %d)", i) + cpy.Testcases = make([]junit.Testcase, 0, len(suite.Testcases)) + for _, test := range suite.Testcases { + tcpy := test + tcpy.Name += fmt.Sprintf(" (retry %d)", i) + cpy.Testcases = append(cpy.Testcases, tcpy) + } + combined.Suites = append(combined.Suites, cpy) + } + } + + return combined +} + +func Main() { + ctx := context.Background() + if len(os.Args) < 2 { + log.Fatalf("expected at least 2 arguments") + } + r := newRunner(os.Args[1]) + args, err := r.sanitizeAndParseArgs(os.Args[2:]) + if err != nil { + log.Fatalf("failed to parse command line options: %v", err) + } + + for retry := 0; retry <= r.retries; retry++ { + var exitErr *exec.ExitError + a := r.newAttempt() + err := a.run(ctx, args) + if err != nil && !errors.As(err, &exitErr) { + log.Fatalf("test run failed with an unexpected error: %v", err) + } + if err := a.recordResult(exitErr); err != nil { + log.Fatalf("failed to record run result: %v", err) + } + if exitErr == nil { + break + } + failures := r.attempts[retry].failures() + if len(failures) == 0 { + log.Fatalf("tests failed but no failures have been detected, not rerunning tests") + } + args = stripRunFromArgs(args) + for i, failure := range failures { + failures[i] = goTestNameToRunFlagRegexp(failure) + } + failureArg := strings.Join(failures, "|") + // -args has special semantics in Go. + argsIdx := slices.Index(args, "-args") + if argsIdx == -1 { + args = append(args, "-run", failureArg) + } else { + args = slices.Insert(args, argsIdx, "-run", failureArg) + } + } + + combinedReport := r.combineAttempts() + f, err := os.Create(r.junitOutputPath) + if err != nil { + log.Fatalf("failed to create junit output file: %v", err) + } + defer f.Close() + enc := xml.NewEncoder(f) + enc.Indent("", " ") + if err := enc.Encode(combinedReport); err != nil { + log.Fatalf("failed to encode junit output: %v", err) + } + + lastAttempt := r.attempts[len(r.attempts)-1] + if lastAttempt.exitErr != nil { + log.Printf("exiting with failure after running %d attempt(s)", len(r.attempts)) + os.Exit(lastAttempt.exitErr.ExitCode()) + } +} + +func stripRunFromArgs(args []string) (argsNoRun []string) { + var skipNext bool + for _, arg := range args { + if skipNext { + skipNext = false + continue + } else if arg == "-run" { + skipNext = true + continue + } else if strings.HasPrefix(arg, "-run=") { + continue + } + argsNoRun = append(argsNoRun, arg) + } + return +} + +func goTestNameToRunFlagRegexp(test string) string { + parts := strings.Split(test, "/") + var sb strings.Builder + for i, p := range parts { + if i > 0 { + sb.WriteByte('/') + } + sb.WriteByte('^') + sb.WriteString(regexp.QuoteMeta(p)) + sb.WriteByte('$') + } + return sb.String() +} diff --git a/tools/testrunner/testrunner_test.go b/tools/testrunner/testrunner_test.go new file mode 100644 index 00000000000..12da5475ae2 --- /dev/null +++ b/tools/testrunner/testrunner_test.go @@ -0,0 +1,134 @@ +package testrunner + +import ( + "slices" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNode(t *testing.T) { + n := node{ + children: map[string]node{ + "a": { + children: map[string]node{ + "b": { + children: make(map[string]node), + }, + }, + }, + "b": { + children: make(map[string]node), + }, + }, + } + + var paths []string + for p := range n.walk() { + paths = append(paths, p) + } + slices.Sort(paths) + require.Equal(t, []string{"a", "a/b", "b"}, paths) +} + +func TestRunnerSanitizeAndParseArgs(t *testing.T) { + t.Run("Passthrough", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"--junitfile", "f", "-foo", "bar"}) + require.NoError(t, err) + require.Equal(t, []string{"-foo", "bar"}, args) + require.Equal(t, "f", r.junitOutputPath) + }) + t.Run("RetriesTwoArgs", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"--junitfile", "f", "-foo", "bar", "-retries", "3"}) + require.NoError(t, err) + require.Equal(t, []string{"-foo", "bar"}, args) + require.Equal(t, "f", r.junitOutputPath) + require.Equal(t, 3, r.retries) + }) + t.Run("RetriesOneArg", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"--junitfile", "f", "-foo", "bar", "-retries=3"}) + require.NoError(t, err) + require.Equal(t, []string{"-foo", "bar"}, args) + require.Equal(t, "f", r.junitOutputPath) + require.Equal(t, 3, r.retries) + }) + t.Run("RetriesTwoArgsInvalid", func(t *testing.T) { + r := newRunner("gotestsum") + _, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "-retries", "invalid"}) + require.ErrorContains(t, err, `strconv.Atoi: parsing "invalid"`) + }) + t.Run("RetriesIncomplete", func(t *testing.T) { + r := newRunner("gotestsum") + _, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "-retries"}) + require.ErrorContains(t, err, "incomplete command line arguments: got -retries flag with no value") + }) + t.Run("RetriesOneArgInvalid", func(t *testing.T) { + r := newRunner("gotestsum") + _, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "-retries=invalid"}) + require.ErrorContains(t, err, `strconv.Atoi: parsing "invalid"`) + }) + t.Run("JuintfileSingleArg", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "--junitfile=foo"}) + require.NoError(t, err) + require.Equal(t, []string{"-foo", "bar"}, args) + require.Equal(t, "foo", r.junitOutputPath) + }) + t.Run("JunitfileIncomplete", func(t *testing.T) { + r := newRunner("gotestsum") + _, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "--junitfile"}) + require.ErrorContains(t, err, "incomplete command line arguments: got --junitfile flag with no value") + }) + t.Run("DoubleDash", func(t *testing.T) { + r := newRunner("gotestsum") + args, err := r.sanitizeAndParseArgs([]string{"-foo", "bar", "--junitfile", "foo", "--", "-retries=3"}) + require.NoError(t, err) + require.Equal(t, []string{"-foo", "bar", "--", "-retries=3"}, args) + require.Equal(t, 0, r.retries) + }) +} + +func TestAttemptRecordResultAndFailures(t *testing.T) { + a := &attempt{ + junitXmlPath: "testdata/junit-attempt-1.xml", + } + err := a.recordResult(nil) + require.NoError(t, err) + require.Len(t, a.suites.Suites, 1) + require.Equal(t, 2, a.suites.Failures) + require.Equal(t, []string{"TestCallbacksSuite/TestWorkflowCallbacks_InvalidArgument"}, a.failures()) +} + +func TestCombineAttempts(t *testing.T) { + r := newRunner("gotestsum") + a1 := r.newAttempt() + require.Equal(t, 1, a1.number) + a1.junitXmlPath = "testdata/junit-attempt-1.xml" + require.NoError(t, a1.recordResult(nil)) + a2 := r.newAttempt() + require.Equal(t, 2, a2.number) + a2.junitXmlPath = "testdata/junit-attempt-2.xml" + require.NoError(t, a2.recordResult(nil)) + report := r.combineAttempts() + + require.Len(t, report.Suites, 2) + require.Equal(t, 4, report.Failures) + require.Equal(t, "go.temporal.io/server/tests (retry 1)", report.Suites[1].Name) + require.Len(t, report.Suites[1].Testcases, 2) + require.Equal(t, "TestCallbacksSuite/TestWorkflowCallbacks_InvalidArgument (retry 1)", report.Suites[1].Testcases[0].Name) +} + +func TestStripRunFromArgs(t *testing.T) { + t.Run("OneArg", func(t *testing.T) { + args := stripRunFromArgs([]string{"-foo", "bar", "-run=A"}) + require.Equal(t, []string{"-foo", "bar"}, args) + }) + + t.Run("TwoArgs", func(t *testing.T) { + args := stripRunFromArgs([]string{"-foo", "bar", "-run", "A"}) + require.Equal(t, []string{"-foo", "bar"}, args) + }) +} From c718bd1241edc024c3bf0ce63af2b0f7751097ce Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Mon, 18 Nov 2024 16:28:05 -0800 Subject: [PATCH 2/6] Revert change to tests/callbacks_test.go --- tests/callbacks_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/callbacks_test.go b/tests/callbacks_test.go index e06e9acbddb..1fa75829224 100644 --- a/tests/callbacks_test.go +++ b/tests/callbacks_test.go @@ -96,7 +96,6 @@ func (s *CallbacksSuite) runNexusCompletionHTTPServer(h *completionHandler, list } func (s *CallbacksSuite) TestWorkflowCallbacks_InvalidArgument() { - s.Equal(1, 2) ctx := testcore.NewContext() taskQueue := testcore.RandomizeStr(s.T().Name()) workflowType := "test" From 3ba1f6b090f47a8a6498a304c11d86c9e1151f74 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Mon, 18 Nov 2024 16:30:01 -0800 Subject: [PATCH 3/6] Remove outdated comment --- tools/testrunner/testrunner.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index 43ce3342fa2..20c22a0db86 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -50,7 +50,6 @@ type attempt struct { func (a *attempt) run(ctx context.Context, args []string) error { args = append([]string{"--junitfile", a.junitXmlPath}, args...) log.Printf("starting test attempt %d with args: %v", a.number, args) - // This must run from the repo root. cmd := exec.CommandContext(ctx, a.runner.gotestsumExecutable, args...) cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout From 04fca1908fd3a686b308bdf7e0c00594c0152f4b Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 19 Nov 2024 06:29:02 -0800 Subject: [PATCH 4/6] Format imports and add license --- Makefile | 2 +- cmd/tools/test-runner/main.go | 22 ++++++++++++++++++++++ go.mod | 2 +- tools/testrunner/testrunner.go | 22 ++++++++++++++++++++++ tools/testrunner/testrunner_test.go | 22 ++++++++++++++++++++++ 5 files changed, 68 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index a18afd0a629..bbb21291220 100644 --- a/Makefile +++ b/Makefile @@ -163,7 +163,7 @@ $(GOLANGCI_LINT): $(LOCALBIN) $(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,$(GOLANGCI_LINT_VERSION)) # Don't get confused, there is a single linter called gci, which is a part of the mega linter we use is called golangci-lint. -GCI_VERSION := v0.13.4 +GCI_VERSION := v0.13.5 GCI := $(LOCALBIN)/gci-$(GCI_VERSION) $(GCI): $(LOCALBIN) $(call go-install-tool,$(GCI),github.com/daixiang0/gci,$(GCI_VERSION)) diff --git a/cmd/tools/test-runner/main.go b/cmd/tools/test-runner/main.go index 68e51529f09..a0c580f65db 100644 --- a/cmd/tools/test-runner/main.go +++ b/cmd/tools/test-runner/main.go @@ -1,3 +1,25 @@ +// The MIT License +// +// Copyright (c) 2024 Temporal Technologies Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package main import ( diff --git a/go.mod b/go.mod index 7326e911bb4..7b2ed68baaa 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,7 @@ require ( github.com/iancoleman/strcase v0.3.0 github.com/jackc/pgx/v5 v5.6.0 github.com/jmoiron/sqlx v1.3.4 + github.com/jstemmer/go-junit-report/v2 v2.1.0 github.com/lib/pq v1.10.9 github.com/mitchellh/mapstructure v1.5.0 github.com/nexus-rpc/sdk-go v0.0.11 @@ -113,7 +114,6 @@ require ( github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/jstemmer/go-junit-report/v2 v2.1.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect github.com/mailru/easyjson v0.7.7 // indirect diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index 20c22a0db86..a15a0387045 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -1,3 +1,25 @@ +// The MIT License +// +// Copyright (c) 2024 Temporal Technologies Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package testrunner import ( diff --git a/tools/testrunner/testrunner_test.go b/tools/testrunner/testrunner_test.go index 12da5475ae2..ff00152ed34 100644 --- a/tools/testrunner/testrunner_test.go +++ b/tools/testrunner/testrunner_test.go @@ -1,3 +1,25 @@ +// The MIT License +// +// Copyright (c) 2024 Temporal Technologies Inc. All rights reserved. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + package testrunner import ( From 237aca868a47ee0635fd27c6a8f6eb5474636ec8 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 19 Nov 2024 10:32:52 -0800 Subject: [PATCH 5/6] Address review comment --- tools/testrunner/testrunner.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index a15a0387045..3b4d664149f 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -104,17 +104,21 @@ func (a attempt) failures() []string { root := node{children: make(map[string]node)} for _, suite := range a.suites.Suites { + if suite.Failures == 0 { + continue + } for _, tc := range suite.Testcases { - if tc.Failure != nil { - n := root - for _, part := range strings.Split(tc.Name, "/") { - child, ok := n.children[part] - if !ok { - child = node{children: make(map[string]node)} - n.children[part] = child - } - n = child + if tc.Failure == nil { + continue + } + n := root + for _, part := range strings.Split(tc.Name, "/") { + child, ok := n.children[part] + if !ok { + child = node{children: make(map[string]node)} + n.children[part] = child } + n = child } } } From 5f32fd3de3ff89e5263e818fa7240816ad75b78f Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Tue, 19 Nov 2024 17:01:11 -0800 Subject: [PATCH 6/6] Add comment per CR --- tools/testrunner/testrunner.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/testrunner/testrunner.go b/tools/testrunner/testrunner.go index 3b4d664149f..683695fcd04 100644 --- a/tools/testrunner/testrunner.go +++ b/tools/testrunner/testrunner.go @@ -125,6 +125,9 @@ func (a attempt) failures() []string { leafFailures := make([]string, 0) + // Walk the tree and find all leaf failures. The way Go test failures are reported in junit is that there's a + // test case per suite and per test in that suite. Filter out any nodes that have children to find the most + // specific failures to rerun. for path, n := range root.walk() { if len(n.children) > 0 { continue