From 7b9530897b94694a97cd4b7fb525f0268eea292b Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Tue, 24 Oct 2023 09:42:29 -0500 Subject: [PATCH 1/7] Move compact snapshot logs from info to verbose level --- output/compact.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/output/compact.go b/output/compact.go index 2c45d264f..05f9e29bf 100644 --- a/output/compact.go +++ b/output/compact.go @@ -48,7 +48,7 @@ func uploadAndSubmitCompactSnapshot(ctx context.Context, s pganalyze_collector.C if collectionOpts.OutputAsJson { debugCompactOutputAsJSON(logger, compressedData) } else if !quiet { - logger.PrintInfo("Collected compact %s snapshot successfully", kind) + logger.PrintVerbose("Collected compact %s snapshot successfully", kind) } return nil } @@ -137,7 +137,7 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection if len(msg) > 0 && collectionOpts.TestRun { logger.PrintInfo(" %s", msg) } else if !quiet { - logger.PrintInfo("Submitted compact %s snapshot successfully", kind) + logger.PrintVerbose("Submitted compact %s snapshot successfully", kind) } return nil From 6637bfde9b4a80c02ba7d194bf845f6bf7068898 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Wed, 25 Oct 2023 11:22:10 -0500 Subject: [PATCH 2/7] log summary of compact snapshots submitted every minute --- output/compact.go | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/output/compact.go b/output/compact.go index 05f9e29bf..5f508ec0d 100644 --- a/output/compact.go +++ b/output/compact.go @@ -9,6 +9,7 @@ import ( "io" "net/http" "net/url" + "sort" "strings" "time" @@ -48,7 +49,7 @@ func uploadAndSubmitCompactSnapshot(ctx context.Context, s pganalyze_collector.C if collectionOpts.OutputAsJson { debugCompactOutputAsJSON(logger, compressedData) } else if !quiet { - logger.PrintVerbose("Collected compact %s snapshot successfully", kind) + logger.PrintInfo("Collected compact %s snapshot successfully", kind) } return nil } @@ -138,7 +139,29 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection logger.PrintInfo(" %s", msg) } else if !quiet { logger.PrintVerbose("Submitted compact %s snapshot successfully", kind) + if time.Now().Sub(lastLog) > time.Minute { + lastLog = time.Now() + details := "" + var keys []string + for k := range callCounts { + keys = append(keys, k) + } + sort.Strings(keys) + for i, kind := range keys { + details += fmt.Sprintf("%d %s", callCounts[kind], kind) + if i < len(keys)-1 { + details += ", " + } + } + logger.PrintInfo("Compact snapshots submitted over the past minute: " + details) + callCounts = make(map[string]uint8) + } else { + callCounts[kind] = callCounts[kind] + 1 + } } return nil } + +var callCounts = make(map[string]uint8) +var lastLog = time.Now() From 61dc1e2b2cfe6a4237be667a20cf649bac1be2c3 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 26 Oct 2023 16:25:30 -0500 Subject: [PATCH 3/7] track stats per-server, shorten log message --- output/compact.go | 22 +++++++++++----------- state/state.go | 4 ++++ 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/output/compact.go b/output/compact.go index 5f508ec0d..2a871c3d7 100644 --- a/output/compact.go +++ b/output/compact.go @@ -139,29 +139,29 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection logger.PrintInfo(" %s", msg) } else if !quiet { logger.PrintVerbose("Submitted compact %s snapshot successfully", kind) - if time.Now().Sub(lastLog) > time.Minute { - lastLog = time.Now() - details := "" + if server.CompactLogTime.IsZero() { + server.CompactLogTime = time.Now() + server.CompactLogStats = make(map[string]uint8) + } else if time.Now().Sub(server.CompactLogTime) > time.Minute { + server.CompactLogTime = time.Now() var keys []string - for k := range callCounts { + for k := range server.CompactLogStats { keys = append(keys, k) } sort.Strings(keys) + details := "" for i, kind := range keys { - details += fmt.Sprintf("%d %s", callCounts[kind], kind) + details += fmt.Sprintf("%d %s", server.CompactLogStats[kind], kind) if i < len(keys)-1 { details += ", " } } - logger.PrintInfo("Compact snapshots submitted over the past minute: " + details) - callCounts = make(map[string]uint8) + logger.PrintInfo("Compact snapshots submitted: " + details) + server.CompactLogStats = make(map[string]uint8) } else { - callCounts[kind] = callCounts[kind] + 1 + server.CompactLogStats[kind] = server.CompactLogStats[kind] + 1 } } return nil } - -var callCounts = make(map[string]uint8) -var lastLog = time.Now() diff --git a/state/state.go b/state/state.go index ddfcc0586..3ad3062ed 100644 --- a/state/state.go +++ b/state/state.go @@ -286,6 +286,10 @@ type Server struct { // differences (see https://groups.google.com/g/golang-nuts/c/eIqkhXh9PLg), // as we access this in high frequency log-related code paths. LogIgnoreFlags uint32 + + // State to track compact snapshot submissions, and log them routinely + CompactLogStats map[string]uint8 + CompactLogTime time.Time } func MakeServer(config config.ServerConfig) *Server { From 5f6300910358c80b7afeabff7608cb03d04c8753 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 26 Oct 2023 16:26:22 -0500 Subject: [PATCH 4/7] fmt --- state/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/state.go b/state/state.go index 3ad3062ed..ecd4e3f45 100644 --- a/state/state.go +++ b/state/state.go @@ -289,7 +289,7 @@ type Server struct { // State to track compact snapshot submissions, and log them routinely CompactLogStats map[string]uint8 - CompactLogTime time.Time + CompactLogTime time.Time } func MakeServer(config config.ServerConfig) *Server { From 707c0f764f14373b600b01a096b03c67d682bda5 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 26 Oct 2023 17:01:00 -0500 Subject: [PATCH 5/7] CI: lint code before running tests to shorten feedback cycle --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5971af871..3a5448ccf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,13 +33,13 @@ jobs: with: submodules: true - - name: Runs tests + - name: Lint + run: | + test $(go fmt ./... | wc -l) -eq 0 + + - name: Run tests run: | make build OUTFILE=pganalyze-collector-linux-amd64 make test DOCKER_BUILDKIT=1 make integration_test shellcheck contrib/install.sh - - - name: Lint - run: | - test $(go fmt ./... | wc -l) -eq 0 From 70be798d74911389669df5a60e1648e2e90764b5 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Wed, 8 Nov 2023 15:34:55 -0600 Subject: [PATCH 6/7] review updates --- output/compact.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/output/compact.go b/output/compact.go index 2a871c3d7..8957d1b30 100644 --- a/output/compact.go +++ b/output/compact.go @@ -143,7 +143,6 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection server.CompactLogTime = time.Now() server.CompactLogStats = make(map[string]uint8) } else if time.Now().Sub(server.CompactLogTime) > time.Minute { - server.CompactLogTime = time.Now() var keys []string for k := range server.CompactLogStats { keys = append(keys, k) @@ -156,7 +155,10 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection details += ", " } } - logger.PrintInfo("Compact snapshots submitted: " + details) + if len(details) > 0 { + logger.PrintInfo("Compact snapshots submitted: " + details) + } + server.CompactLogTime = time.Now() server.CompactLogStats = make(map[string]uint8) } else { server.CompactLogStats[kind] = server.CompactLogStats[kind] + 1 From 651c191085571eef9ec8e7cdf573661ac351f4d5 Mon Sep 17 00:00:00 2001 From: Sean Linsley Date: Thu, 9 Nov 2023 09:32:22 -0600 Subject: [PATCH 7/7] align log events to even minutes --- output/compact.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/output/compact.go b/output/compact.go index 8957d1b30..afc7b1e02 100644 --- a/output/compact.go +++ b/output/compact.go @@ -140,7 +140,7 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection } else if !quiet { logger.PrintVerbose("Submitted compact %s snapshot successfully", kind) if server.CompactLogTime.IsZero() { - server.CompactLogTime = time.Now() + server.CompactLogTime = time.Now().Truncate(time.Minute) server.CompactLogStats = make(map[string]uint8) } else if time.Now().Sub(server.CompactLogTime) > time.Minute { var keys []string @@ -158,7 +158,7 @@ func submitCompactSnapshot(ctx context.Context, server *state.Server, collection if len(details) > 0 { logger.PrintInfo("Compact snapshots submitted: " + details) } - server.CompactLogTime = time.Now() + server.CompactLogTime = time.Now().Truncate(time.Minute) server.CompactLogStats = make(map[string]uint8) } else { server.CompactLogStats[kind] = server.CompactLogStats[kind] + 1