From a4fde176ab7256f1eda58bb3b8f9b336b680b548 Mon Sep 17 00:00:00 2001 From: Dominik Rosiek Date: Wed, 12 Jun 2024 10:46:55 +0200 Subject: [PATCH] feat: use router to detect need for trailing newline Signed-off-by: Dominik Rosiek --- .../logs/collector/common/filelog_receiver.yaml | 15 +++++++++++++-- tests/helm/logs_test.go | 13 ++++++++++--- .../goldenfile/logs_otc/basic.output.yaml | 9 +++++++-- .../goldenfile/logs_otc/debug.output.yaml | 9 +++++++-- .../multiple_multiline.output.yaml | 9 +++++++-- .../logs_otc_windows/configmap/basic.output.yaml | 9 +++++++-- .../logs_otc_windows/configmap/debug.output.yaml | 9 +++++++-- .../daemonset/multiple_multiline.output.yaml | 9 +++++++-- 8 files changed, 65 insertions(+), 17 deletions(-) diff --git a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml index bc2b6ee037..be068b27ce 100644 --- a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml +++ b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml @@ -56,6 +56,7 @@ filelog/containers: ## - k8s.namespace.name ## - log.file.path - type: retain + id: keep-fields fields: - attributes["stream"] - attributes["k8s.pod.name"] @@ -66,6 +67,18 @@ filelog/containers: - attributes["time"] {{ end }} + ## Strip trailing "\n" from the log body + - id: strip-trailing-newline-router +{{- if .Values.sumologic.logs.multiline.enabled }} + default: multiline +{{- else }} + default: merge-multiline-logs +{{- end }} + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router + ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime. ## Uses attributes.log as temporary cotainer for new log ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" } @@ -75,14 +88,12 @@ filelog/containers: parse_to: attributes output: replace-body regex: "^(?P.*)\n$" - if: body matches "^.*\n$" type: regex_parser - id: replace-body type: move from: attributes.log to: body - if: attributes.log != nil {{- if .Values.sumologic.logs.multiline.enabled }} ## Perform multiline detection diff --git a/tests/helm/logs_test.go b/tests/helm/logs_test.go index 4fb3401728..dbb81fb401 100644 --- a/tests/helm/logs_test.go +++ b/tests/helm/logs_test.go @@ -464,6 +464,7 @@ sumologic: Id string Type string Output string + Fields []string } } `yaml:"filelog/containers"` } @@ -472,10 +473,16 @@ sumologic: require.NoError(t, err) keepTimeOperatorFound := false + +operatorLoop: for _, operator := range otelConfig.Receivers.Filelog.Operators { - if operator.Id == "move-time-attribute" { - keepTimeOperatorFound = true - break + if operator.Id == "keep-fields" { + for _, field := range operator.Fields { + if field == "attributes[\"time\"]" { + keepTimeOperatorFound = true + break operatorLoop + } + } } } require.True(t, keepTimeOperatorFound) diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml index 8e22f492cf..2053e36ad8 100644 --- a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml @@ -73,9 +73,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -85,7 +91,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml index b93f466a7a..b744085792 100644 --- a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml @@ -83,9 +83,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -95,7 +101,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml index 8fdc41d84a..a68206bad1 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml @@ -73,9 +73,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -85,7 +91,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml index 4ed2444300..76cb07bc92 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml @@ -52,9 +52,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -64,7 +70,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml index 035d90a571..268c12882a 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml @@ -63,9 +63,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -75,7 +81,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml index 8cbf822959..a28c009e96 100644 --- a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml +++ b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml @@ -52,9 +52,15 @@ data: - attributes["k8s.container.name"] - attributes["k8s.namespace.name"] - attributes["log.file.path"] + id: keep-fields type: retain + - default: multiline + id: strip-trailing-newline-router + routes: + - expr: body matches "^.*\n$" + output: strip-trailing-newline + type: router - id: strip-trailing-newline - if: body matches "^.*\n$" output: replace-body parse_from: body parse_to: attributes @@ -64,7 +70,6 @@ data: type: regex_parser - from: attributes.log id: replace-body - if: attributes.log != nil to: body type: move - default: merge-multiline-logs