feat: use router to detect need for trailing newline

Signed-off-by: Dominik Rosiek <[email protected]>
SumoLogic · Jun 12, 2024 · a4fde17 · a4fde17
1 parent 69f2592
commit a4fde17
Show file tree

Hide file tree

Showing 8 changed files with 65 additions and 17 deletions.
diff --git a/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml b/deploy/helm/sumologic/conf/logs/collector/common/filelog_receiver.yaml
@@ -56,6 +56,7 @@ filelog/containers:
     ## - k8s.namespace.name
     ## - log.file.path
     - type: retain
+      id: keep-fields
       fields:
         - attributes["stream"]
         - attributes["k8s.pod.name"]
@@ -66,6 +67,18 @@ filelog/containers:
         - attributes["time"]
 {{ end }}
 
+    ## Strip trailing "\n" from the log body
+    - id: strip-trailing-newline-router
+{{- if .Values.sumologic.logs.multiline.enabled }}
+      default: multiline
+{{- else }}
+      default: merge-multiline-logs
+{{- end }}
+      routes:
+        - expr: body matches "^.*\n$"
+          output: strip-trailing-newline
+      type: router
+
     ## strip-trailing-newline removes the trailing "\n" from the `log` key. This is required for logs coming from Docker container runtime.
     ## Uses attributes.log as temporary cotainer for new log
     ## Input Body (JSON): { "log": "2001-02-03 04:05:06 very long line that was split by the logging driver\n", "stream": "stdout" }
@@ -75,14 +88,12 @@ filelog/containers:
       parse_to: attributes
       output: replace-body
       regex: "^(?P<log>.*)\n$"
-      if: body matches "^.*\n$"
       type: regex_parser
 
     - id: replace-body
       type: move
       from: attributes.log
       to: body
-      if: attributes.log != nil
 
 {{- if .Values.sumologic.logs.multiline.enabled }}
     ## Perform multiline detection

diff --git a/tests/helm/logs_test.go b/tests/helm/logs_test.go
@@ -464,6 +464,7 @@ sumologic:
 					Id     string
 					Type   string
 					Output string
+					Fields []string
 				}
 			} `yaml:"filelog/containers"`
 		}
@@ -472,10 +473,16 @@ sumologic:
 	require.NoError(t, err)
 
 	keepTimeOperatorFound := false
+
+operatorLoop:
 	for _, operator := range otelConfig.Receivers.Filelog.Operators {
-		if operator.Id == "move-time-attribute" {
-			keepTimeOperatorFound = true
-			break
+		if operator.Id == "keep-fields" {
+			for _, field := range operator.Fields {
+				if field == "attributes[\"time\"]" {
+					keepTimeOperatorFound = true
+					break operatorLoop
+				}
+			}
 		}
 	}
 	require.True(t, keepTimeOperatorFound)

diff --git a/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/basic.output.yaml
@@ -73,9 +73,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -85,7 +91,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs

diff --git a/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc/debug.output.yaml
@@ -83,9 +83,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -95,7 +101,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs

diff --git a/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_daemonset/multiple_multiline.output.yaml
@@ -73,9 +73,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -85,7 +91,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/basic.output.yaml
@@ -52,9 +52,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -64,7 +70,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/configmap/debug.output.yaml
@@ -63,9 +63,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -75,7 +81,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs

diff --git a/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml b/tests/helm/testdata/goldenfile/logs_otc_windows/daemonset/multiple_multiline.output.yaml
@@ -52,9 +52,15 @@ data:
           - attributes["k8s.container.name"]
           - attributes["k8s.namespace.name"]
           - attributes["log.file.path"]
+          id: keep-fields
           type: retain
+        - default: multiline
+          id: strip-trailing-newline-router
+          routes:
+          - expr: body matches "^.*\n$"
+            output: strip-trailing-newline
+          type: router
         - id: strip-trailing-newline
-          if: body matches "^.*\n$"
           output: replace-body
           parse_from: body
           parse_to: attributes
@@ -64,7 +70,6 @@ data:
           type: regex_parser
         - from: attributes.log
           id: replace-body
-          if: attributes.log != nil
           to: body
           type: move
         - default: merge-multiline-logs