Introduce revamped health/readiness

This moves the health code into its own package. It also allows for the enablement of a readiness endpoint (`/readyz`) if a `--readyz` flag is passed to audito-maldito. The health system was also refactored to take a number of components for which we'll be able to output if they're ready or not. These components are then listed as part of the readyz endpoint output. In the future, the intent is to also add a livez endpoint. Signed-off-by: Juan Antonio Osorio <[email protected]>
metal-toolbox · Apr 18, 2023 · 5c08335 · 5c08335
1 parent 826afd5
commit 5c08335
Show file tree

Hide file tree

Showing 11 changed files with 226 additions and 130 deletions.
diff --git a/internal/app/app.go b/internal/app/app.go
@@ -20,6 +20,7 @@ import (
 	"github.com/metal-toolbox/audito-maldito/internal/auditd"
 	"github.com/metal-toolbox/audito-maldito/internal/auditd/dirreader"
 	"github.com/metal-toolbox/audito-maldito/internal/common"
+	"github.com/metal-toolbox/audito-maldito/internal/health"
 	"github.com/metal-toolbox/audito-maldito/internal/journald"
 	"github.com/metal-toolbox/audito-maldito/internal/metrics"
 	"github.com/metal-toolbox/audito-maldito/internal/processors"
@@ -40,11 +41,12 @@ OPTIONS
 var logger *zap.SugaredLogger
 
 //nolint
-func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig *zap.Config) error {
+func Run(ctx context.Context, osArgs []string, h *health.Health, optLoggerConfig *zap.Config) error {
 	var bootID string
 	var auditlogpath string
 	var auditLogDirPath string
 	var enableMetrics bool
+	var enableHealthz bool
 	logLevel := zapcore.DebugLevel // TODO: Switch default back to zapcore.ErrorLevel.
 
 	flagSet := flag.NewFlagSet(osArgs[0], flag.ContinueOnError)
@@ -55,6 +57,7 @@ func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig
 	flagSet.StringVar(&auditLogDirPath, "audit-dir-path", "/var/log/audit", "Path to the Linux audit log directory")
 	flagSet.Var(&logLevel, "log-level", "Set the log level according to zapcore.Level")
 	flagSet.BoolVar(&enableMetrics, "metrics", false, "Enable Prometheus HTTP /metrics server")
+	flagSet.BoolVar(&enableHealthz, "healthz", false, "Enable HTTP health endpoints server")
 	flagSet.Usage = func() {
 		os.Stderr.WriteString(usage)
 		flagSet.PrintDefaults()
@@ -114,11 +117,20 @@ func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig
 		return fmt.Errorf("failed to open audit log file: %w", auditfileerr)
 	}
 
+	server := &http.Server{Addr: ":2112"}
+
 	if enableMetrics {
-		server := &http.Server{Addr: ":2112"}
+		http.Handle("/metrics", promhttp.Handler())
+	}
+
+	if enableHealthz {
+		http.Handle("/readyz", h.ReadyzHandler())
+		// TODO: Add livez endpoint
+	}
+
+	if enableMetrics || enableHealthz {
 		eg.Go(func() error {
-			http.Handle("/metrics", promhttp.Handler())
-			logger.Infoln("Starting HTTP metrics server on :2112")
+			logger.Infoln("Starting HTTP server on :2112")
 			if err := server.ListenAndServe(); err != nil {
 				logger.Errorf("Failed to start HTTP metrics server: %v", err)
 				return err
@@ -139,10 +151,10 @@ func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig
 			auditLogDirPath, err)
 	}
 
-	h.AddReadiness()
+	h.AddReadiness(dirreader.DirReaderComponentName)
 	go func() {
 		<-logDirReader.InitFilesDone()
-		h.OnReady()
+		h.OnReady(dirreader.DirReaderComponentName)
 	}()
 
 	eg.Go(func() error {
@@ -210,7 +222,7 @@ func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig
 			}
 		})
 	} else {
-		h.AddReadiness()
+		h.AddReadiness(journald.JournaldReaderComponentName)
 		eg.Go(func() error {
 			jp := journald.Processor{
 				BootID:    bootID,
@@ -232,7 +244,7 @@ func Run(ctx context.Context, osArgs []string, h *common.Health, optLoggerConfig
 		})
 	}
 
-	h.AddReadiness()
+	h.AddReadiness(auditd.AuditdProcessorComponentName)
 	eg.Go(func() error {
 		ap := auditd.Auditd{
 			After:  time.UnixMicro(int64(lastReadJournalTS)),

diff --git a/internal/auditd/auditd.go b/internal/auditd/auditd.go
@@ -12,6 +12,13 @@ import (
 
 	"github.com/metal-toolbox/audito-maldito/internal/auditd/sessiontracker"
 	"github.com/metal-toolbox/audito-maldito/internal/common"
+	"github.com/metal-toolbox/audito-maldito/internal/health"
+)
+
+const (
+	// AuditdProcessorComponentName is the name of the component
+	// that reads from auditd. This is used in the health check.
+	AuditdProcessorComponentName = "auditd-processor"
 )
 
 // libaudit variables.
@@ -48,7 +55,7 @@ type Auditd struct {
 	// EventW is the auditevent.EventWriter to write events to.
 	EventW *auditevent.EventWriter
 
-	Health *common.Health
+	Health *health.Health
 }
 
 // TODO: Write documentation about creating a splunk query that shows
@@ -82,7 +89,7 @@ func (o *Auditd) Read(ctx context.Context) error {
 	staleDataTicker := time.NewTicker(staleDataCleanupInterval)
 	defer staleDataTicker.Stop()
 
-	o.Health.OnReady()
+	o.Health.OnReady(AuditdProcessorComponentName)
 
 	for {
 		select {

diff --git a/internal/auditd/auditd_good_test.go b/internal/auditd/auditd_good_test.go
@@ -19,6 +19,7 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/metal-toolbox/audito-maldito/internal/common"
+	"github.com/metal-toolbox/audito-maldito/internal/health"
 	"github.com/metal-toolbox/audito-maldito/internal/testtools"
 )
 
@@ -82,7 +83,7 @@ func TestAuditd_Read_GoodRemoteUserLoginFirst(t *testing.T) {
 			Events: events,
 			T:      t,
 		}),
-		Health: common.NewSingleReadinessHealth(),
+		Health: health.NewSingleReadinessHealth(AuditdProcessorComponentName),
 	}
 
 	exited := make(chan error, 1)
@@ -145,7 +146,7 @@ func TestAuditd_Read_GoodAuditdEventsFirst(t *testing.T) {
 			Events: events,
 			T:      t,
 		}),
-		Health: common.NewSingleReadinessHealth(),
+		Health: health.NewSingleReadinessHealth(AuditdProcessorComponentName),
 	}
 
 	exited := make(chan error, 1)

diff --git a/internal/auditd/auditd_test.go b/internal/auditd/auditd_test.go
@@ -18,6 +18,7 @@ import (
 	"github.com/metal-toolbox/audito-maldito/internal/auditd/sessiontracker"
 	fakest "github.com/metal-toolbox/audito-maldito/internal/auditd/sessiontracker/fakes"
 	"github.com/metal-toolbox/audito-maldito/internal/common"
+	"github.com/metal-toolbox/audito-maldito/internal/health"
 	"github.com/metal-toolbox/audito-maldito/internal/testtools"
 )
 
@@ -38,7 +39,7 @@ func TestAuditd_Read_RemoteLoginError(t *testing.T) {
 			Events: events,
 			T:      t,
 		}),
-		Health: common.NewSingleReadinessHealth(),
+		Health: health.NewSingleReadinessHealth(AuditdProcessorComponentName),
 	}
 
 	errs := make(chan error, 1)
@@ -82,7 +83,7 @@ func TestAuditd_Read_ParseAuditLogError(t *testing.T) {
 			Events: events,
 			T:      t,
 		}),
-		Health: common.NewSingleReadinessHealth(),
+		Health: health.NewSingleReadinessHealth(AuditdProcessorComponentName),
 	}
 
 	errs := make(chan error, 1)
@@ -127,7 +128,7 @@ func TestAuditd_Read_AuditEventError(t *testing.T) {
 			Events: events,
 			T:      t,
 		}),
-		Health: common.NewSingleReadinessHealth(),
+		Health: health.NewSingleReadinessHealth(AuditdProcessorComponentName),
 	}
 
 	cancelEventWFn()

diff --git a/internal/auditd/dirreader/dirreader.go b/internal/auditd/dirreader/dirreader.go
@@ -17,6 +17,12 @@ import (
 	"github.com/fsnotify/fsnotify"
 )
 
+const (
+	// DirReaderComponentName is the component name for the dir reader.
+	// This is used for health checks.
+	DirReaderComponentName = "auditlog-dirreader"
+)
+
 // StartLogDirReader creates and starts a LogDirReader for
 // the specified directory path (e.g., "/var/log/audit").
 //

diff --git a/internal/common/health.go b/internal/common/health.go