Merge pull request #807 from MCarlomagno/add-bot-logger-component

Added bot logger component

services/components/components.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/ethereum/go-ethereum/accounts/keystore"
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/forta-network/forta-core-go/clients/agentlogs"
 	"github.com/forta-network/forta-core-go/utils"
 	"github.com/forta-network/forta-node/clients"
 	"github.com/forta-network/forta-node/clients/agentgrpc"
@@ -72,17 +74,22 @@ type BotLifecycleConfig struct {
 	ScannerAddress common.Address
 	MessageClient  clients.MessageClient
 	BotRegistry    registry.BotRegistry
+	Key            *keystore.Key
 }
 
 // BotLifecycle contains the bot lifecycle components.
 type BotLifecycle struct {
 	BotManager   lifecycle.BotLifecycleManager
 	BotClient    containers.BotClient
 	ImageCleanup containers.ImageCleanup
+	BotLogger    lifecycle.BotLogger
 }
 
 // GetBotLifecycleComponents returns the bot lifecycle management components.
-func GetBotLifecycleComponents(ctx context.Context, botLifeConfig BotLifecycleConfig) (BotLifecycle, error) {
+func GetBotLifecycleComponents(
+	ctx context.Context,
+	botLifeConfig BotLifecycleConfig,
+) (BotLifecycle, error) {
 	cfg := botLifeConfig.Config
 	// bot image client is helpful for loading local mode agents from a restricted container registry
 	var (
@@ -120,10 +127,17 @@ func GetBotLifecycleComponents(ctx context.Context, botLifeConfig BotLifecycleCo
 		lifecycleMetrics, botMonitor,
 	)
 	imageCleanup := containers.NewImageCleanup(dockerClient, botLifeConfig.BotRegistry)
+	botLogger := lifecycle.NewBotLogger(
+		botClient,
+		dockerClient,
+		botLifeConfig.Key,
+		agentlogs.NewClient(botLifeConfig.Config.AgentLogsConfig.URL).SendLogs,
+	)
 
 	return BotLifecycle{
 		BotManager:   botManager,
 		BotClient:    botClient,
 		ImageCleanup: imageCleanup,
+		BotLogger:    botLogger,
 	}, nil
 }

services/components/lifecycle/bot_logger.go

@@ -0,0 +1,112 @@
+package lifecycle
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/forta-network/forta-core-go/clients/agentlogs"
+	"github.com/forta-network/forta-core-go/security"
+	"github.com/forta-network/forta-node/clients"
+	"github.com/forta-network/forta-node/clients/docker"
+	"github.com/forta-network/forta-node/services/components/containers"
+	log "github.com/sirupsen/logrus"
+)
+
+// BotLogger manages bots logging.
+type BotLogger interface {
+	SendBotLogs(ctx context.Context) error
+}
+
+type botLogger struct {
+	botClient     containers.BotClient
+	dockerClient  clients.DockerClient
+	key           *keystore.Key
+	prevAgentLogs agentlogs.Agents
+
+	sendAgentLogs func(agents agentlogs.Agents, authToken string) error
+}
+
+var _ BotLogger = &botLogger{}
+
+func NewBotLogger(
+	botClient containers.BotClient,
+	dockerClient clients.DockerClient,
+	key *keystore.Key,
+	sendAgentLogs func(agents agentlogs.Agents, authToken string) error,
+) *botLogger {
+	return &botLogger{
+		botClient:     botClient,
+		dockerClient:  dockerClient,
+		key:           key,
+		sendAgentLogs: sendAgentLogs,
+	}
+}
+
+// adjust these better with auto-upgrade later
+const (
+	defaultAgentLogSendInterval       = time.Minute
+	defaultAgentLogTailLines          = 50
+	defaultAgentLogAvgMaxCharsPerLine = 200
+)
+
+func (bl *botLogger) SendBotLogs(ctx context.Context) error {
+	var (
+		sendLogs agentlogs.Agents
+		keepLogs agentlogs.Agents
+	)
+
+	botContainers, err := bl.botClient.LoadBotContainers(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to load the bot containers: %v", err)
+	}
+
+	for _, container := range botContainers {
+		if container.Labels[docker.LabelFortaSettingsAgentLogsEnable] != "true" {
+			continue
+		}
+		logs, err := bl.dockerClient.GetContainerLogs(
+			ctx, container.ID,
+			strconv.Itoa(defaultAgentLogTailLines),
+			defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+		)
+		if err != nil {
+			log.WithError(err).Warn("failed to get agent container logs")
+			continue
+		}
+
+		agent := &agentlogs.Agent{
+			ID:   container.Labels[docker.LabelFortaBotID],
+			Logs: logs,
+		}
+		// don't send if it's the same with previous logs but keep it for next time
+		// so we can check
+		keepLogs = append(keepLogs, agent)
+		if !bl.prevAgentLogs.Has(agent.ID, logs) {
+			log.WithField("agent", agent.ID).Debug("new agent logs found")
+			sendLogs = append(sendLogs, agent)
+		} else {
+			log.WithField("agent", agent.ID).Debug("no new agent logs")
+		}
+	}
+
+	if len(sendLogs) > 0 {
+		scannerJwt, err := security.CreateScannerJWT(bl.key, map[string]interface{}{
+			"access": "bot_logger",
+		})
+		if err != nil {
+			return fmt.Errorf("failed to create scanner token: %v", err)
+		}
+		if err := bl.sendAgentLogs(sendLogs, scannerJwt); err != nil {
+			return fmt.Errorf("failed to send agent logs: %v", err)
+		}
+		log.WithField("count", len(sendLogs)).Debug("successfully sent new agent logs")
+	} else {
+		log.Debug("no new agent logs were found - not sending")
+	}
+
+	bl.prevAgentLogs = keepLogs
+	return nil
+}

services/components/lifecycle/bot_logger_test.go

@@ -0,0 +1,200 @@
+package lifecycle
+
+import (
+	"context"
+	"errors"
+	"strconv"
+	"testing"
+
+	"github.com/docker/docker/api/types"
+	"github.com/ethereum/go-ethereum/accounts/keystore"
+	"github.com/forta-network/forta-core-go/clients/agentlogs"
+	"github.com/forta-network/forta-core-go/security"
+	"github.com/forta-network/forta-node/clients/docker"
+	mock_clients "github.com/forta-network/forta-node/clients/mocks"
+	mock_containers "github.com/forta-network/forta-node/services/components/containers/mocks"
+	"github.com/golang/mock/gomock"
+	"github.com/stretchr/testify/require"
+	"github.com/stretchr/testify/suite"
+)
+
+func TestSendBotLogsSuite(t *testing.T) {
+	suite.Run(t, &BotLoggerSuite{})
+}
+
+type BotLoggerSuite struct {
+	r *require.Assertions
+
+	botLogger    *botLogger
+	botClient    *mock_containers.MockBotClient
+	dockerClient *mock_clients.MockDockerClient
+	key          *keystore.Key
+	suite.Suite
+}
+
+func (s *BotLoggerSuite) SetupTest() {
+	t := s.T()
+	ctrl := gomock.NewController(s.T())
+	r := s.Require()
+
+	botClient := mock_containers.NewMockBotClient(ctrl)
+	dockerClient := mock_clients.NewMockDockerClient(ctrl)
+
+	dir := t.TempDir()
+	ks := keystore.NewKeyStore(dir, keystore.StandardScryptN, keystore.StandardScryptP)
+
+	_, err := ks.NewAccount("Forta123")
+	r.NoError(err)
+
+	key, err := security.LoadKeyWithPassphrase(dir, "Forta123")
+	r.NoError(err)
+
+	s.botClient = botClient
+	s.dockerClient = dockerClient
+	s.key = key
+	s.r = r
+}
+
+func (s *BotLoggerSuite) TestSendBotLogs() {
+	botLogger := NewBotLogger(
+		s.botClient, s.dockerClient, s.key,
+		func(agents agentlogs.Agents, authToken string) error {
+			s.r.Equal(2, len(agents))
+			s.r.Equal("bot1ID", agents[0].ID)
+			s.r.Equal("bot2ID", agents[1].ID)
+			return nil
+		},
+	)
+	ctx := context.Background()
+
+	mockContainers := []types.Container{
+		{
+			ID:    "bot1",
+			Image: "forta/bot:latest",
+			Labels: map[string]string{
+				docker.LabelFortaSettingsAgentLogsEnable: "true",
+				docker.LabelFortaBotID:                   "bot1ID",
+			},
+		},
+		{
+			ID:    "bot2",
+			Image: "forta/bot:latest",
+			Labels: map[string]string{
+				docker.LabelFortaSettingsAgentLogsEnable: "true",
+				docker.LabelFortaBotID:                   "bot2ID",
+			},
+		},
+	}
+	s.dockerClient.EXPECT().GetContainerLogs(
+		ctx, "bot1",
+		strconv.Itoa(defaultAgentLogTailLines),
+		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+	).Return("some log", nil).Times(1)
+
+	s.dockerClient.EXPECT().GetContainerLogs(
+		ctx, "bot2",
+		strconv.Itoa(defaultAgentLogTailLines),
+		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+	).Return("some log", nil).Times(1)
+
+	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, nil)
+	s.r.NoError(botLogger.SendBotLogs(ctx))
+}
+
+// should fail if there is an error loading
+// bot containers
+func (s *BotLoggerSuite) TestLoadBotContainersError() {
+	botLogger := NewBotLogger(
+		s.botClient, s.dockerClient, s.key,
+		func(agents agentlogs.Agents, authToken string) error {
+			return nil
+		},
+	)
+	ctx := context.Background()
+
+	mockContainers := []types.Container{}
+
+	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, errors.New("test"))
+	s.r.EqualError(botLogger.SendBotLogs(ctx), "failed to load the bot containers: test")
+}
+
+// Should not send agent logs if fails
+// to get container logs but continue processing
+func (s *BotLoggerSuite) TestGetContainerLogsError() {
+	botLogger := NewBotLogger(
+		s.botClient, s.dockerClient, s.key,
+		func(agents agentlogs.Agents, authToken string) error {
+			s.r.Equal(1, len(agents))
+			s.r.Equal("bot2ID", agents[0].ID)
+			s.r.Equal("some log", agents[0].Logs)
+			return nil
+		},
+	)
+	ctx := context.Background()
+
+	mockContainers := []types.Container{
+		{
+			ID:    "bot1",
+			Image: "forta/bot:latest",
+			Labels: map[string]string{
+				docker.LabelFortaSettingsAgentLogsEnable: "true",
+			},
+		},
+		{
+			ID:    "bot2",
+			Image: "forta/bot:latest",
+			Labels: map[string]string{
+				docker.LabelFortaSettingsAgentLogsEnable: "true",
+				docker.LabelFortaBotID:                   "bot2ID",
+			},
+		},
+	}
+
+	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, nil)
+
+	s.dockerClient.EXPECT().GetContainerLogs(
+		ctx, "bot1",
+		strconv.Itoa(defaultAgentLogTailLines),
+		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+	).Return("", errors.New("test")).Times(1)
+
+	s.dockerClient.EXPECT().GetContainerLogs(
+		ctx, "bot2",
+		strconv.Itoa(defaultAgentLogTailLines),
+		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+	).Return("some log", nil).Times(1)
+
+	s.r.NoError(botLogger.SendBotLogs(ctx))
+}
+
+// Fails sending agent logs
+func (s *BotLoggerSuite) TestFailsToSendLogs() {
+	botLogger := NewBotLogger(
+		s.botClient, s.dockerClient, s.key,
+		func(agents agentlogs.Agents, authToken string) error {
+			return errors.New("test")
+		},
+	)
+	ctx := context.Background()
+
+	mockContainers := []types.Container{
+		{
+			ID:    "bot1",
+			Image: "forta/bot:latest",
+			Labels: map[string]string{
+				docker.LabelFortaSettingsAgentLogsEnable: "true",
+				docker.LabelFortaBotID:                   "bot1ID",
+			},
+		},
+	}
+
+	s.botClient.EXPECT().LoadBotContainers(ctx).Return(mockContainers, nil)
+
+	s.dockerClient.EXPECT().GetContainerLogs(
+		ctx, "bot1",
+		strconv.Itoa(defaultAgentLogTailLines),
+		defaultAgentLogAvgMaxCharsPerLine*defaultAgentLogTailLines,
+	).Return("some log", nil).Times(1)
+
+	s.r.EqualError(botLogger.SendBotLogs(ctx), "failed to send agent logs: test")
+}