separate log and agent container (#426)

alpine.multiarch

@@ -1,9 +1,17 @@
 FROM alpine:3.18.2
 
+ENV LOGROTATE_ROTATE=5 \
+    LOGROTATE_SIZE=100M \
+    TZ=Asia/Shanghai
+
 ARG TARGETARCH
 
+ADD hack/logrotate.sh /logrotate.sh
+RUN chmod +x /logrotate.sh
+RUN ln -sf /usr/share/zoneinfo/$TZ /etc/localtime \
+    && echo $TZ > /etc/timezone \
 RUN apk update \
   && apk upgrade \
   && apk add --no-cache \
-    curl jq util-linux bash xfsprogs \
+    curl jq util-linux bash xfsprogs logrotate \
   && rm -rf /var/cache/apk/*

apis/apps/v1alpha1/nebulacluster_common.go

@@ -42,12 +42,13 @@ const (
 	NebulaServiceAccountName  = "nebula-sa"
 	NebulaRoleName            = "nebula-role"
 	NebulaRoleBindingName     = "nebula-rolebinding"
+	LogSidecarContainerName   = "ng-logrotate"
 	AgentSidecarContainerName = "ng-agent"
 	AgentInitContainerName    = "ng-init-agent"
 	DefaultAgentPortGRPC      = 8888
-	agentPortNameGRPC         = "grpc"
-	defaultAgentImage         = "vesoft/nebula-agent"
-	defaultAlpineImage        = "vesoft/nebula-alpine:latest"
+	AgentPortNameGRPC         = "grpc"
+	DefaultAgentImage         = "vesoft/nebula-agent"
+	DefaultAlpineImage        = "vesoft/nebula-alpine:latest"
 
 	ZoneSuffix = "zone"
 )
@@ -271,13 +272,60 @@ func parseStorageRequest(res corev1.ResourceList) (corev1.ResourceRequirements,
 	}, nil
 }
 
+func logVolumeExists(componentType string, volumes []corev1.Volume) bool {
+	logVolName := logVolume(componentType)
+	for _, volume := range volumes {
+		if volume.Name == logVolName {
+			return true
+		}
+	}
+	return false
+}
+
 func GenerateInitAgentContainer(c NebulaClusterComponent) corev1.Container {
 	container := generateAgentContainer(c, true)
 	container.Name = AgentInitContainerName
 
 	return container
 }
 
+func generateLogContainer(c NebulaClusterComponent) corev1.Container {
+	nc := c.GetNebulaCluster()
+	componentType := c.ComponentType().String()
+
+	image := DefaultAlpineImage
+	if nc.Spec.AlpineImage != nil {
+		image = pointer.StringDeref(nc.Spec.AlpineImage, "")
+	}
+
+	cmd := []string{"/bin/sh", "-ecx", "sh /logrotate.sh; crond -f -l 2"}
+	container := corev1.Container{
+		Name:    LogSidecarContainerName,
+		Image:   image,
+		Command: cmd,
+	}
+
+	logRotate := nc.Spec.LogRotate
+	container.Env = []corev1.EnvVar{
+		{
+			Name:  "LOGROTATE_ROTATE",
+			Value: strconv.Itoa(int(logRotate.Rotate)),
+		},
+		{
+			Name:  "LOGROTATE_SIZE",
+			Value: logRotate.Size,
+		},
+	}
+
+	container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
+		Name:      logVolume(componentType),
+		MountPath: "/usr/local/nebula/logs",
+		SubPath:   "logs",
+	})
+
+	return container
+}
+
 func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Container {
 	nc := c.GetNebulaCluster()
 	componentType := c.ComponentType().String()
@@ -288,33 +336,25 @@ func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Containe
 		fmt.Sprintf(" --agent=$(hostname).%s:%d", c.GetServiceFQDN(), DefaultAgentPortGRPC) +
 		" --ratelimit=1073741824 --debug"
 	brCmd := initCmd + " --meta=" + metadAddr
-	logCmd := "sh /logrotate.sh; /etc/init.d/cron start"
-	logfgCmd := "sh /logrotate.sh; exec cron -f"
 
 	if nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled() {
-		initCmd += " --enable_ssl"
-		brCmd += " --enable_ssl"
+		initCmd += " --enable-ssl"
+		brCmd += " --enable-ssl"
 		if nc.InsecureSkipVerify() {
-			initCmd += " --insecure_skip_verify"
-			brCmd += " --insecure_skip_verify"
+			initCmd += " --insecure-skip-verify"
+			brCmd += " --insecure-skip-verify"
 		}
 	}
 
 	if init {
 		cmd = append(cmd, initCmd)
 	} else {
-		if nc.IsLogRotateEnabled() && nc.IsBREnabled() {
-			cmd = append(cmd, fmt.Sprintf(`%s; %s`, logCmd, brCmd))
-		} else if nc.IsLogRotateEnabled() {
-			cmd = append(cmd, logfgCmd)
-		} else if nc.IsBREnabled() {
-			cmd = append(cmd, brCmd)
-		}
+		cmd = append(cmd, brCmd)
 	}
 
 	container := corev1.Container{
 		Name:    AgentSidecarContainerName,
-		Image:   defaultAgentImage,
+		Image:   DefaultAgentImage,
 		Command: cmd,
 	}
 	imagePullPolicy := nc.Spec.ImagePullPolicy
@@ -346,33 +386,13 @@ func generateAgentContainer(c NebulaClusterComponent, init bool) corev1.Containe
 
 		container.Ports = []corev1.ContainerPort{
 			{
-				Name:          agentPortNameGRPC,
+				Name:          AgentPortNameGRPC,
 				ContainerPort: int32(DefaultAgentPortGRPC),
 			},
 		}
 	}
 
-	if nc.IsLogRotateEnabled() {
-		logRotate := nc.Spec.LogRotate
-		container.Env = []corev1.EnvVar{
-			{
-				Name:  "LOGROTATE_ROTATE",
-				Value: strconv.Itoa(int(logRotate.Rotate)),
-			},
-			{
-				Name:  "LOGROTATE_SIZE",
-				Value: logRotate.Size,
-			},
-		}
-
-		container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{
-			Name:      logVolume(componentType),
-			MountPath: "/usr/local/nebula/logs",
-			SubPath:   "logs",
-		})
-	}
-
-	if (nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled()) && nc.IsBREnabled() {
+	if (nc.IsMetadSSLEnabled() || nc.IsClusterSSLEnabled()) && nc.IsBREnabled() && !enableLocalCerts() {
 		certMounts := []corev1.VolumeMount{
 			{
 				Name:      "client-crt",
@@ -412,7 +432,7 @@ NODE_ZONE=$(jq '."topology.kubernetes.io/zone"' -r /node/labels.json)
 echo "NODE_ZONE is ${NODE_ZONE}"
 echo "export NODE_ZONE=${NODE_ZONE}" > /node/zone
 `
-	image := defaultAlpineImage
+	image := DefaultAlpineImage
 	if nc.Spec.AlpineImage != nil {
 		image = pointer.StringDeref(nc.Spec.AlpineImage, "")
 	}
@@ -617,10 +637,14 @@ done
 
 	containers = append(containers, baseContainer)
 
-	if nc.IsBREnabled() || nc.IsLogRotateEnabled() {
+	if nc.IsBREnabled() {
 		agentContainer := generateAgentContainer(c, false)
 		containers = append(containers, agentContainer)
 	}
+	if nc.IsLogRotateEnabled() && logVolumeExists(componentType, c.GenerateVolumes()) {
+		logContainer := generateLogContainer(c)
+		containers = append(containers, logContainer)
+	}
 
 	containers = mergeSidecarContainers(containers, c.ComponentSpec().SidecarContainers())
 
@@ -933,3 +957,9 @@ func separateFlags(config map[string]string) (map[string]string, map[string]stri
 	}
 	return dynamic, static
 }
+
+func enableLocalCerts() bool {
+	return os.Getenv("CA_CERT_PATH") != "" &&
+		os.Getenv("CLIENT_CERT_PATH") != "" &&
+		os.Getenv("CLIENT_KEY_PATH") != ""
+}

apis/apps/v1alpha1/nebulacluster_types.go

@@ -294,7 +294,6 @@ type ConsoleSpec struct {
 
 // AgentContainerSpec defines the desired state of Agent
 type AgentContainerSpec struct {
-
 	// url for docker image
 	// +optional
 	Image string `json:"image,omitempty"`
@@ -306,6 +305,16 @@ type AgentContainerSpec struct {
 	// K8S resources settings.
 	// +optional
 	Resources corev1.ResourceRequirements `json:"resources,omitempty"`
+
+	// Container environment variables.
+	// +optional
+	EnvVars []corev1.EnvVar `json:"env,omitempty"`
+
+	// +optional
+	Volumes []corev1.Volume `json:"volumes,omitempty"`
+
+	// +optional
+	VolumeMounts []corev1.VolumeMount `json:"volumeMounts,omitempty"`
 }
 
 type LicenseSpec struct {

hack/logrotate.sh

@@ -0,0 +1,33 @@
+#!/bin/env bash
+
+ROTATE=5
+SIZE=200M
+
+if [ -n "${LOGROTATE_ROTATE}" ]; then
+  ROTATE=${LOGROTATE_ROTATE}
+fi
+
+if [ -n "${LOGROTATE_SIZE}" ]; then
+  SIZE=${LOGROTATE_SIZE}
+fi
+
+nebula="
+/usr/local/nebula/logs/*.log
+/usr/local/nebula/logs/*.impl
+/usr/local/nebula/logs/*.INFO
+/usr/local/nebula/logs/*.WARNING
+/usr/local/nebula/logs/*.ERROR
+{
+        su root root
+        daily
+        rotate ${ROTATE}
+        copytruncate
+        nocompress
+        missingok
+        notifempty
+        create 644 root root
+        size ${SIZE}
+}
+"
+
+echo "${nebula}" >/etc/logrotate.d/nebula

pkg/controller/component/storaged_updater.go

@@ -154,7 +154,7 @@ func (s *storagedUpdater) RestartPod(nc *v1alpha1.NebulaCluster, ordinal int32)
 
 	updatePod, err := s.clientSet.Pod().GetPod(namespace, updatePodName)
 	if err != nil {
-		klog.Errorf("get pod failed: %v", namespace, updatePodName, err)
+		klog.Errorf("get pod [%s/%s] failed: %v", namespace, updatePodName, err)
 		return err
 	}
 	_, ok := updatePod.Annotations[TransLeaderBeginTime]
@@ -397,6 +397,7 @@ func (s *storagedUpdater) updateRunningPhase(mc nebula.MetaInterface, nc *v1alph
 		return nil
 	}
 
+	// TODO the invoking maybe repeat times
 	for _, space := range spaces {
 		if err := mc.BalanceLeader(*space.Id.SpaceID); err != nil {
 			return err