spidercoordinator: fix auto fetch podCIDRType

charts/spiderpool/README.md

@@ -123,6 +123,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
 | `global.cniConfHostPath`        | the host path of the cni config directory                                   | `/etc/cni/net.d`                     |
 | `global.ipamUNIXSocketHostPath` | the host path of unix domain socket for ipam plugin                         | `/var/run/spidernet/spiderpool.sock` |
 | `global.configName`             | the configmap name                                                          | `spiderpool-conf`                    |
+| `global.ciliumConfigMap`        | the cilium's configMap, default is kube-system/cilium-config                | `kube-system/cilium-config`          |
 
 ### ipam parameters
 

charts/spiderpool/templates/deployment.yaml

@@ -171,6 +171,8 @@ spec:
           value: {{ .Values.multus.enableMultusConfig | quote }}
         - name: SPIDERPOOL_CNI_CONFIG_DIR
           value: {{ .Values.global.cniConfHostPath | quote }}
+        - name: SPIDERPOOL_CILIUM_CONFIGMAP_NAMESPACE_NAME
+          value: {{ .Values.global.ciliumConfigMap | quote }}
         - name: SPIDERPOOL_POD_NAME
           valueFrom:
             fieldRef:

charts/spiderpool/values.yaml

@@ -35,6 +35,9 @@ global:
   ## @param global.configName the configmap name
   configName: "spiderpool-conf"
 
+  ## @param global.ciliumConfigMap the cilium's configMap, default is kube-system/cilium-config
+  ciliumConfigMap: kube-system/cilium-config
+
 ## @section ipam parameters
 ##
 ipam:

cmd/spiderpool-controller/cmd/config.go

@@ -92,6 +92,7 @@ var envInfo = []envConf{
 
 	{"SPIDERPOOL_MULTUS_CONFIG_ENABLED", "false", false, nil, &controllerContext.Cfg.EnableMultusConfig, nil},
 	{"SPIDERPOOL_MULTUS_CONFIG_INFORMER_RESYNC_PERIOD", "60", false, nil, nil, &controllerContext.Cfg.MultusConfigInformerResyncPeriod},
+	{"SPIDERPOOL_CILIUM_CONFIGMAP_NAMESPACE_NAME", "kube-system/cilium-config", false, &controllerContext.Cfg.CiliumConfigName, nil, nil},
 
 	{"SPIDERPOOL_IPPOOL_INFORMER_RESYNC_PERIOD", "300", false, nil, nil, &controllerContext.Cfg.IPPoolInformerResyncPeriod},
 	{"SPIDERPOOL_IPPOOL_INFORMER_WORKERS", "3", true, nil, nil, &controllerContext.Cfg.IPPoolInformerWorkers},
@@ -123,6 +124,8 @@ type Config struct {
 	GopsListenPort    string
 	PyroscopeAddress  string
 	DefaultCniConfDir string
+	// CiliumConfigName is formatted by namespace and name,default is kube-system/cilium-config
+	CiliumConfigName string
 
 	ControllerPodNamespace string
 	ControllerPodName      string

cmd/spiderpool-controller/cmd/daemon.go

@@ -452,6 +452,7 @@ func setupInformers() {
 		LeaderRetryElectGap: time.Duration(controllerContext.Cfg.LeaseRetryGap) * time.Second,
 		ResyncPeriod:        time.Duration(controllerContext.Cfg.CoordinatorInformerResyncPeriod) * time.Second,
 		DefaultCniConfDir:   controllerContext.Cfg.DefaultCniConfDir,
+		CiliumConfigMap:     controllerContext.Cfg.CiliumConfigName,
 	}).SetupInformer(controllerContext.InnerCtx, crdClient, k8sClient, controllerContext.Leader); err != nil {
 		logger.Fatal(err.Error())
 	}

docs/reference/spiderpool-controller.md

@@ -14,17 +14,22 @@ Run the spiderpool controller daemon.
 
 ### ENV
 
-| env                                      | default | description                                                                        |
-|------------------------------------------|---------|------------------------------------------------------------------------------------|
-| SPIDERPOOL_LOG_LEVEL                     | info    | Log level, optional values are "debug", "info", "warn", "error", "fatal", "panic". |
-| SPIDERPOOL_ENABLED_METRIC                | false   | Enable/disable metrics.                                                            |
-| SPIDERPOOL_HEALTH_PORT                   | 5720    | Spiderpool-controller backend HTTP server port.                                    |
-| SPIDERPOOL_METRIC_HTTP_PORT              | 5721    | Metric HTTP server port.                                                           |
-| SPIDERPOOL_WEBHOOK_PORT                  | 5722    | Webhook HTTP server port.                                                          |
-| SPIDERPOOL_CLI_PORT                      | 5723    | Spiderpool-CLI HTTP server port.                                                   |
-| SPIDERPOOL_GOPS_LISTEN_PORT              | 5724    | Port that gops is listening on. Disabled if empty.                                 |
-| SPIDERPOOL_GC_IP_ENABLED                 | true    | Enable/disable IP GC.                                                              |
-| SPIDERPOOL_GC_TERMINATING_POD_IP_ENABLED | true    | Enable/disable IP GC for Terminating pod.                                          |
+| env                                         | default | description                                                                        |
+|---------------------------------------------|---------|------------------------------------------------------------------------------------|
+| SPIDERPOOL_LOG_LEVEL                        | info    | Log level, optional values are "debug", "info", "warn", "error", "fatal", "panic". |
+| SPIDERPOOL_ENABLED_METRIC                   | false   | Enable/disable metrics.                                                            |
+| SPIDERPOOL_ENABLED_DEBUG_METRIC             | false   | Enable spiderpool agent to collect debug level metrics.                            |
+| SPIDERPOOL_METRIC_HTTP_PORT                 | false   | The metrics port of spiderpool agent.                                              |
+| SPIDERPOOL_GOPS_LISTEN_PORT                 | 5724    | The gops port of spiderpool Controller.                                            |
+| SPIDERPOOL_WEBHOOK_PORT                     | 5722    | Webhook HTTP server port.                                                          |
+| SPIDERPOOL_HEALTH_PORT                      | 5720    | The http Port for spiderpoolController, for health checking and http service.      |
+| SPIDERPOOL_GC_IP_ENABLED                    | true    | Enable/disable IP GC.                                                              |
+| SPIDERPOOL_GC_TERMINATING_POD_IP_ENABLED    | true    | Enable/disable IP GC for Terminating pod.                                          |
+| SPIDERPOOL_GC_ADDITIONAL_GRACE_DELAY        | true    | The gc delay seconds after the pod times out of deleting graceful period.          |
+| SPIDERPOOL_GC_DEFAULT_INTERVAL_DURATION     | true    | The gc all interval duration.                                                      |
+| SPIDERPOOL_MULTUS_CONFIG_ENABLED            | true    | Enable/disable SpiderMultusConfig.                                                 |
+| SPIDERPOOL_CNI_CONFIG_DIR                   | true    | The host path of the cni config directory.                                         |
+| SPIDERPOOL_CILIUM_CONFIGMAP_NAMESPACE_NAME  | true    | The cilium's configMap, default is kube-system/cilium-config.                      |
 
 
 ## spiderpool-controller shutdown

pkg/coordinatormanager/coordinator_informer.go

@@ -88,6 +88,7 @@ type CoordinatorController struct {
 	ResyncPeriod        time.Duration
 
 	DefaultCniConfDir string
+	CiliumConfigMap   string
 }
 
 func (cc *CoordinatorController) SetupInformer(
@@ -312,8 +313,16 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 	if err != nil {
 		return client.IgnoreNotFound(err)
 	}
-
 	coordCopy := coord.DeepCopy()
+
+	defer func() {
+		if !reflect.DeepEqual(coordCopy.Status, coord.Status) {
+			if err := cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord)); err != nil {
+				logger.Sugar().Errorf("failed to patch spidercoordinator phase: %v", err.Error())
+			}
+		}
+	}()
+
 	var kubeadmConfig *corev1.ConfigMap
 	if kubeadmConfig, err = cc.ConfigmapLister.ConfigMaps(metav1.NamespaceSystem).Get(kubeadmConfigMap); err != nil {
 		event.EventRecorder.Eventf(
@@ -322,16 +331,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 			"ClusterNotReady",
 			err.Error(),
 		)
-
-		if coordCopy.Status.Phase != NotReady {
-			coordCopy.Status.Phase = NotReady
-			coordCopy.Status.OverlayPodCIDR = []string{}
-			coordCopy.Status.ServiceCIDR = []string{}
-			if err := cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord)); err != nil {
-				return err
-			}
-		}
-
+		setStatus2NoReady(logger, coordCopy)
 		return err
 	}
 
@@ -344,25 +344,32 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 			msg,
 		)
 
-		if coordCopy.Status.Phase != NotReady {
-			coordCopy.Status.Phase = NotReady
-			coordCopy.Status.OverlayPodCIDR = []string{}
-			coordCopy.Status.ServiceCIDR = []string{}
-			if err := cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord)); err != nil {
-				return err
-			}
-		}
-
+		setStatus2NoReady(logger, coordCopy)
 		return errors.New(msg)
 	}
 
 	k8sPodCIDR, k8sServiceCIDR := extractK8sCIDR(kubeadmConfig)
-	switch *coord.Spec.PodCIDRType {
-	case auto:
-		podCidrType := fetchType(cc.DefaultCniConfDir)
+	if *coord.Spec.PodCIDRType == auto {
+		podCidrType, err := fetchType(cc.DefaultCniConfDir)
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				event.EventRecorder.Eventf(
+					coordCopy,
+					corev1.EventTypeWarning,
+					"FoundCNIErr",
+					"failed to found default CNI",
+				)
+			}
+
+			setStatus2NoReady(logger, coordCopy)
+			logger.Sugar().Errorf("failed to auto fetch podCIDRType: %v", err)
+			return err
+		}
 		logger.Sugar().Infof("spidercoordinator change podCIDRType from auto to %v", podCidrType)
 		coord.Spec.PodCIDRType = &podCidrType
-		fallthrough
+	}
+
+	switch *coord.Spec.PodCIDRType {
 	case cluster:
 		if cc.caliCtrlCanncel != nil {
 			cc.caliCtrlCanncel()
@@ -373,7 +380,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 	case calico:
 		var crd apiextensionsv1.CustomResourceDefinition
 		err := cc.APIReader.Get(ctx, types.NamespacedName{Name: calicoIPPoolCRDName}, &crd)
-		if nil != err {
+		if err != nil {
 			if apierrors.IsNotFound(err) {
 				event.EventRecorder.Eventf(
 					coordCopy,
@@ -382,14 +389,8 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 					"Calico needs to be installed first",
 				)
 			}
-			if coordCopy.Status.Phase != NotReady {
-				coordCopy.Status.Phase = NotReady
-				coordCopy.Status.OverlayPodCIDR = []string{}
-				coordCopy.Status.ServiceCIDR = []string{}
-				if err := cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord)); err != nil {
-					return err
-				}
-			}
+
+			setStatus2NoReady(logger, coordCopy)
 			return err
 		}
 
@@ -399,6 +400,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 
 		controller, err := NewCalicoIPPoolController(cc.Manager, coordinatorName)
 		if err != nil {
+			setStatus2NoReady(logger, coordCopy)
 			return err
 		}
 
@@ -421,7 +423,14 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 			cc.caliCtrlCanncel = nil
 		}
 
-		ccm, err := cc.ConfigmapLister.ConfigMaps(metav1.NamespaceSystem).Get(ciliumConfig)
+		ns, name := parseNsAndName(cc.CiliumConfigMap)
+		if ns == "" && name == "" {
+			logger.Sugar().Errorf("invalid ENV %s: %s, unable parse cilium-config configMap", "SPIDERPOOL_CILIUM_CONFIGMAP_NAMESPACE_NAME", cc.CiliumConfigMap)
+			setStatus2NoReady(logger, coordCopy)
+			return fmt.Errorf("invalid ENV %s: %s, unable parse cilium-config configMap", "SPIDERPOOL_CILIUM_CONFIGMAP_NAMESPACE_NAME", cc.CiliumConfigMap)
+		}
+
+		ccm, err := cc.ConfigmapLister.ConfigMaps(ns).Get(name)
 		if err != nil {
 			if apierrors.IsNotFound(err) {
 				event.EventRecorder.Eventf(
@@ -431,14 +440,8 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 					"Cilium needs to be installed first",
 				)
 			}
-			if coordCopy.Status.Phase != NotReady {
-				coordCopy.Status.Phase = NotReady
-				coordCopy.Status.OverlayPodCIDR = []string{}
-				coordCopy.Status.ServiceCIDR = []string{}
-				if err := cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord)); err != nil {
-					return err
-				}
-			}
+
+			setStatus2NoReady(logger, coordCopy)
 			return err
 		}
 
@@ -448,31 +451,32 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 			var podCIDR []string
 			v4, ok := ccm.Data["cluster-pool-ipv4-cidr"]
 			if ok {
-				parts := strings.Split(v4, " ")
-				for _, cidr := range parts {
-					_, _, err := net.ParseCIDR(cidr)
-					if err != nil {
-						continue
-					}
-					podCIDR = append(podCIDR, cidr)
+				v4 = strings.TrimSpace(v4)
+				_, _, err := net.ParseCIDR(v4)
+				if err != nil {
+					logger.Sugar().Errorf("unable to initialize cluster-pool-ipv4-cidr, invalid CIDR address: %v", v4)
+					setStatus2NoReady(logger, coordCopy)
+					return err
 				}
+				podCIDR = append(podCIDR, v4)
 			}
 			v6, ok := ccm.Data["cluster-pool-ipv6-cidr"]
 			if ok {
-				parts := strings.Split(v6, " ")
-				for _, cidr := range parts {
-					_, _, err := net.ParseCIDR(cidr)
-					if err != nil {
-						continue
-					}
-					podCIDR = append(podCIDR, cidr)
+				v4 = strings.TrimSpace(v6)
+				_, _, err := net.ParseCIDR(v6)
+				if err != nil {
+					logger.Sugar().Errorf("unable to initialize cluster-pool-ipv6-cidr, invalid CIDR address: %v", v4)
+					setStatus2NoReady(logger, coordCopy)
+					return err
 				}
+				podCIDR = append(podCIDR, v6)
 			}
 			coordCopy.Status.OverlayPodCIDR = podCIDR
 		case option.IPAMMultiPool:
 			// start controller
 			controller, err := NewCiliumIPPoolController(cc.Manager, coordinatorName)
 			if err != nil {
+				setStatus2NoReady(logger, coordCopy)
 				return err
 			}
 
@@ -493,6 +497,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 			coordCopy.Status.Phase = Synced
 		default:
 			logger.Sugar().Infof("unsupported CIlium IPAM mode: %v", ipam)
+			setStatus2NoReady(logger, coordCopy)
 			return fmt.Errorf("unsupported CIlium IPAM mode: %v", ipam)
 		}
 	case none:
@@ -501,11 +506,7 @@ func (cc *CoordinatorController) syncHandler(ctx context.Context, coordinatorNam
 	}
 
 	coordCopy.Status.ServiceCIDR = k8sServiceCIDR
-	if reflect.DeepEqual(coordCopy.Status, coord.Status) {
-		return nil
-	}
-
-	return cc.Client.Status().Patch(ctx, coordCopy, client.MergeFrom(coord))
+	return err
 }
 
 func extractK8sCIDR(kubeadmConfig *corev1.ConfigMap) ([]string, []string) {
@@ -543,18 +544,33 @@ func extractK8sCIDR(kubeadmConfig *corev1.ConfigMap) ([]string, []string) {
 	return podCIDR, serviceCIDR
 }
 
-func fetchType(cniDir string) string {
+func fetchType(cniDir string) (string, error) {
 	defaultCniName, err := utils.GetDefaultCniName(cniDir)
 	if err != nil {
-		return cluster
+		return "", err
 	}
 
 	switch defaultCniName {
 	case "calico", "k8s-pod-network":
-		return calico
+		return calico, nil
 	case cilium:
-		return cilium
+		return cilium, nil
 	default:
-		return cluster
+		return cluster, nil
+	}
+}
+
+func setStatus2NoReady(logger *zap.Logger, copy *spiderpoolv2beta1.SpiderCoordinator) {
+	logger.Info("set spidercoordinator to NotReady")
+	copy.Status.Phase = NotReady
+	copy.Status.OverlayPodCIDR = []string{}
+}
+
+func parseNsAndName(s string) (ns, name string) {
+	s = strings.TrimSpace(s)
+	r := strings.Split(s, "/")
+	if len(r) != 2 {
+		return "", ""
 	}
+	return r[0], r[1]
 }

pkg/utils/utils.go

@@ -25,7 +25,10 @@ func GetDefaultCniName(cniDir string) (string, error) {
 		return "", err
 	}
 
-	return fetchCniNameFromPath(cniPath)
+	if cniPath != "" {
+		return fetchCniNameFromPath(cniPath)
+	}
+	return "", fmt.Errorf("no cni files found in %s", cniDir)
 }
 
 func findDefaultCNIConf(cniDir string) (string, error) {