Merge pull request #4366 from 0x0034/fix/useClusterConfigurationGetCIDR

fix: fail to get podCIDR and serviceCIDR

Makefile

@@ -279,7 +279,7 @@ unittest-tests: check_test_label
 	@echo "run unittest-tests"
 	$(QUIET) $(ROOT_DIR)/tools/scripts/ginkgo.sh \
 		--cover --coverprofile=./coverage.out --covermode set \
-		--json-report unittestreport.json \
+		--json-report unittestreport.json --label-filter $(E2E_GINKGO_UTLABELS) \
 		-randomize-suites -randomize-all --keep-going  --timeout=1h  -p \
 		-vv  -r $(ROOT_DIR)/pkg $(ROOT_DIR)/cmd
 	$(QUIET) go tool cover -html=./coverage.out -o coverage-all.html

pkg/coordinatormanager/coordinator_informer.go

@@ -14,7 +14,7 @@ import (
 	"time"
 
 	"github.com/cilium/cilium/pkg/ipam/option"
-	v2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
+	"github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
 	"github.com/cilium/cilium/pkg/k8s/client/clientset/versioned"
 	cilium_externalversions "github.com/cilium/cilium/pkg/k8s/client/informers/externalversions"
 	ciliumLister "github.com/cilium/cilium/pkg/k8s/client/listers/cilium.io/v2alpha1"
@@ -465,15 +465,30 @@ func (cc *CoordinatorController) updatePodAndServerCIDR(ctx context.Context, log
 
 	var cm corev1.ConfigMap
 	var k8sPodCIDR, k8sServiceCIDR []string
-	if err := cc.APIReader.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceSystem, Name: "kubeadm-config"}, &cm); err == nil {
-		logger.Sugar().Infof("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config")
-		k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKubeadmConfigMap(&cm)
-		logger.Sugar().Infof("kubeadm-config configMap k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR)
-	} else {
+	// try to get ClusterCIDR from kubeadm-config ConfigMap
+	err = cc.APIReader.Get(ctx, types.NamespacedName{
+		Namespace: metav1.NamespaceSystem,
+		Name:      "kubeadm-config",
+	}, &cm)
+
+	if err == nil {
+		logger.Sugar().Info("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config")
+		k8sPodCIDR, k8sServiceCIDR, err = ExtractK8sCIDRFromKubeadmConfigMap(&cm)
+		if err == nil {
+			// Success to get ClusterCIDR from kubeadm-config
+			logger.Sugar().Infof("Success get CIDR from kubeadm-config: PodCIDR=%v, ServiceCIDR=%v", k8sPodCIDR, k8sServiceCIDR)
+		} else {
+			logger.Sugar().Warnf("Failed get CIDR from kubeadm-config: %v", err)
+		}
+	}
+
+	// if kubeadm-config ConfigMap not found, try to get ClusterCIDR from kube-controller-manager Pod
+	if len(k8sPodCIDR) == 0 || len(k8sServiceCIDR) == 0 {
 		logger.Sugar().Warnf("failed to get kube-system/kubeadm-config: %v, trying to fetch the ClusterCIDR from kube-controller-manager", err)
-		var cmPodList corev1.PodList
-		err = cc.APIReader.List(ctx, &cmPodList, client.MatchingLabels{"component": "kube-controller-manager"})
-		if err != nil {
+		var podList corev1.PodList
+		listOptions := client.MatchingLabels{"component": "kube-controller-manager"}
+
+		if err := cc.APIReader.List(ctx, &podList, listOptions); err != nil {
 			logger.Sugar().Errorf("failed to get kube-controller-manager Pod with label \"component: kube-controller-manager\": %v", err)
 			event.EventRecorder.Eventf(
 				coordCopy,
@@ -485,14 +500,14 @@ func (cc *CoordinatorController) updatePodAndServerCIDR(ctx context.Context, log
 			return coordCopy
 		}
 
-		if len(cmPodList.Items) == 0 {
+		if len(podList.Items) == 0 {
 			errMsg := "No kube-controller-manager pod found, unable to get clusterCIDR"
 			logger.Error(errMsg)
 			setStatus2NoReady(logger, errMsg, coordCopy)
 			return coordCopy
 		}
 
-		k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&cmPodList.Items[0])
+		k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&podList.Items[0])
 		logger.Sugar().Infof("kube-controller-manager k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR)
 	}
 
@@ -763,20 +778,26 @@ func (cc *CoordinatorController) updateServiceCIDR(logger *zap.Logger, coordCopy
 	return nil
 }
 
-func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string) {
+func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string, error) {
+	if cm == nil {
+		return nil, nil, fmt.Errorf("kubeadm configmap is unexpected to nil")
+	}
 	var podCIDR, serviceCIDR []string
 
-	podReg := regexp.MustCompile(`podSubnet: (.*)`)
-	serviceReg := regexp.MustCompile(`serviceSubnet: (.*)`)
-
-	var podSubnets, serviceSubnets []string
-	for _, data := range cm.Data {
-		podSubnets = podReg.FindStringSubmatch(data)
-		serviceSubnets = serviceReg.FindStringSubmatch(data)
+	clusterConfig, exists := cm.Data["ClusterConfiguration"]
+	if !exists {
+		return podCIDR, serviceCIDR, fmt.Errorf("unable to get kubeadm configmap ClusterConfiguration")
 	}
 
-	if len(podSubnets) != 0 {
+	podReg := regexp.MustCompile(`podSubnet:\s*(\S+)`)
+	serviceReg := regexp.MustCompile(`serviceSubnet:\s*(\S+)`)
+
+	podSubnets := podReg.FindStringSubmatch(clusterConfig)
+	serviceSubnets := serviceReg.FindStringSubmatch(clusterConfig)
+
+	if len(podSubnets) > 1 {
 		for _, cidr := range strings.Split(podSubnets[1], ",") {
+			cidr = strings.TrimSpace(cidr)
 			_, _, err := net.ParseCIDR(cidr)
 			if err != nil {
 				continue
@@ -785,8 +806,9 @@ func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []strin
 		}
 	}
 
-	if len(serviceSubnets) != 0 {
+	if len(serviceSubnets) > 1 {
 		for _, cidr := range strings.Split(serviceSubnets[1], ",") {
+			cidr = strings.TrimSpace(cidr)
 			_, _, err := net.ParseCIDR(cidr)
 			if err != nil {
 				continue
@@ -795,7 +817,7 @@ func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []strin
 		}
 	}
 
-	return podCIDR, serviceCIDR
+	return podCIDR, serviceCIDR, nil
 }
 
 func ExtractK8sCIDRFromKCMPod(kcm *corev1.Pod) ([]string, []string) {

pkg/coordinatormanager/coordinator_informer_test.go

@@ -0,0 +1,54 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package coordinatormanager
+
+import (
+	"encoding/json"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
+)
+
+var _ = Describe("Coordinator Manager", Label("coordinatorinformer", "informer_test"), Serial, func() {
+	DescribeTable("should extract CIDRs correctly",
+		func(testName, cmStr string, expectedPodCIDR, expectedServiceCIDR []string, expectError bool) {
+			var cm corev1.ConfigMap
+			err := json.Unmarshal([]byte(cmStr), &cm)
+			Expect(err).NotTo(HaveOccurred(), "Failed to unmarshal configMap: %v\n", err)
+
+			podCIDR, serviceCIDR, err := ExtractK8sCIDRFromKubeadmConfigMap(&cm)
+
+			if expectError {
+				Expect(err).To(HaveOccurred(), "Expected an error but got none")
+			} else {
+				Expect(err).NotTo(HaveOccurred(), "Did not expect an error but got one: %v", err)
+			}
+
+			Expect(podCIDR).To(Equal(expectedPodCIDR), "Pod CIDR does not match")
+			Expect(serviceCIDR).To(Equal(expectedServiceCIDR), "Service CIDR does not match")
+		},
+		Entry("ClusterConfiguration",
+			"ClusterConfiguration",
+			clusterConfigurationJson,
+			[]string{"192.168.165.0/24"},
+			[]string{"245.100.128.0/18"},
+			false,
+		),
+		Entry("No ClusterConfiguration",
+			"No ClusterConfiguration",
+			noClusterConfigurationJson,
+			nil,
+			nil,
+			true,
+		),
+		Entry("No CIDR",
+			"No CIDR",
+			noCIDRJson,
+			nil,
+			nil,
+			false,
+		),
+	)
+
+})

pkg/coordinatormanager/coordinatormanager_suite_test.go

@@ -0,0 +1,61 @@
+// Copyright 2022 Authors of spidernet-io
+// SPDX-License-Identifier: Apache-2.0
+
+package coordinatormanager
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var (
+	clusterConfigurationJson   string
+	noClusterConfigurationJson string
+	noCIDRJson                 string
+)
+
+func TestCoordinatorManager(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CoordinatorManager Suite")
+}
+
+var _ = BeforeSuite(func() {
+	clusterConfigurationJson = `
+	{
+		"apiVersion": "v1",
+		"data": {
+			"ClusterConfiguration": "networking:\n  dnsDomain: cluster.local\n  podSubnet: 192.168.165.0/24\n  serviceSubnet: 245.100.128.0/18"
+		},
+		"kind": "ConfigMap",
+		"metadata": {
+			"name": "kubeadm-config",
+			"namespace": "kube-system"
+		}
+	}`
+	noClusterConfigurationJson = `
+	{
+		"apiVersion": "v1",
+		"data": {
+			"ClusterStatus": "apiEndpoints:\n  anolios79:\n    advertiseAddress: 192.168.165.128\n    bindPort: 6443\napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterStatus\n"
+		},
+		"kind": "ConfigMap",
+		"metadata": {
+			"name": "kubeadm-config",
+			"namespace": "kube-system"
+		}
+	}`
+	noCIDRJson = `
+	{
+		"apiVersion": "v1",
+		"data": {
+			"ClusterConfiguration": "clusterName: spider\ncontrolPlaneEndpoint: spider-control-plane:6443\ncontrollerManager:\n"
+		},
+		"kind": "ConfigMap",
+		"metadata": {
+			"name": "kubeadm-config",
+			"namespace": "kube-system"
+		}
+	}`
+})

test/Makefile.defs

@@ -76,6 +76,8 @@ E2E_GINKGO_LABELS ?=
 
 GINKGO_OPTION ?=
 
+E2E_GINKGO_UTLABELS ?= 
+
 E2E_TIMEOUT ?= 60m
 
 E2E_GINKGO_PROCS ?= 4

test/e2e/spidercoordinator/spidercoordinator_test.go

@@ -464,7 +464,8 @@ var _ = Describe("SpiderCoordinator", Label("spidercoordinator", "overlay"), Ser
 		It("Prioritize getting ClusterCIDR from kubeadm-config", Label("V00009"), func() {
 			GinkgoWriter.Printf("podCIDR and serviceCIDR from spidercoordinator: %v,%v\n", spc.Status.OverlayPodCIDR, spc.Status.ServiceCIDR)
 
-			podCIDR, serviceCIDr := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm)
+			podCIDR, serviceCIDr, err := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm)
+			Expect(err).NotTo(HaveOccurred(), "Failed to extract k8s CIDR from Kubeadm configMap,  error is %v", err)
 			GinkgoWriter.Printf("podCIDR and serviceCIDR from kubeadm-config : %v,%v\n", podCIDR, serviceCIDr)
 
 			Eventually(func() bool {