diff --git a/Makefile b/Makefile index d9cca006e6..1d9df8e7d5 100644 --- a/Makefile +++ b/Makefile @@ -279,7 +279,7 @@ unittest-tests: check_test_label @echo "run unittest-tests" $(QUIET) $(ROOT_DIR)/tools/scripts/ginkgo.sh \ --cover --coverprofile=./coverage.out --covermode set \ - --json-report unittestreport.json \ + --json-report unittestreport.json --label-filter $(E2E_GINKGO_UTLABELS) \ -randomize-suites -randomize-all --keep-going --timeout=1h -p \ -vv -r $(ROOT_DIR)/pkg $(ROOT_DIR)/cmd $(QUIET) go tool cover -html=./coverage.out -o coverage-all.html diff --git a/pkg/coordinatormanager/coordinator_informer.go b/pkg/coordinatormanager/coordinator_informer.go index 0f6da33d98..6e0cb05ea6 100644 --- a/pkg/coordinatormanager/coordinator_informer.go +++ b/pkg/coordinatormanager/coordinator_informer.go @@ -14,7 +14,7 @@ import ( "time" "github.com/cilium/cilium/pkg/ipam/option" - v2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" + "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1" "github.com/cilium/cilium/pkg/k8s/client/clientset/versioned" cilium_externalversions "github.com/cilium/cilium/pkg/k8s/client/informers/externalversions" ciliumLister "github.com/cilium/cilium/pkg/k8s/client/listers/cilium.io/v2alpha1" @@ -465,15 +465,30 @@ func (cc *CoordinatorController) updatePodAndServerCIDR(ctx context.Context, log var cm corev1.ConfigMap var k8sPodCIDR, k8sServiceCIDR []string - if err := cc.APIReader.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceSystem, Name: "kubeadm-config"}, &cm); err == nil { - logger.Sugar().Infof("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config") - k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKubeadmConfigMap(&cm) - logger.Sugar().Infof("kubeadm-config configMap k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR) - } else { + // try to get ClusterCIDR from kubeadm-config ConfigMap + err = cc.APIReader.Get(ctx, types.NamespacedName{ + Namespace: metav1.NamespaceSystem, + Name: "kubeadm-config", + }, &cm) + + if err == nil { + logger.Sugar().Info("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config") + k8sPodCIDR, k8sServiceCIDR, err = ExtractK8sCIDRFromKubeadmConfigMap(&cm) + if err == nil { + // Success to get ClusterCIDR from kubeadm-config + logger.Sugar().Infof("Success get CIDR from kubeadm-config: PodCIDR=%v, ServiceCIDR=%v", k8sPodCIDR, k8sServiceCIDR) + } else { + logger.Sugar().Warnf("Failed get CIDR from kubeadm-config: %v", err) + } + } + + // if kubeadm-config ConfigMap not found, try to get ClusterCIDR from kube-controller-manager Pod + if len(k8sPodCIDR) == 0 || len(k8sServiceCIDR) == 0 { logger.Sugar().Warnf("failed to get kube-system/kubeadm-config: %v, trying to fetch the ClusterCIDR from kube-controller-manager", err) - var cmPodList corev1.PodList - err = cc.APIReader.List(ctx, &cmPodList, client.MatchingLabels{"component": "kube-controller-manager"}) - if err != nil { + var podList corev1.PodList + listOptions := client.MatchingLabels{"component": "kube-controller-manager"} + + if err := cc.APIReader.List(ctx, &podList, listOptions); err != nil { logger.Sugar().Errorf("failed to get kube-controller-manager Pod with label \"component: kube-controller-manager\": %v", err) event.EventRecorder.Eventf( coordCopy, @@ -485,14 +500,14 @@ func (cc *CoordinatorController) updatePodAndServerCIDR(ctx context.Context, log return coordCopy } - if len(cmPodList.Items) == 0 { + if len(podList.Items) == 0 { errMsg := "No kube-controller-manager pod found, unable to get clusterCIDR" logger.Error(errMsg) setStatus2NoReady(logger, errMsg, coordCopy) return coordCopy } - k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&cmPodList.Items[0]) + k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&podList.Items[0]) logger.Sugar().Infof("kube-controller-manager k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR) } @@ -757,20 +772,26 @@ func (cc *CoordinatorController) updateServiceCIDR(logger *zap.Logger, coordCopy return nil } -func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string) { +func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string, error) { + if cm == nil { + return nil, nil, fmt.Errorf("kubeadm configmap is unexpected to nil") + } var podCIDR, serviceCIDR []string - podReg := regexp.MustCompile(`podSubnet: (.*)`) - serviceReg := regexp.MustCompile(`serviceSubnet: (.*)`) - - var podSubnets, serviceSubnets []string - for _, data := range cm.Data { - podSubnets = podReg.FindStringSubmatch(data) - serviceSubnets = serviceReg.FindStringSubmatch(data) + clusterConfig, exists := cm.Data["ClusterConfiguration"] + if !exists { + return podCIDR, serviceCIDR, fmt.Errorf("unable to get kubeadm configmap ClusterConfiguration") } - if len(podSubnets) != 0 { + podReg := regexp.MustCompile(`podSubnet:\s*(\S+)`) + serviceReg := regexp.MustCompile(`serviceSubnet:\s*(\S+)`) + + podSubnets := podReg.FindStringSubmatch(clusterConfig) + serviceSubnets := serviceReg.FindStringSubmatch(clusterConfig) + + if len(podSubnets) > 1 { for _, cidr := range strings.Split(podSubnets[1], ",") { + cidr = strings.TrimSpace(cidr) _, _, err := net.ParseCIDR(cidr) if err != nil { continue @@ -779,8 +800,9 @@ func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []strin } } - if len(serviceSubnets) != 0 { + if len(serviceSubnets) > 1 { for _, cidr := range strings.Split(serviceSubnets[1], ",") { + cidr = strings.TrimSpace(cidr) _, _, err := net.ParseCIDR(cidr) if err != nil { continue @@ -789,7 +811,7 @@ func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []strin } } - return podCIDR, serviceCIDR + return podCIDR, serviceCIDR, nil } func ExtractK8sCIDRFromKCMPod(kcm *corev1.Pod) ([]string, []string) { diff --git a/pkg/coordinatormanager/coordinator_informer_test.go b/pkg/coordinatormanager/coordinator_informer_test.go new file mode 100644 index 0000000000..5dac9062a2 --- /dev/null +++ b/pkg/coordinatormanager/coordinator_informer_test.go @@ -0,0 +1,54 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package coordinatormanager + +import ( + "encoding/json" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" +) + +var _ = Describe("Coordinator Manager", Label("coordinatorinformer", "informer_test"), Serial, func() { + DescribeTable("should extract CIDRs correctly", + func(testName, cmStr string, expectedPodCIDR, expectedServiceCIDR []string, expectError bool) { + var cm corev1.ConfigMap + err := json.Unmarshal([]byte(cmStr), &cm) + Expect(err).NotTo(HaveOccurred(), "Failed to unmarshal configMap: %v\n", err) + + podCIDR, serviceCIDR, err := ExtractK8sCIDRFromKubeadmConfigMap(&cm) + + if expectError { + Expect(err).To(HaveOccurred(), "Expected an error but got none") + } else { + Expect(err).NotTo(HaveOccurred(), "Did not expect an error but got one: %v", err) + } + + Expect(podCIDR).To(Equal(expectedPodCIDR), "Pod CIDR does not match") + Expect(serviceCIDR).To(Equal(expectedServiceCIDR), "Service CIDR does not match") + }, + Entry("ClusterConfiguration", + "ClusterConfiguration", + clusterConfigurationJson, + []string{"192.168.165.0/24"}, + []string{"245.100.128.0/18"}, + false, + ), + Entry("No ClusterConfiguration", + "No ClusterConfiguration", + noClusterConfigurationJson, + nil, + nil, + true, + ), + Entry("No CIDR", + "No CIDR", + noCIDRJson, + nil, + nil, + false, + ), + ) + +}) diff --git a/pkg/coordinatormanager/coordinatormanager_suite_test.go b/pkg/coordinatormanager/coordinatormanager_suite_test.go new file mode 100644 index 0000000000..ff9687ab24 --- /dev/null +++ b/pkg/coordinatormanager/coordinatormanager_suite_test.go @@ -0,0 +1,61 @@ +// Copyright 2022 Authors of spidernet-io +// SPDX-License-Identifier: Apache-2.0 + +package coordinatormanager + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var ( + clusterConfigurationJson string + noClusterConfigurationJson string + noCIDRJson string +) + +func TestCoordinatorManager(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CoordinatorManager Suite") +} + +var _ = BeforeSuite(func() { + clusterConfigurationJson = ` + { + "apiVersion": "v1", + "data": { + "ClusterConfiguration": "networking:\n dnsDomain: cluster.local\n podSubnet: 192.168.165.0/24\n serviceSubnet: 245.100.128.0/18" + }, + "kind": "ConfigMap", + "metadata": { + "name": "kubeadm-config", + "namespace": "kube-system" + } + }` + noClusterConfigurationJson = ` + { + "apiVersion": "v1", + "data": { + "ClusterStatus": "apiEndpoints:\n anolios79:\n advertiseAddress: 192.168.165.128\n bindPort: 6443\napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterStatus\n" + }, + "kind": "ConfigMap", + "metadata": { + "name": "kubeadm-config", + "namespace": "kube-system" + } + }` + noCIDRJson = ` + { + "apiVersion": "v1", + "data": { + "ClusterConfiguration": "clusterName: spider\ncontrolPlaneEndpoint: spider-control-plane:6443\ncontrollerManager:\n" + }, + "kind": "ConfigMap", + "metadata": { + "name": "kubeadm-config", + "namespace": "kube-system" + } + }` +}) diff --git a/test/Makefile.defs b/test/Makefile.defs index edb436cf86..3692acc415 100644 --- a/test/Makefile.defs +++ b/test/Makefile.defs @@ -76,6 +76,8 @@ E2E_GINKGO_LABELS ?= GINKGO_OPTION ?= +E2E_GINKGO_UTLABELS ?= + E2E_TIMEOUT ?= 60m E2E_GINKGO_PROCS ?= 4 diff --git a/test/e2e/spidercoordinator/spidercoordinator_test.go b/test/e2e/spidercoordinator/spidercoordinator_test.go index a36818310b..a4d27ff3bc 100644 --- a/test/e2e/spidercoordinator/spidercoordinator_test.go +++ b/test/e2e/spidercoordinator/spidercoordinator_test.go @@ -464,7 +464,8 @@ var _ = Describe("SpiderCoordinator", Label("spidercoordinator", "overlay"), Ser It("Prioritize getting ClusterCIDR from kubeadm-config", Label("V00009"), func() { GinkgoWriter.Printf("podCIDR and serviceCIDR from spidercoordinator: %v,%v\n", spc.Status.OverlayPodCIDR, spc.Status.ServiceCIDR) - podCIDR, serviceCIDr := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm) + podCIDR, serviceCIDr, err := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm) + Expect(err).NotTo(HaveOccurred(), "Failed to extract k8s CIDR from Kubeadm configMap, error is %v", err) GinkgoWriter.Printf("podCIDR and serviceCIDR from kubeadm-config : %v,%v\n", podCIDR, serviceCIDr) Eventually(func() bool {