Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: fix get null podCIDR and serviceCIDR #4476

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ unittest-tests: check_test_label
@echo "run unittest-tests"
$(QUIET) $(ROOT_DIR)/tools/scripts/ginkgo.sh \
--cover --coverprofile=./coverage.out --covermode set \
--json-report unittestreport.json \
--json-report unittestreport.json --label-filter $(E2E_GINKGO_UTLABELS) \
-randomize-suites -randomize-all --keep-going --timeout=1h -p \
-vv -r $(ROOT_DIR)/pkg $(ROOT_DIR)/cmd
$(QUIET) go tool cover -html=./coverage.out -o coverage-all.html
Expand Down
69 changes: 46 additions & 23 deletions pkg/coordinatormanager/coordinator_informer.go
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@
networkingv1alpha1 "k8s.io/api/networking/v1alpha1"

"github.com/cilium/cilium/pkg/ipam/option"
v2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
"github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
"github.com/cilium/cilium/pkg/k8s/client/clientset/versioned"
cilium_externalversions "github.com/cilium/cilium/pkg/k8s/client/informers/externalversions"
ciliumLister "github.com/cilium/cilium/pkg/k8s/client/listers/cilium.io/v2alpha1"
Expand Down Expand Up @@ -462,15 +462,30 @@

var cm corev1.ConfigMap
var k8sPodCIDR, k8sServiceCIDR []string
if err := cc.APIReader.Get(ctx, types.NamespacedName{Namespace: metav1.NamespaceSystem, Name: "kubeadm-config"}, &cm); err == nil {
logger.Sugar().Infof("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config")
k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKubeadmConfigMap(&cm)
logger.Sugar().Infof("kubeadm-config configMap k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR)
} else {
logger.Sugar().Warn("kube-system/kubeadm-config is no found, trying to fetch the ClusterCIDR from kube-controller-manager Pod")
var cmPodList corev1.PodList
err = cc.APIReader.List(ctx, &cmPodList, client.MatchingLabels{"component": "kube-controller-manager"})
if err != nil {
// try to get ClusterCIDR from kubeadm-config ConfigMap
err = cc.APIReader.Get(ctx, types.NamespacedName{
Namespace: metav1.NamespaceSystem,
Name: "kubeadm-config",
}, &cm)

if err == nil {
logger.Sugar().Info("Trying to fetch the ClusterCIDR from kube-system/kubeadm-config")
k8sPodCIDR, k8sServiceCIDR, err = ExtractK8sCIDRFromKubeadmConfigMap(&cm)
if err == nil {
// Success to get ClusterCIDR from kubeadm-config
logger.Sugar().Infof("Success get CIDR from kubeadm-config: PodCIDR=%v, ServiceCIDR=%v", k8sPodCIDR, k8sServiceCIDR)
} else {
logger.Sugar().Warnf("Failed get CIDR from kubeadm-config: %v", err)
}

Check warning on line 479 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L465-L479

Added lines #L465 - L479 were not covered by tests
}

// if kubeadm-config ConfigMap not found, try to get ClusterCIDR from kube-controller-manager Pod
if len(k8sPodCIDR) == 0 || len(k8sServiceCIDR) == 0 {
logger.Sugar().Warnf("failed to get kube-system/kubeadm-config: %v, trying to fetch the ClusterCIDR from kube-controller-manager", err)
var podList corev1.PodList
listOptions := client.MatchingLabels{"component": "kube-controller-manager"}

if err := cc.APIReader.List(ctx, &podList, listOptions); err != nil {

Check warning on line 488 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L483-L488

Added lines #L483 - L488 were not covered by tests
logger.Sugar().Errorf("failed to get kube-controller-manager Pod with label \"component: kube-controller-manager\": %v", err)
event.EventRecorder.Eventf(
coordCopy,
Expand All @@ -482,14 +497,15 @@
return coordCopy
}

if len(cmPodList.Items) == 0 {
if len(podList.Items) == 0 {

Check warning on line 500 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L500

Added line #L500 was not covered by tests
errMsg := "No kube-controller-manager pod found, unable to get clusterCIDR"
logger.Error(errMsg)
setStatus2NoReady(logger, errMsg, coordCopy)
return coordCopy
}

k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&cmPodList.Items[0])
k8sPodCIDR, k8sServiceCIDR = ExtractK8sCIDRFromKCMPod(&podList.Items[0])
logger.Sugar().Infof("kube-controller-manager k8sPodCIDR %v, k8sServiceCIDR %v", k8sPodCIDR, k8sServiceCIDR)

Check warning on line 508 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L507-L508

Added lines #L507 - L508 were not covered by tests
}

logger.Sugar().Infof("Detect podCIDRType is: %v, try to update podCIDR", podCidrType)
Expand Down Expand Up @@ -765,20 +781,26 @@
return nil
}

func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string) {
func ExtractK8sCIDRFromKubeadmConfigMap(cm *corev1.ConfigMap) ([]string, []string, error) {
if cm == nil {
return nil, nil, fmt.Errorf("kubeadm configmap is unexpected to nil")
}

Check warning on line 787 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L784-L787

Added lines #L784 - L787 were not covered by tests
var podCIDR, serviceCIDR []string

podReg := regexp.MustCompile(`podSubnet: (.*)`)
serviceReg := regexp.MustCompile(`serviceSubnet: (.*)`)

var podSubnets, serviceSubnets []string
for _, data := range cm.Data {
podSubnets = podReg.FindStringSubmatch(data)
serviceSubnets = serviceReg.FindStringSubmatch(data)
clusterConfig, exists := cm.Data["ClusterConfiguration"]
if !exists {
return podCIDR, serviceCIDR, fmt.Errorf("unable to get kubeadm configmap ClusterConfiguration")

Check warning on line 792 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L790-L792

Added lines #L790 - L792 were not covered by tests
}

if len(podSubnets) != 0 {
podReg := regexp.MustCompile(`podSubnet:\s*(\S+)`)
serviceReg := regexp.MustCompile(`serviceSubnet:\s*(\S+)`)

podSubnets := podReg.FindStringSubmatch(clusterConfig)
serviceSubnets := serviceReg.FindStringSubmatch(clusterConfig)

if len(podSubnets) > 1 {

Check warning on line 801 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L795-L801

Added lines #L795 - L801 were not covered by tests
for _, cidr := range strings.Split(podSubnets[1], ",") {
cidr = strings.TrimSpace(cidr)

Check warning on line 803 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L803

Added line #L803 was not covered by tests
_, _, err := net.ParseCIDR(cidr)
if err != nil {
continue
Expand All @@ -787,8 +809,9 @@
}
}

if len(serviceSubnets) != 0 {
if len(serviceSubnets) > 1 {

Check warning on line 812 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L812

Added line #L812 was not covered by tests
for _, cidr := range strings.Split(serviceSubnets[1], ",") {
cidr = strings.TrimSpace(cidr)

Check warning on line 814 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L814

Added line #L814 was not covered by tests
_, _, err := net.ParseCIDR(cidr)
if err != nil {
continue
Expand All @@ -797,7 +820,7 @@
}
}

return podCIDR, serviceCIDR
return podCIDR, serviceCIDR, nil

Check warning on line 823 in pkg/coordinatormanager/coordinator_informer.go

View check run for this annotation

Codecov / codecov/patch

pkg/coordinatormanager/coordinator_informer.go#L823

Added line #L823 was not covered by tests
}

func ExtractK8sCIDRFromKCMPod(kcm *corev1.Pod) ([]string, []string) {
Expand Down
54 changes: 54 additions & 0 deletions pkg/coordinatormanager/coordinator_informer_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
// Copyright 2022 Authors of spidernet-io
// SPDX-License-Identifier: Apache-2.0

package coordinatormanager

import (
"encoding/json"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
corev1 "k8s.io/api/core/v1"
)

var _ = Describe("Coordinator Manager", Label("coordinatorinformer", "informer_test"), Serial, func() {
DescribeTable("should extract CIDRs correctly",
func(testName, cmStr string, expectedPodCIDR, expectedServiceCIDR []string, expectError bool) {
var cm corev1.ConfigMap
err := json.Unmarshal([]byte(cmStr), &cm)
Expect(err).NotTo(HaveOccurred(), "Failed to unmarshal configMap: %v\n", err)

podCIDR, serviceCIDR, err := ExtractK8sCIDRFromKubeadmConfigMap(&cm)

if expectError {
Expect(err).To(HaveOccurred(), "Expected an error but got none")
} else {
Expect(err).NotTo(HaveOccurred(), "Did not expect an error but got one: %v", err)
}

Expect(podCIDR).To(Equal(expectedPodCIDR), "Pod CIDR does not match")
Expect(serviceCIDR).To(Equal(expectedServiceCIDR), "Service CIDR does not match")
},
Entry("ClusterConfiguration",
"ClusterConfiguration",
clusterConfigurationJson,
[]string{"192.168.165.0/24"},
[]string{"245.100.128.0/18"},
false,
),
Entry("No ClusterConfiguration",
"No ClusterConfiguration",
noClusterConfigurationJson,
nil,
nil,
true,
),
Entry("No CIDR",
"No CIDR",
noCIDRJson,
nil,
nil,
false,
),
)

})
61 changes: 61 additions & 0 deletions pkg/coordinatormanager/coordinatormanager_suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// Copyright 2022 Authors of spidernet-io
// SPDX-License-Identifier: Apache-2.0

package coordinatormanager

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var (
clusterConfigurationJson string
noClusterConfigurationJson string
noCIDRJson string
)

func TestCoordinatorManager(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "CoordinatorManager Suite")
}

var _ = BeforeSuite(func() {
clusterConfigurationJson = `
{
"apiVersion": "v1",
"data": {
"ClusterConfiguration": "networking:\n dnsDomain: cluster.local\n podSubnet: 192.168.165.0/24\n serviceSubnet: 245.100.128.0/18"
},
"kind": "ConfigMap",
"metadata": {
"name": "kubeadm-config",
"namespace": "kube-system"
}
}`
noClusterConfigurationJson = `
{
"apiVersion": "v1",
"data": {
"ClusterStatus": "apiEndpoints:\n anolios79:\n advertiseAddress: 192.168.165.128\n bindPort: 6443\napiVersion: kubeadm.k8s.io/v1beta2\nkind: ClusterStatus\n"
},
"kind": "ConfigMap",
"metadata": {
"name": "kubeadm-config",
"namespace": "kube-system"
}
}`
noCIDRJson = `
{
"apiVersion": "v1",
"data": {
"ClusterConfiguration": "clusterName: spider\ncontrolPlaneEndpoint: spider-control-plane:6443\ncontrollerManager:\n"
},
"kind": "ConfigMap",
"metadata": {
"name": "kubeadm-config",
"namespace": "kube-system"
}
}`
})
2 changes: 2 additions & 0 deletions test/Makefile.defs
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ E2E_GINKGO_LABELS ?=

GINKGO_OPTION ?=

E2E_GINKGO_UTLABELS ?=

E2E_TIMEOUT ?= 60m

E2E_GINKGO_PROCS ?= 4
Expand Down
3 changes: 2 additions & 1 deletion test/e2e/spidercoordinator/spidercoordinator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -431,7 +431,8 @@ var _ = Describe("SpiderCoordinator", Label("spidercoordinator", "overlay"), Ser
It("Prioritize getting ClusterCIDR from kubeadm-config", func() {
GinkgoWriter.Printf("podCIDR and serviceCIDR from spidercoordinator: %v,%v\n", spc.Status.OverlayPodCIDR, spc.Status.ServiceCIDR)

podCIDR, serviceCIDr := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm)
podCIDR, serviceCIDr, err := coordinatormanager.ExtractK8sCIDRFromKubeadmConfigMap(cm)
Expect(err).NotTo(HaveOccurred(), "Failed to extract k8s CIDR from Kubeadm configMap, error is %v", err)
GinkgoWriter.Printf("podCIDR and serviceCIDR from kubeadm-config : %v,%v\n", podCIDR, serviceCIDr)

Eventually(func() bool {
Expand Down
Loading