From b14e6108465a9743357fef03d9b2440c4347a6a3 Mon Sep 17 00:00:00 2001 From: Cyclinder Kuo Date: Mon, 23 Dec 2024 18:33:35 +0800 Subject: [PATCH] Fix controller panic in cilium ipam is multi-pool Signed-off-by: Cyclinder Kuo --- .../coordinator_informer.go | 10 +- test/Makefile | 5 +- test/doc/spidercoodinator.md | 1 + test/e2e/common/spiderpool.go | 13 +- .../spidercoordinator_test.go | 93 +++- test/scripts/install-default-cni.sh | 412 ++++++++++-------- 6 files changed, 353 insertions(+), 181 deletions(-) diff --git a/pkg/coordinatormanager/coordinator_informer.go b/pkg/coordinatormanager/coordinator_informer.go index 0f6da33d98..0c636a1b99 100644 --- a/pkg/coordinatormanager/coordinator_informer.go +++ b/pkg/coordinatormanager/coordinator_informer.go @@ -704,18 +704,24 @@ func (cc *CoordinatorController) fetchCiliumIPPools(coordinator *spiderpoolv2bet podCIDR := make([]string, 0, len(ipPoolList)) for _, p := range ipPoolList { - if p.DeletionTimestamp == nil { + if p.DeletionTimestamp != nil { + continue + } + + if p.Spec.IPv4 != nil { for _, cidr := range p.Spec.IPv4.CIDRs { podCIDR = append(podCIDR, string(cidr)) } + } + if p.Spec.IPv6 != nil { for _, cidr := range p.Spec.IPv6.CIDRs { podCIDR = append(podCIDR, string(cidr)) } } } - InformerLogger.Sugar().Debugf("Cilium IPPools CIDR: %v", ipPoolList) + InformerLogger.Sugar().Debugf("Cilium IPPools CIDR: %v", podCIDR) if coordinator.Status.Phase == Synced && reflect.DeepEqual(coordinator.Status.OverlayPodCIDR, podCIDR) { return nil } diff --git a/test/Makefile b/test/Makefile index 361e086377..7b08189046 100644 --- a/test/Makefile +++ b/test/Makefile @@ -228,7 +228,10 @@ setup_kruise: HELM_OPTION=" --wait --timeout 20m --debug --set manager.image.repository=$(E2E_OPENKRUISE_IMAGE) " ; \ HELM_OPTION+=" --version $(E2E_OPENKRUISE_VERSION) " ; \ helm upgrade --install kruise openkruise/kruise $${HELM_OPTION} \ - --kubeconfig $(E2E_KUBECONFIG) || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail" "$(E2E_LOG_FILE)" ; exit 1 ; } ; \ + --kubeconfig $(E2E_KUBECONFIG) || { + kubectl describe pod -n kruise-system --kubeconfig $(E2E_KUBECONFIG) ; \ + KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail" "$(E2E_LOG_FILE)" ; exit 1 ; + } ; \ .PHONY: setup_spiderpool setup_spiderpool: diff --git a/test/doc/spidercoodinator.md b/test/doc/spidercoodinator.md index 60d3ab568f..07e5956266 100644 --- a/test/doc/spidercoodinator.md +++ b/test/doc/spidercoodinator.md @@ -13,3 +13,4 @@ | V00009 | it can get the clusterCIDR from kubeadmConfig or kube-controller-manager pod | p3 | | done | | | V00010 | It can get service cidr from k8s serviceCIDR resoures | p3 | | done | | | V00011 | status should be NotReady if neither kubeadm-config configMap nor kube-controller-manager pod can be found | p3 | | done | | +| V00012 | tesing the switch of cilium ipam mode: multi-pool to cluster-pool | p3 | | done | | diff --git a/test/e2e/common/spiderpool.go b/test/e2e/common/spiderpool.go index c1ac8b309a..b2cdb4441f 100644 --- a/test/e2e/common/spiderpool.go +++ b/test/e2e/common/spiderpool.go @@ -23,8 +23,8 @@ import ( "github.com/spidernet-io/spiderpool/pkg/constant" ip "github.com/spidernet-io/spiderpool/pkg/ip" "github.com/spidernet-io/spiderpool/pkg/types" - corev1 "k8s.io/api/core/v1" + api_errors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -469,6 +469,17 @@ func GenerateExampleIpv4poolObject(ipNum int) (string, *v1.SpiderIPPool) { return v4PoolName, iPv4PoolObj } +func PatchConfigMap(f *frame.Framework, oldcm, newcm *corev1.ConfigMap, opts ...client.PatchOption) error { + mergePatch := client.MergeFrom(oldcm) + d, err := mergePatch.Data(newcm) + GinkgoWriter.Printf("patch configMap: %v. \n", string(d)) + if err != nil { + return fmt.Errorf("failed to generate patch, err is %v", err) + } + + return f.PatchResource(newcm, mergePatch, opts...) +} + func GenerateExampleIpv6poolObject(ipNum int) (string, *v1.SpiderIPPool) { if ipNum < 1 || ipNum > 65533 { GinkgoWriter.Println("the IP range should be between 1 and 65533") diff --git a/test/e2e/spidercoordinator/spidercoordinator_test.go b/test/e2e/spidercoordinator/spidercoordinator_test.go index a36818310b..179ea859ce 100644 --- a/test/e2e/spidercoordinator/spidercoordinator_test.go +++ b/test/e2e/spidercoordinator/spidercoordinator_test.go @@ -382,6 +382,93 @@ var _ = Describe("SpiderCoordinator", Label("spidercoordinator", "overlay"), Ser }) }) + Context("tesing the switch of cilium ipam mode: multi-pool to cluster-pool", func() { + var cm *corev1.ConfigMap + var err error + var clusterIPv4CIDR, clusterIPv6CIDR string + BeforeEach(func() { + if !common.CheckRunOverlayCNI() && !common.CheckCalicoFeatureOn() && !common.CheckCiliumFeatureOn() { + GinkgoWriter.Println("This environment is in underlay mode.") + Skip("This case don't need to run for underlay") + } + + if common.CheckCalicoFeatureOn() && !common.CheckCiliumFeatureOn() { + GinkgoWriter.Println("The environment is calico mode.") + Skip("This case don't need to run for calico") + } + + if common.CheckCiliumFeatureOn() && !common.CheckCalicoFeatureOn() { + GinkgoWriter.Println("The environment is cilium mode.") + } + + cm, err = frame.GetConfigmap("cilium-config", "kube-system") + Expect(err).NotTo(HaveOccurred(), "error to get cilium configmap: %v\n", err) + cmCopy := cm.DeepCopy() + + dataMap := cmCopy.Data + ipam := cm.Data["ipam"] + if ipam != "multi-pool" { + GinkgoWriter.Println("this case only running for cilium ipam multi-pool.") + Skip("") + } + + // update cilim ipam mode to cluster-pool. + dataMap["ipam"] = "cluster-pool" + if frame.Info.IpV4Enabled { + clusterIPv4CIDR = "10.245.0.0/16" + dataMap["cluster-pool-ipv4-cidr"] = clusterIPv4CIDR + } + + if frame.Info.IpV6Enabled { + clusterIPv6CIDR = "fd00:10:245::/112" + dataMap["cluster-pool-ipv6-cidr"] = clusterIPv6CIDR + } + cmCopy.Data = dataMap + + // patch the configMap + Expect( + common.PatchConfigMap(frame, cm, cmCopy)).NotTo( + HaveOccurred(), "error to update configmap: %v\n", err) + + DeferCleanup(func() { + // back ipam to multi-pool + cmCopy.Data["ipam"] = "multi-pool" + // patch the configMap + Expect( + common.PatchConfigMap(frame, cm, cmCopy)).NotTo( + HaveOccurred(), "error to update configmap: %v\n", err) + }) + }) + + It("tesing the switch of cilium ipam mode: multi-pool to cluster-pool", Label("V00012"), func() { + Eventually(func() bool { + spc, err := GetSpiderCoordinator(common.SpidercoodinatorDefaultName) + Expect(err).NotTo(HaveOccurred(), "failed to get SpiderCoordinator, error is %v", err) + GinkgoWriter.Printf("Display the default spider coordinator information, podCIDRType: %v, status: %v \n", *spc.Spec.PodCIDRType, spc.Status) + + if len(spc.Status.OverlayPodCIDR) == 0 || spc.Status.Phase != coordinatormanager.Synced { + GinkgoWriter.Printf("status.overlayPodCIDR status is still synchronizing, status %+v \n", spc.Status.OverlayPodCIDR) + return false + } + + for _, cidr := range spc.Status.OverlayPodCIDR { + if ip.IsIPv4CIDR(cidr) { + if cidr != clusterIPv4CIDR { + return false + } + GinkgoWriter.Printf("ipv4 podCIDR is as expected, value %v=%v \n", cidr, clusterIPv4CIDR) + } else { + if cidr != clusterIPv6CIDR { + return false + } + GinkgoWriter.Printf("ipv6 podCIDR is as expected, value %v=%v \n", cidr, clusterIPv6CIDR) + } + } + return true + }, common.ExecCommandTimeout, common.EventOccurTimeout*2).Should(BeTrue()) + }) + }) + Context("It can get the clusterCIDR from kubeadmConfig and kube-controller-manager pod", func() { var spc *spiderpoolv2beta1.SpiderCoordinator @@ -729,8 +816,10 @@ var _ = Describe("SpiderCoordinator", Label("spidercoordinator", "overlay"), Ser spcCopy.Spec.HostRuleTable = ptr.To(500) Expect(PatchSpiderCoordinator(spcCopy, spc)).NotTo(HaveOccurred()) - GinkgoWriter.Println("delete namespace: ", nsName) - Expect(frame.DeleteNamespace(nsName)).NotTo(HaveOccurred()) + if !CurrentSpecReport().Failed() { + GinkgoWriter.Println("delete namespace: ", nsName) + Expect(frame.DeleteNamespace(nsName)).NotTo(HaveOccurred()) + } }) }) diff --git a/test/scripts/install-default-cni.sh b/test/scripts/install-default-cni.sh index c776853dc2..28fee32f9e 100755 --- a/test/scripts/install-default-cni.sh +++ b/test/scripts/install-default-cni.sh @@ -8,9 +8,12 @@ set -o errexit -o nounset -o pipefail OS=$(uname | tr 'A-Z' 'a-z') SED_COMMAND=sed -CURRENT_FILENAME=$( basename $0 ) -CURRENT_DIR_PATH=$(cd $(dirname $0); pwd) -PROJECT_ROOT_PATH=$( cd ${CURRENT_DIR_PATH}/../.. && pwd ) +CURRENT_FILENAME=$(basename $0) +CURRENT_DIR_PATH=$( + cd $(dirname $0) + pwd +) +PROJECT_ROOT_PATH=$(cd ${CURRENT_DIR_PATH}/../.. && pwd) [ -z "$E2E_CLUSTER_NAME" ] && echo "error, miss E2E_CLUSTER_NAME " && exit 1 [ -z "$E2E_IP_FAMILY" ] && echo "error, miss E2E_IP_FAMILY " && exit 1 @@ -54,161 +57,221 @@ CILIUM_CLUSTER_POD_SUBNET_V6=${CILIUM_CLUSTER_POD_SUBNET_V6:-"fd00:10:244::/112" [ -z "${HTTP_PROXY}" ] || export https_proxy=${HTTP_PROXY} function install_calico() { - cp ${PROJECT_ROOT_PATH}/test/yamls/calico.yaml $CLUSTER_PATH/calico.yaml - if [ -z "${CALICO_VERSION}" ]; then - [ -n "${HTTP_PROXY}" ] && { CALICO_VERSION_INFO=$(curl --retry 3 --retry-delay 5 -x "${HTTP_PROXY}" -s https://api.github.com/repos/projectcalico/calico/releases/latest); echo ${CALICO_VERSION_INFO}; CALICO_VERSION=$(echo ${CALICO_VERSION_INFO} | jq -r '.tag_name'); } - [ -z "${HTTP_PROXY}" ] && { CALICO_VERSION_INFO=$(curl --retry 3 --retry-delay 5 -s https://api.github.com/repos/projectcalico/calico/releases/latest ); echo ${CALICO_VERSION_INFO}; CALICO_VERSION=$(echo ${CALICO_VERSION_INFO} | jq -r '.tag_name'); } - [ "${CALICO_VERSION}" == "null" ] && { echo "failed to get the calico version, will try to use default version."; CALICO_VERSION=${DEFAULT_CALICO_VERSION}; } - else - CALICO_VERSION=${CALICO_VERSION} - fi - echo "install calico version ${CALICO_VERSION}" - [ -n "${HTTP_PROXY}" ] && curl --retry 3 -x "${HTTP_PROXY}" -Lo ${CALICO_YAML} https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml - [ -z "${HTTP_PROXY}" ] && curl --retry 3 -Lo ${CALICO_YAML} https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml - - # set registry - if [ -n "${CALICO_IMAGE_REPO}" ]; then - grep -q -e ".*image:.*docker.io" ${CALICO_YAML} || { echo "failed find image"; exit 1; } - ${SED_COMMAND} -i -E 's?(.*image:.*)(docker.io)(.*)?\1'"${CALICO_IMAGE_REPO}"'\3?g' ${CALICO_YAML} - fi - - # accelerate local cluster , in case that it times out to wait calico ready - IMAGE_LIST=`cat ${CALICO_YAML} | grep "image: " | awk '{print \$2}' | sort | uniq | tr '\n' ' ' | tr '\r' ' ' ` - echo "image: ${IMAGE_LIST}" - for IMAGE in ${IMAGE_LIST} ; do - echo "load calico image ${IMAGE} to kind cluster" - docker pull ${IMAGE} - kind load docker-image ${IMAGE} --name ${E2E_CLUSTER_NAME} - done - - export KUBECONFIG=${E2E_KUBECONFIG} - kubectl apply -f ${CALICO_YAML} - sleep 10 - - kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system - kubectl get po -n kube-system - echo -e "\033[35m Succeed to install Calico \033[0m" - - echo -e "\033[35m Patch Calico \033[0m" - kubectl -n kube-system get cm calico-config -oyaml > ${CALICO_CONFIG} - kubectl -n kube-system get ds calico-node -oyaml > ${CALICO_NODE} - - case ${E2E_IP_FAMILY} in - ipv4) - # set configmap - configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "true", "assign_ipv6": "false"}' --output-format=json) - configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} - ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} - kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { echo "failed to patch calico configmap"; exit 1; } - ;; - ipv6) - # set configmap - configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "false", "assign_ipv6": "true"}' --output-format=json) - configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} - ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} - kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { echo "failed to patch calico configmap"; exit 1; } - - # set calico-node env - grep -q "FELIX_IPV6SUPPORT" ${CALICO_NODE} || { echo "failed find FELIX_IPV6SUPPORT"; exit 1; } - ${SED_COMMAND} -i -E '/FELIX_IPV6SUPPORT/{n;s/value: "false"/value: "true"/}' ${CALICO_NODE} - - grep -q "value: autodetect" ${CALICO_NODE} || { echo "failed find autodetect"; exit 1; } - ${SED_COMMAND} -i '/value: autodetect/a\ - name: IP6\n\ value: autodetect' ${CALICO_NODE} - kubectl -n kube-system patch ds calico-node --patch "$(cat ${CALICO_NODE})" || { echo "failed to patch calico-node"; exit 1; } - ;; - dual) - # set configmap - configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "true", "assign_ipv6": "true"}' --output-format=json) - configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} - ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} - kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { echo "failed to patch calico configmap"; exit 1; } - - # set calico-node env - grep -q "FELIX_IPV6SUPPORT" ${CALICO_NODE} || { echo "failed find FELIX_IPV6SUPPORT"; exit 1; } - ${SED_COMMAND} -i -E '/FELIX_IPV6SUPPORT/{n;s/value: "false"/value: "true"/}' ${CALICO_NODE} - grep -q "value: autodetect" ${CALICO_NODE} || { echo "failed find autodetect"; exit 1; } - ${SED_COMMAND} -i '/value: autodetect/a\ - name: IP6\n\ value: autodetect' ${CALICO_NODE} - kubectl -n kube-system patch ds calico-node --patch "$(cat ${CALICO_NODE})" || { echo "failed to patch calico-node"; exit 1; } - ;; - *) - echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" - exit 1 - esac - # there no default felixconfigurations.crd.projectcalico.org in latest calico version (https://github.com/projectcalico/calico/releases/tag/v3.29.0) - kubectl patch felixconfigurations.crd.projectcalico.org default --type='merge' -p '{"spec":{"chainInsertMode":"Append"}}' || true - - # restart calico pod - kubectl -n kube-system delete pod -l k8s-app=calico-node --force --grace-period=0 && sleep 3 - kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system - kubectl -n kube-system delete pod -l k8s-app=calico-kube-controllers --force --grace-period=0 && sleep 3 - kubectl wait --for=condition=ready -l k8s-app=calico-kube-controllers --timeout=${INSTALL_TIME_OUT} pod -n kube-system - echo -e "\033[35m ===> Succeed to patch calico \033[0m" - - # Update calico's podcidr so that it is inconsistent with the cluster's podcidr. - case ${E2E_IP_FAMILY} in - ipv4) - kubectl patch ippools default-ipv4-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV4POOL_CIDR}"'"}}' --type=merge - ;; - ipv6) - kubectl delete ippools default-ipv4-ippool --force - kubectl patch ippools default-ipv6-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV6POOL_CIDR}"'"}}' --type=merge - ;; - dual) - kubectl patch ippools default-ipv4-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV4POOL_CIDR}"'"}}' --type=merge - kubectl patch ippools default-ipv6-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV6POOL_CIDR}"'"}}' --type=merge - ;; - *) - echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" - exit 1 - esac - - echo -e "\033[35m ===> clean tmp \033[0m" - rm -rf ${DEST_CALICO_YAML_DIR} + cp ${PROJECT_ROOT_PATH}/test/yamls/calico.yaml $CLUSTER_PATH/calico.yaml + if [ -z "${CALICO_VERSION}" ]; then + [ -n "${HTTP_PROXY}" ] && { + CALICO_VERSION_INFO=$(curl --retry 3 --retry-delay 5 -x "${HTTP_PROXY}" -s https://api.github.com/repos/projectcalico/calico/releases/latest) + echo ${CALICO_VERSION_INFO} + CALICO_VERSION=$(echo ${CALICO_VERSION_INFO} | jq -r '.tag_name') + } + [ -z "${HTTP_PROXY}" ] && { + CALICO_VERSION_INFO=$(curl --retry 3 --retry-delay 5 -s https://api.github.com/repos/projectcalico/calico/releases/latest) + echo ${CALICO_VERSION_INFO} + CALICO_VERSION=$(echo ${CALICO_VERSION_INFO} | jq -r '.tag_name') + } + [ "${CALICO_VERSION}" == "null" ] && { + echo "failed to get the calico version, will try to use default version." + CALICO_VERSION=${DEFAULT_CALICO_VERSION} + } + else + CALICO_VERSION=${CALICO_VERSION} + fi + echo "install calico version ${CALICO_VERSION}" + [ -n "${HTTP_PROXY}" ] && curl --retry 3 -x "${HTTP_PROXY}" -Lo ${CALICO_YAML} https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml + [ -z "${HTTP_PROXY}" ] && curl --retry 3 -Lo ${CALICO_YAML} https://raw.githubusercontent.com/projectcalico/calico/${CALICO_VERSION}/manifests/calico.yaml + + # set registry + if [ -n "${CALICO_IMAGE_REPO}" ]; then + grep -q -e ".*image:.*docker.io" ${CALICO_YAML} || { + echo "failed find image" + exit 1 + } + ${SED_COMMAND} -i -E 's?(.*image:.*)(docker.io)(.*)?\1'"${CALICO_IMAGE_REPO}"'\3?g' ${CALICO_YAML} + fi + + # accelerate local cluster , in case that it times out to wait calico ready + IMAGE_LIST=$(cat ${CALICO_YAML} | grep "image: " | awk '{print $2}' | sort | uniq | tr '\n' ' ' | tr '\r' ' ') + echo "image: ${IMAGE_LIST}" + for IMAGE in ${IMAGE_LIST}; do + echo "load calico image ${IMAGE} to kind cluster" + docker pull ${IMAGE} + kind load docker-image ${IMAGE} --name ${E2E_CLUSTER_NAME} + done + + export KUBECONFIG=${E2E_KUBECONFIG} + kubectl apply -f ${CALICO_YAML} + sleep 10 + + kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system + kubectl get po -n kube-system + echo -e "\033[35m Succeed to install Calico \033[0m" + + echo -e "\033[35m Patch Calico \033[0m" + kubectl -n kube-system get cm calico-config -oyaml >${CALICO_CONFIG} + kubectl -n kube-system get ds calico-node -oyaml >${CALICO_NODE} + + case ${E2E_IP_FAMILY} in + ipv4) + # set configmap + configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "true", "assign_ipv6": "false"}' --output-format=json) + configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} + ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} + kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { + echo "failed to patch calico configmap" + exit 1 + } + ;; + ipv6) + # set configmap + configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "false", "assign_ipv6": "true"}' --output-format=json) + configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} + ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} + kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { + echo "failed to patch calico configmap" + exit 1 + } + + # set calico-node env + grep -q "FELIX_IPV6SUPPORT" ${CALICO_NODE} || { + echo "failed find FELIX_IPV6SUPPORT" + exit 1 + } + ${SED_COMMAND} -i -E '/FELIX_IPV6SUPPORT/{n;s/value: "false"/value: "true"/}' ${CALICO_NODE} + + grep -q "value: autodetect" ${CALICO_NODE} || { + echo "failed find autodetect" + exit 1 + } + ${SED_COMMAND} -i '/value: autodetect/a\ - name: IP6\n\ value: autodetect' ${CALICO_NODE} + kubectl -n kube-system patch ds calico-node --patch "$(cat ${CALICO_NODE})" || { + echo "failed to patch calico-node" + exit 1 + } + ;; + dual) + # set configmap + configYaml=$(yq '.data.cni_network_config' ${CALICO_CONFIG} | yq '.plugins[0].ipam = {"type": "calico-ipam", "assign_ipv4": "true", "assign_ipv6": "true"}' --output-format=json) + configYaml=$configYaml yq e '.data.cni_network_config |= strenv(configYaml)' -i ${CALICO_CONFIG} + ${SED_COMMAND} -i 's/"mtu": "__CNI_MTU__"/"mtu": __CNI_MTU__/g' ${CALICO_CONFIG} + kubectl -n kube-system patch cm calico-config --patch "$(cat ${CALICO_CONFIG})" || { + echo "failed to patch calico configmap" + exit 1 + } + + # set calico-node env + grep -q "FELIX_IPV6SUPPORT" ${CALICO_NODE} || { + echo "failed find FELIX_IPV6SUPPORT" + exit 1 + } + ${SED_COMMAND} -i -E '/FELIX_IPV6SUPPORT/{n;s/value: "false"/value: "true"/}' ${CALICO_NODE} + grep -q "value: autodetect" ${CALICO_NODE} || { + echo "failed find autodetect" + exit 1 + } + ${SED_COMMAND} -i '/value: autodetect/a\ - name: IP6\n\ value: autodetect' ${CALICO_NODE} + kubectl -n kube-system patch ds calico-node --patch "$(cat ${CALICO_NODE})" || { + echo "failed to patch calico-node" + exit 1 + } + ;; + *) + echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + ;; + esac + # there no default felixconfigurations.crd.projectcalico.org in latest calico version (https://github.com/projectcalico/calico/releases/tag/v3.29.0) + kubectl patch felixconfigurations.crd.projectcalico.org default --type='merge' -p '{"spec":{"chainInsertMode":"Append"}}' || true + + # restart calico pod + kubectl -n kube-system delete pod -l k8s-app=calico-node --force --grace-period=0 && sleep 3 + kubectl wait --for=condition=ready -l k8s-app=calico-node --timeout=${INSTALL_TIME_OUT} pod -n kube-system + kubectl -n kube-system delete pod -l k8s-app=calico-kube-controllers --force --grace-period=0 && sleep 3 + kubectl wait --for=condition=ready -l k8s-app=calico-kube-controllers --timeout=${INSTALL_TIME_OUT} pod -n kube-system + echo -e "\033[35m ===> Succeed to patch calico \033[0m" + + # Update calico's podcidr so that it is inconsistent with the cluster's podcidr. + case ${E2E_IP_FAMILY} in + ipv4) + kubectl patch ippools default-ipv4-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV4POOL_CIDR}"'"}}' --type=merge + ;; + ipv6) + kubectl delete ippools default-ipv4-ippool --force + kubectl patch ippools default-ipv6-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV6POOL_CIDR}"'"}}' --type=merge + ;; + dual) + kubectl patch ippools default-ipv4-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV4POOL_CIDR}"'"}}' --type=merge + kubectl patch ippools default-ipv6-ippool --patch '{"spec": {"cidr": "'"${CALICO_IPV6POOL_CIDR}"'"}}' --type=merge + ;; + *) + echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + ;; + esac + + echo -e "\033[35m ===> clean tmp \033[0m" + rm -rf ${DEST_CALICO_YAML_DIR} } function install_cilium() { - echo -e "\033[35m ===> Start to install cilium \033[0m" - # cni.exclusive using multus-cni need close - # kubeProxyReplacement Enhance kube-proxy (value probe static default: probe) - # k8sServiceHost api-server address - # k8sServicePort api-service port - # bpf.vlanBypass allow vlan traffic to pass - KUBE_PROXY_REPLACEMENT=false - if [ "$DISABLE_KUBE_PROXY" = "true" ]; then - KUBE_PROXY_REPLACEMENT=true - fi - CILIUM_HELM_OPTIONS=" --set cni.exclusive=false \ + echo -e "\033[35m ===> Start to install cilium \033[0m" + # cni.exclusive using multus-cni need close + # kubeProxyReplacement Enhance kube-proxy (value probe static default: probe) + # k8sServiceHost api-server address + # k8sServicePort api-service port + # bpf.vlanBypass allow vlan traffic to pass + # cilium ipamMode: multi-pool required routingMode=native + KUBE_PROXY_REPLACEMENT=false + if [ "$DISABLE_KUBE_PROXY" = "true" ]; then + KUBE_PROXY_REPLACEMENT=true + fi + CILIUM_HELM_OPTIONS=" --set cni.exclusive=false \ --set kubeProxyReplacement=${KUBE_PROXY_REPLACEMENT} \ --set k8sServiceHost=${E2E_CLUSTER_NAME}-control-plane \ --set k8sServicePort=6443 \ + --set routingMode=native \ + --set ipam.mode=multi-pool \ + --set autoDirectNodeRoutes=true \ --set bpf.vlanBypass={0} " - case ${E2E_IP_FAMILY} in - ipv4) - CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + case ${E2E_IP_FAMILY} in + ipv4) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V4} \ --set ipv4.enabled=true \ - --set ipv6.enabled=false " - ;; - ipv6) - CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv6PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipv6.enabled=false \ + --set ipv4NativeRoutingCIDR=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.cidrs=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.maskSize=26 \ + --set enableIPv4Masquerade=true " + ;; + ipv6) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv6PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv6.cidrs=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv6.maskSize=124 \ --set ipv4.enabled=false \ --set ipv6.enabled=true \ --set ipv6NativeRoutingCIDR=${CILIUM_CLUSTER_POD_SUBNET_V6} \ - --set autoDirectNodeRoutes=true \ - --set enableIPv6Masquerade=true \ - --set routingMode=native " - ;; - dual) - CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set enableIPv6Masquerade=true " + ;; + dual) + CILIUM_HELM_OPTIONS+=" --set ipam.operator.clusterPoolIPv4PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V4} \ --set ipam.operator.clusterPoolIPv6PodCIDRList=${CILIUM_CLUSTER_POD_SUBNET_V6} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.cidrs=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv4.maskSize=26 \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv6.maskSize=124 \ + --set ipam.operator.autoCreateCiliumPodIPPools.default.ipv6.cidrs=${CILIUM_CLUSTER_POD_SUBNET_V6} \ --set ipv4.enabled=true \ - --set ipv6.enabled=true " - ;; - *) - echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" - exit 1 - esac - - CILIUM_HELM_OPTIONS+=" \ + --set ipv6.enabled=true \ + --set enableIPv4Masquerade=true \ + --set enableIPv6Masquerade=true \ + --set ipv4NativeRoutingCIDR=${CILIUM_CLUSTER_POD_SUBNET_V4} \ + --set ipv6NativeRoutingCIDR=${CILIUM_CLUSTER_POD_SUBNET_V6} " + ;; + *) + echo "the value of E2E_IP_FAMILY: ipv4 or ipv6 or dual" + exit 1 + ;; + esac + + CILIUM_HELM_OPTIONS+=" \ --set image.repository=${E2E_CILIUM_IMAGE_REPO}/cilium/cilium \ --set image.useDigest=false \ --set certgen.image.repository=${E2E_CILIUM_IMAGE_REPO}/cilium/certgen \ @@ -223,49 +286,48 @@ function install_cilium() { --set preflight.image.useDigest=false \ --set nodeinit.image.repository=${E2E_CILIUM_IMAGE_REPO}/cilium/startup-script " - echo "CILIUM_HELM_OPTIONS: ${CILIUM_HELM_OPTIONS}" + echo "CILIUM_HELM_OPTIONS: ${CILIUM_HELM_OPTIONS}" + + helm repo remove cilium &>/dev/null || true + helm repo add cilium https://helm.cilium.io + helm repo update + + if [ -n "${CILIUM_VERSION}" ]; then + CILIUM_HELM_OPTIONS+=" --version ${CILIUM_VERSION} " + fi - helm repo remove cilium &>/dev/null || true - helm repo add cilium https://helm.cilium.io - helm repo update + HELM_IMAGES_LIST=$(helm template test cilium/cilium ${CILIUM_HELM_OPTIONS} | grep " image: " | tr -d '"' | awk '{print $2}' | awk -F "@" '{print $1}' | uniq) + [ -z "${HELM_IMAGES_LIST}" ] && echo "can't found image of cilium" && exit 1 + LOCAL_IMAGE_LIST=$(docker images | awk '{printf("%s:%s\n",$1,$2)}') - if [ -n "${CILIUM_VERSION}" ] ; then - CILIUM_HELM_OPTIONS+=" --version ${CILIUM_VERSION} " + for CILIUM_IMAGE in ${HELM_IMAGES_LIST}; do + if ! grep ${CILIUM_IMAGE} <<<${LOCAL_IMAGE_LIST}; then + echo "===> docker pull ${CILIUM_IMAGE} " + docker pull ${CILIUM_IMAGE} fi + echo "===> load image ${CILIUM_IMAGE} to kind..." + kind load docker-image ${CILIUM_IMAGE} --name ${E2E_CLUSTER_NAME} + done - HELM_IMAGES_LIST=` helm template test cilium/cilium ${CILIUM_HELM_OPTIONS} | grep " image: " | tr -d '"'| awk '{print $2}' | awk -F "@" '{print $1}' | uniq ` - [ -z "${HELM_IMAGES_LIST}" ] && echo "can't found image of cilium" && exit 1 - LOCAL_IMAGE_LIST=`docker images | awk '{printf("%s:%s\n",$1,$2)}'` - - for CILIUM_IMAGE in ${HELM_IMAGES_LIST}; do - if ! grep ${CILIUM_IMAGE} <<< ${LOCAL_IMAGE_LIST} ; then - echo "===> docker pull ${CILIUM_IMAGE} " - docker pull ${CILIUM_IMAGE} - fi - echo "===> load image ${CILIUM_IMAGE} to kind..." - kind load docker-image ${CILIUM_IMAGE} --name ${E2E_CLUSTER_NAME} - done - - # Install cilium - helm upgrade --install cilium cilium/cilium --wait -n kube-system --debug --kubeconfig ${E2E_KUBECONFIG} ${CILIUM_HELM_OPTIONS} - - # no matching resources found - sleep 3 - kubectl wait --for=condition=ready -l k8s-app=cilium --timeout=${INSTALL_TIME_OUT} pod -n kube-system \ + # Install cilium + helm upgrade --install cilium cilium/cilium --wait -n kube-system --debug --kubeconfig ${E2E_KUBECONFIG} ${CILIUM_HELM_OPTIONS} + + # no matching resources found + sleep 3 + kubectl wait --for=condition=ready -l k8s-app=cilium --timeout=${INSTALL_TIME_OUT} pod -n kube-system \ --kubeconfig ${E2E_KUBECONFIG} - sleep 10 + sleep 10 - echo -e "\033[35m ===> Succeed to install cilium \033[0m" + echo -e "\033[35m ===> Succeed to install cilium \033[0m" } -if [ "${INSTALL_CALICO}" == "true" ] ; then +if [ "${INSTALL_CALICO}" == "true" ]; then install_calico fi -if [ "${INSTALL_CILIUM}" == "true" ] ; then +if [ "${INSTALL_CILIUM}" == "true" ]; then install_cilium fi - kubectl get po -n kube-system --kubeconfig ${E2E_KUBECONFIG} -owide