CI: run cilium without kube-proxy

test macvlan and ipvlan work with cilium without kube-proxy.

.github/workflows/e2e-init.yaml

@@ -41,6 +41,8 @@ jobs:
             e2e_test_mode: e2e_test_calico
           - e2e_init_mode: e2e_init_cilium
             e2e_test_mode: e2e_test_cilium
+          - e2e_init_mode: e2e_init_cilium_without_kube_proxy
+            e2e_test_mode: e2e_test_cilium_without_kube_proxy
     steps:
       - name: Free disk space
         # https://github.com/actions/virtual-environments/issues/709

Makefile

@@ -318,6 +318,10 @@ e2e_init_calico:
 e2e_init_cilium:
 	$(QUIET)  make e2e_init -e INSTALL_OVERLAY_CNI=true -e INSTALL_CALICO=false -e INSTALL_CILIUM=true -e E2E_SPIDERPOOL_ENABLE_SUBNET=false
 
+.PHONY: e2e_init_cilium_without_kube_proxy
+e2e_init_cilium_without_kube_proxy:
+	$(QUIET)  make e2e_init -e INSTALL_OVERLAY_CNI=true -e INSTALL_CALICO=false -e INSTALL_CILIUM=true DISABLE_KUBE_PROXY=true -e E2E_SPIDERPOOL_ENABLE_SUBNET=false
+
 
 .PHONY: e2e_test
 e2e_test:
@@ -333,8 +337,11 @@ e2e_test_calico:
 
 .PHONY: e2e_test_cilium
 e2e_test_cilium:
-	$(QUIET)  make e2e_test -e INSTALL_OVERLAY_CNI=true -e INSTALL_CALICO=false -e INSTALL_CILIUM=true -e E2E_GINKGO_LABELS=overlay
+	$(QUIET)  make e2e_test -e INSTALL_OVERLAY_CNI=true -e INSTALL_CALICO=false -e INSTALL_CILIUM=true -e E2E_GINKGO_LABELS=kubeProxyReplacement
 
+.PHONY: e2e_test_cilium_without_kube_proxy
+e2e_test_cilium_without_kube_proxy:
+	$(QUIET)  make e2e_test -e INSTALL_OVERLAY_CNI=true -e INSTALL_CALICO=false -e INSTALL_CILIUM=true DISABLE_KUBE_PROXY=true -e E2E_GINKGO_LABELS=kubeProxyReplacement
 
 .PHONY: preview_doc
 preview_doc: PROJECT_DOC_DIR := ${ROOT_DIR}/docs

test/Makefile

@@ -40,6 +40,7 @@ ifeq ($(INSTALL_OVERLAY_CNI),true)
 		CILIUM_VERSION=$(CILIUM_VERSION) \
 		CILIUM_CLUSTER_POD_SUBNET_V4=$(CILIUM_CLUSTER_POD_SUBNET_V4) \
 		CILIUM_CLUSTER_POD_SUBNET_V6=$(CILIUM_CLUSTER_POD_SUBNET_V6) \
+		DISABLE_KUBE_PROXY=$(DISABLE_KUBE_PROXY) \
 		HTTP_PROXY=$(HTTP_PROXY) \
 		$(QUIET) bash scripts/install-default-cni.sh
 endif
@@ -57,6 +58,7 @@ ifeq ($(INSTALL_MULTUS),true)
 		MULTUS_ADDITIONAL_CNI_VLAN100=$(MULTUS_ADDITIONAL_CNI_VLAN100) \
 		MULTUS_ADDITIONAL_CNI_VLAN200=$(MULTUS_ADDITIONAL_CNI_VLAN200) \
 		RELEASE_NAMESPACE=$(RELEASE_NAMESPACE) \
+		DISABLE_KUBE_PROXY=$(DISABLE_KUBE_PROXY) \
 		CLUSTER_PATH=$(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) \
 		E2E_SPIDERPOOL_ENABLE_SUBNET=${E2E_SPIDERPOOL_ENABLE_SUBNET} \
 		scripts/install-multus.sh $(E2E_CLUSTER_NAME) $(E2E_KUBECONFIG)
@@ -155,10 +157,6 @@ setup_kurise:
 
 .PHONY: setup_spiderpool
 setup_spiderpool:
-	@echo "Load Image $(E2E_SPIDERPOOL_TAG) to kind..." ; \
-		for i in $(SPIDERPOOL_IMAGES); do \
-			kind load docker-image $${i}:$(E2E_SPIDERPOOL_TAG) --name $(E2E_CLUSTER_NAME);	\
-		done; \
 		if [ "$(USE_TLS_METHOD)" == "certmanager" ] ; then \
 			echo "USE_TLS_METHOD $(USE_TLS_METHOD) " ; \
 			$(QUIET) IMAGE_CERT_MANAGER="$(IMAGE_CERT_MANAGER_NAME)" scripts/install-cert-manager.sh "$(E2E_CLUSTER_NAME)" "$(CERT_MANAGER_ISSUER_NAME)" ; \
@@ -191,10 +189,6 @@ setup_spiderpool:
 		    else \
 			HELM_OPTION+=" --set multus.multusCNI.defaultCniCRName=$(MULTUS_DEFAULT_CNI_VLAN0) " ; \
 		    fi ; \
-			IMAGE_MULTUS=`helm template $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} | grep ' image: ' | grep multus | tr -d '"' | awk '{print $$2}' | sort | uniq | tr '\n' ' '` ; \
-			echo "MULTUS IMAGE: $${IMAGE_MULTUS} " ; \
-			docker pull $${IMAGE_MULTUS} ; \
-		 	kind load docker-image $${IMAGE_MULTUS}  --name $(E2E_CLUSTER_NAME); \
 			if [ "$(E2E_SPIDERPOOL_ENABLE_MULTUSCONFIG)" == "true" ] ; then \
 				HELM_OPTION+=" --set multus.enableMultusConfig=true " ; \
 			else \
@@ -265,22 +259,31 @@ setup_spiderpool:
 		HELM_OPTION+=" --set sriov.install=true " ; \
 		HELM_OPTION+=" --set rdma.rdmaSharedDevicePlugin.install=true " ; \
 		HELM_OPTION+=" --set rdma.rdmaCni.install=true " ; \
+		HELM_OPTION+=" 	--set spiderpoolAgent.image.registry="" \
+		--set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \
+		--set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \
+		--set spiderpoolController.image.registry="" \
+		--set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
+		--set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \
+		--set spiderpoolInit.image.registry="" \
+		--set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
+		--set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) \
+		--set rdma.rdmaCni.image.registry=$(E2E_RDMA_IMAGE_REPO) \
+		--set rdma.rdmaSharedDevicePlugin.image.registry=$(E2E_RDMA_IMAGE_REPO) \
+		--set sriov.image.registry=$(E2E_SRIOV_IMAGE_REPO)   " \
+		ALL_IMAGES=`helm template $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool $${HELM_OPTION} | grep ' image: ' | tr -d '"' | awk -F 'image: ' '{print $$2}' | sort | uniq | tr '\n' ' '` ; \
+		echo "ALL_IMAGES: $${ALL_IMAGES} " ; \
+		for IMAGE in $${ALL_IMAGES}; do \
+				if ! grep "$${IMAGE}" <<< `docker images | awk '{printf("%s:%s\n",$$1,$$2)}'`; then \
+						echo "==> $${IMAGE} no found, pulling...." ; \
+						docker pull $${IMAGE} ; \
+				fi ; \
+				kind load docker-image $${IMAGE}  --name $(E2E_CLUSTER_NAME); \
+		done ; \
 		echo "setup spiderpool with image $(SPIDERPOOL_AGENT_IMAGE_NAME):$(E2E_SPIDERPOOL_TAG) and $(SPIDERPOOL_CONTROLLER_IMAGE_NAME):$(E2E_SPIDERPOOL_TAG) " ; \
 		set -x ; \
 		helm upgrade --install $(RELEASE_NAME) $(ROOT_DIR)/charts/spiderpool --wait --debug \
 			-n $(RELEASE_NAMESPACE)  \
-			--set spiderpoolAgent.image.registry="" \
-			--set spiderpoolAgent.image.repository=$(SPIDERPOOL_AGENT_IMAGE_NAME) \
-			--set spiderpoolAgent.image.tag=$(E2E_SPIDERPOOL_TAG) \
-			--set spiderpoolController.image.registry="" \
-			--set spiderpoolController.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
-			--set spiderpoolController.image.tag=$(E2E_SPIDERPOOL_TAG) \
-			--set spiderpoolInit.image.registry="" \
-			--set spiderpoolInit.image.repository=$(SPIDERPOOL_CONTROLLER_IMAGE_NAME) \
-			--set spiderpoolInit.image.tag=$(E2E_SPIDERPOOL_TAG) \
-			--set rdma.rdmaCni.image.registry=$(E2E_RDMA_IMAGE_REPO) \
-			--set rdma.rdmaSharedDevicePlugin.image.registry=$(E2E_RDMA_IMAGE_REPO) \
-			--set sriov.image.registry=$(E2E_SRIOV_IMAGE_REPO) \
 			$${HELM_OPTION}  \
 			$(E2E_HELM_ADDITIONAL_OPTIONS) \
 			--kubeconfig $(E2E_KUBECONFIG)  || { KIND_CLUSTER_NAME=$(E2E_CLUSTER_NAME) ./scripts/debugEnv.sh $(E2E_KUBECONFIG) "detail"   ; exit 1 ; } ; \

test/Makefile.defs

@@ -1,7 +1,11 @@
 #============ e2e-kind-config ====================
 
-# iptables or ipvs, default iptables
-E2E_KUBE_PROXY_MODE ?= iptables
+# iptables,ipvs or none, default iptables
+ifeq ($(DISABLE_KUBE_PROXY),true)
+    E2E_KUBE_PROXY_MODE ?= none
+else
+    E2E_KUBE_PROXY_MODE ?= iptables
+endif
 
 E2E_CHINA_IMAGE_REGISTRY ?= false
 HTTP_PROXY ?=
@@ -43,6 +47,8 @@ INSTALL_CALICO ?= true
 
 INSTALL_CILIUM ?= true
 
+DISABLE_KUBE_PROXY ?= false
+
 INSTALL_NETTOOLS ?= false
 
 INSTALL_SPIDERDOCTOR ?= true

test/e2e/coordinator/macvlan-overlay-one/macvlan_overlay_one_test.go

@@ -95,7 +95,7 @@ var _ = Describe("MacvlanOverlayOne", Label("overlay", "one-nic", "coordinator")
 			Expect(err).NotTo(HaveOccurred())
 		})
 
-		It("spiderdoctor connectivity should be succeed", Serial, Label("C00002"), func() {
+		It("spiderdoctor connectivity should be succeed", Serial, Label("C00002"), Label("kubeProxyReplacement"), func() {
 
 			// create task spiderdoctor crd
 			task.Name = name

test/e2e/coordinator/macvlan-underlay-one/macvlan_underlay_one_test.go

@@ -69,7 +69,7 @@ var _ = Describe("MacvlanUnderlayOne", Serial, Label("underlay", "one-interface"
 		time.Sleep(20 * time.Second)
 	})
 
-	It("spiderdoctor connectivity should be succeed", Label("C00001"), func() {
+	It("spiderdoctor connectivity should be succeed", Label("C00001"), Label("kubeProxyReplacement"), func() {
 		// create task spiderdoctor crd
 		task.Name = name
 		// schedule

test/scripts/install-default-cni.sh

@@ -37,6 +37,7 @@ export CALICO_AUTODETECTION_METHOD=${CALICO_AUTODETECTION_METHOD:-"kubernetes-in
 
 E2E_CILIUM_IMAGE_REPO=${E2E_CILIUM_IMAGE_REPO:-"quay.io"}
 CILIUM_VERSION=${CILIUM_VERSION:-""}
+DISABLE_KUBE_PROXY=${DISABLE_KUBE_PROXY:-"false"}
 CILIUM_CLUSTER_POD_SUBNET_V4=${CILIUM_CLUSTER_POD_SUBNET_V4:-"10.244.64.0/18"}
 CILIUM_CLUSTER_POD_SUBNET_V6=${CILIUM_CLUSTER_POD_SUBNET_V6:-"fd00:10:244::/112"}
 
@@ -115,8 +116,9 @@ function install_cilium() {
       # k8sServiceHost api-server address
       # k8sServicePort api-service port
       # bpf.vlanBypass allow vlan traffic to pass
+      KUBE_PROXY_REPLACEMENT=strict
       CILIUM_HELM_OPTIONS=" --set cni.exclusive=false \
-                            --set kubeProxyReplacement=disabled \
+                            --set kubeProxyReplacement=${KUBE_PROXY_REPLACEMENT} \
                             --set k8sServiceHost=${E2E_CLUSTER_NAME}-control-plane \
                             --set k8sServicePort=6443 \
                             --set bpf.vlanBypass=0 "

test/scripts/install-multus.sh

@@ -50,6 +50,7 @@ metadata:
   namespace: <<NAMESPACE>>
 spec:
   cniType: macvlan
+  enableCoordinator: <<ENABLE_COORDINATOR>>
   macvlan:
     master: ["<<MASTER>>"]
     vlanID: <<VLAN>>
@@ -93,9 +94,16 @@ spec:
       exit 1
   esac
 
+  ENABLE_COORDINATOR=true
+  if [ "${DISABLE_KUBE_PROXY}" == "true" ] ; then
+      ENABLE_COORDINATOR=false
+      echo "DISABLE_KUBE_PROXY is true ,  disable coordinator config"
+  fi
+
   echo "${MACVLAN_CR_TEMPLATE}" \
     | sed 's?<<CNI_NAME>>?'""${MULTUS_DEFAULT_CNI_NAME}""'?g' \
     | sed 's?<<NAMESPACE>>?'"${RELEASE_NAMESPACE}"'?g' \
+    | sed 's?<<ENABLE_COORDINATOR>>?'${ENABLE_COORDINATOR}'?g' \
     | sed 's?<<MODE>>?auto?g' \
     | sed 's?<<MASTER>>?eth0?g' \
     | sed 's?<<VLAN>>?0?g' \
@@ -106,6 +114,7 @@ spec:
   echo "${MACVLAN_CR_TEMPLATE}" \
     | sed 's?<<CNI_NAME>>?'""${MULTUS_DEFAULT_CNI_VLAN100}""'?g' \
     | sed 's?<<NAMESPACE>>?'"${RELEASE_NAMESPACE}"'?g' \
+    | sed 's?<<ENABLE_COORDINATOR>>?'${ENABLE_COORDINATOR}'?g' \
     | sed 's?<<MODE>>?auto?g' \
     | sed 's?<<MASTER>>?eth0?g' \
     | sed 's?<<VLAN>>?100?g' \
@@ -116,6 +125,7 @@ spec:
   echo "${MACVLAN_CR_TEMPLATE}" \
     | sed 's?<<CNI_NAME>>?'""${MULTUS_DEFAULT_CNI_VLAN200}""'?g' \
     | sed 's?<<NAMESPACE>>?'"${RELEASE_NAMESPACE}"'?g' \
+    | sed 's?<<ENABLE_COORDINATOR>>?'${ENABLE_COORDINATOR}'?g' \
     | sed 's?<<MODE>>?auto?g' \
     | sed 's?<<MASTER>>?eth0?g' \
     | sed 's?<<VLAN>>?200?g' \