Merge pull request #2549 from Icarus9913/cherrypick

cherrypick 2518, 2514 and 2544 to release v0.7

charts/spiderpool/templates/daemonset.yaml

@@ -154,6 +154,14 @@ spec:
                 - {{ .Values.spiderpoolAgent.binName }}
                 - shutdown
         env:
+        - name: SPIDERPOOL_POD_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.name
+        - name: SPIDERPOOL_POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
         - name: SPIDERPOOL_LOG_LEVEL
           value: {{ .Values.spiderpoolAgent.debug.logLevel | quote }}
         - name: SPIDERPOOL_ENABLED_METRIC

cmd/spiderpool-agent/cmd/config.go

@@ -52,6 +52,8 @@ var envInfo = []envConf{
 	{"SPIDERPOOL_LOG_LEVEL", logutils.LogInfoLevelStr, true, &agentContext.Cfg.LogLevel, nil, nil},
 	{"SPIDERPOOL_ENABLED_METRIC", "false", false, nil, &agentContext.Cfg.EnableMetric, nil},
 	{"SPIDERPOOL_ENABLED_DEBUG_METRIC", "false", false, nil, &agentContext.Cfg.EnableDebugLevelMetric, nil},
+	{"SPIDERPOOL_POD_NAMESPACE", "", true, &agentContext.Cfg.AgentPodNamespace, nil, nil},
+	{"SPIDERPOOL_POD_NAME", "", true, &agentContext.Cfg.AgentPodName, nil, nil},
 	{"SPIDERPOOL_HEALTH_PORT", "5710", true, &agentContext.Cfg.HttpPort, nil, nil},
 	{"SPIDERPOOL_METRIC_HTTP_PORT", "5711", true, &agentContext.Cfg.MetricHttpPort, nil, nil},
 	{"SPIDERPOOL_GOPS_LISTEN_PORT", "5712", false, &agentContext.Cfg.GopsListenPort, nil, nil},
@@ -77,6 +79,8 @@ type Config struct {
 	LogLevel               string
 	EnableMetric           bool
 	EnableDebugLevelMetric bool
+	AgentPodNamespace      string
+	AgentPodName           string
 
 	HttpPort         string
 	MetricHttpPort   string
@@ -90,16 +94,12 @@ type Config struct {
 	MultusClusterNetwork string
 
 	// configmap
-	IpamUnixSocketPath                string   `yaml:"ipamUnixSocketPath"`
-	EnableIPv4                        bool     `yaml:"enableIPv4"`
-	EnableIPv6                        bool     `yaml:"enableIPv6"`
-	EnableStatefulSet                 bool     `yaml:"enableStatefulSet"`
-	EnableSpiderSubnet                bool     `yaml:"enableSpiderSubnet"`
-	ClusterDefaultIPv4IPPool          []string `yaml:"clusterDefaultIPv4IPPool"`
-	ClusterDefaultIPv6IPPool          []string `yaml:"clusterDefaultIPv6IPPool"`
-	ClusterDefaultIPv4Subnet          []string `yaml:"clusterDefaultIPv4Subnet"`
-	ClusterDefaultIPv6Subnet          []string `yaml:"clusterDefaultIPv6Subnet"`
-	ClusterSubnetDefaultFlexibleIPNum int      `yaml:"clusterSubnetDefaultFlexibleIPNumber"`
+	IpamUnixSocketPath                string `yaml:"ipamUnixSocketPath"`
+	EnableIPv4                        bool   `yaml:"enableIPv4"`
+	EnableIPv6                        bool   `yaml:"enableIPv6"`
+	EnableStatefulSet                 bool   `yaml:"enableStatefulSet"`
+	EnableSpiderSubnet                bool   `yaml:"enableSpiderSubnet"`
+	ClusterSubnetDefaultFlexibleIPNum int    `yaml:"clusterSubnetDefaultFlexibleIPNumber"`
 }
 
 type AgentContext struct {

cmd/spiderpool-agent/cmd/daemon.go

@@ -131,14 +131,13 @@ func DaemonMain() {
 
 	logger.Info("Begin to initialize IPAM")
 	ipamConfig := ipam.IPAMConfig{
-		EnableIPv4:               agentContext.Cfg.EnableIPv4,
-		EnableIPv6:               agentContext.Cfg.EnableIPv6,
-		ClusterDefaultIPv4IPPool: agentContext.Cfg.ClusterDefaultIPv4IPPool,
-		ClusterDefaultIPv6IPPool: agentContext.Cfg.ClusterDefaultIPv6IPPool,
-		EnableSpiderSubnet:       agentContext.Cfg.EnableSpiderSubnet,
-		EnableStatefulSet:        agentContext.Cfg.EnableStatefulSet,
-		OperationRetries:         agentContext.Cfg.WaitSubnetPoolMaxRetries,
-		OperationGapDuration:     time.Duration(agentContext.Cfg.WaitSubnetPoolTime) * time.Second,
+		EnableIPv4:           agentContext.Cfg.EnableIPv4,
+		EnableIPv6:           agentContext.Cfg.EnableIPv6,
+		EnableSpiderSubnet:   agentContext.Cfg.EnableSpiderSubnet,
+		EnableStatefulSet:    agentContext.Cfg.EnableStatefulSet,
+		OperationRetries:     agentContext.Cfg.WaitSubnetPoolMaxRetries,
+		OperationGapDuration: time.Duration(agentContext.Cfg.WaitSubnetPoolTime) * time.Second,
+		AgentNamespace:       agentContext.Cfg.AgentPodNamespace,
 	}
 	if len(agentContext.Cfg.MultusClusterNetwork) != 0 {
 		ipamConfig.MultusClusterNetwork = pointer.String(agentContext.Cfg.MultusClusterNetwork)

pkg/ipam/allocate.go

@@ -548,7 +548,9 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 
 			multusNS = netNsName
 			if multusNS == "" {
-				multusNS = pod.Namespace
+				// Reference from Multus source codes: The CRD object of default network should only be defined in multusNamespace
+				// In multus, multusNamespace serves for (clusterNetwork/defaultNetworks)
+				multusNS = i.config.AgentNamespace
 			}
 			multusName = networkName
 		} else {
@@ -572,6 +574,12 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 				}
 			}
 
+			// Refer from the multus-cni source codes, for annotation "k8s.v1.cni.cncf.io/networks" value without Namespace,
+			// we will regard the pod Namespace as the value's namespace
+			if multusNS == "" {
+				multusNS = pod.ObjectMeta.Namespace
+			}
+
 			// impossible
 			if !isFound {
 				return fmt.Errorf("%w: no matched multus object for NIC '%s'. The multus network-attachments: %v", constant.ErrUnknown, nic, podAnno[constant.MultusNetworkAttachmentAnnot])
@@ -581,7 +589,9 @@ func (i *ipam) selectByPod(ctx context.Context, version types.IPVersion, ipPool
 		for index := range ipPool.Spec.MultusName {
 			expectedMultusName := ipPool.Spec.MultusName[index]
 			if !strings.Contains(expectedMultusName, "/") {
-				expectedMultusName = fmt.Sprintf("%s/%s", pod.Namespace, expectedMultusName)
+				// for the ippool.spec.multusName property, if the user doesn't specify the net-attach-def resource namespace,
+				// we'll regard it in the Spiderpool installation namespace
+				expectedMultusName = fmt.Sprintf("%s/%s", i.config.AgentNamespace, expectedMultusName)
 			}
 
 			if strings.Compare(expectedMultusName, fmt.Sprintf("%s/%s", multusNS, multusName)) == 0 {

pkg/ipam/config.go

@@ -15,10 +15,8 @@ import (
 )
 
 type IPAMConfig struct {
-	EnableIPv4               bool
-	EnableIPv6               bool
-	ClusterDefaultIPv4IPPool []string
-	ClusterDefaultIPv6IPPool []string
+	EnableIPv4 bool
+	EnableIPv6 bool
 
 	EnableSpiderSubnet bool
 	EnableStatefulSet  bool
@@ -27,6 +25,7 @@ type IPAMConfig struct {
 	OperationGapDuration time.Duration
 
 	MultusClusterNetwork *string
+	AgentNamespace       string
 }
 
 func setDefaultsForIPAMConfig(config IPAMConfig) IPAMConfig {

pkg/ippoolmanager/ippool_manager.go

@@ -139,6 +139,13 @@ func (im *ipPoolManager) AllocateIP(ctx context.Context, poolName, nic string, p
 }
 
 func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *spiderpoolv2beta1.SpiderIPPool, pod *corev1.Pod) (net.IP, error) {
+	logger := logutils.FromContext(ctx)
+
+	key, err := cache.MetaNamespaceKeyFunc(pod)
+	if err != nil {
+		return nil, err
+	}
+
 	reservedIPs, err := im.rIPManager.AssembleReservedIPs(ctx, *ipPool.Spec.IPVersion)
 	if err != nil {
 		return nil, err
@@ -165,14 +172,20 @@ func (im *ipPoolManager) genRandomIP(ctx context.Context, nic string, ipPool *sp
 
 	availableIPs := spiderpoolip.IPsDiffSet(totalIPs, append(reservedIPs, usedIPs...), false)
 	if len(availableIPs) == 0 {
-		return nil, constant.ErrIPUsedOut
-	}
-	resIP := availableIPs[0]
+		// traverse the usedIPs to find the previous allocated IPs if there be
+		// reference issue: https://github.com/spidernet-io/spiderpool/issues/2517
+		allocatedIPFromRecords, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, key, string(pod.UID))
+		if !hasFound {
+			return nil, constant.ErrIPUsedOut
+		}
 
-	key, err := cache.MetaNamespaceKeyFunc(pod)
-	if err != nil {
-		return nil, err
+		availableIPs, err = spiderpoolip.ParseIPRange(*ipPool.Spec.IPVersion, allocatedIPFromRecords)
+		if nil != err {
+			return nil, err
+		}
+		logger.Sugar().Warnf("find previous IP '%s' from IPPool '%s' recorded IP allocations", allocatedIPFromRecords, ipPool.Name)
 	}
+	resIP := availableIPs[0]
 
 	if allocatedRecords == nil {
 		allocatedRecords = spiderpoolv2beta1.PoolIPAllocations{}

pkg/ippoolmanager/ippool_manager_test.go

@@ -5,6 +5,7 @@ package ippoolmanager_test
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"sync/atomic"
@@ -19,6 +20,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/client-go/tools/cache"
 	"k8s.io/utils/pointer"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
@@ -364,6 +366,60 @@ var _ = Describe("IPPoolManager", Label("ippool_manager_test"), func() {
 				Expect(res.Gateway).To(Equal(gateway))
 				Expect(res.Vlan).To(Equal(vlan))
 			})
+
+			It("allocate IP address from the previous records", func() {
+				mockRIPManager.EXPECT().
+					AssembleReservedIPs(gomock.Eq(ctx), gomock.Eq(constant.IPv4)).
+					Return(nil, nil).
+					Times(1)
+
+				ipVersion := constant.IPv4
+				allocatedIP := "172.18.40.40/24"
+				gateway := "172.18.40.1"
+				vlan := int64(0)
+
+				ip, ipNet, err := net.ParseCIDR(allocatedIP)
+				Expect(err).NotTo(HaveOccurred())
+
+				ipPoolT.Spec.IPVersion = pointer.Int64(ipVersion)
+				ipPoolT.Spec.Subnet = ipNet.String()
+				ipPoolT.Spec.IPs = append(ipPoolT.Spec.IPs, ip.String())
+				ipPoolT.Spec.Gateway = pointer.String(gateway)
+				ipPoolT.Spec.Vlan = pointer.Int64(vlan)
+
+				key, err := cache.MetaNamespaceKeyFunc(podT)
+				Expect(err).NotTo(HaveOccurred())
+
+				records := spiderpoolv2beta1.PoolIPAllocations{
+					ip.String(): spiderpoolv2beta1.PoolIPAllocation{
+						NIC:            nic,
+						NamespacedName: key,
+						PodUID:         string(podT.UID),
+					},
+				}
+				allocatedIPs, err := json.Marshal(records)
+				Expect(err).NotTo(HaveOccurred())
+
+				ipPoolT.Status = spiderpoolv2beta1.IPPoolStatus{
+					AllocatedIPs:     pointer.String(string(allocatedIPs)),
+					TotalIPCount:     pointer.Int64(1),
+					AllocatedIPCount: pointer.Int64(1),
+				}
+
+				err = fakeClient.Create(ctx, ipPoolT)
+				Expect(err).NotTo(HaveOccurred())
+				err = tracker.Add(ipPoolT)
+				Expect(err).NotTo(HaveOccurred())
+
+				res, err := ipPoolManager.AllocateIP(ctx, ipPoolName, nic, podT)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(*res.Nic).To(Equal(nic))
+				Expect(*res.Version).To(Equal(ipVersion))
+				Expect(*res.Address).To(Equal(allocatedIP))
+				Expect(res.IPPool).To(Equal(ipPoolT.Name))
+				Expect(res.Gateway).To(Equal(gateway))
+				Expect(res.Vlan).To(Equal(vlan))
+			})
 		})
 
 		Describe("ReleaseIP", func() {

pkg/ippoolmanager/utils.go

@@ -123,3 +123,17 @@ func (b ByPoolPriority) Less(i, j int) bool {
 
 	return false
 }
+
+// findAllocatedIPFromRecords try to find pod NIC previous allocated IP from the IPPool.Status.AllocatedIPs
+// this function serves for the issue: https://github.com/spidernet-io/spiderpool/issues/2517
+func findAllocatedIPFromRecords(allocatedRecords spiderpoolv2beta1.PoolIPAllocations, nic, namespacedName, podUID string) (previousIP string, hasFound bool) {
+	for tmpIP, poolIPAllocation := range allocatedRecords {
+		if poolIPAllocation.NIC == nic &&
+			poolIPAllocation.NamespacedName == namespacedName &&
+			poolIPAllocation.PodUID == podUID {
+			return tmpIP, true
+		}
+	}
+
+	return "", false
+}

pkg/ippoolmanager/utils_test.go

@@ -11,6 +11,7 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	types2 "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/uuid"
 
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
@@ -290,4 +291,48 @@ var _ = Describe("IPPoolManager-utils", Label("ippool_manager_utils"), func() {
 			Expect(byPoolPriority).Should(Equal(ByPoolPriority{pool2, pool1}))
 		})
 	})
+
+	Context("Test findAllocatedIPFromRecords", func() {
+		var allocatedRecords spiderpoolv2beta1.PoolIPAllocations
+		var ip, nic, namespacedName, podUID string
+
+		BeforeEach(func() {
+			ip = "172.18.40.40"
+			nic = "eth0"
+			namespacedName = "default/testPod"
+			podUID = string(uuid.NewUUID())
+			allocatedRecords = spiderpoolv2beta1.PoolIPAllocations{
+				ip: spiderpoolv2beta1.PoolIPAllocation{
+					NIC:            nic,
+					NamespacedName: namespacedName,
+					PodUID:         podUID,
+				},
+			}
+		})
+
+		It("find previous allocated IP in the records", func() {
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, namespacedName, podUID)
+			Expect(hasFound).To(BeTrue())
+		})
+
+		It("no previous allocated IP in the records", func() {
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "eth1", "kube-system/testPod1", string(uuid.NewUUID()))
+			Expect(hasFound).To(BeFalse())
+		})
+
+		It("no previous allocated IP in records due to different nic", func() {
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, "eth1", namespacedName, podUID)
+			Expect(hasFound).To(BeFalse())
+		})
+
+		It("no previous allocated IP in records due to different NamespacedName", func() {
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, "kube-system/testPod1", podUID)
+			Expect(hasFound).To(BeFalse())
+		})
+
+		It("no previous allocated IP in records due to different podUID", func() {
+			_, hasFound := findAllocatedIPFromRecords(allocatedRecords, nic, namespacedName, string(uuid.NewUUID()))
+			Expect(hasFound).To(BeFalse())
+		})
+	})
 })

test/e2e/common/spiderpool.go

@@ -19,12 +19,10 @@ import (
 	. "github.com/onsi/gomega"
 	frame "github.com/spidernet-io/e2eframework/framework"
 	"github.com/spidernet-io/e2eframework/tools"
-	"github.com/spidernet-io/spiderpool/cmd/spiderpool-agent/cmd"
 	"github.com/spidernet-io/spiderpool/pkg/constant"
 	ip "github.com/spidernet-io/spiderpool/pkg/ip"
 	"github.com/spidernet-io/spiderpool/pkg/types"
 
-	"gopkg.in/yaml.v3"
 	corev1 "k8s.io/api/core/v1"
 	api_errors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -310,39 +308,6 @@ func CheckPodIpRecordInIppool(f *frame.Framework, v4IppoolNameList, v6IppoolName
 	}
 }
 
-func GetClusterDefaultIppool(f *frame.Framework) (v4IppoolList, v6IppoolList []string, e error) {
-	if f == nil {
-		return nil, nil, errors.New("wrong input")
-	}
-
-	configMap, e := f.GetConfigmap(SpiderPoolConfigmapName, SpiderPoolConfigmapNameSpace)
-	if e != nil {
-		return nil, nil, e
-	}
-	GinkgoWriter.Printf("configmap: %+v \n", configMap.Data)
-
-	data, ok := configMap.Data["conf.yml"]
-	if !ok || len(data) == 0 {
-		return nil, nil, errors.New("failed to find cluster default ippool")
-	}
-
-	conf := cmd.Config{}
-	if err := yaml.Unmarshal([]byte(data), &conf); nil != err {
-		GinkgoWriter.Printf("failed to decode yaml config: %v \n", data)
-		return nil, nil, errors.New("failed to find cluster default ippool")
-	}
-	GinkgoWriter.Printf("yaml config: %v \n", conf)
-
-	if conf.EnableIPv4 && len(conf.ClusterDefaultIPv4IPPool) == 0 {
-		return nil, nil, errors.New("IPv4 IPPool is not specified when IPv4 is enabled")
-	}
-	if conf.EnableIPv6 && len(conf.ClusterDefaultIPv6IPPool) == 0 {
-		return nil, nil, errors.New("IPv6 IPPool is not specified when IPv6 is enabled")
-	}
-
-	return conf.ClusterDefaultIPv4IPPool, conf.ClusterDefaultIPv6IPPool, nil
-}
-
 func GetNamespaceDefaultIppool(f *frame.Framework, namespace string) (v4IppoolList, v6IppoolList []string, e error) {
 	ns, err := f.GetNamespace(namespace)
 	if err != nil {