diff --git a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml index 461d27a569..dc6f2df685 100644 --- a/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml +++ b/charts/spiderpool/crds/spiderpool.spidernet.io_spidermultusconfigs.yaml @@ -163,6 +163,7 @@ spec: properties: ibKubernetesEnabled: default: false + description: Enforces ib-sriov-cni to work with ib-kubernetes. type: boolean ippools: description: SpiderpoolPools could specify the IPAM spiderpool @@ -179,17 +180,30 @@ spec: type: object linkState: default: enable + description: 'Enforces link state for the VF. Allowed values: + auto, enable, disable.' enum: - auto - enable - disable type: string pkey: + description: infiniBand pkey for VF, this field is used by ib-kubernetes + to add pkey with guid to InfiniBand subnet manager client e.g. + Mellanox UFM, OpenSM type: string rdmaIsolation: default: true + description: rdmaIsolation enablw RDMA CNI plugin is intended + to be run as a chained CNI plugin. it ensures isolation of RDMA + traffic from other workloads in the system by moving the associated + RDMA interfaces of the provided network interface to the container's + network namespace path. type: boolean resourceName: + description: The SR-IOV RDMA resource name of the SpiderMultusConfig. + the SR-IOV RDMA resource is often reported to kubelet by the + sriov-device-plugin. type: string required: - resourceName @@ -210,11 +224,14 @@ spec: type: array type: object master: + description: name of the host interface to create the link from. type: string type: object ipvlan: properties: bond: + description: Optional bond configuration for the CNI. It must + not be nil if the multiple master interfaces are specified. properties: mode: format: int32 @@ -229,10 +246,6 @@ spec: - mode - name type: object - enableRdma: - default: false - description: enable share rdma for ipvlan - type: boolean ippools: description: SpiderpoolPools could specify the IPAM spiderpool CNI configuration default IPv4&IPv6 pools. @@ -247,15 +260,23 @@ spec: type: array type: object master: + description: The master interface(s) for the CNI configuration. + At least one master interface must be specified. If multiple + master interfaces are specified, the spiderpool will create + a bond device with the bondConfig by the ifacer plugin. items: type: string type: array rdmaResourceName: - description: Resource name of the rdma device-plugin, If it's - empty and enableRdma is true, the value will be auto set by - operator. and the user can also set this value manually. + description: The RDMA resource name of the nic. the RDMA resource + is often reported to kubelet by the k8s-rdma-shared-dev-plugin. + when it is not empty and spiderpool podResourceInject feature + is enabled, spiderpool can automatically inject it into the + container's resources via webhook. type: string vlanID: + description: 'The VLAN ID for the CNI configuration, optional + and must be within the specified range: [0.4096).' format: int32 maximum: 4094 minimum: 0 @@ -266,6 +287,8 @@ spec: macvlan: properties: bond: + description: Optional bond configuration for the CNI. It must + not be nil if the multiple master interfaces are specified. properties: mode: format: int32 @@ -280,10 +303,6 @@ spec: - mode - name type: object - enableRdma: - default: false - description: enable share rdma for macvlan - type: boolean ippools: description: SpiderpoolPools could specify the IPAM spiderpool CNI configuration default IPv4&IPv6 pools. @@ -298,15 +317,23 @@ spec: type: array type: object master: + description: The master interface(s) for the CNI configuration. + At least one master interface must be specified. If multiple + master interfaces are specified, the spiderpool will create + a bond device with the bondConfig by the ifacer plugin. items: type: string type: array rdmaResourceName: - description: Resource name of the rdma device-plugin, If it's - empty and enableRdma is true, the value will be auto set by - operator. and the user can also set this value manually. + description: The RDMA resource name of the nic. the RDMA resource + is often reported to kubelet by the k8s-rdma-shared-dev-plugin. + when it is not empty and spiderpool podResourceInject feature + is enabled, spiderpool can automatically inject it into the + container's resources via webhook. type: string vlanID: + description: 'The VLAN ID for the CNI configuration, optional + and must be within the specified range: [0.4096).' format: int32 maximum: 4094 minimum: 0 @@ -361,6 +388,7 @@ spec: properties: enableRdma: default: false + description: DEPRECATED, use RdmaIsolation flled instead. type: boolean ippools: description: SpiderpoolPools could specify the IPAM spiderpool @@ -376,14 +404,28 @@ spec: type: array type: object maxTxRateMbps: + description: Mbps, 0 = disable rate limiting minimum: 0 type: integer minTxRateMbps: minimum: 0 type: integer + rdmaIsolation: + default: false + description: rdmaIsolation enable RDMA CNI plugin is intended + to be run as a chained CNI plugin. it ensures isolation of RDMA + traffic from other workloads in the system by moving the associated + RDMA interfaces of the provided network interface to the container's + network namespace path. + type: boolean resourceName: + description: The SR-IOV RDMA resource name of the SpiderMultusConfig. + the SR-IOV RDMA resource is often reported to kubelet by the + sriov-device-plugin. type: string vlanID: + description: 'The VLAN ID for the CNI configuration, optional + and must be within the specified range: [0.4096).' format: int32 maximum: 4094 minimum: 0 diff --git a/docs/reference/crd-spidermultusconfig.md b/docs/reference/crd-spidermultusconfig.md index 3ce8f583c2..e294c51ab2 100644 --- a/docs/reference/crd-spidermultusconfig.md +++ b/docs/reference/crd-spidermultusconfig.md @@ -71,6 +71,7 @@ This is the SpiderReservedIP spec for users to configure. | master | the Interfaces on your master, you could specify a single one Interface
or multiple Interfaces to generate one bond Interface | list of strings | required | | | vlanID | vlan ID | int | optional | [0,4094] | | bond | expected bond Interface configurations | [BondConfig](./crd-spidermultusconfig.md#bondconfig) | optional | | +| rdmaResourceName | rdma resource name of the spiderMultusConfig, it often reported to kubelet by the k8s-rdma-shared-dev-plugin. when it is not empty and spiderpool podResourceInject feature is enabled, spiderpool can automatically inject it into the container's resources via webhook | string | optional | | | ippools | the default IPPools in your CNI configurations | [SpiderpoolPools](./crd-spidermultusconfig.md#spiderpoolpools) | optional | | #### SpiderIPvlanCniConfig @@ -80,24 +81,26 @@ This is the SpiderReservedIP spec for users to configure. | master | the Interfaces on your master, you could specify a single one Interface
or multiple Interfaces to generate one bond Interface | list of strings | required | | | vlanID | vlan ID | int | optional | [0,4094] | | bond | expected bond Interface configurations | [BondConfig](./crd-spidermultusconfig.md#bondconfig) | optional | | +| rdmaResourceName | rdma resource name of the spiderMultusConfig, it often reported to kubelet by the k8s-rdma-shared-dev-plugin. when it is not empty and spiderpool podResourceInject feature is enabled, spiderpool can automatically inject it into the container's resources via webhook | string | optional | | | ippools | the default IPPools in your CNI configurations | [SpiderpoolPools](./crd-spidermultusconfig.md#spiderpoolpools) | optional | | #### SpiderSRIOVCniConfig | Field | Description | Schema | Validation | |---------------|-------------------------------------------------------------------------------------------|----------------------------------------------------------------|------------| -| resourceName | this property will create an annotation for Multus net-attach-def to cooperate with SRIOV | string | required | +| resourceName | this property will create an annotation for Multus net-attach-def to cooperate with SRIOV, if spiderpool podResourceInject feature is enabled, spiderpool can automatically inject it into the container's resources via webhook | string | required | | vlanID | vlan ID | int | optional | | minTxRateMbps | change the allowed minimum transmit bandwidth, in Mbps, for the VF. Setting this to 0 disables rate limiting. The min_tx_rate value should be <= max_tx_rate. Support of this feature depends on NICs and drivers | int | optional | | maxTxRateMbps | change the allowed maximum transmit bandwidth, in Mbps, for the VF. Setting this to 0 disables rate limiting | int | optional | -| enableRdma | enable rdma chain cni to isolate the rdma device | bool | optional | +| enableRdma(deprecated) | It will be remove in the furture, use rdmaIsolation instead. | bool | optional | + rdmaIsolation | rdmaIsolation enable RDMA CNI plugin is intended to be run as a chained CNI plugin. it ensures isolation of RDMA traffic from other workloads in the system by moving the associated RDMA interfaces of the provided network interface to the container's network namespace path. | bool | optional | | ippools | the default IPPools in your CNI configurations | [SpiderpoolPools](./crd-spidermultusconfig.md#spiderpoolpools) | optional | #### SpiderIBSRIOVCniConfig | Field | Description | Schema | Validation | |----------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------|------------| -| resourceName | this property will create an annotation for Multus net-attach-def to cooperate with ib-sriov | string | required | +| resourceName | this property will create an annotation for Multus net-attach-def to cooperate with SRIOV, if spiderpool podResourceInject feature is enabled, spiderpool can automatically inject it into the container's resources via webhook | string | required | | pkey | InfiniBand pkey for VF, this field is used by [ib-kubernetes](https://github.com/Mellanox/ib-kubernetes) to add pkey with guid to InfiniBand subnet manager client e.g. [Mellanox UFM](https://www.nvidia.com/en-us/networking/infiniband/ufm/) | string | optional | | linkState | Enforces link state for the VF. Allowed values: auto, enable [default], disable | string | optional | | rdmaIsolation | Enable RDMA network namespace isolation for RDMA workloads, default to true | bool | optional | diff --git a/docs/usage/install/ai/get-started-macvlan-zh_CN.md b/docs/usage/install/ai/get-started-macvlan-zh_CN.md index 6fd0586d3b..5e1edbb8a3 100644 --- a/docs/usage/install/ai/get-started-macvlan-zh_CN.md +++ b/docs/usage/install/ai/get-started-macvlan-zh_CN.md @@ -300,7 +300,6 @@ spidernet.io/shared_cx5_gpu6: 1 spidernet.io/shared_cx5_gpu7: 1 spidernet.io/shared_cx5_gpu8: 1 - #nvidia.com/gpu: 1 EOF $ helm install rdma-tools spiderchart/rdma-tools -f ./values.yaml diff --git a/docs/usage/install/ai/get-started-macvlan.md b/docs/usage/install/ai/get-started-macvlan.md index 8ced42507f..7652971177 100644 --- a/docs/usage/install/ai/get-started-macvlan.md +++ b/docs/usage/install/ai/get-started-macvlan.md @@ -299,7 +299,6 @@ The network planning for the cluster is as follows: spidernet.io/shared_cx5_gpu6: 1 spidernet.io/shared_cx5_gpu7: 1 spidernet.io/shared_cx5_gpu8: 1 - #nvidia.com/gpu: 1 ``` During the creation of the network namespace for the container, Spiderpool will perform connectivity tests on the gateway of the macvlan interface. diff --git a/docs/usage/install/ai/get-started-sriov-zh_CN.md b/docs/usage/install/ai/get-started-sriov-zh_CN.md index 78877db006..e699a4234c 100644 --- a/docs/usage/install/ai/get-started-sriov-zh_CN.md +++ b/docs/usage/install/ai/get-started-sriov-zh_CN.md @@ -360,7 +360,7 @@ Spiderpool 使用了 [sriov-network-operator](https://github.com/k8snetworkplumb cniType: sriov sriov: resourceName: spidernet.io/gpu1sriov - enableRdma: true + rdmaIsolation: true ippools: ipv4: ["gpu1-net11"] EOF @@ -598,10 +598,10 @@ Spiderpool 使用了 [sriov-network-operator](https://github.com/k8snetworkplumb name: ib-sriov namespace: spiderpool spec: - cniType: ib-sriov - ibsriov: - pkey: 1000 - ... + cniType: ib-sriov + ibsriov: + pkey: 1000 + ... EOF ``` diff --git a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go index 1512dc82ec..4a95ac4dee 100644 --- a/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go +++ b/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1/spidermultus_types.go @@ -74,25 +74,25 @@ type MultusCNIConfigSpec struct { type SpiderMacvlanCniConfig struct { // +kubebuilder:validation:Required + // The master interface(s) for the CNI configuration. At least one master interface must be specified. + // If multiple master interfaces are specified, the spiderpool will create a bond device with the bondConfig + // by the ifacer plugin. Master []string `json:"master"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=4094 + // The VLAN ID for the CNI configuration, optional and must be within the specified range: [0.4096). VlanID *int32 `json:"vlanID,omitempty"` // +kubebuilder:validation:Optional + // Optional bond configuration for the CNI. It must not be nil if the multiple master interfaces are specified. Bond *BondConfig `json:"bond,omitempty"` - // +kubebuilder:default=false - // +kubebuilder:validation:Optional - // enable share rdma for macvlan - EnableRdma bool `json:"enableRdma"` - // +kubebuilder:validation:Optional - // Resource name of the rdma device-plugin, If it's empty and enableRdma is true, - // the value will be auto set by operator. and the user can also set this value - // manually. + // The RDMA resource name of the nic. the RDMA resource is often reported to kubelet by the + // k8s-rdma-shared-dev-plugin. when it is not empty and spiderpool podResourceInject feature + // is enabled, spiderpool can automatically inject it into the container's resources via webhook. RdmaResourceName string `json:"rdmaResourceName"` // +kubebuilder:validation:Optional @@ -101,25 +101,25 @@ type SpiderMacvlanCniConfig struct { type SpiderIPvlanCniConfig struct { // +kubebuilder:validation:Required + // The master interface(s) for the CNI configuration. At least one master interface must be specified. + // If multiple master interfaces are specified, the spiderpool will create a bond device with the bondConfig + // by the ifacer plugin. Master []string `json:"master"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=4094 + // The VLAN ID for the CNI configuration, optional and must be within the specified range: [0.4096). VlanID *int32 `json:"vlanID,omitempty"` // +kubebuilder:validation:Optional + // Optional bond configuration for the CNI. It must not be nil if the multiple master interfaces are specified. Bond *BondConfig `json:"bond,omitempty"` - // +kubebuilder:default=false // +kubebuilder:validation:Optional - // enable share rdma for ipvlan - EnableRdma bool `json:"enableRdma"` - - // +kubebuilder:validation:Optional - // Resource name of the rdma device-plugin, If it's empty and enableRdma is true, - // the value will be auto set by operator. and the user can also set this value - // manually. + // The RDMA resource name of the nic. the RDMA resource is often reported to kubelet by the + // k8s-rdma-shared-dev-plugin. when it is not empty and spiderpool podResourceInject feature + // is enabled, spiderpool can automatically inject it into the container's resources via webhook. RdmaResourceName string `json:"rdmaResourceName"` // +kubebuilder:validation:Optional @@ -128,47 +128,73 @@ type SpiderIPvlanCniConfig struct { type SpiderSRIOVCniConfig struct { // +kubebuilder:validation:Required + // The SR-IOV RDMA resource name of the SpiderMultusConfig. the SR-IOV RDMA resource is often + // reported to kubelet by the sriov-device-plugin. ResourceName string `json:"resourceName"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Minimum=0 // +kubebuilder:validation:Maximum=4094 + // The VLAN ID for the CNI configuration, optional and must be within the specified range: [0.4096). VlanID *int32 `json:"vlanID,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Minimum=0 - MinTxRateMbps *int `json:"minTxRateMbps,omitempty"` // Mbps, 0 = disable rate limiting + MinTxRateMbps *int `json:"minTxRateMbps,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:validation:Minimum=0 + // Mbps, 0 = disable rate limiting MaxTxRateMbps *int `json:"maxTxRateMbps,omitempty"` // Mbps, 0 = disable rate limiting // +kubebuilder:default=false // +kubebuilder:validation:Optional + // +kubebuilder:deprecated:reason="This field is deprecated and will be removed in future versions." + // DEPRECATED, use RdmaIsolation flled instead. EnableRdma bool `json:"enableRdma"` + // +kubebuilder:validation:Optional + // +kubebuilder:default=false + // +kubebuilder:validation:Optional + // rdmaIsolation enable RDMA CNI plugin is intended to be run as a chained CNI plugin. + // it ensures isolation of RDMA traffic from other workloads in the system by moving + // the associated RDMA interfaces of the provided network interface to the container's + // network namespace path. + RdmaIsolation bool `json:"rdmaIsolation"` + // +kubebuilder:validation:Optional SpiderpoolConfigPools *SpiderpoolPools `json:"ippools,omitempty"` } type SpiderIBSriovCniConfig struct { // +kubebuilder:validation:Required + // The SR-IOV RDMA resource name of the SpiderMultusConfig. the SR-IOV RDMA resource is often + // reported to kubelet by the sriov-device-plugin. ResourceName string `json:"resourceName"` // +kubebuilder:validation:Optional + // infiniBand pkey for VF, this field is used by ib-kubernetes to add pkey with + // guid to InfiniBand subnet manager client e.g. Mellanox UFM, OpenSM Pkey *string `json:"pkey,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=enable // +kubebuilder:validation:Enum=auto;enable;disable + // Enforces link state for the VF. Allowed values: auto, enable, disable. LinkState *string `json:"linkState,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=true + // +kubebuilder:validation:Optional + // rdmaIsolation enablw RDMA CNI plugin is intended to be run as a chained CNI plugin. + // it ensures isolation of RDMA traffic from other workloads in the system by moving + // the associated RDMA interfaces of the provided network interface to the container's + // network namespace path. RdmaIsolation *bool `json:"rdmaIsolation,omitempty"` // +kubebuilder:validation:Optional // +kubebuilder:default=false + // Enforces ib-sriov-cni to work with ib-kubernetes. IbKubernetesEnabled *bool `json:"ibKubernetesEnabled,omitempty"` // +kubebuilder:validation:Optional @@ -177,6 +203,7 @@ type SpiderIBSriovCniConfig struct { type SpiderIpoibCniConfig struct { // +kubebuilder:validation:Required + // name of the host interface to create the link from. Master string `json:"master,omitempty"` // +kubebuilder:validation:Optional diff --git a/pkg/multuscniconfig/multusconfig_informer.go b/pkg/multuscniconfig/multusconfig_informer.go index 1abab30b15..130c818fb8 100644 --- a/pkg/multuscniconfig/multusconfig_informer.go +++ b/pkg/multuscniconfig/multusconfig_informer.go @@ -418,7 +418,7 @@ func generateNetAttachDef(netAttachName string, multusConf *spiderpoolv2beta1.Sp // SRIOV special annotation anno[constant.ResourceNameAnnot] = multusConfSpec.SriovConfig.ResourceName - if multusConfSpec.SriovConfig.EnableRdma { + if multusConfSpec.SriovConfig.RdmaIsolation { rdmaconf := RdmaNetConf{ Type: "rdma", } diff --git a/pkg/multuscniconfig/multusconfig_validate.go b/pkg/multuscniconfig/multusconfig_validate.go index d72a92e401..4b733c96dd 100644 --- a/pkg/multuscniconfig/multusconfig_validate.go +++ b/pkg/multuscniconfig/multusconfig_validate.go @@ -110,7 +110,7 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } if injectRdmaResource { - if err := ValidateRdmaResouce(multusConfig.Spec.MacvlanConfig.EnableRdma, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil { + if err := ValidateRdmaResouce(multusConfig.Spec.MacvlanConfig.RdmaResourceName != "", multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.MacvlanConfig.RdmaResourceName, multusConfig.Spec.MacvlanConfig.SpiderpoolConfigPools); err != nil { return field.Invalid(macvlanConfigField, *multusConfig.Spec.MacvlanConfig, err.Error()) } } @@ -135,7 +135,7 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } if injectRdmaResource { - if err := ValidateRdmaResouce(multusConfig.Spec.IPVlanConfig.EnableRdma, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IPVlanConfig.RdmaResourceName, multusConfig.Spec.IPVlanConfig.SpiderpoolConfigPools); err != nil { + if err := ValidateRdmaResouce(multusConfig.Spec.IPVlanConfig.RdmaResourceName != "", multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IPVlanConfig.RdmaResourceName, multusConfig.Spec.IPVlanConfig.SpiderpoolConfigPools); err != nil { return field.Invalid(ipvlanConfigField, *multusConfig.Spec.IPVlanConfig, err.Error()) } } @@ -166,7 +166,7 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } if injectRdmaResource { - if err := ValidateRdmaResouce(multusConfig.Spec.SriovConfig.EnableRdma, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.SriovConfig.ResourceName, multusConfig.Spec.SriovConfig.SpiderpoolConfigPools); err != nil { + if err := ValidateRdmaResouce(multusConfig.Spec.SriovConfig.RdmaIsolation, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.SriovConfig.ResourceName, multusConfig.Spec.SriovConfig.SpiderpoolConfigPools); err != nil { return field.Invalid(sriovConfigField, *multusConfig.Spec.SriovConfig, err.Error()) } } @@ -185,7 +185,7 @@ func validateCNIConfig(multusConfig *spiderpoolv2beta1.SpiderMultusConfig) *fiel } if injectRdmaResource { - if err := ValidateRdmaResouce(true, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IbSriovConfig.ResourceName, multusConfig.Spec.IbSriovConfig.SpiderpoolConfigPools); err != nil { + if err := ValidateRdmaResouce(*multusConfig.Spec.IbSriovConfig.RdmaIsolation, multusConfig.Name, multusConfig.Namespace, multusConfig.Spec.IbSriovConfig.ResourceName, multusConfig.Spec.IbSriovConfig.SpiderpoolConfigPools); err != nil { return field.Invalid(ibsriovConfigField, *multusConfig.Spec.IbSriovConfig, err.Error()) } } diff --git a/pkg/multuscniconfig/utils.go b/pkg/multuscniconfig/utils.go index ee8fb0151d..b280a023b2 100644 --- a/pkg/multuscniconfig/utils.go +++ b/pkg/multuscniconfig/utils.go @@ -230,11 +230,11 @@ func ResourceName(smc *spiderpoolv2beta1.SpiderMultusConfig) string { switch *smc.Spec.CniType { case constant.MacvlanCNI: // For Macvlan CNI, return RDMA resource name if RDMA is enabled - if smc.Spec.MacvlanConfig != nil && smc.Spec.MacvlanConfig.EnableRdma { + if smc.Spec.MacvlanConfig != nil { return smc.Spec.MacvlanConfig.RdmaResourceName } case constant.IPVlanCNI: - if smc.Spec.IPVlanConfig != nil && smc.Spec.IPVlanConfig.EnableRdma { + if smc.Spec.IPVlanConfig != nil { return smc.Spec.IPVlanConfig.RdmaResourceName } case constant.SriovCNI: diff --git a/pkg/podmanager/utils.go b/pkg/podmanager/utils.go index 40b45c6677..cf050b0921 100644 --- a/pkg/podmanager/utils.go +++ b/pkg/podmanager/utils.go @@ -181,12 +181,15 @@ func DoValidateRdmaResouce(mc v2beta1.SpiderMultusConfig) error { spec := mc.Spec switch *spec.CniType { case constant.MacvlanCNI: - return multuscniconfig.ValidateRdmaResouce(spec.MacvlanConfig.EnableRdma, mc.Name, mc.Namespace, spec.MacvlanConfig.RdmaResourceName, spec.MacvlanConfig.SpiderpoolConfigPools) + return multuscniconfig.ValidateRdmaResouce(spec.MacvlanConfig.RdmaResourceName != "", mc.Name, mc.Namespace, spec.MacvlanConfig.RdmaResourceName, spec.MacvlanConfig.SpiderpoolConfigPools) case constant.IPVlanCNI: - return multuscniconfig.ValidateRdmaResouce(spec.IPVlanConfig.EnableRdma, mc.Name, mc.Namespace, spec.IPVlanConfig.RdmaResourceName, spec.IPVlanConfig.SpiderpoolConfigPools) + return multuscniconfig.ValidateRdmaResouce(spec.IPVlanConfig.RdmaResourceName != "", mc.Name, mc.Namespace, spec.IPVlanConfig.RdmaResourceName, spec.IPVlanConfig.SpiderpoolConfigPools) case constant.SriovCNI: - return multuscniconfig.ValidateRdmaResouce(spec.SriovConfig.EnableRdma, mc.Name, mc.Namespace, spec.SriovConfig.ResourceName, spec.SriovConfig.SpiderpoolConfigPools) + return multuscniconfig.ValidateRdmaResouce(spec.SriovConfig.RdmaIsolation, mc.Name, mc.Namespace, spec.SriovConfig.ResourceName, spec.SriovConfig.SpiderpoolConfigPools) case constant.IBSriovCNI: + if spec.IbSriovConfig.RdmaIsolation == nil || !*spec.IbSriovConfig.RdmaIsolation { + return fmt.Errorf("spidermultusconfig %s/%s not enable RDMA", mc.Namespace, mc.Name) + } return multuscniconfig.ValidateRdmaResouce(true, mc.Name, mc.Namespace, spec.IbSriovConfig.ResourceName, spec.IbSriovConfig.SpiderpoolConfigPools) case constant.IPoIBCNI: return multuscniconfig.ValidateRdmaResouce(true, mc.Name, mc.Namespace, spec.IpoibConfig.Master, spec.IpoibConfig.SpiderpoolConfigPools) diff --git a/pkg/podmanager/utils_test.go b/pkg/podmanager/utils_test.go index a535a264ee..2e866e62d9 100644 --- a/pkg/podmanager/utils_test.go +++ b/pkg/podmanager/utils_test.go @@ -115,7 +115,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource1", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -131,7 +130,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource2", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -160,7 +158,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource1", }, }, @@ -173,7 +170,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource2", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -199,8 +195,7 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: false, - RdmaResourceName: "spidernet.io/rdma-resource1", + RdmaResourceName: "", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, }, @@ -215,7 +210,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource2", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -247,7 +241,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource1", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -263,7 +256,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To("macvlan"), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma-resource2", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test1"}, @@ -297,7 +289,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "rdma-resource", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, @@ -314,7 +305,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ - EnableRdma: false, RdmaResourceName: "", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, @@ -333,7 +323,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.IPVlanCNI), IPVlanConfig: &v2beta1.SpiderIPvlanCniConfig{ - EnableRdma: true, RdmaResourceName: "rdma-resource", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, @@ -350,7 +339,6 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.IPVlanCNI), IPVlanConfig: &v2beta1.SpiderIPvlanCniConfig{ - EnableRdma: false, RdmaResourceName: "", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, @@ -369,8 +357,8 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.SriovCNI), SriovConfig: &v2beta1.SpiderSRIOVCniConfig{ - EnableRdma: true, - ResourceName: "rdma-resource", + RdmaIsolation: true, + ResourceName: "rdma-resource", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, @@ -386,8 +374,8 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.SriovCNI), SriovConfig: &v2beta1.SpiderSRIOVCniConfig{ - EnableRdma: false, - ResourceName: "", + RdmaIsolation: false, + ResourceName: "", SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, @@ -405,7 +393,8 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { Spec: v2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.IBSriovCNI), IbSriovConfig: &v2beta1.SpiderIBSriovCniConfig{ - ResourceName: "rdma-resource", + ResourceName: "rdma-resource", + RdmaIsolation: ptr.To(true), SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, @@ -422,6 +411,7 @@ var _ = Describe("PodManager utils", Label("pod_manager_utils_test"), func() { CniType: ptr.To(constant.IBSriovCNI), IbSriovConfig: &v2beta1.SpiderIBSriovCniConfig{ ResourceName: "", + SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, diff --git a/test/e2e/podwebhook/podwebhook_test.go b/test/e2e/podwebhook/podwebhook_test.go index eb9a8ec018..f1491974d7 100644 --- a/test/e2e/podwebhook/podwebhook_test.go +++ b/test/e2e/podwebhook/podwebhook_test.go @@ -53,7 +53,6 @@ var _ = Describe("Podwebhook", func() { CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &v2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: true, RdmaResourceName: "spidernet.io/rdma_resource" + "_" + name, SpiderpoolConfigPools: &v2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test-ipv4-pool"}, diff --git a/test/e2e/spidermultus/spidermultus_test.go b/test/e2e/spidermultus/spidermultus_test.go index 07a2c0728f..ed6b5dfbae 100644 --- a/test/e2e/spidermultus/spidermultus_test.go +++ b/test/e2e/spidermultus/spidermultus_test.go @@ -546,8 +546,8 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { Spec: spiderpoolv2beta1.MultusCNIConfigSpec{ CniType: ptr.To(constant.SriovCNI), SriovConfig: &spiderpoolv2beta1.SpiderSRIOVCniConfig{ - ResourceName: "spidernet.io/mellanoxrdma", - EnableRdma: true, + ResourceName: "spidernet.io/mellanoxrdma", + RdmaIsolation: true, }, }, } @@ -778,7 +778,6 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: true, RdmaResourceName: "test", SpiderpoolConfigPools: &spiderpoolv2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, @@ -809,8 +808,7 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: false, - RdmaResourceName: "test", + RdmaResourceName: "", SpiderpoolConfigPools: &spiderpoolv2beta1.SpiderpoolPools{ IPv4IPPool: []string{"test"}, }, @@ -840,7 +838,6 @@ var _ = Describe("test spidermultus", Label("SpiderMultusConfig"), func() { CniType: ptr.To(constant.MacvlanCNI), MacvlanConfig: &spiderpoolv2beta1.SpiderMacvlanCniConfig{ Master: []string{common.NIC1}, - EnableRdma: true, RdmaResourceName: "test", }, EnableCoordinator: ptr.To(true),