Skip to content

Commit

Permalink
add kubevirt e2e
Browse files Browse the repository at this point in the history 
Signed-off-by: Icarus9913 <[email protected]>
  • Loading branch information
@Icarus9913
Icarus9913 committed Oct 19, 2023
1 parent 7a76a2b commit 20459b9
Show file tree
Hide file tree
Showing 21 changed files with 882 additions and 38 deletions.
1 change: 1 addition & 0 deletions .github/codespell-ignorewords
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,3 +4,4 @@ keypair
cyclinder
shouldnot
Requestor
passt
4 changes: 3 additions & 1 deletion .github/workflows/e2e-init.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,8 @@ jobs:
-e SPIDERPOOL_AGENT_IMAGE_NAME=spiderpool-agent-race \
-e SPIDERPOOL_CONTROLLER_IMAGE_NAME=spiderpool-controller-race \
-e E2E_IP_FAMILY=${{ inputs.ip_family }} -e PYROSCOPE_LOCAL_PORT="" \
-e E2E_KIND_IMAGE_TAG=${{ inputs.k8s_version }}
-e E2E_KIND_IMAGE_TAG=${{ inputs.k8s_version }} \
-e INSTALL_KUBEVIRT=true
- name: Run e2e Test
id: run_e2e
Expand All @@ -137,6 +138,7 @@ jobs:
make ${{ matrix.e2e_test_mode }} -e E2E_CLUSTER_NAME=${{ env.E2E_CLUSTER_NAME }} \
-e E2E_GINKGO_LABELS=${E2E_LABELS} \
-e E2E_TIMEOUT=${{ env.E2E_TIME_OUT }} \
-e INSTALL_MULTUS=true \
-e E2E_IP_FAMILY=${{ inputs.ip_family }} || RESULT=1
if ((RESULT==0)) ; then
echo "RUN_E2E_PASS=true" >> $GITHUB_ENV
Expand Down
37 changes: 21 additions & 16 deletions charts/spiderpool/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,22 +126,27 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \

### ipam parameters

| Name | Description | Value |
| -------------------------------------- | ------------------------------------------------------------------------------------------------ | ------- |
| `ipam.enableIPv4` | enable ipv4 | `true` |
| `ipam.enableIPv6` | enable ipv6 | `true` |
| `ipam.enableStatefulSet` | the network mode | `true` |
| `ipam.enableKubevirtStaticIP` | the feature to keep kubevirt vm pod static IP | `true` |
| `ipam.enableSpiderSubnet` | SpiderSubnet feature gate. | `true` |
| `ipam.subnetDefaultFlexibleIPNumber` | the default flexible IP number of SpiderSubnet feature auto-created IPPools | `1` |
| `ipam.gc.enabled` | enable retrieve IP in spiderippool CR | `true` |
| `ipam.gc.gcAll.intervalInSecond` | the gc all interval duration | `600` |
| `ipam.gc.GcDeletingTimeOutPod.enabled` | enable retrieve IP for the pod who times out of deleting graceful period | `true` |
| `ipam.gc.GcDeletingTimeOutPod.delay` | the gc delay seconds after the pod times out of deleting graceful period | `0` |
| `grafanaDashboard.install` | install grafanaDashboard for spiderpool. This requires the grafana operator CRDs to be available | `false` |
| `grafanaDashboard.namespace` | the grafanaDashboard namespace. Default to the namespace of helm instance | `""` |
| `grafanaDashboard.annotations` | the additional annotations of spiderpool grafanaDashboard | `{}` |
| `grafanaDashboard.labels` | the additional label of spiderpool grafanaDashboard | `{}` |
| Name | Description | Value |
| -------------------------------------- | --------------------------------------------------------------------------- | ------ |
| `ipam.enableIPv4` | enable ipv4 | `true` |
| `ipam.enableIPv6` | enable ipv6 | `true` |
| `ipam.enableStatefulSet` | the network mode | `true` |
| `ipam.enableKubevirtStaticIP` | the feature to keep kubevirt vm pod static IP | `true` |
| `ipam.enableSpiderSubnet` | SpiderSubnet feature gate. | `true` |
| `ipam.subnetDefaultFlexibleIPNumber` | the default flexible IP number of SpiderSubnet feature auto-created IPPools | `1` |
| `ipam.gc.enabled` | enable retrieve IP in spiderippool CR | `true` |
| `ipam.gc.gcAll.intervalInSecond` | the gc all interval duration | `600` |
| `ipam.gc.GcDeletingTimeOutPod.enabled` | enable retrieve IP for the pod who times out of deleting graceful period | `true` |
| `ipam.gc.GcDeletingTimeOutPod.delay` | the gc delay seconds after the pod times out of deleting graceful period | `0` |

### grafanaDashboard parameters

| Name | Description | Value |
| ------------------------------ | ------------------------------------------------------------------------------------------------ | ------- |
| `grafanaDashboard.install` | install grafanaDashboard for spiderpool. This requires the grafana operator CRDs to be available | `false` |
| `grafanaDashboard.namespace` | the grafanaDashboard namespace. Default to the namespace of helm instance | `""` |
| `grafanaDashboard.annotations` | the additional annotations of spiderpool grafanaDashboard | `{}` |
| `grafanaDashboard.labels` | the additional label of spiderpool grafanaDashboard | `{}` |

### coordinator parameters

Expand Down
2 changes: 1 addition & 1 deletion cmd/spiderpool-agent/cmd/coordinator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,6 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
if ok {
_, err := kubevirtMgr.GetVMIMByName(ctx, pod.Namespace, vmimName, false)
if nil != err {
// TODO (Icarus9913): should we still cancel the IP conflict detection for no VMIM pod ?
if apierrors.IsNotFound(err) {
logger.Sugar().Warnf("no kubevirt vm pod '%s/%s' corresponding VirtualMachineInstanceMigration '%s/%s' found, still execute IP conflict detection",
pod.Namespace, pod.Name, pod.Namespace, vmimName)
Expand All @@ -90,6 +89,7 @@ func (g *_unixGetCoordinatorConfig) Handle(params daemonset.GetCoordinatorConfig
}
} else {
// cancel IP conflict detection because there's a moment the old vm pod still running during the vm live migration phase
logger.Sugar().Infof("cancel IP conflict detection for live migration new pod '%s/%s'", pod.Namespace, pod.Name)
detectIPConflict = false
}
}
Expand Down
14 changes: 8 additions & 6 deletions cmd/spiderpool-agent/cmd/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,12 +343,14 @@ func initAgentServiceManagers(ctx context.Context) {
}
agentContext.StsManager = statefulSetManager

logger.Debug("Begin to initialize Kubevirt manager")
kubevirtManager := kubevirtmanager.NewKubevirtManager(
agentContext.CRDManager.GetClient(),
agentContext.CRDManager.GetAPIReader(),
)
agentContext.KubevirtManager = kubevirtManager
if agentContext.Cfg.EnableKubevirtStaticIP {
logger.Debug("Begin to initialize Kubevirt manager")
kubevirtManager := kubevirtmanager.NewKubevirtManager(
agentContext.CRDManager.GetClient(),
agentContext.CRDManager.GetAPIReader(),
)
agentContext.KubevirtManager = kubevirtManager
}

logger.Debug("Begin to initialize Endpoint manager")
endpointManager, err := workloadendpointmanager.NewWorkloadEndpointManager(
Expand Down
14 changes: 8 additions & 6 deletions cmd/spiderpool-controller/cmd/daemon.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -262,12 +262,14 @@ func initControllerServiceManagers(ctx context.Context) {
}
controllerContext.StsManager = statefulSetManager

logger.Debug("Begin to initialize Kubevirt manager")
kubevirtManager := kubevirtmanager.NewKubevirtManager(
controllerContext.CRDManager.GetClient(),
controllerContext.CRDManager.GetAPIReader(),
)
controllerContext.KubevirtManager = kubevirtManager
if controllerContext.Cfg.EnableKubevirtStaticIP {
logger.Debug("Begin to initialize Kubevirt manager")
kubevirtManager := kubevirtmanager.NewKubevirtManager(
controllerContext.CRDManager.GetClient(),
controllerContext.CRDManager.GetAPIReader(),
)
controllerContext.KubevirtManager = kubevirtManager
}

logger.Debug("Begin to initialize Endpoint manager")
endpointManager, err := workloadendpointmanager.NewWorkloadEndpointManager(
Expand Down
1 change: 1 addition & 0 deletions docs/mkdocs.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ nav:
- Plugin ifacer: usage/ifacer.md
- node-based topology: usage/network-topology.md
- RDMA: usage/rdma.md
- Kubevirt: usage/kubevirt.md
- FAQ: usage/debug.md
- Concepts:
- Architecture: concepts/arch.md
Expand Down
174 changes: 174 additions & 0 deletions docs/usage/kubevirt-zh_CN.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
# Kubevirt

**简体中文** | [**English**](./kubevirt.md)

## 介绍

*Spiderpool 能保证 kubevirt vm 的 Pod 在重启、重建场景下，持续获取到相同的 IP 地址。*

## Kubevirt 网络搭配

Spiderpool underlay 网络解决方案可给 Kubevirt 赋予介入 underlay 的能力:

1. 对于 Kubevirt 的 Passt 网络模式，可搭配 Spiderpool macvlan 集成方案使用。在该网络模式下，**支持** Service Mesh 的所有功能，不过只能使用**单网卡**，且不支持热迁移。

2. 对于 Kubevirt 的 Bridge 网络模式，可搭配 OVS CNI 使用。在该网络模式下，**不支持** Service Mesh 功能，可使用**多网卡**，不支持热迁移。

## Kubevirt VM 固定地址

Kubevirt VM 会在以下一些场景中会出现固定地址的使用：

1. VM 的热迁移，期望迁移过后的 VM 仍能继承之前的 IP 地址。

2. VM 资源对应的 Pod 出现了重启的情况。

3. VM 资源对应的 VMI(VirtualMachineInstance) 资源被删除的情景。

此外，Kubevirt VM 固定 IP 地址与 StatefulSet 的表现形式是不一样的：

1. 对于 VM ，Pod 重启前后，其 Pod 的名字是会发生变化的，但是其对应的 VMI 不论重启与否，其名字都不会发生变化。因此，我们将会以 VM 为单位来记录其固定的 IP 地址(我们的 SpiderEndpoint 资源将会继承使用 VM 资源的命名空间以及名字)。

2. 对于 StatefulSet，Pod 副本重启前后，其 Pod 名保持不变，我们 Spiderpool 会因此以 Pod 为单位来记录其固定的 IP 地址。

> Notice: 该功能默认开启。若开启，无任何限制， VM 可通过有限 IP 地址集合的 IP 池来固化 IP 的范围，但是，无论 VM 是否使用固定的 IP 池，它的 Pod 都可以持续分到相同 IP。 若关闭，VM 对应的 Pod 将被当作无状态对待，使用 Helm 安装 Spiderpool 时，可通过`--set ipam.enableKubevirtStaticIP=false` 关闭。
## 实施要求

1. 一套 Kubernetes 集群。

2. 已安装 [Helm](https://helm.sh/docs/intro/install/)。

## 步骤

以下流程将会演示 Kubevirt 的 Passt 网络模式搭配 macvlan CNI 以使得 VM 获得 underlay 接入能力，并通过 spiderpool 实现分配固定 IP 的功能。

> Notice：当前 macvlan 和 ipvlan 并不适用于 Kubevirt 的 bridge 网络模式，因为对于 bridge 网络模式会将 pod 网卡的 MAC 地址移动到VM，使得 Pod 使用另一个不同的地址。而 macvlan 和 ipvlan CNI 要求 Pod 的网卡接口具有原始 MAC 地址。
### 安装 Spiderpool

请参考 [Macvlan Quick Start](./install/underlay/get-started-macvlan-zh_CN.md)

### 创建 Kubevirt VM 应用

以下的示例 Yaml 中， 会创建 1 个 Kubevirt VM 应用 ，其中：

- `v1.multus-cni.io/default-network`：为应用选择一张默认网卡的 CNI 配置。

```bash
apiVersion: kubevirt.io/v1
kind: VirtualMachine
metadata:
name: vm-cirros
labels:
kubevirt.io/vm: vm-cirros
spec:
runStrategy: Always
template:
metadata:
annotations:
v1.multus-cni.io/default-network: kube-system/macvlan-ens192
labels:
kubevirt.io/vm: vm-cirros
spec:
domain:
devices:
disks:
- name: containerdisk
disk:
bus: virtio
- name: cloudinitdisk
disk:
bus: virtio
interfaces:
- name: default
passt: {}
resources:
requests:
memory: 64M
networks:
- name: default
pod: {}
volumes:
- name: containerdisk
containerDisk:
image: quay.io/kubevirt/cirros-container-disk-demo
- name: cloudinitdisk
cloudInitNoCloud:
userData: |
#!/bin/sh
echo 'printed from cloud-init userdata'
```

最终，在 Kubevirt VM 应用被创建时，Spiderpool 会从指定 IPPool 中随机选择一个 IP 来与应用形成绑定关系。

```bash
~# kubectl get spiderippool
NAME VERSION SUBNET ALLOCATED-IP-COUNT TOTAL-IP-COUNT DEFAULT
test-ippool 4 10.6.0.0/16 1 10 false

~# kubectl get po -l vm.kubevirt.io/name=vm-cirros -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
virt-launcher-vm-cirros-rg6fs 2/2 Running 0 3m43s 10.6.168.105 node2 <none> 1/1
```

重启 Kubevirt VM Pod, 观察到新的 Pod 的 IP 不会变化，符合预期。

```bash
~# kubectl delete pod virt-launcher-vm-cirros-rg6fs
pod "virt-launcher-vm-cirros-rg6fs" deleted

~# kubectl get po -l vm.kubevirt.io/name=vm-cirros -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
virt-launcher-vm-cirros-d68l2 2/2 Running 0 1m21s 10.6.168.105 node2 <none> 1/1
```

重启 Kubevirt VMI，观察到后续新的 Pod 的IP 也不会变化，符合预期。

```bash
~# kubectl delete vmi vm-cirros
virtualmachineinstance.kubevirt.io "vm-cirros" deleted

~# kubectl get po -l vm.kubevirt.io/name=vm-cirros -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
virt-launcher-vm-cirros-jjgrl 2/2 Running 0 104s 10.6.168.105 node2 <none> 1/1
```

VM 也可与其他 underlay Pod 的通信。

```bash
~# kubectl get po -o wide
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
daocloud-2048-5855b45f44-bvmdr 1/1 Running 0 5m55s 10.6.168.108 spider-worker <none> <none>

~# kubectl virtctl console vm-cirros
$ ping -c 1 10.6.168.108
PING 10.6.168.108 (10.6.168.108): 56 data bytes
64 bytes from 10.6.168.108: seq=0 ttl=255 time=70.554 ms

--- 10.6.168.108 ping statistics ---
1 packets transmitted, 1 packets received, 0% packet loss
round-trip min/avg/max = 70.554/70.554/70.554 ms
```

VM 也可访问 cluster IP。

```bash
~# kubectl get svc -o wide
NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR
daocloud-2048-svc ClusterIP 10.233.36.38 <none> 80/TCP 3m50s app=daocloud-2048

~# curl -I 10.233.36.38:80
HTTP/1.1 200 OK
Server: nginx/1.10.1
Date: Tue, 17 Oct 2023 06:50:04 GMT
Content-Type: text/html
Content-Length: 4090
Last-Modified: Tue, 17 Oct 2023 06:40:53 GMT
Connection: keep-alive
ETag: "652e2c75-ffa"
Accept-Ranges: bytes
```

## 总结

Spiderpool 能保证 Kubevirt VM Pod 在重启、重建场景下，持续获取到相同的 IP 地址，能很好的满足 Kubevirt 虚拟机的固定 IP 需求。并可配合 macvlan 或 OVS CNI 与 Kubevirt 的多种网络模式实现 VM underlay 接入能力。
3 changes: 3 additions & 0 deletions docs/usage/kubevirt.md
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Kubevirt

**English** ｜ [**简体中文**](./kubevirt-zh_CN.md)
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ require (
kubevirt.io/api v1.0.0
sigs.k8s.io/controller-runtime v0.16.1
sigs.k8s.io/controller-tools v0.11.4
sigs.k8s.io/yaml v1.3.0
)

require (
Expand Down Expand Up @@ -179,5 +180,4 @@ require (
kubevirt.io/controller-lifecycle-operator-sdk/api v0.0.0-20220329064328-f3cc58c6ed90 // indirect
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
)
2 changes: 1 addition & 1 deletion pkg/ipam/release.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ func (i *ipam) releaseForAllNICs(ctx context.Context, uid, nic string, endpoint
}
}

// Check whether the kubevirt VM pod needs to keep its IP allocation.
// Check whether the kubevirt VM pod needs to keep its IP allocation.
if i.config.EnableKubevirtStaticIP && endpoint.Status.OwnerControllerType == constant.KindKubevirtVMI {
isValidVMPod, err := i.kubevirtManager.IsValidVMPod(ctx, endpoint.Namespace, endpoint.Status.OwnerControllerType, endpoint.Status.OwnerControllerName)
if nil != err {
Expand Down
10 changes: 10 additions & 0 deletions test/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ ifeq ($(INSTALL_MULTUS),true)
MULTUS_DEFAULT_CNI_VLAN200=$(MULTUS_DEFAULT_CNI_VLAN200) \
MULTUS_ADDITIONAL_CNI_VLAN100=$(MULTUS_ADDITIONAL_CNI_VLAN100) \
MULTUS_ADDITIONAL_CNI_VLAN200=$(MULTUS_ADDITIONAL_CNI_VLAN200) \
MULTUS_KUBEVIRT_CNI_VLAN30=$(MULTUS_KUBEVIRT_CNI_VLAN30) \
MULTUS_KUBEVIRT_CNI_VLAN40=$(MULTUS_KUBEVIRT_CNI_VLAN40) \
INSTALL_KUBEVIRT=$(INSTALL_KUBEVIRT) \
RELEASE_NAMESPACE=$(RELEASE_NAMESPACE) \
CLUSTER_PATH=$(CLUSTER_DIR)/$(E2E_CLUSTER_NAME) \
E2E_SPIDERPOOL_ENABLE_SUBNET=${E2E_SPIDERPOOL_ENABLE_SUBNET} \
Expand All @@ -75,6 +78,13 @@ ifeq ($(INSTALL_SPIDERDOCTOR),true)
E2E_SPIDERDOCTOR_IMAGE_REPO=$(E2E_SPIDERDOCTOR_IMAGE_REPO) \
HTTP_PROXY=$(HTTP_PROXY) \
$(QUIET) bash scripts/install-spiderdoctor.sh
endif
ifeq ($(INSTALL_KUBEVIRT),true)
@echo -e "\033[35m [Step 10] Install kubevirt \033[0m"
E2E_KUBECONFIG=$(E2E_KUBECONFIG) \
E2E_CLUSTER_NAME=$(E2E_CLUSTER_NAME) \
HTTP_PROXY=$(HTTP_PROXY) \
$(QUIET) bash scripts/install-kubevirt.sh
endif
@ echo "wait for the cluster ready" ; \
TEST_IMAGE_NAME=$(TEST_IMAGE_NAME) \
Expand Down
4 changes: 4 additions & 0 deletions test/Makefile.defs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,8 @@ INSTALL_NETTOOLS ?= false

INSTALL_SPIDERDOCTOR ?= true

INSTALL_KUBEVIRT ?= false

CALICO_VERSION ?= v3.25.0

CNI_PACKAGE_VERSION ?= v1.3.0
Expand Down Expand Up @@ -113,6 +115,8 @@ MULTUS_DEFAULT_CNI_CILIUM := cilium
MULTUS_DEFAULT_CNI_VLAN0 := macvlan-vlan0
MULTUS_DEFAULT_CNI_VLAN100 := macvlan-vlan100
MULTUS_DEFAULT_CNI_VLAN200 := macvlan-vlan200
MULTUS_KUBEVIRT_CNI_VLAN30 := kubevirt-macvlan-vlan30
MULTUS_KUBEVIRT_CNI_VLAN40 := kubevirt-macvlan-vlan40

ifeq ($(E2E_CHINA_IMAGE_REGISTRY),true)
E2E_MULTUS_IMAGE_REGISTER ?= ghcr.m.daocloud.io
Expand Down
5 changes: 3 additions & 2 deletions test/doc/kubevirt.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@

| Case ID | Title | Priority | Smoke | Status | Other |
|---------|-------------------------------------------------------------------------------|----------|-------|--------|-------|
| F00001 | Succeed to keep static IP for kubevirt VM/VMI after restarting the VM/VMI pod | P1 | | | |
| F00002 | Succeed to keep static IP for the kubevirt VM live migration | P1 | | | |
| F00001 | Succeed to keep static IP for kubevirt VM/VMI after restarting the VM/VMI pod | P1 | | done | |
| F00002 | Succeed to keep static IP for the kubevirt VM live migration | P1 | | done | |
| F00003 | Succeed to allocation multiple NICs | P1 | | done | |
Loading

0 comments on commit 20459b9

Please sign in to comment.