Skip to content

Commit

Permalink
Merge pull request #2516 from cyclinder/init_pod/wait_spideragent
Browse files Browse the repository at this point in the history 
init-pod: add readiness for wait spiderpool-agent pods ready
  • Loading branch information
@weizhoublue
weizhoublue authored Nov 2, 2023
2 parents 80472ec + c14e776 commit 5d26a95
Show file tree
Hide file tree
Showing 9 changed files with 86 additions and 32 deletions.
48 changes: 25 additions & 23 deletions charts/spiderpool/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -368,29 +368,31 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \

### spiderpoolInit parameters

| Name | Description | Value |
| ------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
| `spiderpoolInit.name` | the init job for installing default spiderippool | `spiderpool-init` |
| `spiderpoolInit.binName` | the binName name of spiderpoolInit | `/usr/bin/spiderpool-init` |
| `spiderpoolInit.hostnetwork` | enable hostnetwork mode of spiderpoolInit pod. Notice, if no CNI available before spiderpool installation, must enable this | `true` |
| `spiderpoolInit.image.registry` | the image registry of spiderpoolInit | `ghcr.io` |
| `spiderpoolInit.image.repository` | the image repository of spiderpoolInit | `spidernet-io/spiderpool/spiderpool-controller` |
| `spiderpoolInit.image.pullPolicy` | the image pullPolicy of spiderpoolInit | `IfNotPresent` |
| `spiderpoolInit.image.digest` | the image digest of spiderpoolInit, which takes preference over tag | `""` |
| `spiderpoolInit.image.tag` | the image tag of spiderpoolInit, overrides the image tag whose default is the chart appVersion. | `""` |
| `spiderpoolInit.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolInit | `[]` |
| `spiderpoolInit.priorityClassName` | the priority Class Name for spiderpoolInit | `system-node-critical` |
| `spiderpoolInit.affinity` | the affinity of spiderpoolInit | `{}` |
| `spiderpoolInit.extraArgs` | the additional arguments of spiderpoolInit container | `[]` |
| `spiderpoolInit.resources.limits.cpu` | the cpu limit of spiderpoolInit pod | `200m` |
| `spiderpoolInit.resources.limits.memory` | the memory limit of spiderpoolInit pod | `256Mi` |
| `spiderpoolInit.resources.requests.cpu` | the cpu requests of spiderpoolInit pod | `100m` |
| `spiderpoolInit.resources.requests.memory` | the memory requests of spiderpoolInit pod | `128Mi` |
| `spiderpoolInit.extraEnv` | the additional environment variables of spiderpoolInit container | `[]` |
| `spiderpoolInit.securityContext` | the security Context of spiderpoolInit pod | `{}` |
| `spiderpoolInit.podAnnotations` | the additional annotations of spiderpoolInit pod | `{}` |
| `spiderpoolInit.podLabels` | the additional label of spiderpoolInit pod | `{}` |
| `spiderpoolInit.serviceAccount.annotations` | the annotations of spiderpoolInit service account | `{}` |
| Name | Description | Value |
| ------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------- |
| `spiderpoolInit.name` | the init job for installing default spiderippool | `spiderpool-init` |
| `spiderpoolInit.binName` | the binName name of spiderpoolInit | `/usr/bin/spiderpool-init` |
| `spiderpoolInit.hostnetwork` | enable hostnetwork mode of spiderpoolInit pod. Notice, if no CNI available before spiderpool installation, must enable this | `true` |
| `spiderpoolInit.image.registry` | the image registry of spiderpoolInit | `ghcr.io` |
| `spiderpoolInit.image.repository` | the image repository of spiderpoolInit | `spidernet-io/spiderpool/spiderpool-controller` |
| `spiderpoolInit.image.pullPolicy` | the image pullPolicy of spiderpoolInit | `IfNotPresent` |
| `spiderpoolInit.image.digest` | the image digest of spiderpoolInit, which takes preference over tag | `""` |
| `spiderpoolInit.image.tag` | the image tag of spiderpoolInit, overrides the image tag whose default is the chart appVersion. | `""` |
| `spiderpoolInit.image.imagePullSecrets` | the image imagePullSecrets of spiderpoolInit | `[]` |
| `spiderpoolInit.priorityClassName` | the priority Class Name for spiderpoolInit | `system-node-critical` |
| `spiderpoolInit.affinity` | the affinity of spiderpoolInit | `{}` |
| `spiderpoolInit.extraArgs` | the additional arguments of spiderpoolInit container | `[]` |
| `spiderpoolInit.resources.limits.cpu` | the cpu limit of spiderpoolInit pod | `200m` |
| `spiderpoolInit.resources.limits.memory` | the memory limit of spiderpoolInit pod | `256Mi` |
| `spiderpoolInit.resources.requests.cpu` | the cpu requests of spiderpoolInit pod | `100m` |
| `spiderpoolInit.resources.requests.memory` | the memory requests of spiderpoolInit pod | `128Mi` |
| `spiderpoolInit.readinessProbe.failureThreshold` | the failure threshold of startup probe for spiderpool-init health checking | `30` |
| `spiderpoolInit.readinessProbe.periodSeconds` | the period seconds of startup probe for spiderpool-init health checking | `10` |
| `spiderpoolInit.extraEnv` | the additional environment variables of spiderpoolInit container | `[]` |
| `spiderpoolInit.securityContext` | the security Context of spiderpoolInit pod | `{}` |
| `spiderpoolInit.podAnnotations` | the additional annotations of spiderpoolInit pod | `{}` |
| `spiderpoolInit.podLabels` | the additional label of spiderpoolInit pod | `{}` |
| `spiderpoolInit.serviceAccount.annotations` | the annotations of spiderpoolInit service account | `{}` |

### sriov network operator parameters

Expand Down
2 changes: 1 addition & 1 deletion charts/spiderpool/templates/daemonset.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,7 +108,7 @@ spec:
- "/bin/sh"
- "-c"
- |
BIN_LIST="multus"
ITEM="multus"
rm -f /host/opt/cni/bin/${ITEM}.old || true
( [ -f "/host/opt/cni/bin/${ITEM}" ] && mv /host/opt/cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}.old ) || true
cp /usr/src/multus-cni/bin/${ITEM} /host/opt/cni/bin/${ITEM}
Expand Down
12 changes: 12 additions & 0 deletions charts/spiderpool/templates/pod.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,16 @@ spec:
args:
{{- toYaml . | trim | nindent 4 }}
{{- end }}
readinessProbe:
exec:
command:
- cat
- "/etc/spiderpool/ready"
initialDelaySeconds: 15
periodSeconds: {{ .Values.spiderpoolInit.readinessProbe.periodSeconds }}
successThreshold: 1
failureThreshold: {{ .Values.spiderpoolInit.readinessProbe.failureThreshold }}
timeoutSeconds: 1
{{- with .Values.spiderpoolInit.resources }}
resources:
{{- toYaml . | trim | nindent 6 }}
Expand Down Expand Up @@ -132,6 +142,8 @@ spec:
{{- if eq .Values.multus.multusCNI.defaultCniCRName "" }}
- name: SPIDERPOOL_INIT_DEFAULT_CNI_DIR
value: {{ .Values.global.cniConfHostPath | quote }}
- name: SPIDERPOOL_INIT_READINESS_FILE
value: "/etc/spiderpool/ready"
volumeMounts:
- name: cni
mountPath: {{ .Values.global.cniConfHostPath }}
Expand Down
7 changes: 7 additions & 0 deletions charts/spiderpool/values.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -801,6 +801,13 @@ spiderpoolInit:
## @param spiderpoolInit.resources.requests.memory the memory requests of spiderpoolInit pod
memory: 128Mi

readinessProbe:
## @param spiderpoolInit.readinessProbe.failureThreshold the failure threshold of startup probe for spiderpool-init health checking
failureThreshold: 30

## @param spiderpoolInit.readinessProbe.periodSeconds the period seconds of startup probe for spiderpool-init health checking
periodSeconds: 10

## @param spiderpoolInit.extraEnv the additional environment variables of spiderpoolInit container
extraEnv: []

Expand Down
7 changes: 4 additions & 3 deletions cmd/spiderpool-init/cmd/client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,9 +188,9 @@ func (c *CoreClient) WaitPodListReady(ctx context.Context, namespace string, lab
var err error
noReady := true
for noReady {
if err = c.List(ctx, &podList, client.MatchingLabels(labels), client.InNamespace(namespace)); err != nil && !apierrors.IsNotFound(err) {
logger.Sugar().Errorf("failed to get multus pod lists: %v", err)
return err
if err = c.List(ctx, &podList, client.MatchingLabels(labels), client.InNamespace(namespace)); err != nil {
logger.Sugar().Errorf("failed to get spiderAgent pods: %v, retrying...", err)
continue
}

if podList.Items == nil {
Expand All @@ -214,5 +214,6 @@ func (c *CoreClient) WaitPodListReady(ctx context.Context, namespace string, lab
time.Sleep(interval)
}
}

return nil
}
9 changes: 9 additions & 0 deletions cmd/spiderpool-init/cmd/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,11 +51,13 @@ const (
ENVDefaultCNIName = "SPIDERPOOL_INIT_DEFAULT_CNI_NAME"
ENVDefaultCNINamespace = "SPIDERPOOL_INIT_DEFAULT_CNI_NAMESPACE"
ENVDefaultMultusConfigMap = "SPIDERPOOL_INIT_MULTUS_CONFIGMAP"
ENVDefaultReadinessFile = "SPIDERPOOL_INIT_READINESS_FILE"
)

var (
legacyCalicoCniName = "k8s-pod-network"
calicoCniName = "calico"
readinessFileName = "/etc/spiderpool/ready"
)

type InitDefaultConfig struct {
Expand Down Expand Up @@ -91,6 +93,9 @@ type InitDefaultConfig struct {
DefaultCNIName string
DefaultCNINamespace string
MultusConfigMap string

// readiness
ReadinessFile string
}

func NewInitDefaultConfig() InitDefaultConfig {
Expand Down Expand Up @@ -282,6 +287,10 @@ func parseENVAsDefault() InitDefaultConfig {
config.DefaultCNIName = strings.ReplaceAll(os.Getenv(ENVDefaultCNIName), "\"", "")
config.DefaultCNINamespace = strings.ReplaceAll(os.Getenv(ENVDefaultCNINamespace), "\"", "")
config.MultusConfigMap = strings.ReplaceAll(os.Getenv(ENVDefaultMultusConfigMap), "\"", "")
config.ReadinessFile = strings.ReplaceAll(os.Getenv(ENVDefaultReadinessFile), "\"", "")
if config.ReadinessFile == "" {
config.ReadinessFile = readinessFileName
}

logger.Sugar().Infof("Init default config: %+v", config)

Expand Down
21 changes: 19 additions & 2 deletions cmd/spiderpool-init/cmd/multus.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,8 @@ import (
"context"
"encoding/json"
"fmt"
"time"
"os"
"path"

"github.com/spidernet-io/spiderpool/pkg/multuscniconfig"
"github.com/spidernet-io/spiderpool/pkg/utils"
Expand Down Expand Up @@ -84,6 +85,23 @@ func InitMultusDefaultCR(ctx context.Context, config *InitDefaultConfig, client
if err = restartSpiderAgent(ctx, client, config.AgentName, config.DefaultCNINamespace); err != nil {
return err
}

logger.Sugar().Infof("successfully restart spiderpool-agent")
return nil
}

func makeReadinessReady(config *InitDefaultConfig) error {
// tell readness by writing to the file that the spiderpool is ready
readinessDir := path.Dir(config.ReadinessFile)
err := os.MkdirAll(readinessDir, 0644)
if err != nil {
return err
}

if err = os.WriteFile(config.ReadinessFile, []byte("ready"), 0777); err != nil {
return err
}
logger.Sugar().Infof("success to make spiderpool-init pod's readiness to ready")
return nil
}

Expand Down Expand Up @@ -122,7 +140,6 @@ func restartSpiderAgent(ctx context.Context, client *CoreClient, name, ns string
return err
}

time.Sleep(3 * time.Second)
if err = client.WaitPodListReady(ctx, ns, spiderAgent.Spec.Template.Labels); err != nil {
return err
}
Expand Down
8 changes: 6 additions & 2 deletions cmd/spiderpool-init/cmd/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,11 @@ func Execute() {
logger.Fatal(err.Error())
}

logger.Info("Finish init")
// Wait for helm --wait.
if err = makeReadinessReady(&config); err != nil {
logger.Fatal(err.Error())
}

// helm wait
time.Sleep(300 * time.Second)
logger.Info("Finish init")
}
4 changes: 3 additions & 1 deletion test/scripts/install-multus.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,9 @@ EOF
esac
}

kubectl wait --for=condition=ready -l app.kubernetes.io/name=spiderpool --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || ( kubectl describe po -n kube-system -l app.kubernetes.io/name=spiderpool --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )
kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || \
( kubectl get pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} ; \
kubectl logs -n kube-system spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )

Install::MultusCR
Install::SpiderpoolCR
Expand Down

0 comments on commit 5d26a95

Please sign in to comment.