Fix issues in e2e test

.github/workflows/e2e.yml

@@ -57,9 +57,16 @@ jobs:
           export E2E_DOCKER_CONFIG_JSON_SECRET=`cat ~/.docker/config.json| base64 -w 0`
           make e2e E2EARGS="-v=5 -skip-features 'pv expansion|custom config for dynamic|tools for exporter'"
         env:
+          E2E_AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          E2E_AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          E2E_BR_IMAGE: reg.vesoft-inc.com/cloud-dev/br-ent
+          E2E_BR_VERSION: v3.7.0
+          E2E_GOOGLE_APPLICATION_CREDENTIALS: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }}
           E2E_OPERATOR_IMAGE: reg.vesoft-inc.com/ci/nebula-operator:ci-e2e
           E2E_OPERATOR_INSTALL: "true"
-          E2E_NC_VERSION: v3.6.0
+          E2E_NC_VERSION: v3.7.0
+          E2E_NC_AGENT_IMAGE: reg.vesoft-inc.com/cloud-dev/nebula-agent
+          E2E_NC_AGENT_VERSION: v3.6.0
           E2E_NC_GRAPHD_IMAGE: reg.vesoft-inc.com/vesoft-ent/nebula-graphd-ent
           E2E_NC_METAD_IMAGE: reg.vesoft-inc.com/vesoft-ent/nebula-metad-ent
           E2E_NC_STORAGED_IMAGE: reg.vesoft-inc.com/vesoft-ent/nebula-storaged-ent

go.mod

@@ -53,6 +53,24 @@ require (
 	github.com/antlr/antlr4/runtime/Go/antlr v1.4.10 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/aws/aws-sdk-go v1.44.178 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.25.0 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.27.0 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 // indirect
+	github.com/aws/smithy-go v1.20.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect

go.sum

@@ -63,6 +63,42 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a h1:idn718Q4
 github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY=
 github.com/aws/aws-sdk-go v1.44.178 h1:4igreoWPEA7xVLnOeSXLhDXTsTSPKQONZcQ3llWAJw0=
 github.com/aws/aws-sdk-go v1.44.178/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go-v2 v1.25.0 h1:sv7+1JVJxOu/dD/sz/csHX7jFqmP001TIY7aytBWDSQ=
+github.com/aws/aws-sdk-go-v2 v1.25.0/go.mod h1:G104G1Aho5WqF+SR3mDIobTABQzpYV0WxMsKxlMggOA=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0 h1:2UO6/nT1lCZq1LqM67Oa4tdgP1CvL1sLSxvuD+VrOeE=
+github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.0/go.mod h1:5zGj2eA85ClyedTDK+Whsu+w9yimnVIZvhvBKrDquM8=
+github.com/aws/aws-sdk-go-v2/config v1.27.0 h1:J5sdGCAHuWKIXLeXiqr8II/adSvetkx0qdZwdbXXpb0=
+github.com/aws/aws-sdk-go-v2/config v1.27.0/go.mod h1:cfh8v69nuSUohNFMbIISP2fhmblGmYEOKs5V53HiHnk=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.0 h1:lMW2x6sKBsiAJrpi1doOXqWFyEPoE886DTb1X0wb7So=
+github.com/aws/aws-sdk-go-v2/credentials v1.17.0/go.mod h1:uT41FIH8cCIxOdUYIL0PYyHlL1NoneDuDSCwg5VE/5o=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0 h1:xWCwjjvVz2ojYTP4kBKUuUh9ZrXfcAXpflhOUUeXg1k=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.15.0/go.mod h1:j3fACuqXg4oMTQOR2yY7m0NmJY0yBK4L4sLsRXq1Ins=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0 h1:NPs/EqVO+ajwOoq56EfcGKa3L3ruWuazkIw1BqxwOPw=
+github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.0/go.mod h1:D+duLy2ylgatV+yTlQ8JTuLfDD0BnFvnQRc+o6tbZ4M=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0 h1:ks7KGMVUMoDzcxNWUlEdI+/lokMFD136EL6DWmUOV80=
+github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.0/go.mod h1:hL6BWM/d/qz113fVitZjbXR0E+RCTU1+x+1Idyn5NgE=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0 h1:TkbRExyKSVHELwG9gz2+gql37jjec2R5vus9faTomwE=
+github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.0/go.mod h1:T3/9xMKudHhnj8it5EqIrhvv11tVZqWYkKcot+BFStc=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0 h1:a33HuFlO0KsveiP90IUJh8Xr/cx9US2PqkSroaLc+o8=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.0/go.mod h1:SxIkWpByiGbhbHYTo9CMTUnx2G4p4ZQMrDPcRRy//1c=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.0 h1:UiSyK6ent6OKpkMJN3+k5HZ4sk4UfchEaaW5wv7SblQ=
+github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.0/go.mod h1:l7kzl8n8DXoRyFz5cIMG70HnPauWa649TUhgw8Rq6lo=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0 h1:SHN/umDLTmFTmYfI+gkanz6da3vK8Kvj/5wkqnTHbuA=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.0/go.mod h1:l8gPU5RYGOFHJqWEpPMoRTP0VoaWQSkJdKo+hwWnnDA=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0 h1:l5puwOHr7IxECuPMIuZG7UKOzAnF24v6t4l+Z5Moay4=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.0/go.mod h1:Oov79flWa/n7Ni+lQC3z+VM7PoRM47omRqbJU9B5Y7E=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0 h1:jZAdMD1ioZdqirzzVVRhpHHWJmcGGCn8JqDYBs5nmYA=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.50.0/go.mod h1:1o/W6JFUuREj2ExoQ21vHJgO7wakvjhol91M9eknFgs=
+github.com/aws/aws-sdk-go-v2/service/sso v1.19.0 h1:u6OkVDxtBPnxPkZ9/63ynEe+8kHbtS5IfaC4PzVxzWM=
+github.com/aws/aws-sdk-go-v2/service/sso v1.19.0/go.mod h1:YqbU3RS/pkDVu+v+Nwxvn0i1WB0HkNWEePWbmODEbbs=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0 h1:6DL0qu5+315wbsAEEmzK+P9leRwNbkp+lGjPC+CEvb8=
+github.com/aws/aws-sdk-go-v2/service/ssooidc v1.22.0/go.mod h1:olUAyg+FaoFaL/zFaeQQONjOZ9HXoxgvI/c7mQTYz7M=
+github.com/aws/aws-sdk-go-v2/service/sts v1.27.0 h1:cjTRjh700H36MQ8M0LnDn33W3JmwC77mdxIIyPWCdpM=
+github.com/aws/aws-sdk-go-v2/service/sts v1.27.0/go.mod h1:nXfOBMWPokIbOY+Gi7a1psWMSvskUCemZzI+SMB7Akc=
+github.com/aws/smithy-go v1.20.0 h1:6+kZsCXZwKxZS9RfISnPc4EXlHoyAkm2hPuM8X2BrrQ=
+github.com/aws/smithy-go v1.20.0/go.mod h1:uo5RKksAl4PzhqaAbjd4rLgFoq5koTsQKYuGe7dklGc=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=

pkg/kube/cronbackup.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"reflect"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/util/retry"
@@ -31,8 +32,11 @@ import (
 )
 
 type NebulaCronBackup interface {
+	CreateCronBackup(ncb *v1alpha1.NebulaCronBackup) error
 	GetCronBackup(namespace, name string) (*v1alpha1.NebulaCronBackup, error)
 	UpdateCronBackupStatus(cronBackup *v1alpha1.NebulaCronBackup) error
+	UpdateCronBackup(cronBackup *v1alpha1.NebulaCronBackup) error
+	DeleteCronBackup(namespace, name string) error
 }
 
 type cronBackupClient struct {
@@ -43,6 +47,17 @@ func NewCronNebulaBackup(client client.Client) NebulaCronBackup {
 	return &cronBackupClient{client: client}
 }
 
+func (c *cronBackupClient) CreateCronBackup(ncb *v1alpha1.NebulaCronBackup) error {
+	if err := c.client.Create(context.TODO(), ncb); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			klog.Infof("NebulaCronBackup %s/%s already exists", ncb.Namespace, ncb.Name)
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 func (c *cronBackupClient) GetCronBackup(namespace, name string) (*v1alpha1.NebulaCronBackup, error) {
 	cronBackup := &v1alpha1.NebulaCronBackup{}
 	err := c.client.Get(context.TODO(), types.NamespacedName{
@@ -56,6 +71,41 @@ func (c *cronBackupClient) GetCronBackup(namespace, name string) (*v1alpha1.Nebu
 	return cronBackup, nil
 }
 
+func (c *cronBackupClient) UpdateCronBackup(cronBackup *v1alpha1.NebulaCronBackup) error {
+	ns := cronBackup.Namespace
+	cronBackupName := cronBackup.Name
+	cronBackupSpec := cronBackup.Spec.DeepCopy()
+	cronBackupLabels := cronBackup.GetLabels()
+	annotations := cronBackup.GetAnnotations()
+
+	return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
+		// Update the set with the latest resource version for the next poll
+		cronBackupClone, err := c.GetCronBackup(ns, cronBackupName)
+		if err != nil {
+			utilruntime.HandleError(fmt.Errorf("get NebulaCronBackup [%s/%s] failed: %v", ns, cronBackupName, err))
+			return err
+		}
+
+		if reflect.DeepEqual(cronBackupSpec, cronBackupClone.Spec) &&
+			reflect.DeepEqual(cronBackupLabels, cronBackupClone.Labels) &&
+			reflect.DeepEqual(annotations, cronBackupClone.Annotations) {
+			return nil
+		}
+
+		cronBackup = cronBackupClone.DeepCopy()
+		cronBackup.Spec = *cronBackupSpec
+		cronBackup.SetLabels(cronBackupLabels)
+		cronBackup.SetAnnotations(annotations)
+		updateErr := c.client.Update(context.TODO(), cronBackup)
+		if updateErr == nil {
+			klog.Infof("NebulaCronBackup [%s/%s] updated successfully", ns, cronBackupName)
+			return nil
+		}
+		klog.Errorf("update NebulaCronBackup [%s/%s] failed: %v", ns, cronBackupName, updateErr)
+		return updateErr
+	})
+}
+
 func (c *cronBackupClient) UpdateCronBackupStatus(cronBackup *v1alpha1.NebulaCronBackup) error {
 	ns := cronBackup.GetNamespace()
 	cbName := cronBackup.GetName()
@@ -83,3 +133,18 @@ func (c *cronBackupClient) UpdateCronBackupStatus(cronBackup *v1alpha1.NebulaCro
 		return updateErr
 	})
 }
+
+func (c *cronBackupClient) DeleteCronBackup(namespace, name string) error {
+	cronBackup, err := c.GetCronBackup(namespace, name)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if err := c.client.Delete(context.TODO(), cronBackup); err != nil {
+		return err
+	}
+	klog.Infof("NebulaCronBackup [%s/%s] deleted successfully", namespace, name)
+	return nil
+}

pkg/kube/nebularestore.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -44,8 +45,10 @@ type RestoreUpdateStatus struct {
 }
 
 type NebulaRestore interface {
+	CreateNebulaRestore(nr *v1alpha1.NebulaRestore) error
 	GetNebulaRestore(namespace, name string) (*v1alpha1.NebulaRestore, error)
 	UpdateNebulaRestoreStatus(restore *v1alpha1.NebulaRestore, condition *v1alpha1.RestoreCondition, newStatus *RestoreUpdateStatus) error
+	DeleteNebulaRestore(namespace, name string) error
 }
 
 type restoreClient struct {
@@ -56,6 +59,17 @@ func NewNebulaRestore(client client.Client) NebulaRestore {
 	return &restoreClient{client: client}
 }
 
+func (r *restoreClient) CreateNebulaRestore(nr *v1alpha1.NebulaRestore) error {
+	if err := r.client.Create(context.TODO(), nr); err != nil {
+		if apierrors.IsAlreadyExists(err) {
+			klog.Infof("NebulaRestore %s/%s already exists", nr.Namespace, nr.Name)
+			return nil
+		}
+		return err
+	}
+	return nil
+}
+
 func (r *restoreClient) GetNebulaRestore(namespace, name string) (*v1alpha1.NebulaRestore, error) {
 	restore := &v1alpha1.NebulaRestore{}
 	err := r.client.Get(context.TODO(), types.NamespacedName{
@@ -97,6 +111,21 @@ func (r *restoreClient) UpdateNebulaRestoreStatus(restore *v1alpha1.NebulaRestor
 	})
 }
 
+func (r *restoreClient) DeleteNebulaRestore(namespace, name string) error {
+	restore, err := r.GetNebulaRestore(namespace, name)
+	if apierrors.IsNotFound(err) {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	if err := r.client.Delete(context.TODO(), restore); err != nil {
+		return err
+	}
+	klog.Infof("NebulaRestore [%s/%s] deleted successfully", namespace, name)
+	return nil
+}
+
 func updateRestoreStatus(status *v1alpha1.RestoreStatus, newStatus *RestoreUpdateStatus) bool {
 	if newStatus == nil {
 		return false

pkg/kube/node.go

@@ -27,6 +27,7 @@ import (
 
 type Node interface {
 	GetNode(nodeName string) (*corev1.Node, error)
+	ListAllNodes() ([]corev1.Node, error)
 }
 
 type nodeClient struct {
@@ -46,3 +47,12 @@ func (pd *nodeClient) GetNode(nodeName string) (*corev1.Node, error) {
 	}
 	return node, nil
 }
+
+func (pd *nodeClient) ListAllNodes() ([]corev1.Node, error) {
+	nodeList := &corev1.NodeList{}
+	if err := pd.kubecli.List(context.TODO(), nodeList, &client.ListOptions{}); err != nil {
+		return nil, err
+	}
+
+	return nodeList.Items, nil
+}

pkg/util/extender/unstructured.go

@@ -183,8 +183,7 @@ func ObjectEqual(newUnstruct, oldUnstruct *unstructured.Unstructured) bool {
 	annotations := map[string]string{}
 	for k, v := range oldUnstruct.GetAnnotations() {
 		if k == annotation.AnnLastAppliedConfigKey ||
-			k == annotation.AnnRestartTimestamp ||
-			k == annotation.AnnLastReplicas {
+			k == annotation.AnnRestartTimestamp {
 			continue
 		}
 		annotations[k] = v

tests/e2e/config/config.go

@@ -58,6 +58,9 @@ type CommonConfig struct {
 	// DockerConfigJsonSecret is the docker config file.
 	// export E2E_DOCKER_CONFIG_JSON_SECRET=`cat ~/.docker/config.json| base64 -w 0`
 	DockerConfigJsonSecret Base64Value `env:"E2E_DOCKER_CONFIG_JSON_SECRET"`
+	AWSAccessKey           Base64Value `env:"E2E_AWS_ACCESS_KEY_ID"`
+	AWSSecretKey           Base64Value `env:"E2E_AWS_SECRET_ACCESS_KEY"`
+	GSSecret               Base64Value `env:"E2E_GOOGLE_APPLICATION_CREDENTIALS"`
 }
 
 type ClusterConfig struct {
@@ -75,6 +78,10 @@ type OperatorConfig struct {
 type NebulaClusterConfig struct {
 	ChartPath         string   `env:"E2E_NC_CHART_PATH,notEmpty,required" envDefault:"../../charts/nebula-cluster"`
 	Version           string   `env:"E2E_NC_VERSION"`
+	AgentImage        string   `env:"E2E_NC_AGENT_IMAGE"`
+	AgentVersion      string   `env:"E2E_NC_AGENT_VERSION"`
+	BrImage           string   `env:"E2E_BR_IMAGE"`
+	BrVersion         string   `env:"E2E_BR_VERSION"`
 	GraphdImage       string   `env:"E2E_NC_GRAPHD_IMAGE"`
 	MetadImage        string   `env:"E2E_NC_METAD_IMAGE"`
 	StoragedImage     string   `env:"E2E_NC_STORAGED_IMAGE"`