Add support for timeout for restic backup and restore

Signed-off-by: Md. Ishtiaq Islam <[email protected]>

go.mod

@@ -23,7 +23,7 @@ require (
 	k8s.io/utils v0.0.0-20240502163921-fe8a2dddb1d0
 	kmodules.xyz/client-go v0.30.14
 	kmodules.xyz/offshoot-api v0.29.4
-	kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a
+	kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1
 	sigs.k8s.io/controller-runtime v0.18.4
 	sigs.k8s.io/yaml v1.4.0
 )

go.sum

@@ -809,8 +809,8 @@ kmodules.xyz/offshoot-api v0.29.4 h1:WQV2BIUIoVKKiqZNmZ4gAy367jEdwBhEl3dFCLZM1qA
 kmodules.xyz/offshoot-api v0.29.4/go.mod h1:e+NQ0s4gW/YTPWBWEfdISZcmk+tlTq8IjvP5SLdqvko=
 kmodules.xyz/prober v0.29.0 h1:Ex7m4F9rH7uWNNJlLgP63ROOM+nUATJkC2L5OQ7nwMg=
 kmodules.xyz/prober v0.29.0/go.mod h1:UtK+HKyI1lFLEKX+HFLyOCVju6TO93zv3kwGpzqmKOo=
-kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a h1:CiTlSi3CIfTY+gbIBFm4oMmbhTknYvrRGWn4LZC82wA=
-kubestash.dev/apimachinery v0.12.1-0.20240912114724-e2698888fc6a/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc=
+kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1 h1:1aiK375epNLvl2CzSieF56sA25YsQnODBGLaDN/Vx3Q=
+kubestash.dev/apimachinery v0.12.1-0.20240918082744-9cbda1e7f2b1/go.mod h1:gtVSpHtK8LvS26+rKyLTnZvijvSSCdfG83n6GL6+Kwc=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=

pkg/trigger.go

@@ -100,8 +100,9 @@ func triggerBackup(backupConfig *coreapi.BackupConfiguration, session coreapi.Se
 				Kind:     coreapi.ResourceKindBackupConfiguration,
 				Name:     backupConfig.Name,
 			},
-			Session:   session.Name,
-			RetryLeft: 0,
+			Session:       session.Name,
+			RetryLeft:     0,
+			BackupTimeout: session.BackupTimeout,
 		},
 	}
 

vendor/kubestash.dev/apimachinery/apis/constant.go

@@ -75,7 +75,6 @@ const (
 	KubeStashAppRefKind      = "kubestash.com/app-ref-kind"
 	KubeStashAppRefNamespace = "kubestash.com/app-ref-namespace"
 	KubeStashAppRefName      = "kubestash.com/app-ref-name"
-	KubeDBAppVersion         = "kubedb.com/db-version"
 )
 
 // Keys for structure logging
@@ -153,3 +152,9 @@ const (
 	SnapshotVersionV1 = "v1"
 	DirRepository     = "repository"
 )
+
+// Annotations
+const (
+	AnnKubeDBAppVersion          = "kubedb.com/db-version"
+	AnnRestoreSessionBeneficiary = "restoresession.kubestash.com/beneficiary"
+)

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupconfiguration_types.go

@@ -132,10 +132,10 @@ type SessionConfig struct {
 	// +optional
 	RetryConfig *RetryConfig `json:"retryConfig,omitempty"`
 
-	// Timeout specifies the maximum duration of backup. BackupSession will be considered Failed
-	// if backup does not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// BackupTimeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
 	// +optional
-	Timeout *metav1.Duration `json:"timeout,omitempty"`
+	BackupTimeout *metav1.Duration `json:"backupTimeout,omitempty"`
 
 	// SessionHistoryLimit specifies how many backup Jobs and associate resources KubeStash should keep for debugging purpose.
 	// The default value is 1.

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_helpers.go

@@ -243,3 +243,14 @@ func (b *BackupSession) checkFailureInRetentionPolicy() (bool, string) {
 	}
 	return false, ""
 }
+
+func (b *BackupSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if b.Spec.BackupTimeout == nil || b.Status.BackupDeadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if b.Status.BackupDeadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: b.Status.BackupDeadline.Sub(currentTime.Time)}, nil
+}

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/backupsession_types.go

@@ -63,6 +63,11 @@ type BackupSessionSpec struct {
 	// If this set to non-zero, KubeStash will create a new BackupSession if the current one fails.
 	// +optional
 	RetryLeft int32 `json:"retryLeft,omitempty"`
+
+	// BackupTimeout specifies the maximum duration of backup. Backup will be considered Failed
+	// if backup tasks do not complete within this time limit. By default, KubeStash don't set any timeout for backup.
+	// +optional
+	BackupTimeout *metav1.Duration `json:"backupTimeout,omitempty"`
 }
 
 // BackupSessionStatus defines the observed state of BackupSession
@@ -75,10 +80,10 @@ type BackupSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies the deadline of backup. BackupSession will be
-	// considered Failed if backup does not complete within this deadline
+	// BackupDeadline specifies the deadline of backup. Backup will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"sessionDeadline,omitempty"`
+	BackupDeadline *metav1.Time `json:"backupDeadline,omitempty"`
 
 	// TotalSnapshots specifies the total number of snapshots created for this backupSession.
 	// +optional

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_helpers.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"fmt"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kmapi "kmodules.xyz/client-go/api/v1"
 	"kubestash.dev/apimachinery/apis"
@@ -41,8 +42,7 @@ func (rs *RestoreSession) CalculatePhase() RestorePhase {
 	}
 
 	if cutil.IsConditionTrue(rs.Status.Conditions, TypeMetricsPushed) &&
-		(cutil.IsConditionTrue(rs.Status.Conditions, TypeDeadlineExceeded) ||
-			cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
+		(cutil.IsConditionFalse(rs.Status.Conditions, TypePreRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypePostRestoreHooksExecutionSucceeded) ||
 			cutil.IsConditionFalse(rs.Status.Conditions, TypeRestoreExecutorEnsured)) {
 		return RestoreFailed
@@ -181,3 +181,14 @@ func (rs *RestoreSession) GetDataSourceNamespace() string {
 	}
 	return rs.Spec.DataSource.Namespace
 }
+
+func (rs *RestoreSession) GetRemainingTimeoutDuration() (*metav1.Duration, error) {
+	if rs.Spec.RestoreTimeout == nil || rs.Status.RestoreDeadline == nil {
+		return nil, nil
+	}
+	currentTime := metav1.Now()
+	if rs.Status.RestoreDeadline.Before(&currentTime) {
+		return nil, fmt.Errorf("deadline exceeded")
+	}
+	return &metav1.Duration{Duration: rs.Status.RestoreDeadline.Sub(currentTime.Time)}, nil
+}

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/restoresession_types.go

@@ -64,10 +64,10 @@ type RestoreSessionSpec struct {
 	// +optional
 	Hooks *RestoreHooks `json:"hooks,omitempty"`
 
-	// Timeout specifies a duration that KubeStash should wait for the session execution to be completed.
-	// If the session execution does not finish within this time period, KubeStash will consider this session as a failure.
+	// RestoreTimeout specifies a duration that KubeStash should wait for the restore to be completed.
+	// If the restore tasks do not finish within this time period, KubeStash will consider this restore as a failure.
 	// +optional
-	Timeout *metav1.Duration `json:"timeout,omitempty"`
+	RestoreTimeout *metav1.Duration `json:"restoreTimeout,omitempty"`
 
 	// ManifestOptions provide options to select particular manifest object to restore
 	// +optional
@@ -98,6 +98,18 @@ type ManifestRestoreOptions struct {
 	// MSSQLServer specifies the options for selecting particular MSSQLServer components to restore in manifest restore
 	// +optional
 	MSSQLServer *MSSQLServerManifestOptions `json:"msSQLServer,omitempty"`
+
+	// Druid specifies the options for selecting particular Druid components to restore in manifest restore
+	// +optional
+	Druid *DruidManifestOptions `json:"druid,omitempty"`
+
+	// ZooKeeper specifies the options for selecting particular ZooKeeper components to restore in manifest restore
+	// +optional
+	ZooKeeper *KubeDBManifestOptions `json:"zooKeeper,omitempty"`
+
+	// Redis specifies the options for selecting particular Redis components to restore in manifest restore
+	// +optional
+	Redis *KubeDBManifestOptions `json:"redis,omitempty"`
 }
 
 type MSSQLServerManifestOptions struct {
@@ -126,6 +138,50 @@ type MSSQLServerManifestOptions struct {
 	TLSIssuerRef *core.TypedLocalObjectReference `json:"tlsIssuerRef,omitempty"`
 }
 
+type DruidManifestOptions struct {
+	// DB specifies whether to restore the DB manifest or not
+	// +optional
+	DB bool `json:"db,omitempty"`
+
+	// DBName specifies the new name of the DB yaml after restore
+	// +optional
+	DBName string `json:"dbName,omitempty"`
+
+	// AuthSecret specifies whether to restore the AuthSecret manifest or not
+	// +optional
+	AuthSecret bool `json:"authSecret,omitempty"`
+
+	// AuthSecretName specifies new name of the AuthSecret yaml after restore
+	// +optional
+	AuthSecretName string `json:"authSecretName,omitempty"`
+
+	// ConfigSecret specifies whether to restore the ConfigSecret manifest or not
+	// +optional
+	ConfigSecret bool `json:"configSecret,omitempty"`
+
+	// ConfigSecretName specifies new name of the ConfigSecret yaml after restore
+	// +optional
+	ConfigSecretName string `json:"configSecretName,omitempty"`
+
+	// DeepStorageSecret specifies whether to restore the DeepStorageSecret manifest or not
+	// +optional
+	DeepStorageSecret bool `json:"deepStorageSecret,omitempty"`
+
+	// MetadataStorage specifies new configuration of the Metadata Storage after restore
+	// +optional
+	MetadataStorage bool `json:"metadataStorage,omitempty"`
+
+	// +optional
+	MetadataStorageRef *kmapi.ObjectReference `json:"metadataStorageRef,omitempty"`
+
+	// ZooKeeper specifies new configuration of the Metadata Storage after restore
+	// +optional
+	Zookeeper bool `json:"zookeeper,omitempty"`
+
+	// +optional
+	ZookeeperRef *kmapi.ObjectReference `json:"zookeeperRef,omitempty"`
+}
+
 type KubeDBManifestOptions struct {
 	// DB specifies whether to restore the DB manifest or not
 	// +optional
@@ -151,6 +207,10 @@ type KubeDBManifestOptions struct {
 	// +optional
 	ConfigSecretName string `json:"configSecretName,omitempty"`
 
+	// InitScript specifies whether to restore the InitScript manifest or not
+	// +optional
+	InitScript bool `json:"initScript,omitempty"`
+
 	// TLSIssuerRef specifies the name of the IssuerRef used for TLS configurations for both client and server
 	// +optional
 	TLSIssuerRef *core.TypedLocalObjectReference `json:"tlsIssuerRef,omitempty"`
@@ -222,10 +282,10 @@ type RestoreSessionStatus struct {
 	// +optional
 	Duration string `json:"duration,omitempty"`
 
-	// Deadline specifies a timestamp till this session is valid. If the session does not complete within this deadline,
-	// it will be considered as failed.
+	// RestoreDeadline specifies the deadline of restore. Restore will be
+	// considered Failed if it does not complete within this deadline
 	// +optional
-	Deadline *metav1.Time `json:"deadline,omitempty"`
+	RestoreDeadline *metav1.Time `json:"restoreDeadline,omitempty"`
 
 	// TotalComponents represents the number of total components for this RestoreSession
 	// +optional

vendor/kubestash.dev/apimachinery/apis/core/v1alpha1/types.go

@@ -217,9 +217,6 @@ type RetryConfig struct {
 }
 
 const (
-	TypeDeadlineExceeded                 = "DeadlineExceeded"
-	ReasonFailedToCompleteWithinDeadline = "FailedToCompleteWithinDeadline"
-
 	// TypeMetricsPushed indicates whether Metrics are pushed or not
 	TypeMetricsPushed               = "MetricsPushed"
 	ReasonSuccessfullyPushedMetrics = "SuccessfullyPushedMetrics"

vendor/kubestash.dev/apimachinery/crds/core.kubestash.com_backupbatches.yaml

@@ -110,6 +110,12 @@ spec:
                   description: BatchSession specifies the session configuration for
                     the targets.
                   properties:
+                    backupTimeout:
+                      description: BackupTimeout specifies the maximum duration of
+                        backup. Backup will be considered Failed if backup tasks do
+                        not complete within this time limit. By default, KubeStash
+                        don't set any timeout for backup.
+                      type: string
                     hooks:
                       description: Hooks specifies the backup hooks that should be
                         executed before and/or after the backup.
@@ -36194,12 +36200,6 @@ spec:
                             type: array
                         type: object
                       type: array
-                    timeout:
-                      description: Timeout specifies the maximum duration of backup.
-                        BackupSession will be considered Failed if backup does not
-                        complete within this time limit. By default, KubeStash don't
-                        set any timeout for backup.
-                      type: string
                   type: object
                 type: array
               targets: