[Feature] Add Backup Retries Until (#1873)

Author: ajanikow

CHANGELOG.md

@@ -12,6 +12,7 @@
 - (Feature) (Platform) Docs and Installer improvements
 - (Feature) Promote RestartPolicy Always Feature
 - (Maintenance) Update Dependencies
+- (Feature) Backup Retry Until and BackupPolicy Until Propagation feature
 
 ## [1.2.47](https://github.com/arangodb/kube-arangodb/tree/1.2.47) (2025-03-28)
 - (Bugfix) Use Profile Annotations

README.md

@@ -164,6 +164,7 @@ Flags:
       --deployment.feature.all                                 Enable ALL Features
       --deployment.feature.async-backup-creation               Create backups asynchronously to avoid blocking the operator and reaching the timeout - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.backup-cleanup                      Cleanup imported backups if required - Required ArangoDB >= 3.8.0
+      --deployment.feature.backup-policy-until-propagation     Sets Until field in the Backup based on next schedule time - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.deployment-spec-defaults-restore    Restore defaults from last accepted state of deployment - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.enforced-resign-leadership          Enforce ResignLeadership and ensure that Leaders are moved from restarted DBServer - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.ephemeral-volumes                   Enables ephemeral volumes for apps and tmp directory - Required ArangoDB >= 3.8.0

docs/api/ArangoBackup.V1.md

@@ -10,39 +10,45 @@ title: ArangoBackup V1
 
 ### .spec.backoff.iterations
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L31)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L38)</sup>
 
 Iterations defines number of iterations before reaching MaxDelay. Default to 5
 
+Default Value: `5`
+
 ***
 
 ### .spec.backoff.max_delay
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L29)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L35)</sup>
 
 MaxDelay defines maximum delay in seconds. Default to 600
 
+Default Value: `600`
+
 ***
 
 ### .spec.backoff.max_iterations
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L33)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L40)</sup>
 
 MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit)
 
 ***
 
 ### .spec.backoff.min_delay
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L27)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L32)</sup>
 
 MinDelay defines minimum delay in seconds. Default to 30
 
+Default Value: `30`
+
 ***
 
 ### .spec.deployment.name
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L54)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L55)</sup>
 
 Name of the ArangoDeployment Custom Resource within same namespace as ArangoBackup Custom Resource.
 
@@ -52,7 +58,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.download.autoDelete
 
-Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L84)</sup>
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L85)</sup>
 
 AutoDelete removes the ArangoBackup resource (which removes the backup from the cluster) after successful upload
 
@@ -62,7 +68,7 @@ Default Value: `false`
 
 ### .spec.download.credentialsSecretName
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L81)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L82)</sup>
 
 CredentialsSecretName is the name of the secret used while accessing repository
 
@@ -75,7 +81,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.download.id
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L92)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L93)</sup>
 
 ID of the ArangoBackup to be downloaded
 
@@ -85,7 +91,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.download.repositoryURL
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L77)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L78)</sup>
 
 RepositoryURL is the URL path for file storage
 Same repositoryURL needs to be defined in `credentialsSecretName` if protocol is other than local.
@@ -106,15 +112,15 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.lifetime
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L47)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L48)</sup>
 
 Lifetime is the time after which the backup will be deleted. Format: "1.5h" or "2h45m".
 
 ***
 
 ### .spec.options.allowInconsistent
 
-Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L66)</sup>
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L67)</sup>
 
 AllowInconsistent flag for Backup creation request.
 If this value is set to true, backup is taken even if we are not able to acquire lock.
@@ -127,7 +133,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.options.timeout
 
-Type: `number` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L61)</sup>
+Type: `number` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L62)</sup>
 
 Timeout for Backup creation request in seconds. Works only when AsyncBackupCreation feature is set to false.
 
@@ -149,7 +155,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.upload.autoDelete
 
-Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L84)</sup>
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L85)</sup>
 
 AutoDelete removes the ArangoBackup resource (which removes the backup from the cluster) after successful upload
 
@@ -159,7 +165,7 @@ Default Value: `false`
 
 ### .spec.upload.credentialsSecretName
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L81)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L82)</sup>
 
 CredentialsSecretName is the name of the secret used while accessing repository
 
@@ -172,7 +178,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.upload.repositoryURL
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L77)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L78)</sup>
 
 RepositoryURL is the URL path for file storage
 Same repositoryURL needs to be defined in `credentialsSecretName` if protocol is other than local.

docs/api/ArangoBackupPolicy.V1.md

@@ -51,34 +51,40 @@ Links:
 
 ### .spec.template.backoff.iterations
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L31)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L38)</sup>
 
 Iterations defines number of iterations before reaching MaxDelay. Default to 5
 
+Default Value: `5`
+
 ***
 
 ### .spec.template.backoff.max_delay
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L29)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L35)</sup>
 
 MaxDelay defines maximum delay in seconds. Default to 600
 
+Default Value: `600`
+
 ***
 
 ### .spec.template.backoff.max_iterations
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L33)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L40)</sup>
 
 MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit)
 
 ***
 
 ### .spec.template.backoff.min_delay
 
-Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L27)</sup>
+Type: `integer` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec_backoff.go#L32)</sup>
 
 MinDelay defines minimum delay in seconds. Default to 30
 
+Default Value: `30`
+
 ***
 
 ### .spec.template.lifetime
@@ -91,7 +97,7 @@ Lifetime is the time after which the backup will be deleted. Format: "1.5h" or "
 
 ### .spec.template.options.allowInconsistent
 
-Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L66)</sup>
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L67)</sup>
 
 AllowInconsistent flag for Backup creation request.
 If this value is set to true, backup is taken even if we are not able to acquire lock.
@@ -104,7 +110,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.template.options.timeout
 
-Type: `number` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L61)</sup>
+Type: `number` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L62)</sup>
 
 Timeout for Backup creation request in seconds. Works only when AsyncBackupCreation feature is set to false.
 
@@ -116,7 +122,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.template.upload.autoDelete
 
-Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L84)</sup>
+Type: `boolean` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L85)</sup>
 
 AutoDelete removes the ArangoBackup resource (which removes the backup from the cluster) after successful upload
 
@@ -126,7 +132,7 @@ Default Value: `false`
 
 ### .spec.template.upload.credentialsSecretName
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L81)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L82)</sup>
 
 CredentialsSecretName is the name of the secret used while accessing repository
 
@@ -139,7 +145,7 @@ This field is **immutable**: can't be changed after backup creation
 
 ### .spec.template.upload.repositoryURL
 
-Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L77)</sup>
+Type: `string` <sup>[\[ref\]](https://github.com/arangodb/kube-arangodb/blob/1.2.47/pkg/apis/backup/v1/backup_spec.go#L78)</sup>
 
 RepositoryURL is the URL path for file storage
 Same repositoryURL needs to be defined in `credentialsSecretName` if protocol is other than local.

docs/cli/arangodb_operator.md

@@ -47,6 +47,7 @@ Flags:
       --deployment.feature.all                                 Enable ALL Features
       --deployment.feature.async-backup-creation               Create backups asynchronously to avoid blocking the operator and reaching the timeout - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.backup-cleanup                      Cleanup imported backups if required - Required ArangoDB >= 3.8.0
+      --deployment.feature.backup-policy-until-propagation     Sets Until field in the Backup based on next schedule time - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.deployment-spec-defaults-restore    Restore defaults from last accepted state of deployment - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.enforced-resign-leadership          Enforce ResignLeadership and ensure that Leaders are moved from restarted DBServer - Required ArangoDB >= 3.8.0 (default true)
       --deployment.feature.ephemeral-volumes                   Enables ephemeral volumes for apps and tmp directory - Required ArangoDB >= 3.8.0

pkg/apis/backup/v1/backup_spec.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2024 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2025 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -41,6 +41,7 @@ type ArangoBackupSpec struct {
 	// +doc/immutable: can't be changed after backup creation
 	PolicyName *string `json:"policyName,omitempty"`
 
+	// Backoff define the operation backoff policy
 	Backoff *ArangoBackupSpecBackOff `json:"backoff,omitempty"`
 
 	// Lifetime is the time after which the backup will be deleted. Format: "1.5h" or "2h45m".

pkg/apis/backup/v1/backup_spec_backoff.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2025 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -20,17 +20,26 @@
 
 package v1
 
-import "time"
+import (
+	"time"
+
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
 
 type ArangoBackupSpecBackOff struct {
 	// MinDelay defines minimum delay in seconds. Default to 30
+	// +doc/default: 30
 	MinDelay *int `json:"min_delay,omitempty"`
 	// MaxDelay defines maximum delay in seconds. Default to 600
+	// +doc/default: 600
 	MaxDelay *int `json:"max_delay,omitempty"`
 	// Iterations defines number of iterations before reaching MaxDelay. Default to 5
+	// +doc/default: 5
 	Iterations *int `json:"iterations,omitempty"`
 	// MaxIterations defines maximum number of iterations after backoff will be disabled. Default to nil (no limit)
 	MaxIterations *int `json:"max_iterations,omitempty"`
+	// Until defines the deadline of the retry mechanism in UTC
+	Until *meta.Time `json:"until,omitempty"`
 }
 
 func (a *ArangoBackupSpecBackOff) GetMaxDelay() int {

pkg/apis/backup/v1/backup_status_backoff.go

@@ -1,7 +1,7 @@
 //
 // DISCLAIMER
 //
-// Copyright 2016-2023 ArangoDB GmbH, Cologne, Germany
+// Copyright 2016-2025 ArangoDB GmbH, Cologne, Germany
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -52,7 +52,21 @@ func (a *ArangoBackupStatusBackOff) GetNext() meta.Time {
 }
 
 func (a *ArangoBackupStatusBackOff) ShouldBackoff(spec *ArangoBackupSpecBackOff) bool {
-	return spec == nil || spec.MaxIterations == nil || a.GetIterations() < *spec.MaxIterations
+	if spec != nil {
+		if u := spec.Until; u != nil && !u.IsZero() {
+			if time.Now().After(u.Time) {
+				return false
+			}
+		}
+
+		if u := spec.MaxIterations; u != nil {
+			if a.GetIterations() >= *u {
+				return false
+			}
+		}
+	}
+
+	return true
 }
 
 func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *ArangoBackupStatusBackOff {
@@ -64,8 +78,18 @@ func (a *ArangoBackupStatusBackOff) Backoff(spec *ArangoBackupSpecBackOff) *Aran
 		}
 	}
 
+	next := time.Now().Add(spec.Backoff(a.GetIterations()))
+
+	if spec != nil {
+		if u := spec.Until; u != nil && !u.IsZero() {
+			if next.After(u.Time) {
+				next = u.Time
+			}
+		}
+	}
+
 	return &ArangoBackupStatusBackOff{
 		Iterations: a.GetIterations() + 1,
-		Next:       meta.Time{Time: time.Now().Add(spec.Backoff(a.GetIterations()))},
+		Next:       meta.Time{Time: next},
 	}
 }

pkg/apis/backup/v1/zz_generated.deepcopy.go

@@ -4,6 +4,7 @@ v1:
       spec:
         properties:
           backoff:
+            description: Backoff define the operation backoff policy
             properties:
               iterations:
                 description: Iterations defines number of iterations before reaching MaxDelay. Default to 5
@@ -21,6 +22,10 @@ v1:
                 description: MinDelay defines minimum delay in seconds. Default to 30
                 format: int32
                 type: integer
+              until:
+                description: Until defines the deadline of the retry mechanism in UTC
+                format: date-time
+                type: string
             type: object
           deployment:
             description: Deployment describes the deployment which should have a backup