fix: operator picks (#1463)

* feat(crd): add crd mode for operators (#1329)

* feat(operator): add base for zitadel operator

* fix(operator): changed pipeline to release operator

* fix(operator): fmt with only one parameter

* fix(operator): corrected workflow job name

* fix(zitadelctl): added restore and backuplist command

* fix(zitadelctl): scale for restore

* chore(container): use scratch for deploy container

* fix(zitadelctl): limit image to scratch

* fix(migration): added migration scripts for newer version

* fix(operator): changed handling of kubeconfig in operator logic

* fix(operator): changed handling of secrets in operator logic

* fix(operator): use new version of zitadel

* fix(operator): added path for migrations

* fix(operator): delete doublets of migration scripts

* fix(operator): delete subpaths and integrate logic into init container

* fix(operator): corrected path in dockerfile for local migrations

* fix(operator): added migrations for cockroachdb-secure

* fix(operator): delete logic for ambassador module

* fix(operator): added read and write secret commands

* fix(operator): correct and align operator pipeline with zitadel pipeline

* fix(operator): correct yaml error in operator pipeline

* fix(operator): correct action name in operator pipeline

* fix(operator): correct case-sensitive filename in operator pipeline

* fix(operator): upload artifacts from buildx output

* fix(operator): corrected attribute spelling error

* fix(operator): combined jobs for operator binary and image

* fix(operator): added missing comma in operator pipeline

* fix(operator): added codecov for operator image

* fix(operator): added codecov for operator image

* fix(testing): code changes for testing and several unit-tests (#1009)

* fix(operator): usage of interface of kubernetes client for testing and several unit-tests

* fix(operator): several unit-tests

* fix(operator): several unit-tests

* fix(operator): changed order for the operator logic

* fix(operator): added version of zitadelctl from semantic release

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): corrected function call with version of zitadelctl

* fix(operator): add check output to operator release pipeline

* fix(operator): set --short length everywhere to 12

* fix(operator): zitadel setup in job instead of exec with several unit tests

* fix(operator): fixes to combine newest zitadel and testing branch

* fix(operator): corrected path in Dockerfile

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): fixed unit-test that was ignored during changes

* fix(operator): corrected Dockerfile to correctly use env variable

* fix(operator): quickfix takeoff deployment

* fix(operator): corrected the clusterrolename in the applied artifacts

* fix: update secure migrations

* fix(operator): migrations (#1057)

* fix(operator): copied migrations from orbos repository

* fix(operator): newest migrations

* chore: use cockroach-secure

* fix: rename migration

* fix: remove insecure cockroach migrations

Co-authored-by: Stefan Benz <[email protected]>

* fix: finalize labels

* fix(operator): cli logging concurrent and fixe deployment of operator during restore

* fix: finalize labels and cli commands

* fix: restore

* chore: cockroachdb is always secure

* chore: use orbos consistent-labels latest commit

* test: make tests compatible with new labels

* fix: default to sa token for start command

* fix: use cockroachdb v12.02

* fix: don't delete flyway user

* test: fix migration test

* fix: use correct table qualifiers

* fix: don't alter sequence ownership

* fix: upgrade flyway

* fix: change ownership of all dbs and tables to admin user

* fix: change defaultdb user

* fix: treat clientid status codes >= 400 as errors

* fix: reconcile specified ZITADEL version, not binary version

* fix: add ca-certs

* fix: use latest orbos code

* fix: use orbos with fixed race condition

* fix: use latest ORBOS code

* fix: use latest ORBOS code

* fix: make migration and scaling around restoring work

* fix(operator): move zitadel operator

* chore(migrations): include owner change migration

* feat(db): add code base for database operator

* fix(db): change used image registry for database operator

* fix(db): generated mock

* fix(db): add accidentally ignored file

* fix(db): add cockroachdb backup image to pipeline

* fix(db): correct pipeline and image versions

* fix(db): correct version of used orbos

* fix(db): correct database import

* fix(db): go mod tidy

* fix(db): use new version for orbos

* fix(migrations): include migrations into zitadelctl binary (#1211)

* fix(db): use statik to integrate migrations into binary

* fix(migrations): corrections unit tests and pipeline for integrated migrations into zitadelctl binary

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations): correction in dockerfile for pipeline build

* fix(migrations):  dockerfile changes for cache optimization

* fix(database): correct used part-of label in database operator

* fix(database): correct used selectable label in zitadel operator

* fix(operator): correct lables for user secrets in zitadel operator

* fix(operator): correct lables for service test in zitadel operator

* fix: don't enable database features for user operations (#1227)

* fix: don't enable database features for user operations

* fix: omit database feature for connection info adapter

* fix: use latest orbos version

* fix(crd): corrected logic to get database connection and other info

* fix(crd): corrected yaml tags and start for zitadel operator

* fix(crd): move some dependencies and use consistent structure

* fix(crd): corrected unit-tests

* fix(crd): corrected main files for debug starts

* chore(pipeline): use correct version for zitadelctl build

* fix(crd): correct calculating of current db state for zitadel operator

* fix(crd): use binary version for deployment of crd mode operators

* fix(crd): add gitops attribute for reconciling

* fix(crd): corrected crd with newest version

* fix(migration): collect cleanup functions and only use them if all jobs are successful

* fix(zitadelctl): import gcp auth to connect to gke cluster

* feat: Add read and writesecret options for crd mode (#1435)

* fix: don't require orbconfig for crd mode

* test: pass

* fix(zitadelctl): import gcp auth to connect to gke cluster

* feat: add read and writesecret option for crd mode

* test: fix

* fix: make all crd secrets writable

* fix: use in-cluster configs for in-cluster operators

* chore: remove unnecessary debug files

Co-authored-by: Stefan Benz <[email protected]>

* fix: Crdoperatormerge review (#1385)

* fix: don't require orbconfig for crd mode

* test: pass

* fix(zitadelctl): import gcp auth to connect to gke cluster

* fix: ensure caos-system namespace

* fix: apply orbconfig at takeoff

* docs: improve help for creating an orbconfig

* docs: describe orbconfig properties

* docs: add --gitops to help message example

* fix(pipeline): correct upload of artifacts in dev releases

* test: pass

Co-authored-by: Stefan Benz <[email protected]>

* fix(test): corrected falsely merged tests

* chore: update orbos library

* fix: only handle exactly named and namespaced crd resource

* fix: print errors, check correct crd namespace

* fix: validate bucket secret

* chore: compile

* fix(operator): corrected secret handling when unused secrets are not defined

* fix(operator): corrected handling of jobs

* fix: dont print logs when readsecret path is provided

* fix(operator): corrected handling of jobs and sort for mounted volumes

* fix(operator): sort for volumes

* fix(operator): change orboos import to newest release

Co-authored-by: Florian Forster <[email protected]>
Co-authored-by: Elio Bischof <[email protected]>

(cherry picked from commit fa9bd5a8e7a5032b21272552fbc4cdf1630db049)

* fix(operator): Standard timeout handling (#1458)

* fix: always use standard time.Duration

* fix: give backup and restore more time

* fix: give backup and restore jobs more time

(cherry picked from commit 7468b7d1e818102dcd58dbb6d3647f49277a4e65)

* fix go mod

Co-authored-by: Stefan Benz <[email protected]>
Co-authored-by: Elio Bischof <[email protected]>

operator/database/kinds/backups/backups.go

@@ -28,6 +28,8 @@ func GetQueryAndDestroyFuncs(
 	operator.QueryFunc,
 	operator.DestroyFunc,
 	map[string]*secret.Secret,
+	map[string]*secret.Existing,
+	bool,
 	error,
 ) {
 	switch desiredTree.Common.Kind {
@@ -50,7 +52,7 @@ func GetQueryAndDestroyFuncs(
 			features,
 		)(monitor, desiredTree, currentTree)
 	default:
-		return nil, nil, nil, errors.Errorf("unknown database kind %s", desiredTree.Common.Kind)
+		return nil, nil, nil, nil, false, errors.Errorf("unknown database kind %s", desiredTree.Common.Kind)
 	}
 }
 

operator/database/kinds/backups/bucket/adapt.go

@@ -2,6 +2,7 @@ package bucket
 
 import (
 	"github.com/caos/orbos/mntr"
+	"github.com/caos/orbos/pkg/helper"
 	"github.com/caos/orbos/pkg/kubernetes"
 	"github.com/caos/orbos/pkg/kubernetes/resources/secret"
 	"github.com/caos/orbos/pkg/labels"
@@ -32,28 +33,36 @@ func AdaptFunc(
 	version string,
 	features []string,
 ) operator.AdaptFunc {
-	return func(monitor mntr.Monitor, desired *tree.Tree, current *tree.Tree) (queryFunc operator.QueryFunc, destroyFunc operator.DestroyFunc, secrets map[string]*secretpkg.Secret, err error) {
+	return func(
+		monitor mntr.Monitor,
+		desired *tree.Tree,
+		current *tree.Tree,
+	) (
+		operator.QueryFunc,
+		operator.DestroyFunc,
+		map[string]*secretpkg.Secret,
+		map[string]*secretpkg.Existing,
+		bool,
+		error,
+	) {
 
 		internalMonitor := monitor.WithField("component", "backup")
 
 		desiredKind, err := ParseDesiredV0(desired)
 		if err != nil {
-			return nil, nil, nil, errors.Wrap(err, "parsing desired state failed")
+			return nil, nil, nil, nil, false, errors.Wrap(err, "parsing desired state failed")
 		}
 		desired.Parsed = desiredKind
 
+		secrets, existing := getSecretsMap(desiredKind)
+
 		if !monitor.IsVerbose() && desiredKind.Spec.Verbose {
 			internalMonitor.Verbose()
 		}
 
 		destroyS, err := secret.AdaptFuncToDestroy(namespace, secretName)
 		if err != nil {
-			return nil, nil, nil, err
-		}
-
-		queryS, err := secret.AdaptFuncToEnsure(namespace, labels.MustForName(componentLabels, secretName), map[string]string{secretKey: desiredKind.Spec.ServiceAccountJSON.Value})
-		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, nil, false, err
 		}
 
 		_, destroyB, err := backup.AdaptFunc(
@@ -74,7 +83,7 @@ func AdaptFunc(
 			version,
 		)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, nil, false, err
 		}
 
 		_, destroyR, err := restore.AdaptFunc(
@@ -93,7 +102,7 @@ func AdaptFunc(
 			version,
 		)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, nil, false, err
 		}
 
 		_, destroyC, err := clean.AdaptFunc(
@@ -110,7 +119,7 @@ func AdaptFunc(
 			version,
 		)
 		if err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, nil, false, err
 		}
 
 		destroyers := make([]operator.DestroyFunc, 0)
@@ -133,6 +142,11 @@ func AdaptFunc(
 		}
 
 		return func(k8sClient kubernetes.ClientInt, queried map[string]interface{}) (operator.EnsureFunc, error) {
+
+				if err := desiredKind.validateSecrets(); err != nil {
+					return nil, err
+				}
+
 				currentDB, err := coreDB.ParseQueriedForDatabase(queried)
 				if err != nil {
 					return nil, err
@@ -143,6 +157,16 @@ func AdaptFunc(
 					databases = []string{}
 				}
 
+				value, err := helper.GetSecretValue(k8sClient, desiredKind.Spec.ServiceAccountJSON, desiredKind.Spec.ExistingServiceAccountJSON)
+				if err != nil {
+					return nil, err
+				}
+
+				queryS, err := secret.AdaptFuncToEnsure(namespace, labels.MustForName(componentLabels, secretName), map[string]string{secretKey: value})
+				if err != nil {
+					return nil, err
+				}
+
 				queryB, _, err := backup.AdaptFunc(
 					internalMonitor,
 					name,
@@ -201,30 +225,53 @@ func AdaptFunc(
 				}
 
 				queriers := make([]operator.QueryFunc, 0)
+				cleanupQueries := make([]operator.QueryFunc, 0)
 				if databases != nil && len(databases) != 0 {
 					for _, feature := range features {
 						switch feature {
-						case backup.Normal, backup.Instant:
+						case backup.Normal:
 							queriers = append(queriers,
 								operator.ResourceQueryToZitadelQuery(queryS),
 								queryB,
 							)
+						case backup.Instant:
+							queriers = append(queriers,
+								operator.ResourceQueryToZitadelQuery(queryS),
+								queryB,
+							)
+							cleanupQueries = append(cleanupQueries,
+								operator.EnsureFuncToQueryFunc(backup.GetCleanupFunc(monitor, namespace, name)),
+							)
 						case clean.Instant:
 							queriers = append(queriers,
+								operator.ResourceQueryToZitadelQuery(queryS),
 								queryC,
 							)
+							cleanupQueries = append(cleanupQueries,
+								operator.EnsureFuncToQueryFunc(clean.GetCleanupFunc(monitor, namespace, name)),
+							)
 						case restore.Instant:
 							queriers = append(queriers,
+								operator.ResourceQueryToZitadelQuery(queryS),
 								queryR,
 							)
+							cleanupQueries = append(cleanupQueries,
+								operator.EnsureFuncToQueryFunc(restore.GetCleanupFunc(monitor, namespace, name)),
+							)
 						}
 					}
 				}
 
+				for _, cleanup := range cleanupQueries {
+					queriers = append(queriers, cleanup)
+				}
+
 				return operator.QueriersToEnsureFunc(internalMonitor, false, queriers, k8sClient, queried)
 			},
 			operator.DestroyersToDestroyFunc(internalMonitor, destroyers),
-			getSecretsMap(desiredKind),
+			secrets,
+			existing,
+			false,
 			nil
 	}
 }

operator/database/kinds/backups/bucket/adapt_test.go

@@ -1,6 +1,8 @@
 package bucket
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
@@ -13,7 +15,6 @@ import (
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	corev1 "k8s.io/api/core/v1"
-	"testing"
 )
 
 func TestBucket_Secrets(t *testing.T) {
@@ -60,7 +61,7 @@ func TestBucket_Secrets(t *testing.T) {
 		"serviceaccountjson": saJson,
 	}
 
-	_, _, secrets, err := AdaptFunc(
+	_, _, secrets, existing, _, err := AdaptFunc(
 		backupName,
 		namespace,
 		componentLabels,
@@ -78,6 +79,7 @@ func TestBucket_Secrets(t *testing.T) {
 	assert.NoError(t, err)
 	for key, value := range allSecrets {
 		assert.Contains(t, secrets, key)
+		assert.Contains(t, existing, key)
 		assert.Equal(t, value, secrets[key].Value)
 	}
 }
@@ -131,7 +133,7 @@ func TestBucket_AdaptBackup(t *testing.T) {
 
 	SetBackup(client, namespace, k8sLabels, saJson)
 
-	query, _, _, err := AdaptFunc(
+	query, _, _, _, _, err := AdaptFunc(
 		backupName,
 		namespace,
 		componentLabels,
@@ -205,7 +207,7 @@ func TestBucket_AdaptInstantBackup(t *testing.T) {
 
 	SetInstantBackup(client, namespace, backupName, k8sLabels, saJson)
 
-	query, _, _, err := AdaptFunc(
+	query, _, _, _, _, err := AdaptFunc(
 		backupName,
 		namespace,
 		componentLabels,
@@ -280,7 +282,7 @@ func TestBucket_AdaptRestore(t *testing.T) {
 
 	SetRestore(client, namespace, backupName, k8sLabels, saJson)
 
-	query, _, _, err := AdaptFunc(
+	query, _, _, _, _, err := AdaptFunc(
 		backupName,
 		namespace,
 		componentLabels,
@@ -316,6 +318,15 @@ func TestBucket_AdaptClean(t *testing.T) {
 	namespace := "testNs"
 
 	componentLabels := labels.MustForComponent(labels.MustForAPI(labels.MustForOperator("testProd", "testOp", "testVersion"), "BucketBackup", "v0"), "testComponent")
+	k8sLabels := map[string]string{
+		"app.kubernetes.io/component":  "testComponent",
+		"app.kubernetes.io/managed-by": "testOp",
+		"app.kubernetes.io/name":       "backup-serviceaccountjson",
+		"app.kubernetes.io/part-of":    "testProd",
+		"app.kubernetes.io/version":    "testVersion",
+		"caos.ch/apiversion":           "v0",
+		"caos.ch/kind":                 "BucketBackup",
+	}
 
 	timestamp := "test"
 	nodeselector := map[string]string{"test": "test"}
@@ -344,9 +355,9 @@ func TestBucket_AdaptClean(t *testing.T) {
 		return nil
 	}
 
-	SetClean(client, namespace, backupName)
+	SetClean(client, namespace, backupName, k8sLabels, saJson)
 
-	query, _, _, err := AdaptFunc(
+	query, _, _, _, _, err := AdaptFunc(
 		backupName,
 		namespace,
 		componentLabels,

operator/database/kinds/backups/bucket/backup/adapt.go

@@ -1,9 +1,10 @@
 package backup
 
 import (
-	"github.com/caos/zitadel/operator"
 	"time"
 
+	"github.com/caos/zitadel/operator"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	"github.com/caos/orbos/pkg/kubernetes/resources/cronjob"
@@ -13,18 +14,18 @@ import (
 )
 
 const (
-	defaultMode        int32         = 256
-	certPath                         = "/cockroach/cockroach-certs"
-	secretPath                       = "/secrets/sa.json"
-	backupPath                       = "/cockroach"
-	backupNameEnv                    = "BACKUP_NAME"
-	cronJobNamePrefix                = "backup-"
-	internalSecretName               = "client-certs"
-	image                            = "ghcr.io/caos/zitadel-crbackup"
-	rootSecretName                   = "cockroachdb.client.root"
-	timeout            time.Duration = 60
-	Normal                           = "backup"
-	Instant                          = "instantbackup"
+	defaultMode        int32 = 256
+	certPath                 = "/cockroach/cockroach-certs"
+	secretPath               = "/secrets/sa.json"
+	backupPath               = "/cockroach"
+	backupNameEnv            = "BACKUP_NAME"
+	cronJobNamePrefix        = "backup-"
+	internalSecretName       = "client-certs"
+	image                    = "ghcr.io/caos/zitadel-crbackup"
+	rootSecretName           = "cockroachdb.client.root"
+	timeout                  = 15 * time.Minute
+	Normal                   = "backup"
+	Instant                  = "instantbackup"
 )
 
 func AdaptFunc(
@@ -119,7 +120,6 @@ func AdaptFunc(
 			queriers = append(queriers,
 				operator.EnsureFuncToQueryFunc(checkDBReady),
 				operator.ResourceQueryToZitadelQuery(queryJ),
-				operator.EnsureFuncToQueryFunc(getCleanupFunc(monitor, jobDef.Namespace, jobDef.Name)),
 			)
 		}
 	}

operator/database/kinds/backups/bucket/backup/adapt_test.go

@@ -1,6 +1,8 @@
 package backup
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
@@ -10,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	macherrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"testing"
 )
 
 func TestBackup_AdaptInstantBackup1(t *testing.T) {
@@ -60,8 +61,6 @@ func TestBackup_AdaptInstantBackup1(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,
@@ -134,8 +133,6 @@ func TestBackup_AdaptInstantBackup2(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,

operator/database/kinds/backups/bucket/backup/cleanup.go

@@ -7,15 +7,19 @@ import (
 	"github.com/pkg/errors"
 )
 
-func getCleanupFunc(monitor mntr.Monitor, namespace string, name string) operator.EnsureFunc {
+func GetCleanupFunc(
+	monitor mntr.Monitor,
+	namespace string,
+	backupName string,
+) operator.EnsureFunc {
 	return func(k8sClient kubernetes.ClientInt) error {
 		monitor.Info("waiting for backup to be completed")
-		if err := k8sClient.WaitUntilJobCompleted(namespace, name, timeout); err != nil {
+		if err := k8sClient.WaitUntilJobCompleted(namespace, GetJobName(backupName), timeout); err != nil {
 			monitor.Error(errors.Wrap(err, "error while waiting for backup to be completed"))
 			return err
 		}
 		monitor.Info("backup is completed, cleanup")
-		if err := k8sClient.DeleteJob(namespace, name); err != nil {
+		if err := k8sClient.DeleteJob(namespace, GetJobName(backupName)); err != nil {
 			monitor.Error(errors.Wrap(err, "error while trying to cleanup backup"))
 			return err
 		}

operator/database/kinds/backups/bucket/backup/cleanup_test.go

@@ -1,12 +1,13 @@
 package backup
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/mntr"
 	kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
 	"github.com/golang/mock/gomock"
 	"github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 func TestBackup_Cleanup1(t *testing.T) {
@@ -15,12 +16,12 @@ func TestBackup_Cleanup1(t *testing.T) {
 	name := "test"
 	namespace := "testNs"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }
 
@@ -30,11 +31,11 @@ func TestBackup_Cleanup2(t *testing.T) {
 	name := "test2"
 	namespace := "testNs2"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }

operator/database/kinds/backups/bucket/clean/adapt.go

@@ -1,9 +1,10 @@
 package clean
 
 import (
-	"github.com/caos/zitadel/operator"
 	"time"
 
+	"github.com/caos/zitadel/operator"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	"github.com/caos/orbos/pkg/kubernetes/resources/job"
@@ -12,16 +13,16 @@ import (
 )
 
 const (
-	Instant                          = "clean"
-	defaultMode                      = int32(256)
-	certPath                         = "/cockroach/cockroach-certs"
-	secretPath                       = "/secrets/sa.json"
-	internalSecretName               = "client-certs"
-	image                            = "ghcr.io/caos/zitadel-crbackup"
-	rootSecretName                   = "cockroachdb.client.root"
-	jobPrefix                        = "backup-"
-	jobSuffix                        = "-clean"
-	timeout            time.Duration = 60
+	Instant            = "clean"
+	defaultMode        = int32(256)
+	certPath           = "/cockroach/cockroach-certs"
+	secretPath         = "/secrets/sa.json"
+	internalSecretName = "client-certs"
+	image              = "ghcr.io/caos/zitadel-crbackup"
+	rootSecretName     = "cockroachdb.client.root"
+	jobPrefix          = "backup-"
+	jobSuffix          = "-clean"
+	timeout            = 60 * time.Second
 )
 
 func AdaptFunc(
@@ -71,7 +72,6 @@ func AdaptFunc(
 	queriers := []operator.QueryFunc{
 		operator.EnsureFuncToQueryFunc(checkDBReady),
 		operator.ResourceQueryToZitadelQuery(queryJ),
-		operator.EnsureFuncToQueryFunc(getCleanupFunc(monitor, jobDef.Namespace, jobDef.Name)),
 	}
 
 	return func(k8sClient kubernetes.ClientInt, queried map[string]interface{}) (operator.EnsureFunc, error) {

operator/database/kinds/backups/bucket/clean/adapt_test.go

@@ -1,6 +1,8 @@
 package clean
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
@@ -10,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	macherrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"testing"
 )
 
 func TestBackup_Adapt1(t *testing.T) {
@@ -49,8 +50,6 @@ func TestBackup_Adapt1(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,
@@ -109,8 +108,6 @@ func TestBackup_Adapt2(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,

operator/database/kinds/backups/bucket/clean/cleanup.go

@@ -7,19 +7,19 @@ import (
 	"github.com/pkg/errors"
 )
 
-func getCleanupFunc(
+func GetCleanupFunc(
 	monitor mntr.Monitor,
 	namespace string,
-	jobName string,
+	backupName string,
 ) operator.EnsureFunc {
 	return func(k8sClient kubernetes.ClientInt) error {
 		monitor.Info("waiting for clean to be completed")
-		if err := k8sClient.WaitUntilJobCompleted(namespace, jobName, 60); err != nil {
+		if err := k8sClient.WaitUntilJobCompleted(namespace, GetJobName(backupName), timeout); err != nil {
 			monitor.Error(errors.Wrap(err, "error while waiting for clean to be completed"))
 			return err
 		}
 		monitor.Info("clean is completed, cleanup")
-		if err := k8sClient.DeleteJob(namespace, jobName); err != nil {
+		if err := k8sClient.DeleteJob(namespace, GetJobName(backupName)); err != nil {
 			monitor.Error(errors.Wrap(err, "error while trying to cleanup clean"))
 			return err
 		}

operator/database/kinds/backups/bucket/clean/cleanup_test.go

@@ -15,12 +15,12 @@ func TestBackup_Cleanup1(t *testing.T) {
 	name := "test"
 	namespace := "testNs"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }
 
@@ -30,11 +30,11 @@ func TestBackup_Cleanup2(t *testing.T) {
 	name := "test2"
 	namespace := "testNs2"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }

operator/database/kinds/backups/bucket/desired.go

@@ -1,7 +1,9 @@
 package bucket
 
 import (
-	secret2 "github.com/caos/orbos/pkg/secret"
+	"fmt"
+
+	"github.com/caos/orbos/pkg/secret"
 	"github.com/caos/orbos/pkg/tree"
 	"github.com/pkg/errors"
 )
@@ -12,14 +14,15 @@ type DesiredV0 struct {
 }
 
 type Spec struct {
-	Verbose            bool
-	Cron               string          `yaml:"cron,omitempty"`
-	Bucket             string          `yaml:"bucket,omitempty"`
-	ServiceAccountJSON *secret2.Secret `yaml:"serviceAccountJSON,omitempty"`
+	Verbose                    bool
+	Cron                       string           `yaml:"cron,omitempty"`
+	Bucket                     string           `yaml:"bucket,omitempty"`
+	ServiceAccountJSON         *secret.Secret   `yaml:"serviceAccountJSON,omitempty"`
+	ExistingServiceAccountJSON *secret.Existing `yaml:"existingServiceAccountJSON,omitempty"`
 }
 
 func (s *Spec) IsZero() bool {
-	if (s.ServiceAccountJSON == nil || s.ServiceAccountJSON.IsZero()) &&
+	if ((s.ServiceAccountJSON == nil || s.ServiceAccountJSON.IsZero()) && (s.ExistingServiceAccountJSON == nil || s.ExistingServiceAccountJSON.IsZero())) &&
 		!s.Verbose &&
 		s.Cron == "" &&
 		s.Bucket == "" {
@@ -40,3 +43,10 @@ func ParseDesiredV0(desiredTree *tree.Tree) (*DesiredV0, error) {
 
 	return desiredKind, nil
 }
+
+func (d *DesiredV0) validateSecrets() error {
+	if err := secret.ValidateSecret(d.Spec.ServiceAccountJSON, d.Spec.ExistingServiceAccountJSON); err != nil {
+		return fmt.Errorf("validating api key failed: %w", err)
+	}
+	return nil
+}

operator/database/kinds/backups/bucket/mock.go

@@ -65,7 +65,18 @@ func SetClean(
 	k8sClient *kubernetesmock.MockClientInt,
 	namespace string,
 	backupName string,
+	labels map[string]string,
+	saJson string,
 ) {
+	k8sClient.EXPECT().ApplySecret(&corev1.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      secretName,
+			Namespace: namespace,
+			Labels:    labels,
+		},
+		StringData: map[string]string{secretKey: saJson},
+		Type:       "Opaque",
+	}).Times(1).Return(nil)
 
 	k8sClient.EXPECT().ApplyJob(gomock.Any()).Times(1).Return(nil)
 	k8sClient.EXPECT().GetJob(namespace, clean.GetJobName(backupName)).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, clean.GetJobName(backupName)))

operator/database/kinds/backups/bucket/restore/adapt.go

@@ -1,9 +1,10 @@
 package restore
 
 import (
-	"github.com/caos/zitadel/operator"
 	"time"
 
+	"github.com/caos/zitadel/operator"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	"github.com/caos/orbos/pkg/kubernetes/resources/job"
@@ -12,16 +13,16 @@ import (
 )
 
 const (
-	Instant                          = "restore"
-	defaultMode                      = int32(256)
-	certPath                         = "/cockroach/cockroach-certs"
-	secretPath                       = "/secrets/sa.json"
-	jobPrefix                        = "backup-"
-	jobSuffix                        = "-restore"
-	image                            = "ghcr.io/caos/zitadel-crbackup"
-	internalSecretName               = "client-certs"
-	rootSecretName                   = "cockroachdb.client.root"
-	timeout            time.Duration = 60
+	Instant            = "restore"
+	defaultMode        = int32(256)
+	certPath           = "/cockroach/cockroach-certs"
+	secretPath         = "/secrets/sa.json"
+	jobPrefix          = "backup-"
+	jobSuffix          = "-restore"
+	image              = "ghcr.io/caos/zitadel-crbackup"
+	internalSecretName = "client-certs"
+	rootSecretName     = "cockroachdb.client.root"
+	timeout            = 15 * time.Minute
 )
 
 func AdaptFunc(
@@ -79,7 +80,6 @@ func AdaptFunc(
 	queriers := []operator.QueryFunc{
 		operator.EnsureFuncToQueryFunc(checkDBReady),
 		operator.ResourceQueryToZitadelQuery(queryJ),
-		operator.EnsureFuncToQueryFunc(getCleanupFunc(monitor, jobdef.Namespace, jobdef.Name)),
 	}
 
 	return func(k8sClient kubernetes.ClientInt, queried map[string]interface{}) (operator.EnsureFunc, error) {

operator/database/kinds/backups/bucket/restore/adapt_test.go

@@ -1,6 +1,8 @@
 package restore
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/mntr"
 	"github.com/caos/orbos/pkg/kubernetes"
 	kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
@@ -10,7 +12,6 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	macherrs "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"testing"
 )
 
 func TestBackup_Adapt1(t *testing.T) {
@@ -54,8 +55,6 @@ func TestBackup_Adapt1(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,
@@ -121,8 +120,6 @@ func TestBackup_Adapt2(t *testing.T) {
 
 	client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
 	client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
-	client.EXPECT().WaitUntilJobCompleted(jobDef.Namespace, jobDef.Name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil)
 
 	query, _, err := AdaptFunc(
 		monitor,

operator/database/kinds/backups/bucket/restore/cleanup.go

@@ -7,15 +7,19 @@ import (
 	"github.com/pkg/errors"
 )
 
-func getCleanupFunc(monitor mntr.Monitor, namespace, jobName string) operator.EnsureFunc {
+func GetCleanupFunc(
+	monitor mntr.Monitor,
+	namespace,
+	backupName string,
+) operator.EnsureFunc {
 	return func(k8sClient kubernetes.ClientInt) error {
 		monitor.Info("waiting for restore to be completed")
-		if err := k8sClient.WaitUntilJobCompleted(namespace, jobName, timeout); err != nil {
+		if err := k8sClient.WaitUntilJobCompleted(namespace, GetJobName(backupName), timeout); err != nil {
 			monitor.Error(errors.Wrap(err, "error while waiting for restore to be completed"))
 			return err
 		}
 		monitor.Info("restore is completed, cleanup")
-		if err := k8sClient.DeleteJob(namespace, jobName); err != nil {
+		if err := k8sClient.DeleteJob(namespace, GetJobName(backupName)); err != nil {
 			monitor.Error(errors.Wrap(err, "error while trying to cleanup restore"))
 			return err
 		}

operator/database/kinds/backups/bucket/restore/cleanup_test.go

@@ -15,12 +15,12 @@ func TestBackup_Cleanup1(t *testing.T) {
 	name := "test"
 	namespace := "testNs"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }
 
@@ -30,11 +30,11 @@ func TestBackup_Cleanup2(t *testing.T) {
 	name := "test2"
 	namespace := "testNs2"
 
-	cleanupFunc := getCleanupFunc(monitor, namespace, name)
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(nil)
-	client.EXPECT().DeleteJob(namespace, name).Times(1)
+	cleanupFunc := GetCleanupFunc(monitor, namespace, name)
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
+	client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
 	assert.NoError(t, cleanupFunc(client))
 
-	client.EXPECT().WaitUntilJobCompleted(namespace, name, timeout).Times(1).Return(errors.New("fail"))
+	client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(errors.New("fail"))
 	assert.Error(t, cleanupFunc(client))
 }

operator/database/kinds/backups/bucket/secrets.go

@@ -4,8 +4,12 @@ import (
 	"github.com/caos/orbos/pkg/secret"
 )
 
-func getSecretsMap(desiredKind *DesiredV0) map[string]*secret.Secret {
-	secrets := make(map[string]*secret.Secret, 0)
+func getSecretsMap(desiredKind *DesiredV0) (map[string]*secret.Secret, map[string]*secret.Existing) {
+
+	var (
+		secrets  = make(map[string]*secret.Secret, 0)
+		existing = make(map[string]*secret.Existing, 0)
+	)
 	if desiredKind.Spec == nil {
 		desiredKind.Spec = &Spec{}
 	}
@@ -13,7 +17,14 @@ func getSecretsMap(desiredKind *DesiredV0) map[string]*secret.Secret {
 	if desiredKind.Spec.ServiceAccountJSON == nil {
 		desiredKind.Spec.ServiceAccountJSON = &secret.Secret{}
 	}
-	secrets["serviceaccountjson"] = desiredKind.Spec.ServiceAccountJSON
 
-	return secrets
+	if desiredKind.Spec.ExistingServiceAccountJSON == nil {
+		desiredKind.Spec.ExistingServiceAccountJSON = &secret.Existing{}
+	}
+
+	sakey := "serviceaccountjson"
+	secrets[sakey] = desiredKind.Spec.ServiceAccountJSON
+	existing[sakey] = desiredKind.Spec.ExistingServiceAccountJSON
+
+	return secrets, existing
 }

operator/database/kinds/backups/bucket/secrets_test.go

@@ -1,22 +1,26 @@
 package bucket
 
 import (
+	"testing"
+
 	"github.com/caos/orbos/pkg/secret"
 	"github.com/stretchr/testify/assert"
-	"testing"
 )
 
 func TestBucket_getSecretsFull(t *testing.T) {
-	secrets := getSecretsMap(&desired)
+	secrets, existing := getSecretsMap(&desired)
 	assert.Equal(t, desired.Spec.ServiceAccountJSON, secrets["serviceaccountjson"])
+	assert.Equal(t, desired.Spec.ExistingServiceAccountJSON, existing["serviceaccountjson"])
 }
 
 func TestBucket_getSecretsEmpty(t *testing.T) {
-	secrets := getSecretsMap(&desiredWithoutSecret)
+	secrets, existing := getSecretsMap(&desiredWithoutSecret)
 	assert.Equal(t, &secret.Secret{}, secrets["serviceaccountjson"])
+	assert.Equal(t, &secret.Existing{}, existing["serviceaccountjson"])
 }
 
 func TestBucket_getSecretsNil(t *testing.T) {
-	secrets := getSecretsMap(&desiredNil)
+	secrets, existing := getSecretsMap(&desiredNil)
 	assert.Equal(t, &secret.Secret{}, secrets["serviceaccountjson"])
+	assert.Equal(t, &secret.Existing{}, existing["serviceaccountjson"])
 }