mirror of
https://github.com/andwati/zitadel.git
synced 2026-11-19 11:45:12 +00:00
feat(crdb): use crdb native backup and s3 backup added (#1915)
* fix(zitadelctl): implement takedown command * fix(zitadelctl): correct destroy flow * fix(zitadelctl): correct backup commands to read crds beforehand * fix: add of destroyfile * fix: clean for userlist * fix: change backup and restore to crdb native * fix: timeout for delete pvc for cockroachdb * fix: corrected unit tests * fix: add ignored file for scale * fix: correct handling of gitops in backup command * feat: add s3 backup kind * fix: backuplist for s3 and timeout for pv deletion * fix(database): fix nil pointer with binary version * fix(database): cleanup of errors which cam with merging of the s3 logic * fix: correct unit tests * fix: cleanup monitor output Co-authored-by: Elio Bischof <[email protected]> * fix: backup imagepullpolixy to ifnotpresent Co-authored-by: Elio Bischof <[email protected]>
This commit is contained in:
co-authored by
Elio Bischof
parent
591a460450
commit
425a8b5fd5
@@ -0,0 +1,154 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/caos/zitadel/operator"
|
||||
|
||||
"github.com/caos/orbos/mntr"
|
||||
"github.com/caos/orbos/pkg/kubernetes"
|
||||
"github.com/caos/orbos/pkg/kubernetes/resources/cronjob"
|
||||
"github.com/caos/orbos/pkg/kubernetes/resources/job"
|
||||
"github.com/caos/orbos/pkg/labels"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMode int32 = 256
|
||||
certPath = "/cockroach/cockroach-certs"
|
||||
accessKeyIDPath = "/secrets/accessaccountkey"
|
||||
secretAccessKeyPath = "/secrets/secretaccesskey"
|
||||
sessionTokenPath = "/secrets/sessiontoken"
|
||||
backupNameEnv = "BACKUP_NAME"
|
||||
cronJobNamePrefix = "backup-"
|
||||
internalSecretName = "client-certs"
|
||||
rootSecretName = "cockroachdb.client.root"
|
||||
timeout = 15 * time.Minute
|
||||
Normal = "backup"
|
||||
Instant = "instantbackup"
|
||||
)
|
||||
|
||||
func AdaptFunc(
|
||||
monitor mntr.Monitor,
|
||||
backupName string,
|
||||
namespace string,
|
||||
componentLabels *labels.Component,
|
||||
checkDBReady operator.EnsureFunc,
|
||||
bucketName string,
|
||||
cron string,
|
||||
accessKeyIDName string,
|
||||
accessKeyIDKey string,
|
||||
secretAccessKeyName string,
|
||||
secretAccessKeyKey string,
|
||||
sessionTokenName string,
|
||||
sessionTokenKey string,
|
||||
region string,
|
||||
endpoint string,
|
||||
timestamp string,
|
||||
nodeselector map[string]string,
|
||||
tolerations []corev1.Toleration,
|
||||
dbURL string,
|
||||
dbPort int32,
|
||||
features []string,
|
||||
image string,
|
||||
) (
|
||||
queryFunc operator.QueryFunc,
|
||||
destroyFunc operator.DestroyFunc,
|
||||
err error,
|
||||
) {
|
||||
|
||||
command := getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
)
|
||||
|
||||
jobSpecDef := getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
image,
|
||||
command,
|
||||
)
|
||||
|
||||
destroyers := []operator.DestroyFunc{}
|
||||
queriers := []operator.QueryFunc{}
|
||||
|
||||
cronJobDef := getCronJob(
|
||||
namespace,
|
||||
labels.MustForName(componentLabels, GetJobName(backupName)),
|
||||
cron,
|
||||
jobSpecDef,
|
||||
)
|
||||
|
||||
destroyCJ, err := cronjob.AdaptFuncToDestroy(cronJobDef.Namespace, cronJobDef.Name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
queryCJ, err := cronjob.AdaptFuncToEnsure(cronJobDef)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
jobDef := getJob(
|
||||
namespace,
|
||||
labels.MustForName(componentLabels, cronJobNamePrefix+backupName),
|
||||
jobSpecDef,
|
||||
)
|
||||
|
||||
destroyJ, err := job.AdaptFuncToDestroy(jobDef.Namespace, jobDef.Name)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
queryJ, err := job.AdaptFuncToEnsure(jobDef)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for _, feature := range features {
|
||||
switch feature {
|
||||
case Normal:
|
||||
destroyers = append(destroyers,
|
||||
operator.ResourceDestroyToZitadelDestroy(destroyCJ),
|
||||
)
|
||||
queriers = append(queriers,
|
||||
operator.EnsureFuncToQueryFunc(checkDBReady),
|
||||
operator.ResourceQueryToZitadelQuery(queryCJ),
|
||||
)
|
||||
case Instant:
|
||||
destroyers = append(destroyers,
|
||||
operator.ResourceDestroyToZitadelDestroy(destroyJ),
|
||||
)
|
||||
queriers = append(queriers,
|
||||
operator.EnsureFuncToQueryFunc(checkDBReady),
|
||||
operator.ResourceQueryToZitadelQuery(queryJ),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
return func(k8sClient kubernetes.ClientInt, queried map[string]interface{}) (operator.EnsureFunc, error) {
|
||||
return operator.QueriersToEnsureFunc(monitor, false, queriers, k8sClient, queried)
|
||||
},
|
||||
operator.DestroyersToDestroyFunc(monitor, destroyers),
|
||||
nil
|
||||
}
|
||||
|
||||
func GetJobName(backupName string) string {
|
||||
return cronJobNamePrefix + backupName
|
||||
}
|
||||
@@ -0,0 +1,404 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/caos/orbos/mntr"
|
||||
"github.com/caos/orbos/pkg/kubernetes"
|
||||
kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
|
||||
"github.com/caos/orbos/pkg/labels"
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
macherrs "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
)
|
||||
|
||||
func TestBackup_AdaptInstantBackup1(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
|
||||
features := []string{Instant}
|
||||
monitor := mntr.Monitor{}
|
||||
namespace := "testNs"
|
||||
|
||||
bucketName := "testBucket"
|
||||
cron := "testCron"
|
||||
timestamp := "test"
|
||||
nodeselector := map[string]string{"test": "test"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey", Operator: "testOp"}}
|
||||
backupName := "testName"
|
||||
version := "testVersion"
|
||||
accessKeyIDName := "testAKIN"
|
||||
accessKeyIDKey := "testAKIK"
|
||||
secretAccessKeyName := "testSAKN"
|
||||
secretAccessKeyKey := "testSAKK"
|
||||
sessionTokenName := "testSTN"
|
||||
sessionTokenKey := "testSTK"
|
||||
region := "region"
|
||||
endpoint := "endpoint"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
jobName := GetJobName(backupName)
|
||||
componentLabels := labels.MustForComponent(labels.MustForAPI(labels.MustForOperator("testProd2", "testOp2", "testVersion2"), "testKind2", "testVersion2"), "testComponent")
|
||||
nameLabels := labels.MustForName(componentLabels, jobName)
|
||||
|
||||
checkDBReady := func(k8sClient kubernetes.ClientInt) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
jobDef := getJob(
|
||||
namespace,
|
||||
nameLabels,
|
||||
getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
version,
|
||||
getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
|
||||
client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
|
||||
|
||||
query, _, err := AdaptFunc(
|
||||
monitor,
|
||||
backupName,
|
||||
namespace,
|
||||
componentLabels,
|
||||
checkDBReady,
|
||||
bucketName,
|
||||
cron,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
region,
|
||||
endpoint,
|
||||
timestamp,
|
||||
nodeselector,
|
||||
tolerations,
|
||||
dbURL,
|
||||
dbPort,
|
||||
features,
|
||||
version,
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
queried := map[string]interface{}{}
|
||||
ensure, err := query(client, queried)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ensure(client))
|
||||
}
|
||||
|
||||
func TestBackup_AdaptInstantBackup2(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
|
||||
features := []string{Instant}
|
||||
monitor := mntr.Monitor{}
|
||||
namespace := "testNs2"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
bucketName := "testBucket2"
|
||||
cron := "testCron2"
|
||||
timestamp := "test2"
|
||||
nodeselector := map[string]string{"test2": "test2"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey2", Operator: "testOp2"}}
|
||||
backupName := "testName2"
|
||||
version := "testVersion2"
|
||||
accessKeyIDName := "testAKIN2"
|
||||
accessKeyIDKey := "testAKIK2"
|
||||
secretAccessKeyName := "testSAKN2"
|
||||
secretAccessKeyKey := "testSAKK2"
|
||||
sessionTokenName := "testSTN2"
|
||||
sessionTokenKey := "testSTK2"
|
||||
region := "region2"
|
||||
endpoint := "endpoint2"
|
||||
jobName := GetJobName(backupName)
|
||||
componentLabels := labels.MustForComponent(labels.MustForAPI(labels.MustForOperator("testProd2", "testOp2", "testVersion2"), "testKind2", "testVersion2"), "testComponent")
|
||||
nameLabels := labels.MustForName(componentLabels, jobName)
|
||||
|
||||
checkDBReady := func(k8sClient kubernetes.ClientInt) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
jobDef := getJob(
|
||||
namespace,
|
||||
nameLabels,
|
||||
getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
version,
|
||||
getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
client.EXPECT().ApplyJob(jobDef).Times(1).Return(nil)
|
||||
client.EXPECT().GetJob(jobDef.Namespace, jobDef.Name).Times(1).Return(nil, macherrs.NewNotFound(schema.GroupResource{"batch", "jobs"}, jobName))
|
||||
|
||||
query, _, err := AdaptFunc(
|
||||
monitor,
|
||||
backupName,
|
||||
namespace,
|
||||
componentLabels,
|
||||
checkDBReady,
|
||||
bucketName,
|
||||
cron,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
region,
|
||||
endpoint,
|
||||
timestamp,
|
||||
nodeselector,
|
||||
tolerations,
|
||||
dbURL,
|
||||
dbPort,
|
||||
features,
|
||||
version,
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
queried := map[string]interface{}{}
|
||||
ensure, err := query(client, queried)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ensure(client))
|
||||
}
|
||||
|
||||
func TestBackup_AdaptBackup1(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
|
||||
features := []string{Normal}
|
||||
monitor := mntr.Monitor{}
|
||||
namespace := "testNs"
|
||||
bucketName := "testBucket"
|
||||
cron := "testCron"
|
||||
timestamp := "test"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
nodeselector := map[string]string{"test": "test"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey", Operator: "testOp"}}
|
||||
backupName := "testName"
|
||||
version := "testVersion"
|
||||
accessKeyIDName := "testAKIN"
|
||||
accessKeyIDKey := "testAKIK"
|
||||
secretAccessKeyName := "testSAKN"
|
||||
secretAccessKeyKey := "testSAKK"
|
||||
sessionTokenName := "testSTN"
|
||||
sessionTokenKey := "testSTK"
|
||||
region := "region"
|
||||
endpoint := "endpoint"
|
||||
jobName := GetJobName(backupName)
|
||||
componentLabels := labels.MustForComponent(labels.MustForAPI(labels.MustForOperator("testProd2", "testOp2", "testVersion2"), "testKind2", "testVersion2"), "testComponent")
|
||||
nameLabels := labels.MustForName(componentLabels, jobName)
|
||||
|
||||
checkDBReady := func(k8sClient kubernetes.ClientInt) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
jobDef := getCronJob(
|
||||
namespace,
|
||||
nameLabels,
|
||||
cron,
|
||||
getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
version,
|
||||
getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
client.EXPECT().ApplyCronJob(jobDef).Times(1).Return(nil)
|
||||
|
||||
query, _, err := AdaptFunc(
|
||||
monitor,
|
||||
backupName,
|
||||
namespace,
|
||||
componentLabels,
|
||||
checkDBReady,
|
||||
bucketName,
|
||||
cron,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
region,
|
||||
endpoint,
|
||||
timestamp,
|
||||
nodeselector,
|
||||
tolerations,
|
||||
dbURL,
|
||||
dbPort,
|
||||
features,
|
||||
version,
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
queried := map[string]interface{}{}
|
||||
ensure, err := query(client, queried)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ensure(client))
|
||||
}
|
||||
|
||||
func TestBackup_AdaptBackup2(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
|
||||
features := []string{Normal}
|
||||
monitor := mntr.Monitor{}
|
||||
namespace := "testNs2"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
bucketName := "testBucket2"
|
||||
cron := "testCron2"
|
||||
timestamp := "test2"
|
||||
nodeselector := map[string]string{"test2": "test2"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey2", Operator: "testOp2"}}
|
||||
backupName := "testName2"
|
||||
version := "testVersion2"
|
||||
accessKeyIDName := "testAKIN2"
|
||||
accessKeyIDKey := "testAKIK2"
|
||||
secretAccessKeyName := "testSAKN2"
|
||||
secretAccessKeyKey := "testSAKK2"
|
||||
sessionTokenName := "testSTN2"
|
||||
sessionTokenKey := "testSTK2"
|
||||
region := "region2"
|
||||
endpoint := "endpoint2"
|
||||
jobName := GetJobName(backupName)
|
||||
componentLabels := labels.MustForComponent(labels.MustForAPI(labels.MustForOperator("testProd2", "testOp2", "testVersion2"), "testKind2", "testVersion2"), "testComponent")
|
||||
nameLabels := labels.MustForName(componentLabels, jobName)
|
||||
|
||||
checkDBReady := func(k8sClient kubernetes.ClientInt) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
jobDef := getCronJob(
|
||||
namespace,
|
||||
nameLabels,
|
||||
cron,
|
||||
getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
version,
|
||||
getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
),
|
||||
),
|
||||
)
|
||||
|
||||
client.EXPECT().ApplyCronJob(jobDef).Times(1).Return(nil)
|
||||
|
||||
query, _, err := AdaptFunc(
|
||||
monitor,
|
||||
backupName,
|
||||
namespace,
|
||||
componentLabels,
|
||||
checkDBReady,
|
||||
bucketName,
|
||||
cron,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
region,
|
||||
endpoint,
|
||||
timestamp,
|
||||
nodeselector,
|
||||
tolerations,
|
||||
dbURL,
|
||||
dbPort,
|
||||
features,
|
||||
version,
|
||||
)
|
||||
|
||||
assert.NoError(t, err)
|
||||
queried := map[string]interface{}{}
|
||||
ensure, err := query(client, queried)
|
||||
assert.NoError(t, err)
|
||||
assert.NoError(t, ensure(client))
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/caos/orbos/mntr"
|
||||
"github.com/caos/orbos/pkg/kubernetes"
|
||||
"github.com/caos/zitadel/operator"
|
||||
)
|
||||
|
||||
func GetCleanupFunc(
|
||||
monitor mntr.Monitor,
|
||||
namespace string,
|
||||
backupName string,
|
||||
) operator.EnsureFunc {
|
||||
return func(k8sClient kubernetes.ClientInt) error {
|
||||
monitor.Info("waiting for backup to be completed")
|
||||
if err := k8sClient.WaitUntilJobCompleted(namespace, GetJobName(backupName), timeout); err != nil {
|
||||
return fmt.Errorf("error while waiting for backup to be completed: %w", err)
|
||||
}
|
||||
monitor.Info("backup is completed, cleanup")
|
||||
if err := k8sClient.DeleteJob(namespace, GetJobName(backupName)); err != nil {
|
||||
return fmt.Errorf("error while trying to cleanup backup: %w", err)
|
||||
}
|
||||
monitor.Info("cleanup backup is completed")
|
||||
return nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"github.com/caos/orbos/mntr"
|
||||
kubernetesmock "github.com/caos/orbos/pkg/kubernetes/mock"
|
||||
"github.com/golang/mock/gomock"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBackup_Cleanup1(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
monitor := mntr.Monitor{}
|
||||
name := "test"
|
||||
namespace := "testNs"
|
||||
|
||||
cleanupFunc := GetCleanupFunc(monitor, namespace, name)
|
||||
client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
|
||||
client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
|
||||
assert.NoError(t, cleanupFunc(client))
|
||||
|
||||
client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(fmt.Errorf("fail"))
|
||||
assert.Error(t, cleanupFunc(client))
|
||||
}
|
||||
|
||||
func TestBackup_Cleanup2(t *testing.T) {
|
||||
client := kubernetesmock.NewMockClientInt(gomock.NewController(t))
|
||||
monitor := mntr.Monitor{}
|
||||
name := "test2"
|
||||
namespace := "testNs2"
|
||||
|
||||
cleanupFunc := GetCleanupFunc(monitor, namespace, name)
|
||||
client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(nil)
|
||||
client.EXPECT().DeleteJob(namespace, GetJobName(name)).Times(1)
|
||||
assert.NoError(t, cleanupFunc(client))
|
||||
|
||||
client.EXPECT().WaitUntilJobCompleted(namespace, GetJobName(name), timeout).Times(1).Return(fmt.Errorf("fail"))
|
||||
assert.Error(t, cleanupFunc(client))
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func getBackupCommand(
|
||||
timestamp string,
|
||||
bucketName string,
|
||||
backupName string,
|
||||
certsFolder string,
|
||||
accessKeyIDPath string,
|
||||
secretAccessKeyPath string,
|
||||
sessionTokenPath string,
|
||||
region string,
|
||||
endpoint string,
|
||||
dbURL string,
|
||||
dbPort int32,
|
||||
) string {
|
||||
|
||||
backupCommands := make([]string, 0)
|
||||
if timestamp != "" {
|
||||
backupCommands = append(backupCommands, "export "+backupNameEnv+"="+timestamp)
|
||||
} else {
|
||||
backupCommands = append(backupCommands, "export "+backupNameEnv+"=$(date +%Y-%m-%dT%H:%M:%SZ)")
|
||||
}
|
||||
|
||||
parameters := []string{
|
||||
"AWS_ACCESS_KEY_ID=$(cat " + accessKeyIDPath + ")",
|
||||
"AWS_SECRET_ACCESS_KEY=$(cat " + secretAccessKeyPath + ")",
|
||||
"AWS_SESSION_TOKEN=$(cat " + sessionTokenPath + ")",
|
||||
"AWS_ENDPOINT=" + endpoint,
|
||||
}
|
||||
if region != "" {
|
||||
parameters = append(parameters, "AWS_REGION="+region)
|
||||
}
|
||||
|
||||
backupCommands = append(backupCommands,
|
||||
strings.Join([]string{
|
||||
"cockroach",
|
||||
"sql",
|
||||
"--certs-dir=" + certsFolder,
|
||||
"--host=" + dbURL,
|
||||
"--port=" + strconv.Itoa(int(dbPort)),
|
||||
"-e",
|
||||
"\"BACKUP TO \\\"s3://" + bucketName + "/" + backupName + "/${" + backupNameEnv + "}?" + strings.Join(parameters, "&") + "\\\";\"",
|
||||
}, " ",
|
||||
),
|
||||
)
|
||||
|
||||
return strings.Join(backupCommands, " && ")
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"github.com/stretchr/testify/assert"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBackup_Command1(t *testing.T) {
|
||||
timestamp := ""
|
||||
bucketName := "test"
|
||||
backupName := "test"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
region := "region"
|
||||
endpoint := "endpoint"
|
||||
|
||||
cmd := getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
)
|
||||
equals := "export " + backupNameEnv + "=$(date +%Y-%m-%dT%H:%M:%SZ) && cockroach sql --certs-dir=" + certPath + " --host=testDB --port=80 -e \"BACKUP TO \\\"s3://test/test/${BACKUP_NAME}?AWS_ACCESS_KEY_ID=$(cat " + accessKeyIDPath + ")&AWS_SECRET_ACCESS_KEY=$(cat " + secretAccessKeyPath + ")&AWS_SESSION_TOKEN=$(cat " + sessionTokenPath + ")&AWS_ENDPOINT=endpoint&AWS_REGION=region\\\";\""
|
||||
assert.Equal(t, equals, cmd)
|
||||
}
|
||||
|
||||
func TestBackup_Command2(t *testing.T) {
|
||||
timestamp := "test"
|
||||
bucketName := "test"
|
||||
backupName := "test"
|
||||
dbURL := "testDB"
|
||||
dbPort := int32(80)
|
||||
region := "region"
|
||||
endpoint := "endpoint"
|
||||
|
||||
cmd := getBackupCommand(
|
||||
timestamp,
|
||||
bucketName,
|
||||
backupName,
|
||||
certPath,
|
||||
accessKeyIDPath,
|
||||
secretAccessKeyPath,
|
||||
sessionTokenPath,
|
||||
region,
|
||||
endpoint,
|
||||
dbURL,
|
||||
dbPort,
|
||||
)
|
||||
equals := "export " + backupNameEnv + "=test && cockroach sql --certs-dir=" + certPath + " --host=testDB --port=80 -e \"BACKUP TO \\\"s3://test/test/${BACKUP_NAME}?AWS_ACCESS_KEY_ID=$(cat " + accessKeyIDPath + ")&AWS_SECRET_ACCESS_KEY=$(cat " + secretAccessKeyPath + ")&AWS_SESSION_TOKEN=$(cat " + sessionTokenPath + ")&AWS_ENDPOINT=endpoint&AWS_REGION=region\\\";\""
|
||||
assert.Equal(t, equals, cmd)
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"github.com/caos/orbos/pkg/labels"
|
||||
"github.com/caos/zitadel/operator/helpers"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
"k8s.io/api/batch/v1beta1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func getCronJob(
|
||||
namespace string,
|
||||
nameLabels *labels.Name,
|
||||
cron string,
|
||||
jobSpecDef batchv1.JobSpec,
|
||||
) *v1beta1.CronJob {
|
||||
return &v1beta1.CronJob{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: nameLabels.Name(),
|
||||
Namespace: namespace,
|
||||
Labels: labels.MustK8sMap(nameLabels),
|
||||
},
|
||||
Spec: v1beta1.CronJobSpec{
|
||||
Schedule: cron,
|
||||
ConcurrencyPolicy: v1beta1.ForbidConcurrent,
|
||||
JobTemplate: v1beta1.JobTemplateSpec{
|
||||
Spec: jobSpecDef,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func getJob(
|
||||
namespace string,
|
||||
nameLabels *labels.Name,
|
||||
jobSpecDef batchv1.JobSpec,
|
||||
) *batchv1.Job {
|
||||
return &batchv1.Job{
|
||||
ObjectMeta: v1.ObjectMeta{
|
||||
Name: nameLabels.Name(),
|
||||
Namespace: namespace,
|
||||
Labels: labels.MustK8sMap(nameLabels),
|
||||
},
|
||||
Spec: jobSpecDef,
|
||||
}
|
||||
}
|
||||
|
||||
func getJobSpecDef(
|
||||
nodeselector map[string]string,
|
||||
tolerations []corev1.Toleration,
|
||||
accessKeyIDName string,
|
||||
accessKeyIDKey string,
|
||||
secretAccessKeyName string,
|
||||
secretAccessKeyKey string,
|
||||
sessionTokenName string,
|
||||
sessionTokenKey string,
|
||||
backupName string,
|
||||
image string,
|
||||
command string,
|
||||
) batchv1.JobSpec {
|
||||
return batchv1.JobSpec{
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
NodeSelector: nodeselector,
|
||||
Tolerations: tolerations,
|
||||
Containers: []corev1.Container{{
|
||||
Name: backupName,
|
||||
Image: image,
|
||||
Command: []string{
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
command,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{{
|
||||
Name: internalSecretName,
|
||||
MountPath: certPath,
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
SubPath: accessKeyIDKey,
|
||||
MountPath: accessKeyIDPath,
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
SubPath: secretAccessKeyKey,
|
||||
MountPath: secretAccessKeyPath,
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
SubPath: sessionTokenKey,
|
||||
MountPath: sessionTokenPath,
|
||||
}},
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
}},
|
||||
Volumes: []corev1.Volume{{
|
||||
Name: internalSecretName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: rootSecretName,
|
||||
DefaultMode: helpers.PointerInt32(defaultMode),
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: accessKeyIDName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secretAccessKeyName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: sessionTokenName,
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
package backup
|
||||
|
||||
import (
|
||||
"github.com/caos/zitadel/operator/common"
|
||||
"github.com/caos/zitadel/operator/helpers"
|
||||
"github.com/stretchr/testify/assert"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBackup_JobSpec1(t *testing.T) {
|
||||
nodeselector := map[string]string{"test": "test"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey", Operator: "testOp"}}
|
||||
backupName := "testName"
|
||||
version := "testVersion"
|
||||
command := "test"
|
||||
accessKeyIDName := "testAKIN"
|
||||
accessKeyIDKey := "testAKIK"
|
||||
secretAccessKeyName := "testSAKN"
|
||||
secretAccessKeyKey := "testSAKK"
|
||||
sessionTokenName := "testSTN"
|
||||
sessionTokenKey := "testSTK"
|
||||
|
||||
equals := batchv1.JobSpec{
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
NodeSelector: nodeselector,
|
||||
Tolerations: tolerations,
|
||||
Containers: []corev1.Container{{
|
||||
Name: backupName,
|
||||
Image: common.BackupImage.Reference("", version),
|
||||
Command: []string{
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
command,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{{
|
||||
Name: internalSecretName,
|
||||
MountPath: certPath,
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
SubPath: accessKeyIDKey,
|
||||
MountPath: accessKeyIDPath,
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
SubPath: secretAccessKeyKey,
|
||||
MountPath: secretAccessKeyPath,
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
SubPath: sessionTokenKey,
|
||||
MountPath: sessionTokenPath,
|
||||
}},
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
}},
|
||||
Volumes: []corev1.Volume{{
|
||||
Name: internalSecretName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: rootSecretName,
|
||||
DefaultMode: helpers.PointerInt32(defaultMode),
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: accessKeyIDName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secretAccessKeyName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: sessionTokenName,
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, equals, getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
common.BackupImage.Reference("", version),
|
||||
command))
|
||||
}
|
||||
|
||||
func TestBackup_JobSpec2(t *testing.T) {
|
||||
nodeselector := map[string]string{"test2": "test2"}
|
||||
tolerations := []corev1.Toleration{
|
||||
{Key: "testKey2", Operator: "testOp2"}}
|
||||
backupName := "testName2"
|
||||
version := "testVersion2"
|
||||
command := "test2"
|
||||
accessKeyIDName := "testAKIN2"
|
||||
accessKeyIDKey := "testAKIK2"
|
||||
secretAccessKeyName := "testSAKN2"
|
||||
secretAccessKeyKey := "testSAKK2"
|
||||
sessionTokenName := "testSTN2"
|
||||
sessionTokenKey := "testSTK2"
|
||||
|
||||
equals := batchv1.JobSpec{
|
||||
Template: corev1.PodTemplateSpec{
|
||||
Spec: corev1.PodSpec{
|
||||
RestartPolicy: corev1.RestartPolicyNever,
|
||||
NodeSelector: nodeselector,
|
||||
Tolerations: tolerations,
|
||||
Containers: []corev1.Container{{
|
||||
Name: backupName,
|
||||
Image: common.BackupImage.Reference("", version),
|
||||
Command: []string{
|
||||
"/bin/bash",
|
||||
"-c",
|
||||
command,
|
||||
},
|
||||
VolumeMounts: []corev1.VolumeMount{{
|
||||
Name: internalSecretName,
|
||||
MountPath: certPath,
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
SubPath: accessKeyIDKey,
|
||||
MountPath: accessKeyIDPath,
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
SubPath: secretAccessKeyKey,
|
||||
MountPath: secretAccessKeyPath,
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
SubPath: sessionTokenKey,
|
||||
MountPath: sessionTokenPath,
|
||||
}},
|
||||
ImagePullPolicy: corev1.PullIfNotPresent,
|
||||
}},
|
||||
Volumes: []corev1.Volume{{
|
||||
Name: internalSecretName,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: rootSecretName,
|
||||
DefaultMode: helpers.PointerInt32(defaultMode),
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: accessKeyIDKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: accessKeyIDName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: secretAccessKeyKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: secretAccessKeyName,
|
||||
},
|
||||
},
|
||||
}, {
|
||||
Name: sessionTokenKey,
|
||||
VolumeSource: corev1.VolumeSource{
|
||||
Secret: &corev1.SecretVolumeSource{
|
||||
SecretName: sessionTokenName,
|
||||
},
|
||||
},
|
||||
}},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
assert.Equal(t, equals, getJobSpecDef(
|
||||
nodeselector,
|
||||
tolerations,
|
||||
accessKeyIDName,
|
||||
accessKeyIDKey,
|
||||
secretAccessKeyName,
|
||||
secretAccessKeyKey,
|
||||
sessionTokenName,
|
||||
sessionTokenKey,
|
||||
backupName,
|
||||
common.BackupImage.Reference("", version),
|
||||
command,
|
||||
))
|
||||
}
|
||||
Reference in New Issue
Block a user