From c6d29fc20122813d545f37f9117522711bc2bc87 Mon Sep 17 00:00:00 2001 From: adlerhurst Date: Thu, 11 May 2023 10:04:35 +0200 Subject: [PATCH 1/4] fix(eventstore): new column to test clock_timestamp() --- cmd/setup/10.go | 6 ++--- cmd/setup/{ => 10}/10_create_temp_table.sql | 0 cmd/setup/{ => 10}/10_fill_table.sql | 0 cmd/setup/{ => 10}/10_update.sql | 0 cmd/setup/11.go | 26 +++++++++++++++++++++ cmd/setup/11.sql | 1 + cmd/setup/config.go | 1 + cmd/setup/setup.go | 3 +++ 8 files changed, 34 insertions(+), 3 deletions(-) rename cmd/setup/{ => 10}/10_create_temp_table.sql (100%) rename cmd/setup/{ => 10}/10_fill_table.sql (100%) rename cmd/setup/{ => 10}/10_update.sql (100%) create mode 100644 cmd/setup/11.go create mode 100644 cmd/setup/11.sql diff --git a/cmd/setup/10.go b/cmd/setup/10.go index fd8683f49..1661ccf7d 100644 --- a/cmd/setup/10.go +++ b/cmd/setup/10.go @@ -13,11 +13,11 @@ import ( ) var ( - //go:embed 10_create_temp_table.sql + //go:embed 10/10_create_temp_table.sql correctCreationDate10CreateTable string - //go:embed 10_fill_table.sql + //go:embed 10/10_fill_table.sql correctCreationDate10FillTable string - //go:embed 10_update.sql + //go:embed 10/10_update.sql correctCreationDate10Update string ) diff --git a/cmd/setup/10_create_temp_table.sql b/cmd/setup/10/10_create_temp_table.sql similarity index 100% rename from cmd/setup/10_create_temp_table.sql rename to cmd/setup/10/10_create_temp_table.sql diff --git a/cmd/setup/10_fill_table.sql b/cmd/setup/10/10_fill_table.sql similarity index 100% rename from cmd/setup/10_fill_table.sql rename to cmd/setup/10/10_fill_table.sql diff --git a/cmd/setup/10_update.sql b/cmd/setup/10/10_update.sql similarity index 100% rename from cmd/setup/10_update.sql rename to cmd/setup/10/10_update.sql diff --git a/cmd/setup/11.go b/cmd/setup/11.go new file mode 100644 index 000000000..97f6bf668 --- /dev/null +++ b/cmd/setup/11.go @@ -0,0 +1,26 @@ +package setup + +import ( + "context" + _ "embed" + + "github.com/zitadel/zitadel/internal/database" +) + +var ( + //go:embed 11.sql + addEventCreatedAt string +) + +type AddEventCreatedAt struct { + dbClient *database.DB +} + +func (mig *AddEventCreatedAt) Execute(ctx context.Context) error { + _, err := mig.dbClient.ExecContext(ctx, addEventCreatedAt) + return err +} + +func (mig *AddEventCreatedAt) String() string { + return "11_event_created_at" +} diff --git a/cmd/setup/11.sql b/cmd/setup/11.sql new file mode 100644 index 000000000..28b49e474 --- /dev/null +++ b/cmd/setup/11.sql @@ -0,0 +1 @@ +ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ /*NOT NULL*/ DEFAULT clock_timestamp(); \ No newline at end of file diff --git a/cmd/setup/config.go b/cmd/setup/config.go index dc75d5404..5e9819e4c 100644 --- a/cmd/setup/config.go +++ b/cmd/setup/config.go @@ -66,6 +66,7 @@ type Steps struct { s8AuthTokens *AuthTokenIndexes s9EventstoreIndexes2 *EventstoreIndexesNew CorrectCreationDate *CorrectCreationDate + s11AddEventCreatedAt *AddEventCreatedAt } type encryptionKeyConfig struct { diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index 918f08dc2..31df9f265 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -91,6 +91,7 @@ func Setup(config *Config, steps *Steps, masterKey string) { steps.s8AuthTokens = &AuthTokenIndexes{dbClient: dbClient} steps.s9EventstoreIndexes2 = New09(dbClient) steps.CorrectCreationDate.dbClient = dbClient + steps.s11AddEventCreatedAt = &AddEventCreatedAt{dbClient: dbClient} err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil) logging.OnError(err).Fatal("unable to start projections") @@ -128,6 +129,8 @@ func Setup(config *Config, steps *Steps, masterKey string) { logging.OnError(err).Fatal("unable to migrate step 9") err = migration.Migrate(ctx, eventstoreClient, steps.CorrectCreationDate) logging.OnError(err).Fatal("unable to migrate step 10") + err = migration.Migrate(ctx, eventstoreClient, steps.s11AddEventCreatedAt) + logging.OnError(err).Fatal("unable to migrate step 11") for _, repeatableStep := range repeatableSteps { err = migration.Migrate(ctx, eventstoreClient, repeatableStep) From 5a3d09d3a30000ddfbacd14340ec9768a7dd3e5a Mon Sep 17 00:00:00 2001 From: adlerhurst Date: Thu, 11 May 2023 10:31:33 +0200 Subject: [PATCH 2/4] fix(eventstore): use creation_date for existing columns --- cmd/setup/11.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/setup/11.sql b/cmd/setup/11.sql index 28b49e474..39f3e8cb0 100644 --- a/cmd/setup/11.sql +++ b/cmd/setup/11.sql @@ -1 +1,5 @@ -ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ /*NOT NULL*/ DEFAULT clock_timestamp(); \ No newline at end of file +ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ; + +UPDATE eventstore.events SET created_at = creation_date WHERE created_at IS NULL; + +ALTER TABLE eventstore.events ALTER COLUMN created_at SET NOT NULL DEFAULT clock_timestamp(); \ No newline at end of file From b9a3fac3d2f48a6e924ebddc6681e25821209e18 Mon Sep 17 00:00:00 2001 From: adlerhurst Date: Thu, 11 May 2023 11:14:42 +0200 Subject: [PATCH 3/4] fix(eventstore): backfill column --- cmd/setup/11.sql | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/setup/11.sql b/cmd/setup/11.sql index 39f3e8cb0..bb62b5f9b 100644 --- a/cmd/setup/11.sql +++ b/cmd/setup/11.sql @@ -1,5 +1,15 @@ -ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ; +BEGIN; +-- create table with empty created_at +ALTER TABLE eventstore.events ADD COLUMN created_at TIMESTAMPTZ DEFAULT NULL; +COMMIT; +BEGIN; +-- backfill created_at UPDATE eventstore.events SET created_at = creation_date WHERE created_at IS NULL; +COMMIT; -ALTER TABLE eventstore.events ALTER COLUMN created_at SET NOT NULL DEFAULT clock_timestamp(); \ No newline at end of file +BEGIN; +-- set column rules +ALTER TABLE eventstore.events ALTER COLUMN created_at SET DEFAULT clock_timestamp(); +ALTER TABLE eventstore.events ALTER COLUMN created_at SET NOT NULL; +COMMIT; \ No newline at end of file From 4d7a733e11a80fbaeb5c36d213a8b9a1b5938287 Mon Sep 17 00:00:00 2001 From: adlerhurst Date: Thu, 11 May 2023 11:53:52 +0200 Subject: [PATCH 4/4] fix(step11): execute step 10 to make sure events are in correct order --- cmd/setup/11.go | 6 ++++++ cmd/setup/setup.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/setup/11.go b/cmd/setup/11.go index 97f6bf668..4745c7199 100644 --- a/cmd/setup/11.go +++ b/cmd/setup/11.go @@ -13,10 +13,16 @@ var ( ) type AddEventCreatedAt struct { + step10 *CorrectCreationDate dbClient *database.DB } func (mig *AddEventCreatedAt) Execute(ctx context.Context) error { + // execute step 10 again because events created after the first execution of step 10 + // could still have the wrong ordering of sequences and creation date + if err := mig.step10.Execute(ctx); err != nil { + return err + } _, err := mig.dbClient.ExecContext(ctx, addEventCreatedAt) return err } diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index 31df9f265..59b7c5d6b 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -91,7 +91,7 @@ func Setup(config *Config, steps *Steps, masterKey string) { steps.s8AuthTokens = &AuthTokenIndexes{dbClient: dbClient} steps.s9EventstoreIndexes2 = New09(dbClient) steps.CorrectCreationDate.dbClient = dbClient - steps.s11AddEventCreatedAt = &AddEventCreatedAt{dbClient: dbClient} + steps.s11AddEventCreatedAt = &AddEventCreatedAt{dbClient: dbClient, step10: steps.CorrectCreationDate} err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil) logging.OnError(err).Fatal("unable to start projections")