andwati/zitadel
1
0
Fork 0
mirror of https://github.com/andwati/zitadel.git synced 2026-11-06 14:22:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(logging): add streams (#11435)

Browse Source 
# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>
This commit is contained in:
Tim Möhlmann
2026-02-04 11:51:43 +01:00
committed by GitHub
co-authored by Silvan
parent b99271755b
commit 11dbb1b277
112 changed files with 2679 additions and 981 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/mirror/auth.go
+14 -11
View File
@@ -11,8 +11,8 @@ import (
"github.com/jackc/pgx/v5/stdlib"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/database"
)
@@ -24,7 +24,10 @@ func authCmd() *cobra.Command {
ZITADEL needs to be initialized and set up with the --for-mirror flag
Only auth requests are mirrored`,
RunE: func(cmd *cobra.Command, args []string) (err error) {
config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror auth command failed")
}()
config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
if err != nil {
return err
}
@@ -43,11 +46,11 @@ Only auth requests are mirrored`,
func copyAuth(ctx context.Context, config *Migration) {
sourceClient, err := database.Connect(config.Source, false)
logging.OnError(err).Fatal("unable to connect to source database")
logging.OnError(ctx, err).Fatal("unable to connect to source database")
defer sourceClient.Close()
destClient, err := database.Connect(config.Destination, false)
logging.OnError(err).Fatal("unable to connect to destination database")
logging.OnError(ctx, err).Fatal("unable to connect to destination database")
defer destClient.Close()
copyAuthRequests(ctx, sourceClient, destClient, config.MaxAuthRequestAge)
@@ -56,12 +59,12 @@ func copyAuth(ctx context.Context, config *Migration) {
func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthRequestAge time.Duration) {
start := time.Now()
logging.Info("creating index on auth.auth_requests.change_date to speed up copy in source database")
logging.Info(ctx, "creating index on auth.auth_requests.change_date to speed up copy in source database")
_, err := source.ExecContext(ctx, "CREATE INDEX CONCURRENTLY IF NOT EXISTS auth_requests_change_date ON auth.auth_requests (change_date)")
logging.OnError(err).Fatal("unable to create index on auth.auth_requests.change_date")
logging.OnError(ctx, err).Fatal("unable to create index on auth.auth_requests.change_date")
sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire connection")
logging.OnError(ctx, err).Fatal("unable to acquire connection")
defer sourceConn.Close()
r, w := io.Pipe()
@@ -78,7 +81,7 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthReq
}()
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire connection")
logging.OnError(ctx, err).Fatal("unable to acquire connection")
defer destConn.Close()
var affected int64
@@ -97,7 +100,7 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthReq
return err
})
logging.OnError(err).Fatal("unable to copy auth requests to destination")
logging.OnError(<-errs).Fatal("unable to copy auth requests from source")
logging.WithFields("took", time.Since(start), "count", affected).Info("auth requests migrated")
logging.OnError(ctx, err).Fatal("unable to copy auth requests to destination")
logging.OnError(ctx, <-errs).Fatal("unable to copy auth requests from source")
logging.Info(ctx, "auth requests migrated", "took", time.Since(start), "count", affected)
}
cmd/mirror/config.go
+39 -24
View File
@@ -1,16 +1,17 @@
package mirror
import (
"context"
_ "embed"
"fmt"
"time"
"github.com/mitchellh/mapstructure"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
old_logging "github.com/zitadel/logging" //nolint:staticcheck
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/cmd/hooks"
"github.com/zitadel/zitadel/internal/actions"
internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -28,7 +29,7 @@ type Migration struct {
EventBulkSize uint32
MaxAuthRequestAge time.Duration
Log *logging.Config
Log *old_logging.Config
Machine *id.Config
Instrumentation instrumentation.Config
Metrics instrumentation.LegacyMetricConfig
@@ -39,39 +40,50 @@ var (
defaultConfig []byte
)
func mustNewMigrationConfig(ctx context.Context, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
func newMigrationConfig(cmd *cobra.Command, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
config := new(Migration)
mustNewConfig(v, config)
shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
err := newConfig(v, config)
if err != nil {
return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
return nil, nil, err
}
// Legacy logger
err = config.Log.SetLogger()
shutdown, err := startInstrumentation(cmd, config.Instrumentation, config.Log)
if err != nil {
return nil, nil, fmt.Errorf("unable to set logger: %w", err)
return nil, nil, err
}
id.Configure(config.Machine)
return config, shutdown, nil
}
func mustNewProjectionsConfig(v *viper.Viper) *ProjectionsConfig {
func newProjectionsConfig(cmd *cobra.Command, v *viper.Viper) (*ProjectionsConfig, instrumentation.ShutdownFunc, error) {
config := new(ProjectionsConfig)
mustNewConfig(v, config)
err := config.Log.SetLogger()
logging.OnError(err).Fatal("unable to set logger")
err := newConfig(v, config)
if err != nil {
return nil, nil, err
}
shutdown, err := startInstrumentation(cmd, config.Instrumentation, config.Log)
if err != nil {
return nil, nil, err
}
id.Configure(config.Machine)
return config
return config, shutdown, nil
}
func mustNewConfig(v *viper.Viper, config any) {
func startInstrumentation(cmd *cobra.Command, cfg instrumentation.Config, logConfig *old_logging.Config) (instrumentation.ShutdownFunc, error) {
cfg.Log.SetLegacyConfig(logConfig)
shutdown, err := instrumentation.Start(cmd.Context(), cfg)
if err != nil {
return nil, fmt.Errorf("unable to start instrumentation: %w", err)
}
// Legacy logger
err = logConfig.SetLogger()
if err != nil {
return nil, fmt.Errorf("unable to set logger: %w", err)
}
cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
return shutdown, nil
}
func newConfig(v *viper.Viper, config any) error {
err := v.Unmarshal(config,
viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
hooks.SliceTypeStringDecode[*domain.CustomMessageText],
@@ -91,5 +103,8 @@ func mustNewConfig(v *viper.Viper, config any) {
mapstructure.TextUnmarshallerHookFunc(),
)),
)
logging.OnError(err).Fatal("unable to read default config")
if err != nil {
return fmt.Errorf("unable to decode config: %w", err)
}
return nil
}
cmd/mirror/defaults.yaml
+30 -4
View File
@@ -46,14 +46,36 @@ Instrumentation:
Log:
# Log lines lower than this level are not emitted.
Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
# Streams enable logging for specific parts of the application.
Streams: # ZITADEL_INSTRUMENTATION_LOG_STREAMS (comma separated list)
- runtime # General runtime logs, such as startup and shutdown messages.
- request # Logs for incoming API and HTTP requests.
- event_handler # Logs for event handling in projections.
- queue # Logs for the job queue processing.
# - event_pusher # Logs for event pushing to the database. Warning: contains sensitive information.
# Mask replaces sensitive information with Value in logs matched by Key
Mask:
# Keys are the attribute keys to be masked in logs.
# Keys are unqualified attribute names and apply to all attributes with the specified name,
# regardless of their position in the attribute hierarchy.
# Eg. "some_key" matches "some_key" and "parent.some_key" etc.
# If the matched log attribute is a nested object or array,
# the entire structure is replaced with the specified Value.
Keys: # ZITADEL_INSTRUMENTATION_LOG_MASK_KEYS (comma separated list)
# - "first_name"
# - "last_name"
# Value is the string that replaces the original value of masked attributes.
Value: "****" # ZITADEL_INSTRUMENTATION_LOG_MASK_VALUE
# Enable printing structured logs to standard error in the specified format.
# When disabled, the legacy Log configuration is used.
# Important: when legacy Log is customized, please add a format and customize before upgrading to v5
# The following formats are supported:
# - "disabled": Disables logging
# - "disabled": Disables logging (or fallback to legacy Log config)
# - "text": Logs are printed on StdErr as human-readable text
# - "json": Logs are printed on StdErr as JSON objects
# - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
# - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting. Need to configure GoogleProjectID below!
Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
# - "gcp": JSON formatted logs compatible with Google Cloud Platform logging.
# - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting.
Format: "disabled" # ZITADEL_INSTRUMENTATION_LOG_STDERR
# Adds the source file and line number where the log line was emitted.
# Not to be confused with the source of an error.
AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
@@ -63,6 +85,10 @@ Instrumentation:
ReportLocation: true # ZITADEL_INSTRUMENTATION_LOG_ERRORS_REPORTLOCATION
# Adds stack traces to logged errors.
StackTrace: false # ZITADEL_INSTRUMENTATION_LOG_ERRORS_STACKTRACE
# Exporter configures where the OTEL formatted logs are exported to.
# You typically want to disable this unless you have a OTEL collector setup.
# Enabling stdOut or stdErr here will result in duplicate logs (of different format)
# if there's also a Format set above.
Exporter:
# The following exporter types are supported:
# - "none": Disables OTEL log exporter
cmd/mirror/event_store.go
+40 -32
View File
@@ -13,8 +13,8 @@ import (
"github.com/shopspring/decimal"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
db "github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/database/dialect"
"github.com/zitadel/zitadel/internal/id"
@@ -34,7 +34,11 @@ func eventstoreCmd() *cobra.Command {
ZITADEL needs to be initialized and set up with the --for-mirror flag
Migrate only copies events2 and unique constraints`,
RunE: func(cmd *cobra.Command, args []string) (err error) {
config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror eventstore command failed")
}()
config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
if err != nil {
return err
}
@@ -54,49 +58,48 @@ Migrate only copies events2 and unique constraints`,
func copyEventstore(ctx context.Context, config *Migration) {
sourceClient, err := db.Connect(config.Source, false)
logging.OnError(err).Fatal("unable to connect to source database")
logging.OnError(ctx, err).Fatal("unable to connect to source database")
defer sourceClient.Close()
destClient, err := db.Connect(config.Destination, false)
logging.OnError(err).Fatal("unable to connect to destination database")
logging.OnError(ctx, err).Fatal("unable to connect to destination database")
defer destClient.Close()
copyEvents(ctx, sourceClient, destClient, config.EventBulkSize)
copyUniqueConstraints(ctx, sourceClient, destClient)
}
func positionQuery(db *db.DB) string {
func positionQuery(db *db.DB) (string, error) {
switch db.Type() {
case dialect.DatabaseTypePostgres:
return "SELECT EXTRACT(EPOCH FROM clock_timestamp())"
return "SELECT EXTRACT(EPOCH FROM clock_timestamp())", nil
case dialect.DatabaseTypeCockroach:
return "SELECT cluster_logical_timestamp()"
return "SELECT cluster_logical_timestamp()", nil
default:
logging.WithFields("db_type", db.Type()).Fatal("database type not recognized")
return ""
return "", errors.New("database type not recognized")
}
}
func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
logging.Info("starting to copy events")
logging.Info(ctx, "starting to copy events")
start := time.Now()
reader, writer := io.Pipe()
migrationID, err := id.SonyFlakeGenerator().Next()
logging.OnError(err).Fatal("unable to generate migration id")
logging.OnError(ctx, err).Fatal("unable to generate migration id")
sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire source connection")
logging.OnError(ctx, err).Fatal("unable to acquire source connection")
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire dest connection")
logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
destinationES := eventstore.NewEventstoreFromOne(postgres.New(dest, &postgres.Config{
MaxRetries: 3,
}))
previousMigration, err := queryLastSuccessfulMigration(ctx, destinationES, source.DatabaseName())
logging.OnError(err).Fatal("unable to query latest successful migration")
logging.OnError(ctx, err).Fatal("unable to query latest successful migration")
var maxPosition decimal.Decimal
err = source.QueryRowContext(ctx,
@@ -105,9 +108,8 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
},
"SELECT MAX(position) FROM eventstore.events2 "+instanceClause(),
)
logging.OnError(err).Fatal("unable to query max position from source")
logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")
logging.OnError(ctx, err).Fatal("unable to query max position from source")
logging.Info(ctx, "start event migration", "from", previousMigration.Position, "to", maxPosition)
nextPos := make(chan bool, 1)
pos := make(chan decimal.Decimal, 1)
@@ -140,10 +142,10 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
if err != nil {
return zerrors.ThrowUnknownf(err, "MIGRA-KTuSq", "unable to copy events from source during iteration %d", i)
}
logging.WithFields("batch_count", i).Info("batch of events copied")
logging.Info(ctx, "batch of events copied", "batch_count", i)
if tag.RowsAffected() < int64(bulkSize) {
logging.WithFields("batch_count", i).Info("last batch of events copied")
logging.Info(ctx, "last batch of events copied", "batch_count", i)
return nil
}
@@ -162,12 +164,18 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
defer close(pos)
for range nextPos {
var position decimal.Decimal
err := dest.QueryRowContext(
query, err := positionQuery(dest)
if err != nil {
errs <- zerrors.ThrowUnknown(err, "MIGRA-Hy6t3", "unable to generate position query")
return
}
err = dest.QueryRowContext(
ctx,
func(row *sql.Row) error {
return row.Scan(&position)
},
positionQuery(dest),
query,
)
if err != nil {
errs <- zerrors.ThrowUnknown(err, "MIGRA-kMyPH", "unable to query next position")
@@ -187,7 +195,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
pgErr := new(pgconn.PgError)
errors.As(err, &pgErr)
logging.WithError(err).WithField("pg_err_details", pgErr.Detail).Error("unable to copy events into destination")
logging.WithError(ctx, err).Error("unable to copy events into destination", "pg_err_details", pgErr.Detail)
return zerrors.ThrowUnknown(err, "MIGRA-DTHi7", "unable to copy events into destination")
}
@@ -197,7 +205,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
close(errs)
writeCopyEventsDone(ctx, destinationES, migrationID, source.DatabaseName(), maxPosition, errs)
logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated")
logging.Info(ctx, "events migrated", "took", time.Since(start), "count", eventCount)
}
func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position decimal.Decimal, errs <-chan error) {
@@ -208,24 +216,24 @@ func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, sou
err := errors.Join(joinedErrs...)
if err != nil {
logging.WithError(err).Error("unable to mirror events")
logging.WithError(ctx, err).Error("unable to mirror events")
err := writeMigrationFailed(ctx, es, id, source, err)
logging.OnError(err).Fatal("unable to write failed event")
logging.OnError(ctx, err).Fatal("unable to write failed event")
return
}
err = writeMigrationSucceeded(ctx, es, id, source, position)
logging.OnError(err).Fatal("unable to write failed event")
logging.OnError(ctx, err).Fatal("unable to write succeeded event")
}
func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
logging.Info("starting to copy unique constraints")
logging.Info(ctx, "starting to copy unique constraints")
start := time.Now()
reader, writer := io.Pipe()
errs := make(chan error, 1)
sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire source connection")
logging.OnError(ctx, err).Fatal("unable to acquire source connection")
go func() {
err := sourceConn.Raw(func(driverConn interface{}) error {
@@ -243,7 +251,7 @@ func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
}()
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire dest connection")
logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
var eventCount int64
err = destConn.Raw(func(driverConn interface{}) error {
@@ -265,7 +273,7 @@ func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
return err
})
logging.OnError(err).Fatal("unable to copy unique constraints to destination")
logging.OnError(<-errs).Fatal("unable to copy unique constraints from source")
logging.WithFields("took", time.Since(start), "count", eventCount).Info("unique constraints migrated")
logging.OnError(ctx, err).Fatal("unable to copy unique constraints to destination")
logging.OnError(ctx, <-errs).Fatal("unable to copy unique constraints from source")
logging.Info(ctx, "unique constraints migrated", "took", time.Since(start), "count", eventCount)
}
cmd/mirror/mirror.go
+17 -12
View File
@@ -5,13 +5,12 @@ import (
_ "embed"
"errors"
"fmt"
"log/slog"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/cmd/key"
)
@@ -37,24 +36,28 @@ Order of execution:
3. mirror event store tables
4. recompute projections
5. verify`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
err := viper.MergeConfig(bytes.NewBuffer(defaultConfig))
logging.OnError(err).Fatal("unable to read default config")
PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror (sub)command failed")
}()
err = viper.MergeConfig(bytes.NewBuffer(defaultConfig))
if err != nil {
return fmt.Errorf("unable to read default config: %w", err)
}
for _, file := range *configFiles {
viper.SetConfigFile(file)
err := viper.MergeInConfig()
logging.WithFields("file", file).OnError(err).Warn("unable to read config file")
logging.OnError(cmd.Context(), err).Error("unable to read config file")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) (err error) {
defer func() {
if err != nil {
slog.Error("zitadel mirror command failed", "err", err)
}
logging.OnError(cmd.Context(), err).Error("zitadel mirror command failed")
}()
config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
if err != nil {
return fmt.Errorf("unable to create migration config: %w", err)
}
@@ -62,7 +65,10 @@ Order of execution:
err = errors.Join(err, shutdown(cmd.Context()))
}()
projectionConfig := mustNewProjectionsConfig(viper.GetViper())
projectionConfig, _, err := newProjectionsConfig(cmd, viper.GetViper())
if err != nil {
return fmt.Errorf("unable to create projections config: %w", err)
}
masterKey, err := key.MasterKey(cmd)
if err != nil {
@@ -72,7 +78,6 @@ Order of execution:
copySystem(cmd.Context(), config)
copyAuth(cmd.Context(), config)
copyEventstore(cmd.Context(), config)
projections(cmd.Context(), projectionConfig, masterKey)
return nil
},
cmd/mirror/projections.go
+54 -39
View File
@@ -3,6 +3,7 @@ package mirror
import (
"context"
"database/sql"
"errors"
"fmt"
"net/http"
"sync"
@@ -10,8 +11,10 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
old_logging "github.com/zitadel/logging" //nolint:staticcheck
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/cmd/encryption"
"github.com/zitadel/zitadel/cmd/key"
"github.com/zitadel/zitadel/cmd/tls"
@@ -51,13 +54,24 @@ func projectionsCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "projections",
Short: "calls the projections synchronously",
Run: func(cmd *cobra.Command, args []string) {
config := mustNewProjectionsConfig(viper.GetViper())
RunE: func(cmd *cobra.Command, args []string) (err error) {
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror projections command failed")
}()
config, shutdown, err := newProjectionsConfig(cmd, viper.GetViper())
if err != nil {
return fmt.Errorf("unable to create projections config: %w", err)
}
defer func() {
err = errors.Join(err, shutdown(cmd.Context()))
}()
masterKey, err := key.MasterKey(cmd)
logging.OnError(err).Fatal("unable to read master key")
if err != nil {
return fmt.Errorf("unable to read master key: %w", err)
}
projections(cmd.Context(), config, masterKey)
return nil
},
}
@@ -67,18 +81,19 @@ func projectionsCmd() *cobra.Command {
}
type ProjectionsConfig struct {
Destination database.Config
Projections projection.Config
Notifications handlers.WorkerConfig
EncryptionKeys *encryption.EncryptionKeyConfig
SystemAPIUsers map[string]*internal_authz.SystemAPIUser
Eventstore *eventstore.Config
Caches *connector.CachesConfig
Instrumentation instrumentation.Config
Destination database.Config
Projections projection.Config
Notifications handlers.WorkerConfig
EncryptionKeys *encryption.EncryptionKeyConfig
SystemAPIUsers map[string]*internal_authz.SystemAPIUser
Eventstore *eventstore.Config
Caches *connector.CachesConfig
Admin admin_es.Config
Auth auth_es.Config
Log *logging.Config
Log *old_logging.Config
Machine *id.Config
ExternalPort uint16
@@ -104,21 +119,21 @@ func projections(
ctx context.Context,
config *ProjectionsConfig,
masterKey string,
) {
logging.Info("starting to fill projections")
) error {
logging.Info(ctx, "starting to fill projections")
start := time.Now()
client, err := database.Connect(config.Destination, false)
logging.OnError(err).Fatal("unable to connect to database")
logging.OnError(ctx, err).Fatal("unable to connect to database")
keyStorage, err := crypto_db.NewKeyStorage(client, masterKey)
logging.OnError(err).Fatal("cannot start key storage")
logging.OnError(ctx, err).Fatal("cannot start key storage")
keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
logging.OnError(err).Fatal("unable to read encryption keys")
logging.OnError(ctx, err).Fatal("unable to read encryption keys")
staticStorage, err := config.AssetStorage.NewStorage(client.DB)
logging.OnError(err).Fatal("unable create static storage")
logging.OnError(ctx, err).Fatal("unable create static storage")
newEventstore := new_es.NewEventstore(client)
config.Eventstore.Querier = old_es.NewPostgres(client)
@@ -133,7 +148,7 @@ func projections(
sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
cacheConnectors, err := connector.StartConnectors(config.Caches, client)
logging.OnError(err).Fatal("unable to start caches")
logging.OnError(ctx, err).Fatal("unable to start caches")
queries, err := query.StartQueries(
ctx,
@@ -162,10 +177,10 @@ func projections(
config.SystemAPIUsers,
false,
)
logging.OnError(err).Fatal("unable to start queries")
logging.OnError(ctx, err).Fatal("unable to start queries")
authZRepo, err := authz.Start(queries, es, client, keys.OIDC, config.ExternalSecure)
logging.OnError(err).Fatal("unable to start authz repo")
logging.OnError(ctx, err).Fatal("unable to start authz repo")
webAuthNConfig := &webauthn.Config{
DisplayName: config.WebAuthNName,
@@ -202,10 +217,10 @@ func projections(
nil,
nil,
)
logging.OnError(err).Fatal("unable to start commands")
logging.OnError(ctx, err).Fatal("unable to start commands")
err = projection.Create(ctx, client, es, config.Projections, keys.OIDC, keys.SAML, config.SystemAPIUsers)
logging.OnError(err).Fatal("unable to start projections")
logging.OnError(ctx, err).Fatal("unable to start projections")
i18n.MustLoadSupportedLanguagesFromDir()
@@ -235,13 +250,13 @@ func projections(
config.Auth.Spooler.Client = client
config.Auth.Spooler.Eventstore = es
authView, err := auth_view.StartView(config.Auth.Spooler.Client, keys.OIDC, queries, config.Auth.Spooler.Eventstore)
logging.OnError(err).Fatal("unable to start auth view")
logging.OnError(ctx, err).Fatal("unable to start auth view")
auth_handler.Register(ctx, config.Auth.Spooler, authView, queries)
config.Admin.Spooler.Client = client
config.Admin.Spooler.Eventstore = es
adminView, err := admin_view.StartView(config.Admin.Spooler.Client)
logging.OnError(err).Fatal("unable to start admin view")
logging.OnError(ctx, err).Fatal("unable to start admin view")
admin_handler.Register(ctx, config.Admin.Spooler, adminView, staticStorage)
@@ -252,7 +267,7 @@ func projections(
go func() {
for instance := range failedInstances {
logging.WithFields("instance", instance).Error("projection failed")
logging.WithError(ctx, errors.New("projection failed for instance")).Error("projection failed", "instance", instance)
}
}()
@@ -263,64 +278,65 @@ func projections(
existingInstances := queryInstanceIDs(ctx, client)
for i, instance := range existingInstances {
instances <- instance
logging.WithFields("id", instance, "index", fmt.Sprintf("%d/%d", i, len(existingInstances))).Info("instance queued for projection")
logging.Info(ctx, "instance queued for projection", "instance", instance, "index", fmt.Sprintf("%d/%d", i, len(existingInstances)))
}
close(instances)
wg.Wait()
close(failedInstances)
logging.WithFields("took", time.Since(start)).Info("projections executed")
logging.Info(ctx, "projections executed", "took", time.Since(start))
return nil
}
func execProjections(ctx context.Context, instances <-chan string, failedInstances chan<- string, wg *sync.WaitGroup) {
for instance := range instances {
logging.WithFields("instance", instance).Info("starting projections")
ctx = internal_authz.WithInstanceID(ctx, instance)
logging.Info(ctx, "starting projections")
err := projection.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger failed")
logging.WithError(ctx, err).Error("trigger failed")
failedInstances <- instance
continue
}
err = projection.ProjectInstanceFields(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
logging.WithError(ctx, err).Error("trigger fields failed")
failedInstances <- instance
continue
}
err = admin_handler.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger admin handler failed")
logging.WithError(ctx, err).Error("trigger admin handler failed")
failedInstances <- instance
continue
}
err = projection.ProjectInstanceFields(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
logging.WithError(ctx, err).Error("trigger fields failed")
failedInstances <- instance
continue
}
err = auth_handler.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger auth handler failed")
logging.WithError(ctx, err).Error("trigger auth handler failed")
failedInstances <- instance
continue
}
err = notification.ProjectInstance(ctx)
if err != nil {
logging.WithFields("instance", instance).WithError(err).Info("trigger notification failed")
logging.WithError(ctx, err).Error("trigger notification failed")
failedInstances <- instance
continue
}
logging.WithFields("instance", instance).Info("projections done")
logging.Info(ctx, "projections done")
}
wg.Done()
}
@@ -348,7 +364,6 @@ func queryInstanceIDs(ctx context.Context, source *database.DB) []string {
},
"SELECT DISTINCT instance_id FROM eventstore.events2 WHERE instance_id <> '' AND aggregate_type = 'instance' AND event_type = 'instance.added' AND instance_id NOT IN (SELECT instance_id FROM eventstore.events2 WHERE instance_id <> '' AND aggregate_type = 'instance' AND event_type = 'instance.removed')",
)
logging.OnError(err).Fatal("unable to query instances")
logging.OnError(ctx, err).Fatal("unable to query instances")
return instances
}
cmd/mirror/system.go
+19 -16
View File
@@ -10,8 +10,8 @@ import (
"github.com/jackc/pgx/v5/stdlib"
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/database"
)
@@ -23,7 +23,10 @@ func systemCmd() *cobra.Command {
ZITADEL needs to be initialized
Only keys and assets are mirrored`,
RunE: func(cmd *cobra.Command, args []string) (err error) {
config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror system command failed")
}()
config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
if err != nil {
return err
}
@@ -42,11 +45,11 @@ Only keys and assets are mirrored`,
func copySystem(ctx context.Context, config *Migration) {
sourceClient, err := database.Connect(config.Source, false)
logging.OnError(err).Fatal("unable to connect to source database")
logging.OnError(ctx, err).Fatal("unable to connect to source database")
defer sourceClient.Close()
destClient, err := database.Connect(config.Destination, false)
logging.OnError(err).Fatal("unable to connect to destination database")
logging.OnError(ctx, err).Fatal("unable to connect to destination database")
defer destClient.Close()
copyAssets(ctx, sourceClient, destClient)
@@ -54,11 +57,11 @@ func copySystem(ctx context.Context, config *Migration) {
}
func copyAssets(ctx context.Context, source, dest *database.DB) {
logging.Info("starting to copy assets")
logging.Info(ctx, "starting to copy assets")
start := time.Now()
sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire source connection")
logging.OnError(ctx, err).Fatal("unable to acquire source connection")
defer sourceConn.Close()
r, w := io.Pipe()
@@ -76,7 +79,7 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
}()
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire dest connection")
logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
defer destConn.Close()
var assetCount int64
@@ -95,17 +98,17 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
return err
})
logging.OnError(err).Fatal("unable to copy assets to destination")
logging.OnError(<-errs).Fatal("unable to copy assets from source")
logging.WithFields("took", time.Since(start), "count", assetCount).Info("assets migrated")
logging.OnError(ctx, err).Fatal("unable to copy assets to destination")
logging.OnError(ctx, <-errs).Fatal("unable to copy assets from source")
logging.Info(ctx, "assets migrated", "took", time.Since(start), "count", assetCount)
}
func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
logging.Info("starting to copy encryption keys")
logging.Info(ctx, "starting to copy encryption keys")
start := time.Now()
sourceConn, err := source.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire source connection")
logging.OnError(ctx, err).Fatal("unable to acquire source connection")
defer sourceConn.Close()
r, w := io.Pipe()
@@ -123,7 +126,7 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
}()
destConn, err := dest.Conn(ctx)
logging.OnError(err).Fatal("unable to acquire dest connection")
logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
defer destConn.Close()
var keyCount int64
@@ -142,7 +145,7 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
return err
})
logging.OnError(err).Fatal("unable to copy encryption keys to destination")
logging.OnError(<-errs).Fatal("unable to copy encryption keys from source")
logging.WithFields("took", time.Since(start), "count", keyCount).Info("encryption keys migrated")
logging.OnError(ctx, err).Fatal("unable to copy encryption keys to destination")
logging.OnError(ctx, <-errs).Fatal("unable to copy encryption keys from source")
logging.Info(ctx, "encryption keys migrated", "took", time.Since(start), "count", keyCount)
}
cmd/mirror/verify.go
+13 -10
View File
@@ -10,8 +10,8 @@ import (
"github.com/spf13/cobra"
"github.com/spf13/viper"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
cryptoDatabase "github.com/zitadel/zitadel/internal/crypto/database"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/query/projection"
@@ -22,7 +22,10 @@ func verifyCmd() *cobra.Command {
Use: "verify",
Short: "counts if source and dest have the same amount of entries",
RunE: func(cmd *cobra.Command, args []string) (err error) {
config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
defer func() {
logging.OnError(cmd.Context(), err).Error("zitadel mirror verify command failed")
}()
config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
if err != nil {
return err
}
@@ -45,11 +48,11 @@ var schemas = []string{
func verifyMigration(ctx context.Context, config *Migration) {
sourceClient, err := database.Connect(config.Source, false)
logging.OnError(err).Fatal("unable to connect to source database")
logging.OnError(ctx, err).Fatal("unable to connect to source database")
defer sourceClient.Close()
destClient, err := database.Connect(config.Destination, false)
logging.OnError(err).Fatal("unable to connect to destination database")
logging.OnError(ctx, err).Fatal("unable to connect to destination database")
defer destClient.Close()
for _, schema := range schemas {
@@ -57,12 +60,12 @@ func verifyMigration(ctx context.Context, config *Migration) {
sourceCount := countEntries(ctx, sourceClient, table)
destCount := countEntries(ctx, destClient, table)
entry := logging.WithFields("table", table, "dest", destCount, "source", sourceCount)
logger := logging.FromCtx(ctx).With("table", table, "dest", destCount, "source", sourceCount)
if sourceCount == destCount {
entry.Debug("equal count")
logger.DebugContext(ctx, "equal count")
continue
}
entry.WithField("diff", destCount-sourceCount).Info("unequal count")
logger.InfoContext(ctx, "unequal count", "diff", destCount-sourceCount)
}
}
}
@@ -83,7 +86,7 @@ func getTables(ctx context.Context, dest *database.DB, schema string) (tables []
"SELECT CONCAT(schemaname, '.', tablename) FROM pg_tables WHERE schemaname = $1",
schema,
)
logging.WithFields("schema", schema).OnError(err).Fatal("unable to query tables")
logging.OnError(ctx, err).Fatal("unable to query tables")
return tables
}
@@ -103,7 +106,7 @@ func getViews(ctx context.Context, dest *database.DB, schema string) (tables []s
"SELECT CONCAT(schemaname, '.', viewname) FROM pg_views WHERE schemaname = $1",
schema,
)
logging.WithFields("schema", schema).OnError(err).Fatal("unable to query views")
logging.OnError(ctx, err).Fatal("unable to query views")
return tables
}
@@ -125,7 +128,7 @@ func countEntries(ctx context.Context, client *database.DB, table string) (count
},
fmt.Sprintf("SELECT COUNT(*) FROM %s %s", table, instanceClause),
)
logging.WithFields("table", table, "db", client.DatabaseName()).OnError(err).Error("unable to count")
logging.OnError(ctx, err).Fatal("unable to count")
return count
}