feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

cmd/setup/setup.go

@@ -6,7 +6,6 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
@@ -17,8 +16,8 @@ import (
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/key"
@@ -64,9 +63,7 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel setup command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel setup command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -84,7 +81,7 @@ Requirements:
 				return fmt.Errorf("unable to bind \"for-mirror\" flag: %w", err)
 			}
 
-			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -92,15 +89,16 @@ Requirements:
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
 
-			steps := MustNewSteps(viper.New())
+			steps, err := NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
 
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
 				return fmt.Errorf("no master key provided: %w", err)
 			}
-
-			Setup(cmd.Context(), config, steps, masterKey)
-			return nil
+			return Setup(cmd.Context(), config, steps, masterKey)
 		},
 	}
 
@@ -127,9 +125,8 @@ func bindForMirror(cmd *cobra.Command) error {
 	return viper.BindPFlag("ForMirror", cmd.Flags().Lookup("for-mirror"))
 }
 
-func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) {
-	logging.Info("setup started")
-
+func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) (err error) {
+	logging.Info(ctx, "setup started")
 	var setupErr error
 	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
 
@@ -137,15 +134,14 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		stop()
 
 		if setupErr == nil {
-			logging.Info("setup completed")
+			logging.Info(ctx, "setup completed")
 			return
 		}
 
-		if setupErr != nil && !errors.Is(setupErr, context.Canceled) {
+		if !errors.Is(setupErr, context.Canceled) {
 			// If Setup failed for some other reason than the context being cancelled,
 			// then this could be a fatal error we should not retry
-			logging.WithFields("error", setupErr).Fatal("setup failed, skipping cleanup")
-			return
+			logging.OnError(ctx, setupErr).Fatal("setup failed, skipping cleanup")
 		}
 
 		// if we're in the middle of long-running setup, run cleanup before exiting
@@ -155,12 +151,13 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 		defer cleanupCancel()
 
-		Cleanup(cleanupCtx, config)
+		err = Cleanup(cleanupCtx, config)
+		logging.OnError(ctx, err).Error("setup cleanup failed")
 	}()
 
 	i18n.MustLoadSupportedLanguagesFromDir()
 	dbClient, err := database.Connect(config.Database, false)
-	logging.OnError(err).Fatal("unable to connect to database")
+	logging.OnError(ctx, err).Fatal("unable to connect to database")
 
 	config.Eventstore.Querier = old_es.NewPostgres(dbClient)
 	esV3 := new_es.NewEventstore(dbClient)
@@ -168,7 +165,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	config.Eventstore.Searcher = esV3
 	eventstoreClient := eventstore.NewEventstore(config.Eventstore)
 
-	logging.OnError(err).Fatal("unable to start eventstore")
+	logging.OnError(ctx, err).Fatal("unable to start eventstore")
 	eventstoreV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(dbClient, &es_v4_pg.Config{
 		MaxRetries: config.Eventstore.MaxRetries,
 	}))
@@ -252,7 +249,9 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s69CacheTablesLogged = &CacheTablesLogged{dbClient: dbClient}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil, nil)
-	logging.OnError(err).Fatal("unable to start projections")
+	if err != nil {
+		return fmt.Errorf("unable to create projections: %w", err)
+	}
 
 	for _, step := range []migration.Migration{
 		steps.s14NewEventsTable,
@@ -383,6 +382,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 			return
 		}
 	}
+	return nil
 }
 
 func executeMigration(ctx context.Context, eventstoreClient *eventstore.Eventstore, step migration.Migration, errorMsg string) error {
@@ -403,8 +403,8 @@ func executeMigration(ctx context.Context, eventstoreClient *eventstore.Eventsto
 			"hint", pgErr.Hint,
 		)
 	}
-	logging.WithFields(logFields...).WithError(err).Error(errorMsg)
-	return fmt.Errorf("%s: %w", errorMsg, err)
+	logging.WithError(ctx, err).Error(errorMsg, logFields...)
+	return err
 }
 
 // readStmt reads a single file from the embedded FS,
@@ -455,10 +455,10 @@ func startCommandsQueries(
 	*auth_view.View,
 ) {
 	keyStorage, err := cryptoDB.NewKeyStorage(dbClient, masterKey)
-	logging.OnError(err).Fatal("unable to start key storage")
+	logging.OnError(ctx, err).Fatal("unable to start key storage")
 
 	keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
-	logging.OnError(err).Fatal("unable to ensure encryption keys")
+	logging.OnError(ctx, err).Fatal("unable to ensure encryption keys")
 
 	err = projection.Create(
 		ctx,
@@ -473,13 +473,13 @@ func startCommandsQueries(
 		keys.SAML,
 		config.SystemAPIUsers,
 	)
-	logging.OnError(err).Fatal("unable to start projections")
+	logging.OnError(ctx, err).Fatal("unable to start projections")
 
 	staticStorage, err := config.AssetStorage.NewStorage(dbClient.DB)
-	logging.OnError(err).Fatal("unable to start asset storage")
+	logging.OnError(ctx, err).Fatal("unable to start asset storage")
 
 	adminView, err := admin_view.StartView(dbClient)
-	logging.OnError(err).Fatal("unable to start admin view")
+	logging.OnError(ctx, err).Fatal("unable to start admin view")
 	admin_handler.Register(ctx,
 		admin_handler.Config{
 			Client:                dbClient,
@@ -494,7 +494,7 @@ func startCommandsQueries(
 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
 
 	cacheConnectors, err := connector.StartConnectors(config.Caches, dbClient)
-	logging.OnError(err).Fatal("unable to start caches")
+	logging.OnError(ctx, err).Fatal("unable to start caches")
 
 	queries, err := query.StartQueries(
 		ctx,
@@ -523,10 +523,10 @@ func startCommandsQueries(
 		nil, // not needed for projections
 		false,
 	)
-	logging.OnError(err).Fatal("unable to start queries")
+	logging.OnError(ctx, err).Fatal("unable to start queries")
 
 	authView, err := auth_view.StartView(dbClient, keys.OIDC, queries, eventstoreClient)
-	logging.OnError(err).Fatal("unable to start admin view")
+	logging.OnError(ctx, err).Fatal("unable to start auth view")
 	auth_handler.Register(ctx,
 		auth_handler.Config{
 			Client:                dbClient,
@@ -539,7 +539,7 @@ func startCommandsQueries(
 	)
 
 	authZRepo, err := authz.Start(queries, eventstoreClient, dbClient, keys.OIDC, config.ExternalSecure)
-	logging.OnError(err).Fatal("unable to start authz repo")
+	logging.OnError(ctx, err).Fatal("unable to start authz repo")
 	permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
 		return internal_authz.CheckPermission(ctx, authZRepo, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
 	}
@@ -577,12 +577,12 @@ func startCommandsQueries(
 		nil,
 		nil,
 	)
-	logging.OnError(err).Fatal("unable to start commands")
+	logging.OnError(ctx, err).Fatal("unable to start commands")
 
 	q, err := queue.NewQueue(&queue.Config{
 		Client: dbClient,
 	})
-	logging.OnError(err).Fatal("unable to init queue")
+	logging.OnError(ctx, err).Fatal("unable to init queue")
 
 	notify_handler.Register(
 		ctx,
@@ -616,28 +616,28 @@ func initProjections(
 ) error {
 	for _, p := range projection.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("projection migration failed")
+			logging.WithError(ctx, err).Error("projection migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range admin_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("admin schema migration failed")
+			logging.WithError(ctx, err).Error("admin schema migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range auth_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("auth schema migration failed")
+			logging.WithError(ctx, err).Error("auth schema migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range notify_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("notification migration failed")
+			logging.WithError(ctx, err).Error("notification migration failed", "name", p.String())
 			return err
 		}
 	}