feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

cmd/initialise/config.go

@@ -1,34 +1,54 @@
 package initialise
 
 import (
-	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	"errors"
+	"fmt"
 
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/id"
 )
 
 type Config struct {
-	Database database.Config
-	Machine  *id.Config
-	Log      *logging.Config
+	Instrumentation instrumentation.Config
+	Database        database.Config
+	Machine         *id.Config
+	Log             *old_logging.Config
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
 			database.DecodeHook(false),
+			mapstructure.StringToTimeDurationHookFunc(),
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read config: %w", err)
+	}
+
+	config.Instrumentation.Log.SetLegacyConfig(config.Log)
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
 
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
+	if err != nil {
+		err = errors.Join(err, shutdown(cmd.Context()))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
-	return config
+	return config, shutdown, nil
 }

cmd/initialise/init.go

@@ -3,13 +3,13 @@ package initialise
 import (
 	"context"
 	"embed"
+	"errors"
 	"fmt"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -47,11 +47,15 @@ The user provided by flags needs privileges to
 `,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel init command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel init command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
-			config := MustNewConfig(viper.GetViper())
 
 			return InitAll(cmd.Context(), config)
 		},
@@ -79,7 +83,7 @@ func InitAll(ctx context.Context, config *Config) error {
 }
 
 func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {
-	logging.Info("initialization started")
+	logging.Info(ctx, "initialization started")
 
 	err := ReadStmts()
 	if err != nil {

cmd/initialise/verify_database.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -27,11 +28,19 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel init verify database command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyDatabase(config.Database.DatabaseName()))
-			logging.OnError(err).Fatal("unable to initialize the database")
+			return initialise(cmd.Context(), config.Database, VerifyDatabase(config.Database.DatabaseName()))
 		},
 	}
 }
@@ -46,10 +55,10 @@ func VerifyDatabase(databaseName string) func(context.Context, *database.DB) err
 			return fmt.Errorf("unable to get current database: %w", err)
 		}
 		if currentDatabase == databaseName {
-			logging.WithFields("database", databaseName).Info("database is same as config.database.postgres.admin.ExistingDatabase, skipping creation")
+			logging.Info(ctx, "database is same as config.database.postgres.admin.ExistingDatabase, skipping creation", "database", databaseName)
 			return nil
 		}
-		logging.WithFields("database", databaseName).Info("verify database")
+		logging.Info(ctx, "verify database", "database", databaseName)
 
 		return exec(ctx, db, fmt.Sprintf(databaseStmt, databaseName), []string{dbAlreadyExistsCode})
 	}

cmd/initialise/verify_grant.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -22,11 +23,19 @@ func newGrant() *cobra.Command {
 Prerequisites:
 - postgreSQL
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify grant command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyGrant(config.Database.DatabaseName(), config.Database.Username()))
-			logging.OnError(err).Fatal("unable to set grant")
+			return initialise(cmd.Context(), config.Database, VerifyGrant(config.Database.DatabaseName(), config.Database.Username()))
 		},
 	}
 }
@@ -41,10 +50,10 @@ func VerifyGrant(databaseName, username string) func(context.Context, *database.
 			return fmt.Errorf("unable to get current user: %w", err)
 		}
 		if currentUser == username {
-			logging.WithFields("username", username).Info("config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping grant")
+			logging.Info(ctx, "config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping grant", "username", username)
 			return nil
 		}
-		logging.WithFields("user", username, "database", databaseName).Info("verify grant")
+		logging.Info(ctx, "verify grant", "user", username, "database", databaseName)
 
 		return exec(ctx, db, fmt.Sprintf(grantStmt, databaseName, username), nil)
 	}

cmd/initialise/verify_user.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -27,11 +28,19 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify user command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyUser(config.Database.Username(), config.Database.Password()))
-			logging.OnError(err).Fatal("unable to init user")
+			return initialise(cmd.Context(), config.Database, VerifyUser(config.Database.Username(), config.Database.Password()))
 		},
 	}
 }
@@ -46,11 +55,10 @@ func VerifyUser(username, password string) func(context.Context, *database.DB) e
 			return fmt.Errorf("unable to get current user: %w", err)
 		}
 		if currentUser == username {
-			logging.WithFields("username", username).Info("config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping create user")
+			logging.Info(ctx, "config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping create user", "username", username)
 			return nil
 		}
-		logging.WithFields("username", username).Info("verify user")
-
+		logging.Info(ctx, "verify user", "username", username)
 		if password != "" {
 			createUserStmt += " WITH PASSWORD '" + password + "'"
 		}

cmd/initialise/verify_zitadel.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	es_v3 "github.com/zitadel/zitadel/internal/eventstore/v3"
 )
@@ -23,10 +24,18 @@ func newZitadel() *cobra.Command {
 Prerequisites:
 - postgreSQL with user and database
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
-			err := verifyZitadel(cmd.Context(), config.Database)
-			logging.OnError(err).Fatal("unable to init zitadel")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify zitadel command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+			return verifyZitadel(cmd.Context(), config.Database)
 		},
 	}
 }
@@ -43,32 +52,32 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
 	}
 	defer conn.Close()
 
-	logging.WithFields().Info("verify system")
+	logging.Info(ctx, "verify system")
 	if err := exec(ctx, conn, fmt.Sprintf(createSystemStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify encryption keys")
+	logging.Info(ctx, "verify encryption keys")
 	if err := createEncryptionKeys(ctx, conn); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify projections")
+	logging.Info(ctx, "verify projections")
 	if err := exec(ctx, conn, fmt.Sprintf(createProjectionsStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify eventstore")
+	logging.Info(ctx, "verify eventstore")
 	if err := exec(ctx, conn, fmt.Sprintf(createEventstoreStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify events tables")
+	logging.Info(ctx, "verify events tables")
 	if err := createEvents(ctx, conn); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify unique constraints")
+	logging.Info(ctx, "verify unique constraints")
 	if err := exec(ctx, conn, createUniqueConstraints, nil); err != nil {
 		return err
 	}
@@ -77,7 +86,7 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
 }
 
 func verifyZitadel(ctx context.Context, config database.Config) error {
-	logging.WithFields("database", config.DatabaseName()).Info("verify zitadel")
+	logging.Info(ctx, "verify zitadel", "database", config.DatabaseName())
 
 	db, err := database.Connect(config, false)
 	if err != nil {
@@ -98,7 +107,7 @@ func createEncryptionKeys(ctx context.Context, db database.Beginner) error {
 	}
 	if _, err = tx.Exec(createEncryptionKeysStmt); err != nil {
 		rollbackErr := tx.Rollback()
-		logging.OnError(rollbackErr).Error("rollback failed")
+		logging.WithError(ctx, rollbackErr).Error("rollback failed")
 		return err
 	}
 
@@ -113,7 +122,7 @@ func createEvents(ctx context.Context, conn *sql.Conn) (err error) {
 	defer func() {
 		if err != nil {
 			rollbackErr := tx.Rollback()
-			logging.OnError(rollbackErr).Error("rollback failed")
+			logging.WithError(ctx, rollbackErr).Error("rollback failed")
 			return
 		}
 		err = tx.Commit()