feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>

cmd/start/config.go

@@ -1,16 +1,17 @@
 package start
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
 	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -42,7 +43,7 @@ import (
 
 type Config struct {
 	Instrumentation     instrumentation.Config
-	Log                 *logging.Config
+	Log                 *old_logging.Config
 	Port                uint16
 	ExternalPort        uint16
 	ExternalDomain      string
@@ -95,7 +96,7 @@ type QuotasConfig struct {
 	Execution *logstore.EmitterConfig
 }
 
-func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
+func NewConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config, err := readConfig(v)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to read config: %w", err)
@@ -103,16 +104,18 @@ func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.Sh
 
 	config.Instrumentation.Trace.SetLegacyConfig(config.Tracing)
 	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	config.Instrumentation.Log.SetLegacyConfig(config.Log)
 	config.Instrumentation.Profile.SetLegacyConfig(config.Profiler)
-	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
 	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
 
 	// Legacy logger
 	err = config.Log.SetLogger()
 	if err != nil {
-		err = errors.Join(err, shutdown(ctx))
+		err = errors.Join(err, shutdown(cmd.Context()))
 		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
 	}
 
@@ -123,7 +126,7 @@ func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.Sh
 
 	err = config.SystemDefaults.Validate()
 	if err != nil {
-		err = errors.Join(err, shutdown(ctx))
+		err = errors.Join(err, shutdown(cmd.Context()))
 		return nil, nil, fmt.Errorf("system defaults config invalid: %w", err)
 	}
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.

cmd/start/flags.go

@@ -4,8 +4,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/tls"
 )
@@ -23,6 +23,7 @@ func init() {
 func startFlags(cmd *cobra.Command) {
 	cmd.Flags().AddFlagSet(startFlagSet)
 	logging.OnError(
+		cmd.Context(),
 		viper.BindPFlags(startFlagSet),
 	).Fatal("start flags")
 

cmd/start/start.go

@@ -6,7 +6,6 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"log/slog"
 	"math"
 	"net/http"
 	"os"
@@ -21,7 +20,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 	"github.com/zitadel/oidc/v3/pkg/op"
 	"github.com/zitadel/saml/pkg/provider"
 	"golang.org/x/net/http2"
@@ -29,6 +27,7 @@ import (
 	"golang.org/x/text/language"
 
 	new_domain "github.com/zitadel/zitadel/backend/v3/domain"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	v3_postgres "github.com/zitadel/zitadel/backend/v3/storage/database/dialect/postgres"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/encryption"
@@ -131,16 +130,14 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start command failed")
 			}()
 
 			err = cmd_tls.ModeFromFlag(cmd)
 			if err != nil {
 				return err
 			}
-			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -342,6 +339,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	notification.Start(ctx)
 
 	execution.Register(
+		ctx,
 		config.Executions,
 		q,
 		keys.Target,
@@ -359,7 +357,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	}
 
 	// the scheduler / periodic jobs need to be started after the queue already runs
-	if err = serviceping.Start(config.ServicePing, q); err != nil {
+	if err = serviceping.Start(ctx, config.ServicePing, q); err != nil {
 		return err
 	}
 
@@ -750,7 +748,7 @@ func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls
 	errCh := make(chan error)
 
 	go func() {
-		logging.Infof("server is listening on %s", lis.Addr().String())
+		logging.Info(ctx, "server is listening", "address", lis.Addr().String())
 		if tlsConfig != nil {
 			// we don't need to pass the files here, because we already initialized the TLS config on the server
 			errCh <- http1Server.ServeTLS(lis, "", "")
@@ -776,7 +774,7 @@ func shutdownServer(ctx context.Context, server *http.Server) error {
 	if err != nil {
 		return fmt.Errorf("could not shutdown gracefully: %w", err)
 	}
-	logging.New().Info("server shutdown gracefully")
+	logging.Info(ctx, "server shutdown gracefully")
 	return nil
 }
 

cmd/start/start_from_init.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
@@ -28,9 +28,7 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start-from-init command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start-from-init command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -43,30 +41,42 @@ Requirements:
 				return fmt.Errorf("no master key provided: %w", err)
 			}
 
-			initCtx, cancel := context.WithCancel(cmd.Context())
-			initialise.InitAll(initCtx, initialise.MustNewConfig(viper.GetViper()))
-			cancel()
-
-			err = setup.BindInitProjections(cmd)
-			if err != nil {
-				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
-			}
-
-			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			initConfig, shutdown, err := initialise.NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
 			defer func() {
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
+			initCtx, cancel := context.WithCancel(cmd.Context())
+			defer cancel()
 
-			setupSteps := setup.MustNewSteps(viper.New())
+			err = initialise.InitAll(initCtx, initConfig)
+			if err != nil {
+				return err
+			}
 
-			setupCtx, cancel := context.WithCancel(cmd.Context())
-			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
-			cancel()
+			err = setup.BindInitProjections(cmd)
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
 
-			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			setupConfig, _, err := setup.NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+
+			setupSteps, err := setup.NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
+
+			err = setup.Setup(cmd.Context(), setupConfig, setupSteps, masterKey)
+			if err != nil {
+				return err
+			}
+
+			startConfig, _, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}

cmd/start/start_from_setup.go

@@ -1,13 +1,12 @@
 package start
 
 import (
-	"context"
 	"errors"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
 	"github.com/zitadel/zitadel/cmd/tls"
@@ -27,9 +26,7 @@ Requirements:
 `,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start-from-setup command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start-from-setup command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -47,7 +44,7 @@ Requirements:
 				return err
 			}
 
-			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			setupConfig, shutdown, err := setup.NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -55,13 +52,17 @@ Requirements:
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
 
-			setupSteps := setup.MustNewSteps(viper.New())
+			setupSteps, err := setup.NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
 
-			setupCtx, cancel := context.WithCancel(cmd.Context())
-			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
-			cancel()
+			err = setup.Setup(cmd.Context(), setupConfig, setupSteps, masterKey)
+			if err != nil {
+				return err
+			}
 
-			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			startConfig, _, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}