feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

cmd/start/config.go

@@ -1,12 +1,16 @@
 package start
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -34,12 +38,10 @@ import (
 	"github.com/zitadel/zitadel/internal/query/projection"
 	"github.com/zitadel/zitadel/internal/serviceping"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
-	profiler "github.com/zitadel/zitadel/internal/telemetry/profiler/config"
-	tracing "github.com/zitadel/zitadel/internal/telemetry/tracing/config"
 )
 
 type Config struct {
+	Instrumentation     instrumentation.Config
 	Log                 *logging.Config
 	Port                uint16
 	ExternalPort        uint16
@@ -53,9 +55,9 @@ type Config struct {
 	WebAuthNName        string
 	Database            database.Config
 	Caches              *connector.CachesConfig
-	Tracing             tracing.Config
-	Metrics             metrics.Config
-	Profiler            profiler.Config
+	Tracing             *instrumentation.LegacyTraceConfig
+	Metrics             *instrumentation.LegacyMetricConfig
+	Profiler            *instrumentation.LegacyProfileConfig
 	Projections         projection.Config
 	Notifications       handlers.WorkerConfig
 	Executions          execution.WorkerConfig
@@ -93,21 +95,26 @@ type QuotasConfig struct {
 	Execution *logstore.EmitterConfig
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config, err := readConfig(v)
-	logging.OnError(err).Fatal("unable to read config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read config: %w", err)
+	}
 
+	config.Instrumentation.Trace.SetLegacyConfig(config.Tracing)
+	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	config.Instrumentation.Profile.SetLegacyConfig(config.Profiler)
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+
+	// Legacy logger
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Tracing.NewTracer()
-	logging.OnError(err).Fatal("unable to set tracer")
-
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
-
-	err = config.Profiler.NewProfiler()
-	logging.OnError(err).Fatal("unable to set profiler")
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 	if config.Actions != nil {
@@ -115,12 +122,14 @@ func MustNewConfig(v *viper.Viper) *Config {
 	}
 
 	err = config.SystemDefaults.Validate()
-	logging.OnError(err).Fatal("system defaults config invalid")
-
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("system defaults config invalid: %w", err)
+	}
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.
 	config.DefaultInstance.RolePermissionMappings = config.InternalAuthZ.RolePermissionMappings
 
-	return config
+	return config, shutdown, nil
 }
 
 func readConfig(v *viper.Viper) (*Config, error) {

cmd/start/start.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/tls"
 	_ "embed"
+	"errors"
 	"fmt"
+	"log/slog"
 	"math"
 	"net/http"
 	"os"
@@ -127,12 +129,25 @@ func New(server chan<- *Server) *cobra.Command {
 		Long: `starts ZITADEL.
 Requirements:
 - postgreSQL`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			err := cmd_tls.ModeFromFlag(cmd)
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start command failed", "err", err)
+				}
+			}()
+
+			err = cmd_tls.ModeFromFlag(cmd)
 			if err != nil {
 				return err
 			}
-			config := MustNewConfig(viper.GetViper())
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
 				return err

cmd/start/start_from_init.go

@@ -2,10 +2,12 @@ package start
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
@@ -24,31 +26,52 @@ Last ZITADEL starts.
 
 Requirements:
 - postgreSQL`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-init command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return fmt.Errorf("invalid tlsMode: %w", err)
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return fmt.Errorf("no master key provided: %w", err)
+			}
 
 			initCtx, cancel := context.WithCancel(cmd.Context())
 			initialise.InitAll(initCtx, initialise.MustNewConfig(viper.GetViper()))
 			cancel()
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}
 

cmd/start/start_from_setup.go

@@ -2,10 +2,11 @@ package start
 
 import (
 	"context"
+	"errors"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
@@ -24,27 +25,48 @@ Requirements:
 - database
 - database is initialized
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-setup command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return err
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return err
+			}
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return err
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}