feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

internal/api/api.go

@@ -12,10 +12,12 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
 	"github.com/zitadel/logging"
+	"go.opentelemetry.io/otel"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server"
@@ -26,7 +28,6 @@ import (
 	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 	instance_pb "github.com/zitadel/zitadel/pkg/grpc/instance/v2"
@@ -131,7 +132,12 @@ func New(
 	if err != nil {
 		return nil, err
 	}
-	api.connectOTELInterceptor, err = otelconnect.NewInterceptor()
+	api.connectOTELInterceptor, err = otelconnect.NewInterceptor(
+		otelconnect.WithTracerProvider(otel.GetTracerProvider()),
+		otelconnect.WithMeterProvider(otel.GetMeterProvider()),
+		otelconnect.WithPropagator(otel.GetTextMapPropagator()),
+		otelconnect.WithoutServerPeerAttributes(),
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -199,6 +205,7 @@ func (a *API) registerConnectServer(service server.ConnectServer) {
 		connect_middleware.CallDurationHandler(),
 		a.connectOTELInterceptor,
 		connect_middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+		connect_middleware.LogHandler(grpc_api.Probes...),
 		connect_middleware.NoCacheInterceptor(),
 		connect_middleware.InstanceInterceptor(a.queries, a.externalDomain, a.translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName, instance_pb.InstanceService_ServiceDesc.ServiceName),
 		connect_middleware.AccessStorageInterceptor(a.accessInterceptor.AccessService()),

internal/api/assets/asset.go

@@ -79,7 +79,7 @@ type ErrorHandler func(w http.ResponseWriter, r *http.Request, err error, defaul
 func DefaultErrorHandler(translator *i18n.Translator) func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 	return func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 		logging.WithFields("uri", r.RequestURI).WithError(err).Warn("error occurred on asset api")
-		code, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+		code, ok := http_util.ZitadelErrorToHTTPStatusCode(r.Context(), err)
 		if !ok {
 			code = defaultCode
 		}

internal/api/grpc/gerrors/zitadel_errors.go

@@ -1,10 +1,13 @@
 package gerrors
 
 import (
+	"context"
 	"errors"
+	"log/slog"
 
 	"connectrpc.com/connect"
 	"github.com/jackc/pgx/v5/pgconn"
+	slogctx "github.com/veqryn/slog-context"
 	"github.com/zitadel/logging"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -16,16 +19,15 @@ import (
 	"github.com/zitadel/zitadel/pkg/grpc/message"
 )
 
-func ZITADELToGRPCError(err error) error {
+func ZITADELToGRPCError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err, "code", code)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -38,20 +40,21 @@ func ZITADELToGRPCError(err error) error {
 	return s.Err()
 }
 
-func ZITADELToConnectError(err error) error {
+func ZITADELToConnectError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
 	connectError := new(connect.Error)
 	if errors.As(err, &connectError) {
+		// Connect error may be returned by other middlewares,
+		// so we assume it's a client error and log as warning.
+		slogctx.FromCtx(ctx).WarnContext(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
 		return err
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -62,44 +65,62 @@ func ZITADELToConnectError(err error) error {
 	return cErr
 }
 
-func ExtractZITADELError(err error) (c codes.Code, msg, id string, ok bool) {
+func ExtractZITADELError(err error) (code codes.Code, msg, id string) {
 	if err == nil {
-		return codes.OK, "", "", false
+		return codes.OK, "", ""
 	}
+	code, msg, id, _ = extractError(err)
+	return code, msg, id
+}
+
+func extractError(err error) (c codes.Code, msg, id string, lvl slog.Level) {
 	connErr := new(pgconn.ConnectError)
 	if ok := errors.As(err, &connErr); ok {
-		return codes.Internal, "db connection error", "", true
+		return codes.Internal, "db connection error", "", slog.LevelError
 	}
-	zitadelErr := new(zerrors.ZitadelError)
-	if ok := errors.As(err, &zitadelErr); !ok {
-		return codes.Unknown, err.Error(), "", false
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return codes.Unknown, err.Error(), "", slog.LevelError
 	}
-	switch {
-	case zerrors.IsErrorAlreadyExists(err):
-		return codes.AlreadyExists, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsDeadlineExceeded(err):
-		return codes.DeadlineExceeded, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsInternal(err):
-		return codes.Internal, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsErrorInvalidArgument(err):
-		return codes.InvalidArgument, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsNotFound(err):
-		return codes.NotFound, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPermissionDenied(err):
-		return codes.PermissionDenied, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPreconditionFailed(err):
-		return codes.FailedPrecondition, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnauthenticated(err):
-		return codes.Unauthenticated, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnavailable(err):
-		return codes.Unavailable, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnimplemented(err):
-		return codes.Unimplemented, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsResourceExhausted(err):
-		return codes.ResourceExhausted, zitadelErr.GetMessage(), zitadelErr.GetID(), true
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		c, lvl = codes.AlreadyExists, slog.LevelWarn
+	case zerrors.KindDeadlineExceeded:
+		c, lvl = codes.DeadlineExceeded, slog.LevelError
+	case zerrors.KindInternal:
+		c, lvl = codes.Internal, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		c, lvl = codes.InvalidArgument, slog.LevelWarn
+	case zerrors.KindNotFound:
+		c, lvl = codes.NotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		c, lvl = codes.PermissionDenied, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		c, lvl = codes.FailedPrecondition, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		c, lvl = codes.Unauthenticated, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		c, lvl = codes.Unavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		c, lvl = codes.Unimplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		c, lvl = codes.ResourceExhausted, slog.LevelError
+	case zerrors.KindDataLoss:
+		c, lvl = codes.DataLoss, slog.LevelError
+	case zerrors.KindCanceled:
+		c, lvl = codes.Canceled, slog.LevelWarn
+	case zerrors.KindAborted:
+		c, lvl = codes.Aborted, slog.LevelError
+	case zerrors.KindOutOfRange:
+		c, lvl = codes.OutOfRange, slog.LevelWarn
+	case zerrors.KindUnknown:
+		c, lvl = codes.Unknown, slog.LevelError
 	default:
-		return codes.Unknown, err.Error(), "", false
+		c, lvl = codes.Unknown, slog.LevelError
 	}
+	return c, msg, id, lvl
 }
 
 func getErrorInfo(id, key string, err error) protoadapt.MessageV1 {

internal/api/grpc/gerrors/zitadel_errors_test.go

@@ -3,6 +3,7 @@ package gerrors
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -41,7 +42,7 @@ func TestCaosToGRPCError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := ZITADELToGRPCError(tt.args.err); (err != nil) != tt.wantErr {
+			if err := ZITADELToGRPCError(t.Context(), tt.args.err); (err != nil) != tt.wantErr {
 				t.Errorf("ZITADELToGRPCError() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
@@ -96,7 +97,7 @@ func Test_getErrorInfo(t *testing.T) {
 	}
 }
 
-func Test_Extract(t *testing.T) {
+func Test_extractError(t *testing.T) {
 	type args struct {
 		err error
 	}
@@ -106,7 +107,7 @@ func Test_Extract(t *testing.T) {
 		wantC   codes.Code
 		wantMsg string
 		wantID  string
-		wantOk  bool
+		wantLvl slog.Level
 	}{
 		{
 			"already exists",
@@ -114,7 +115,7 @@ func Test_Extract(t *testing.T) {
 			codes.AlreadyExists,
 			"already exists",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"deadline exceeded",
@@ -122,7 +123,7 @@ func Test_Extract(t *testing.T) {
 			codes.DeadlineExceeded,
 			"deadline exceeded",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"internal error",
@@ -130,7 +131,7 @@ func Test_Extract(t *testing.T) {
 			codes.Internal,
 			"internal error",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"invalid argument",
@@ -138,7 +139,7 @@ func Test_Extract(t *testing.T) {
 			codes.InvalidArgument,
 			"invalid argument",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"not found",
@@ -146,7 +147,7 @@ func Test_Extract(t *testing.T) {
 			codes.NotFound,
 			"not found",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"permission denied",
@@ -154,7 +155,7 @@ func Test_Extract(t *testing.T) {
 			codes.PermissionDenied,
 			"permission denied",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"precondition failed",
@@ -162,7 +163,7 @@ func Test_Extract(t *testing.T) {
 			codes.FailedPrecondition,
 			"precondition failed",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unauthenticated",
@@ -170,7 +171,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unauthenticated,
 			"unauthenticated",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unavailable",
@@ -178,7 +179,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unavailable,
 			"unavailable",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unimplemented",
@@ -186,7 +187,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unimplemented,
 			"unimplemented",
 			"id",
-			true,
+			slog.LevelInfo,
 		},
 		{
 			"exhausted",
@@ -194,7 +195,7 @@ func Test_Extract(t *testing.T) {
 			codes.ResourceExhausted,
 			"exhausted",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unknown",
@@ -202,12 +203,12 @@ func Test_Extract(t *testing.T) {
 			codes.Unknown,
 			"unknown",
 			"",
-			false,
+			slog.LevelError,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotC, gotMsg, gotID, gotOk := ExtractZITADELError(tt.args.err)
+			gotC, gotMsg, gotID := ExtractZITADELError(tt.args.err)
 			if gotC != tt.wantC {
 				t.Errorf("extract() gotC = %v, want %v", gotC, tt.wantC)
 			}
@@ -217,9 +218,6 @@ func Test_Extract(t *testing.T) {
 			if gotID != tt.wantID {
 				t.Errorf("extract() gotID = %v, want %v", gotID, tt.wantID)
 			}
-			if gotOk != tt.wantOk {
-				t.Errorf("extract() gotOk = %v, want %v", gotOk, tt.wantOk)
-			}
 		})
 	}
 }

internal/api/grpc/server/connect_middleware/activity_interceptor.go

@@ -19,7 +19,7 @@ func ActivityInterceptor() connect.UnaryInterceptorFunc {
 			ctx = activityInfoFromGateway(ctx, req.Header()).SetMethod(req.Spec().Procedure).IntoContext(ctx)
 			resp, err := handler(ctx, req)
 			if isResourceAPI(req.Spec().Procedure) {
-				code, _, _, _ := gerrors.ExtractZITADELError(err)
+				code, _, _ := gerrors.ExtractZITADELError(err)
 				ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 				activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 			}

internal/api/grpc/server/connect_middleware/error_interceptor.go

@@ -19,5 +19,5 @@ func ErrorHandler() connect.UnaryInterceptorFunc {
 
 func toConnectError(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc) (connect.AnyResponse, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToConnectError(err) // TODO !
+	return resp, gerrors.ZITADELToConnectError(ctx, err)
 }

internal/api/grpc/server/connect_middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req connect.AnyRequest, handler connec
 func addInstanceByDomain(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (connect.AnyResponse, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr))

internal/api/grpc/server/connect_middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package connect_middleware
+
+import (
+	"connectrpc.com/connect"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredPrefix ...string) connect.UnaryInterceptorFunc {
+	return func(next connect.UnaryFunc) connect.UnaryFunc {
+		return logging.NewConnectInterceptor(next, ignoredPrefix...)
+	}
+}

internal/api/grpc/server/connect_middleware/metrics_interceptor.go

@@ -10,8 +10,8 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"google.golang.org/grpc/codes"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/gateway.go

@@ -20,10 +20,10 @@ import (
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	client_middleware "github.com/zitadel/zitadel/internal/api/grpc/client/middleware"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -209,7 +209,7 @@ func addInterceptors(
 	handler = http_mw.CallDurationHandler(handler)
 	handler = http_mw.CORSInterceptor(handler)
 	handler = http_mw.RobotsTagHandler(handler)
-	handler = http_mw.DefaultTelemetryHandler(handler)
+	handler = http_mw.DefaultTraceHandler(handler)
 	handler = http_mw.ActivityHandler(handler)
 	// For some non-obvious reason, the exhaustedCookieInterceptor sends the SetCookie header
 	// only if it follows the http_mw.DefaultTelemetryHandler
@@ -218,6 +218,7 @@ func addInterceptors(
 		metrics.MetricTypeTotalCount,
 		metrics.MetricTypeStatusCode,
 	}, http_utils.Probes...)(handler)
+	handler = http_mw.LogHandler("grpc_gateway", http_utils.Probes...)(handler)
 	return handler
 }
 

internal/api/grpc/server/middleware/activity_interceptor.go

@@ -18,7 +18,7 @@ func ActivityInterceptor() grpc.UnaryServerInterceptor {
 		ctx = activityInfoFromGateway(ctx).SetMethod(info.FullMethod).IntoContext(ctx)
 		resp, err := handler(ctx, req)
 		if isResourceAPI(info.FullMethod) {
-			code, _, _, _ := gerrors.ExtractZITADELError(err)
+			code, _, _ := gerrors.ExtractZITADELError(err)
 			ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 			activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 		}

internal/api/grpc/server/middleware/error_interceptor.go

@@ -17,5 +17,5 @@ func ErrorHandler() grpc.UnaryServerInterceptor {
 
 func toGRPCError(ctx context.Context, req interface{}, handler grpc.UnaryHandler) (interface{}, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToGRPCError(err)
+	return resp, gerrors.ZITADELToGRPCError(ctx, err)
 }

internal/api/grpc/server/middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req interface{}, handler grpc.UnaryHan
 func addInstanceByDomain(ctx context.Context, req interface{}, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (interface{}, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, status.Error(codes.NotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr).Error())

internal/api/grpc/server/middleware/log_interceptor.go

@@ -0,0 +1,11 @@
+package middleware
+
+import (
+	"google.golang.org/grpc"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return logging.NewGrpcInterceptor(ignoredMethodSuffixes...)
+}

internal/api/grpc/server/middleware/metrics_interceptor.go

@@ -9,8 +9,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/status"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/server.go

@@ -11,6 +11,7 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
@@ -19,7 +20,6 @@ import (
 	"github.com/zitadel/zitadel/internal/logstore"
 	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
 )
 
@@ -71,6 +71,7 @@ func CreateServer(
 			grpc_middleware.ChainUnaryServer(
 				middleware.CallDurationHandler(),
 				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+				middleware.LogHandler(grpc_api.Probes...),
 				middleware.NoCacheInterceptor(),
 				middleware.InstanceInterceptor(queries, externalDomain, translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
 				middleware.AccessStorageInterceptor(accessSvc),

internal/api/grpc/user/v2/integration_test/intent_test.go

@@ -12,13 +12,13 @@ import (
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/sink"
 	"github.com/zitadel/zitadel/pkg/grpc/object/v2"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"google.golang.org/protobuf/types/known/structpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestServer_StartIdentityProviderIntent(t *testing.T) {
@@ -631,7 +631,7 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 						DisplayName:       gu.Ptr("displayname"),
 					},
 					Email: &user.SetHumanEmail{
-						Email: "[email protected]",
+						Email:        "[email protected]",
 						Verification: &user.SetHumanEmail_SendCode{SendCode: &user.SendEmailVerificationCode{}},
 					},
 				},
@@ -1037,4 +1037,4 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 			assert.EqualExportedValues(t, tt.want, got)
 		})
 	}
-}
+}

internal/api/http/error.go

@@ -1,49 +1,77 @@
 package http
 
 import (
-	"errors"
+	"context"
+	"log/slog"
 	"net/http"
 
+	slogctx "github.com/veqryn/slog-context"
+
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func ZitadelErrorToHTTPStatusCode(err error) (statusCode int, ok bool) {
+func ZitadelErrorToHTTPStatusCode(ctx context.Context, err error) (statusCode int, ok bool) {
 	if err == nil {
 		return http.StatusOK, true
 	}
-	//nolint:errorlint
-	switch err.(type) {
-	case *zerrors.AlreadyExistsError:
-		return http.StatusConflict, true
-	case *zerrors.DeadlineExceededError:
-		return http.StatusGatewayTimeout, true
-	case *zerrors.InternalError:
-		return http.StatusInternalServerError, true
-	case *zerrors.InvalidArgumentError:
-		return http.StatusBadRequest, true
-	case *zerrors.NotFoundError:
-		return http.StatusNotFound, true
-	case *zerrors.PermissionDeniedError:
-		return http.StatusForbidden, true
-	case *zerrors.PreconditionFailedError:
-		// use the same code as grpc-gateway:
-		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
-		return http.StatusBadRequest, true
-	case *zerrors.UnauthenticatedError:
-		return http.StatusUnauthorized, true
-	case *zerrors.UnavailableError:
-		return http.StatusServiceUnavailable, true
-	case *zerrors.UnimplementedError:
-		return http.StatusNotImplemented, true
-	case *zerrors.ResourceExhaustedError:
-		return http.StatusTooManyRequests, true
-	default:
-		c := new(zerrors.ZitadelError)
-		if errors.As(err, &c) {
-			return ZitadelErrorToHTTPStatusCode(errors.Unwrap(err))
-		}
+	statusCode, key, id, lvl := extractError(err)
+	msg := key
+	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
+	if statusCode == statusUnknown {
 		return http.StatusInternalServerError, false
 	}
+	return statusCode, true
+}
+
+const statusUnknown = 0
+
+func extractError(err error) (statusCode int, msg, id string, lvl slog.Level) {
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return statusUnknown, err.Error(), "", slog.LevelError
+	}
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		statusCode, lvl = http.StatusConflict, slog.LevelError
+	case zerrors.KindDeadlineExceeded:
+		statusCode, lvl = http.StatusGatewayTimeout, slog.LevelError
+	case zerrors.KindInternal:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindNotFound:
+		statusCode, lvl = http.StatusNotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		statusCode, lvl = http.StatusForbidden, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		// use the same code as grpc-gateway:
+		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		statusCode, lvl = http.StatusUnauthorized, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		statusCode, lvl = http.StatusServiceUnavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		statusCode, lvl = http.StatusNotImplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		statusCode, lvl = http.StatusTooManyRequests, slog.LevelError
+	case zerrors.KindCanceled:
+		statusCode, lvl = 499, slog.LevelWarn
+	case zerrors.KindDataLoss:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindOutOfRange:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindAborted:
+		statusCode, lvl = http.StatusConflict, slog.LevelWarn
+	case zerrors.KindUnknown:
+		fallthrough
+	default:
+		statusCode, lvl = statusUnknown, slog.LevelError
+	}
+	return statusCode, msg, id, lvl
 }
 
 func HTTPStatusCodeToZitadelError(parent error, statusCode int, id string, message string) error {

internal/api/http/error_test.go

@@ -3,6 +3,7 @@ package http
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 
@@ -118,7 +119,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 			wantOk:         true,
 		},
 		{
-			name: "no caos/zitadel error",
+			name: "no zitadel error",
 			args: args{
 				err: errors.New("error"),
 			},
@@ -128,7 +129,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(tt.args.err)
+			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(t.Context(), tt.args.err)
 			if gotStatusCode != tt.wantStatusCode {
 				t.Errorf("ZitadelErrorToHTTPStatusCode() gotStatusCode = %v, want %v", gotStatusCode, tt.wantStatusCode)
 			}
@@ -272,9 +273,9 @@ func TestHTTPStatusCodeToZitadelError(t *testing.T) {
 				statusCode: 1000,
 				id:         "id",
 				message:    "message",
-				parent:     errors.New("parent error"),
+				parent:     io.EOF,
 			},
-			wantErr: zerrors.ThrowUnknown(nil, "id", "message"),
+			wantErr: zerrors.ThrowUnknown(io.EOF, "id", "message"),
 		},
 	}
 	for _, tt := range tests {

internal/api/http/middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		return logging.NewHandler(h, service, ignoredPrefix...)
+	}
+}

internal/api/http/middleware/metrics_interceptor.go

@@ -3,11 +3,11 @@ package middleware
 import (
 	"net/http"
 
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethods ...string) func(http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
-		return metrics.NewMetricsHandler(handler, metricTypes, ignoredMethods...)
+		return metrics.NewHandler(handler, metricTypes, ignoredMethods...)
 	}
 }

internal/api/http/middleware/telemetry_interceptor.go

@@ -1,18 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/telemetry"
-)
-
-func DefaultTelemetryHandler(handler http.Handler) http.Handler {
-	return TelemetryHandler(http_utils.Probes...)(handler)
-}
-
-func TelemetryHandler(ignoredMethods ...string) func(http.Handler) http.Handler {
-	return func(handler http.Handler) http.Handler {
-		return telemetry.TelemetryHandler(handler, ignoredMethods...)
-	}
-}

internal/api/http/middleware/trace_interceptor.go

@@ -0,0 +1,18 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
+	http_utils "github.com/zitadel/zitadel/internal/api/http"
+)
+
+func DefaultTraceHandler(handler http.Handler) http.Handler {
+	return TraceHandler(http_utils.Probes...)(handler)
+}
+
+func TraceHandler(ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(handler http.Handler) http.Handler {
+		return tracing.NewHandler(handler, ignoredPrefix...)
+	}
+}

internal/api/http/request_context.go

@@ -3,6 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"net"
 	"net/url"
 )
@@ -67,6 +68,22 @@ func (r *DomainCtx) Origin() string {
 	return fmt.Sprintf("%s://%s", r.Protocol, host)
 }
 
+var _ slog.LogValuer = (*DomainCtx)(nil)
+
+func (r *DomainCtx) LogValue() slog.Value {
+	values := make([]slog.Attr, 0, 3)
+	if r.InstanceHost != "" {
+		values = append(values, slog.String("instance_host", r.InstanceHost))
+	}
+	if r.PublicHost != "" {
+		values = append(values, slog.String("public_host", r.PublicHost))
+	}
+	if r.Protocol != "" {
+		values = append(values, slog.String("protocol", r.Protocol))
+	}
+	return slog.GroupValue(values...)
+}
+
 func DomainContext(ctx context.Context) *DomainCtx {
 	o, ok := ctx.Value(domainCtx).(*DomainCtx)
 	if !ok {

internal/api/oidc/auth_request.go

@@ -44,7 +44,7 @@ const (
 func (o *OPStorage) CreateAuthRequest(ctx context.Context, req *oidc.AuthRequest, userID string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -167,7 +167,7 @@ func (o *OPStorage) audienceFromProjectID(ctx context.Context, projectID string)
 func (o *OPStorage) AuthRequestByID(ctx context.Context, id string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -206,7 +206,7 @@ func (o *OPStorage) decryptGrant(grant string) (string, error) {
 func (o *OPStorage) SaveAuthCode(ctx context.Context, id, code string) (err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -249,7 +249,7 @@ func (o *OPStorage) TerminateSession(ctx context.Context, userID, clientID strin
 func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessions []command.HumanSignOutSession, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	userAgentID, ok := middleware.UserAgentIDFromCtx(ctx)
@@ -276,7 +276,7 @@ func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessio
 func (o *OPStorage) TerminateSessionFromRequest(ctx context.Context, endSessionRequest *op.EndSessionRequest) (redirectURI string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -505,7 +505,7 @@ func (o *OPStorage) revokeTokenV1(ctx context.Context, token, userID, clientID s
 func (o *OPStorage) GetRefreshTokenInfo(ctx context.Context, clientID string, token string) (userID string, tokenID string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/client.go

@@ -36,8 +36,6 @@ const (
 	ClaimUserGroups                 = ScopeUserGroups
 	ScopeCustomUserGroups           = "urn:zitadel:iam:user:groups"
 	ClaimCustomUserGroups           = ScopeCustomUserGroups
-
-	oidcCtx = "oidc"
 )
 
 // GetClientByClientID implements the op.Storage interface to retrieve an OIDC client by its ID.
@@ -46,7 +44,7 @@ const (
 func (o *OPStorage) GetClientByClientID(ctx context.Context, id string) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	client, err := o.query.ActiveOIDCClientByID(ctx, id, false)
@@ -188,7 +186,7 @@ func userinfoClaims(userInfo *oidc.UserInfo) func(c *actions.FieldConfig) interf
 func (s *Server) VerifyClient(ctx context.Context, r *op.Request[op.ClientCredentials]) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/error.go

@@ -1,6 +1,7 @@
 package oidc
 
 import (
+	"context"
 	"errors"
 
 	"github.com/zitadel/oidc/v3/pkg/oidc"
@@ -15,7 +16,7 @@ import (
 // When err is already of the correct type is passed as-is.
 // If the err is a Zitadel error, it is transformed with a proper HTTP status code.
 // Unknown errors are treated as internal server errors.
-func oidcError(err error) error {
+func oidcError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
@@ -37,7 +38,7 @@ func oidcError(err error) error {
 		errors.As(err, &zError)
 	}
 
-	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	newOidcErr := oidc.ErrServerError
 	if statusCode < 500 {
 		newOidcErr = oidc.ErrInvalidRequest

internal/api/oidc/error_test.go

@@ -56,7 +56,7 @@ func Test_oidcError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := oidcError(tt.err)
+			err := oidcError(t.Context(), tt.err)
 			require.ErrorIs(t, err, tt.wantErr)
 		})
 	}

internal/api/oidc/introspect.go

@@ -20,7 +20,7 @@ import (
 func (s *Server) Introspect(ctx context.Context, r *op.Request[op.IntrospectionRequest]) (resp *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	ctx, cancel := context.WithCancel(ctx)

internal/api/oidc/key.go

@@ -150,7 +150,7 @@ func (k *publicKeyCache) getKey(ctx context.Context, keyID string) (_ *cachedPub
 func (k *publicKeyCache) verifySignature(ctx context.Context, jws *jose.JSONWebSignature, checkKeyExpiry bool) (_ []byte, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/op.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zitadel/oidc/v3/pkg/oidc"
 	"github.com/zitadel/oidc/v3/pkg/op"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/assets"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -21,7 +22,6 @@ import (
 	"github.com/zitadel/zitadel/internal/domain/federatedlogout"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -173,7 +173,8 @@ func NewServer(
 		op.WithFallbackLogger(fallbackLogger),
 		op.WithHTTPMiddleware(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("oidc"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/oidc/server.go

@@ -117,7 +117,7 @@ func (s *Server) Ready(ctx context.Context, r *op.Request[struct{}]) (_ *op.Resp
 func (s *Server) Discovery(ctx context.Context, r *op.Request[struct{}]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	restrictions, err := s.query.GetInstanceRestrictions(ctx)

internal/api/oidc/token_client_credentials.go

@@ -16,7 +16,7 @@ func (s *Server) ClientCredentialsExchange(ctx context.Context, r *op.ClientRequ
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 	client, ok := r.Client.(*clientCredentialsClient)
 	if !ok {

internal/api/oidc/token_code.go

@@ -19,7 +19,7 @@ func (s *Server) CodeExchange(ctx context.Context, r *op.ClientRequest[oidc.Acce
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_device.go

@@ -18,7 +18,7 @@ func (s *Server) DeviceToken(ctx context.Context, r *op.ClientRequest[oidc.Devic
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_exchange.go

@@ -32,7 +32,7 @@ func init() {
 func (s *Server) TokenExchange(ctx context.Context, r *op.ClientRequest[oidc.TokenExchangeRequest]) (_ *op.Response, err error) {
 	resp, err := s.tokenExchange(ctx, r)
 	if err != nil {
-		return nil, oidcError(err)
+		return nil, oidcError(ctx, err)
 	}
 	return resp, nil
 }

internal/api/oidc/token_jwt_profile.go

@@ -18,7 +18,7 @@ func (s *Server) JWTProfile(ctx context.Context, r *op.Request[oidc.JWTProfileGr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	user, err := s.verifyJWTProfile(ctx, r.Data)

internal/api/oidc/token_refresh.go

@@ -18,7 +18,7 @@ func (s *Server) RefreshToken(ctx context.Context, r *op.ClientRequest[oidc.Refr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/userinfo.go

@@ -30,7 +30,7 @@ import (
 func (s *Server) UserInfo(ctx context.Context, r *op.Request[oidc.UserInfoRequest]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	token, err := s.verifyAccessToken(ctx, r.Data.AccessToken)

internal/api/saml/provider.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/zitadel/saml/pkg/provider"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
@@ -17,7 +18,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -71,7 +71,8 @@ func NewProvider(
 	options := []provider.Option{
 		provider.WithHttpInterceptors(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("saml"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/scim/serrors/errors.go

@@ -127,7 +127,8 @@ func ThrowPayloadTooLarge(parent error) error {
 }
 
 func IsScimOrZitadelError(err error) bool {
-	return IsScimError(err) || zerrors.IsZitadelError(err)
+	_, zok := zerrors.AsZitadelError(err)
+	return IsScimError(err) || zok
 }
 
 func IsScimError(err error) bool {
@@ -174,7 +175,7 @@ func MapToScimError(ctx context.Context, translator *i18n.Translator, err error)
 		}
 	}
 
-	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	if !ok {
 		statusCode = http.StatusInternalServerError
 	}

internal/api/ui/console/console.go

@@ -114,7 +114,12 @@ func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, call
 	handler.Use(security, limitingAccessInterceptor.WithoutLimiting().Handle)
 
 	env := handler.NewRoute().Path(envRequestPath).Subrouter()
-	env.Use(callDurationInterceptor, middleware.TelemetryHandler(), instanceHandler)
+	env.Use(
+		callDurationInterceptor,
+		middleware.TraceHandler(),
+		middleware.LogHandler("console"),
+		instanceHandler,
+	)
 	env.HandleFunc("", func(w http.ResponseWriter, r *http.Request) {
 		url := http_util.BuildOrigin(r.Host, externalSecure)
 		ctx := r.Context()

internal/api/ui/login/external_provider_handler.go

@@ -1611,5 +1611,5 @@ func WrapIdPError(err error) *IdPError {
 	if errors.As(err, &zErr) {
 		id = zErr.ID
 	}
-	return &IdPError{err: zerrors.CreateZitadelError(err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
+	return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
 }

internal/api/ui/login/login.go

@@ -128,7 +128,18 @@ func CreateLogin(
 	cacheInterceptor := createCacheInterceptor(config.Cache.MaxAge, config.Cache.SharedMaxAge, assetCache)
 	security := middleware.SecurityHeaders(csp(), login.cspErrorHandler)
 
-	login.router = CreateRouter(login, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor, accessHandler)
+	login.router = CreateRouter(login,
+		middleware.TraceHandler(IgnoreInstanceEndpoints...),
+		middleware.LogHandler("login_v1", IgnoreInstanceEndpoints...),
+		oidcInstanceHandler,
+		samlInstanceHandler,
+		csrfInterceptor,
+		cacheInterceptor,
+		security,
+		userAgentCookie,
+		issuerInterceptor,
+		accessHandler,
+	)
 	login.renderer = CreateRenderer(HandlerPrefix, staticStorage, config.LanguageCookieName)
 	login.parser = form.NewParser()