feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

internal/actions/http_module_test.go

@@ -3,7 +3,6 @@ package actions
 import (
 	"bytes"
 	"context"
-	"errors"
 	"io"
 	"net"
 	"net/http"
@@ -92,13 +91,13 @@ func Test_isHostBlocked(t *testing.T) {
 			name: "looked up failure",
 			fields: fields{
 				lookup: func(host string) ([]net.IP, error) {
-					return nil, errors.New("some error")
+					return nil, io.EOF
 				},
 			},
 			args: args{
 				address: mustNewURL(t, "https://test2.com/hodor"),
 			},
-			want: zerrors.ThrowInternal(nil, "ACTIO-4m9s2", "lookup failed"),
+			want: zerrors.ThrowInternal(io.EOF, "ACTIO-4m9s2", "lookup failed"),
 		},
 	}
 	for _, tt := range tests {

internal/api/api.go

@@ -12,10 +12,12 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
 	"github.com/zitadel/logging"
+	"go.opentelemetry.io/otel"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/health"
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server"
@@ -26,7 +28,6 @@ import (
 	"github.com/zitadel/zitadel/internal/crypto"
 	"github.com/zitadel/zitadel/internal/i18n"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/telemetry/tracing"
 	"github.com/zitadel/zitadel/internal/zerrors"
 	instance_pb "github.com/zitadel/zitadel/pkg/grpc/instance/v2"
@@ -131,7 +132,12 @@ func New(
 	if err != nil {
 		return nil, err
 	}
-	api.connectOTELInterceptor, err = otelconnect.NewInterceptor()
+	api.connectOTELInterceptor, err = otelconnect.NewInterceptor(
+		otelconnect.WithTracerProvider(otel.GetTracerProvider()),
+		otelconnect.WithMeterProvider(otel.GetMeterProvider()),
+		otelconnect.WithPropagator(otel.GetTextMapPropagator()),
+		otelconnect.WithoutServerPeerAttributes(),
+	)
 	if err != nil {
 		return nil, err
 	}
@@ -199,6 +205,7 @@ func (a *API) registerConnectServer(service server.ConnectServer) {
 		connect_middleware.CallDurationHandler(),
 		a.connectOTELInterceptor,
 		connect_middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+		connect_middleware.LogHandler(grpc_api.Probes...),
 		connect_middleware.NoCacheInterceptor(),
 		connect_middleware.InstanceInterceptor(a.queries, a.externalDomain, a.translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName, instance_pb.InstanceService_ServiceDesc.ServiceName),
 		connect_middleware.AccessStorageInterceptor(a.accessInterceptor.AccessService()),

internal/api/assets/asset.go

@@ -79,7 +79,7 @@ type ErrorHandler func(w http.ResponseWriter, r *http.Request, err error, defaul
 func DefaultErrorHandler(translator *i18n.Translator) func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 	return func(w http.ResponseWriter, r *http.Request, err error, defaultCode int) {
 		logging.WithFields("uri", r.RequestURI).WithError(err).Warn("error occurred on asset api")
-		code, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+		code, ok := http_util.ZitadelErrorToHTTPStatusCode(r.Context(), err)
 		if !ok {
 			code = defaultCode
 		}

internal/api/grpc/gerrors/zitadel_errors.go

@@ -1,10 +1,13 @@
 package gerrors
 
 import (
+	"context"
 	"errors"
+	"log/slog"
 
 	"connectrpc.com/connect"
 	"github.com/jackc/pgx/v5/pgconn"
+	slogctx "github.com/veqryn/slog-context"
 	"github.com/zitadel/logging"
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
@@ -16,16 +19,15 @@ import (
 	"github.com/zitadel/zitadel/pkg/grpc/message"
 )
 
-func ZITADELToGRPCError(err error) error {
+func ZITADELToGRPCError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err, "code", code)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -38,20 +40,21 @@ func ZITADELToGRPCError(err error) error {
 	return s.Err()
 }
 
-func ZITADELToConnectError(err error) error {
+func ZITADELToConnectError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
 	connectError := new(connect.Error)
 	if errors.As(err, &connectError) {
+		// Connect error may be returned by other middlewares,
+		// so we assume it's a client error and log as warning.
+		slogctx.FromCtx(ctx).WarnContext(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
 		return err
 	}
-	code, key, id, ok := ExtractZITADELError(err)
-	if !ok {
-		return status.Convert(err).Err()
-	}
+	code, key, id, lvl := extractError(err)
 	msg := key
 	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
 
 	errorInfo := getErrorInfo(id, key, err)
 
@@ -62,44 +65,62 @@ func ZITADELToConnectError(err error) error {
 	return cErr
 }
 
-func ExtractZITADELError(err error) (c codes.Code, msg, id string, ok bool) {
+func ExtractZITADELError(err error) (code codes.Code, msg, id string) {
 	if err == nil {
-		return codes.OK, "", "", false
+		return codes.OK, "", ""
 	}
+	code, msg, id, _ = extractError(err)
+	return code, msg, id
+}
+
+func extractError(err error) (c codes.Code, msg, id string, lvl slog.Level) {
 	connErr := new(pgconn.ConnectError)
 	if ok := errors.As(err, &connErr); ok {
-		return codes.Internal, "db connection error", "", true
+		return codes.Internal, "db connection error", "", slog.LevelError
 	}
-	zitadelErr := new(zerrors.ZitadelError)
-	if ok := errors.As(err, &zitadelErr); !ok {
-		return codes.Unknown, err.Error(), "", false
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return codes.Unknown, err.Error(), "", slog.LevelError
 	}
-	switch {
-	case zerrors.IsErrorAlreadyExists(err):
-		return codes.AlreadyExists, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsDeadlineExceeded(err):
-		return codes.DeadlineExceeded, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsInternal(err):
-		return codes.Internal, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsErrorInvalidArgument(err):
-		return codes.InvalidArgument, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsNotFound(err):
-		return codes.NotFound, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPermissionDenied(err):
-		return codes.PermissionDenied, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsPreconditionFailed(err):
-		return codes.FailedPrecondition, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnauthenticated(err):
-		return codes.Unauthenticated, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnavailable(err):
-		return codes.Unavailable, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsUnimplemented(err):
-		return codes.Unimplemented, zitadelErr.GetMessage(), zitadelErr.GetID(), true
-	case zerrors.IsResourceExhausted(err):
-		return codes.ResourceExhausted, zitadelErr.GetMessage(), zitadelErr.GetID(), true
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		c, lvl = codes.AlreadyExists, slog.LevelWarn
+	case zerrors.KindDeadlineExceeded:
+		c, lvl = codes.DeadlineExceeded, slog.LevelError
+	case zerrors.KindInternal:
+		c, lvl = codes.Internal, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		c, lvl = codes.InvalidArgument, slog.LevelWarn
+	case zerrors.KindNotFound:
+		c, lvl = codes.NotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		c, lvl = codes.PermissionDenied, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		c, lvl = codes.FailedPrecondition, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		c, lvl = codes.Unauthenticated, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		c, lvl = codes.Unavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		c, lvl = codes.Unimplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		c, lvl = codes.ResourceExhausted, slog.LevelError
+	case zerrors.KindDataLoss:
+		c, lvl = codes.DataLoss, slog.LevelError
+	case zerrors.KindCanceled:
+		c, lvl = codes.Canceled, slog.LevelWarn
+	case zerrors.KindAborted:
+		c, lvl = codes.Aborted, slog.LevelError
+	case zerrors.KindOutOfRange:
+		c, lvl = codes.OutOfRange, slog.LevelWarn
+	case zerrors.KindUnknown:
+		c, lvl = codes.Unknown, slog.LevelError
 	default:
-		return codes.Unknown, err.Error(), "", false
+		c, lvl = codes.Unknown, slog.LevelError
 	}
+	return c, msg, id, lvl
 }
 
 func getErrorInfo(id, key string, err error) protoadapt.MessageV1 {

internal/api/grpc/gerrors/zitadel_errors_test.go

@@ -3,6 +3,7 @@ package gerrors
 import (
 	"errors"
 	"fmt"
+	"log/slog"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -41,7 +42,7 @@ func TestCaosToGRPCError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			if err := ZITADELToGRPCError(tt.args.err); (err != nil) != tt.wantErr {
+			if err := ZITADELToGRPCError(t.Context(), tt.args.err); (err != nil) != tt.wantErr {
 				t.Errorf("ZITADELToGRPCError() error = %v, wantErr %v", err, tt.wantErr)
 			}
 		})
@@ -96,7 +97,7 @@ func Test_getErrorInfo(t *testing.T) {
 	}
 }
 
-func Test_Extract(t *testing.T) {
+func Test_extractError(t *testing.T) {
 	type args struct {
 		err error
 	}
@@ -106,7 +107,7 @@ func Test_Extract(t *testing.T) {
 		wantC   codes.Code
 		wantMsg string
 		wantID  string
-		wantOk  bool
+		wantLvl slog.Level
 	}{
 		{
 			"already exists",
@@ -114,7 +115,7 @@ func Test_Extract(t *testing.T) {
 			codes.AlreadyExists,
 			"already exists",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"deadline exceeded",
@@ -122,7 +123,7 @@ func Test_Extract(t *testing.T) {
 			codes.DeadlineExceeded,
 			"deadline exceeded",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"internal error",
@@ -130,7 +131,7 @@ func Test_Extract(t *testing.T) {
 			codes.Internal,
 			"internal error",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"invalid argument",
@@ -138,7 +139,7 @@ func Test_Extract(t *testing.T) {
 			codes.InvalidArgument,
 			"invalid argument",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"not found",
@@ -146,7 +147,7 @@ func Test_Extract(t *testing.T) {
 			codes.NotFound,
 			"not found",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"permission denied",
@@ -154,7 +155,7 @@ func Test_Extract(t *testing.T) {
 			codes.PermissionDenied,
 			"permission denied",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"precondition failed",
@@ -162,7 +163,7 @@ func Test_Extract(t *testing.T) {
 			codes.FailedPrecondition,
 			"precondition failed",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unauthenticated",
@@ -170,7 +171,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unauthenticated,
 			"unauthenticated",
 			"id",
-			true,
+			slog.LevelWarn,
 		},
 		{
 			"unavailable",
@@ -178,7 +179,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unavailable,
 			"unavailable",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unimplemented",
@@ -186,7 +187,7 @@ func Test_Extract(t *testing.T) {
 			codes.Unimplemented,
 			"unimplemented",
 			"id",
-			true,
+			slog.LevelInfo,
 		},
 		{
 			"exhausted",
@@ -194,7 +195,7 @@ func Test_Extract(t *testing.T) {
 			codes.ResourceExhausted,
 			"exhausted",
 			"id",
-			true,
+			slog.LevelError,
 		},
 		{
 			"unknown",
@@ -202,12 +203,12 @@ func Test_Extract(t *testing.T) {
 			codes.Unknown,
 			"unknown",
 			"",
-			false,
+			slog.LevelError,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotC, gotMsg, gotID, gotOk := ExtractZITADELError(tt.args.err)
+			gotC, gotMsg, gotID := ExtractZITADELError(tt.args.err)
 			if gotC != tt.wantC {
 				t.Errorf("extract() gotC = %v, want %v", gotC, tt.wantC)
 			}
@@ -217,9 +218,6 @@ func Test_Extract(t *testing.T) {
 			if gotID != tt.wantID {
 				t.Errorf("extract() gotID = %v, want %v", gotID, tt.wantID)
 			}
-			if gotOk != tt.wantOk {
-				t.Errorf("extract() gotOk = %v, want %v", gotOk, tt.wantOk)
-			}
 		})
 	}
 }

internal/api/grpc/server/connect_middleware/activity_interceptor.go

@@ -19,7 +19,7 @@ func ActivityInterceptor() connect.UnaryInterceptorFunc {
 			ctx = activityInfoFromGateway(ctx, req.Header()).SetMethod(req.Spec().Procedure).IntoContext(ctx)
 			resp, err := handler(ctx, req)
 			if isResourceAPI(req.Spec().Procedure) {
-				code, _, _, _ := gerrors.ExtractZITADELError(err)
+				code, _, _ := gerrors.ExtractZITADELError(err)
 				ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 				activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 			}

internal/api/grpc/server/connect_middleware/error_interceptor.go

@@ -19,5 +19,5 @@ func ErrorHandler() connect.UnaryInterceptorFunc {
 
 func toConnectError(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc) (connect.AnyResponse, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToConnectError(err) // TODO !
+	return resp, gerrors.ZITADELToConnectError(ctx, err)
 }

internal/api/grpc/server/connect_middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req connect.AnyRequest, handler connec
 func addInstanceByDomain(ctx context.Context, req connect.AnyRequest, handler connect.UnaryFunc, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (connect.AnyResponse, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, connect.NewError(connect.CodeNotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr))

internal/api/grpc/server/connect_middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package connect_middleware
+
+import (
+	"connectrpc.com/connect"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredPrefix ...string) connect.UnaryInterceptorFunc {
+	return func(next connect.UnaryFunc) connect.UnaryFunc {
+		return logging.NewConnectInterceptor(next, ignoredPrefix...)
+	}
+}

internal/api/grpc/server/connect_middleware/metrics_interceptor.go

@@ -10,8 +10,8 @@ import (
 	"go.opentelemetry.io/otel/attribute"
 	"google.golang.org/grpc/codes"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/gateway.go

@@ -20,10 +20,10 @@ import (
 	"google.golang.org/protobuf/encoding/protojson"
 	"google.golang.org/protobuf/proto"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	client_middleware "github.com/zitadel/zitadel/internal/api/grpc/client/middleware"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -209,7 +209,7 @@ func addInterceptors(
 	handler = http_mw.CallDurationHandler(handler)
 	handler = http_mw.CORSInterceptor(handler)
 	handler = http_mw.RobotsTagHandler(handler)
-	handler = http_mw.DefaultTelemetryHandler(handler)
+	handler = http_mw.DefaultTraceHandler(handler)
 	handler = http_mw.ActivityHandler(handler)
 	// For some non-obvious reason, the exhaustedCookieInterceptor sends the SetCookie header
 	// only if it follows the http_mw.DefaultTelemetryHandler
@@ -218,6 +218,7 @@ func addInterceptors(
 		metrics.MetricTypeTotalCount,
 		metrics.MetricTypeStatusCode,
 	}, http_utils.Probes...)(handler)
+	handler = http_mw.LogHandler("grpc_gateway", http_utils.Probes...)(handler)
 	return handler
 }
 

internal/api/grpc/server/middleware/activity_interceptor.go

@@ -18,7 +18,7 @@ func ActivityInterceptor() grpc.UnaryServerInterceptor {
 		ctx = activityInfoFromGateway(ctx).SetMethod(info.FullMethod).IntoContext(ctx)
 		resp, err := handler(ctx, req)
 		if isResourceAPI(info.FullMethod) {
-			code, _, _, _ := gerrors.ExtractZITADELError(err)
+			code, _, _ := gerrors.ExtractZITADELError(err)
 			ctx = ainfo.ActivityInfoFromContext(ctx).SetGRPCStatus(code).IntoContext(ctx)
 			activity.TriggerGRPCWithContext(ctx, activity.ResourceAPI)
 		}

internal/api/grpc/server/middleware/error_interceptor.go

@@ -17,5 +17,5 @@ func ErrorHandler() grpc.UnaryServerInterceptor {
 
 func toGRPCError(ctx context.Context, req interface{}, handler grpc.UnaryHandler) (interface{}, error) {
 	resp, err := handler(ctx, req)
-	return resp, gerrors.ZITADELToGRPCError(err)
+	return resp, gerrors.ZITADELToGRPCError(ctx, err)
 }

internal/api/grpc/server/middleware/instance_interceptor.go

@@ -77,8 +77,8 @@ func addInstanceByID(ctx context.Context, req interface{}, handler grpc.UnaryHan
 func addInstanceByDomain(ctx context.Context, req interface{}, handler grpc.UnaryHandler, verifier authz.InstanceVerifier, translator *i18n.Translator, domain string) (interface{}, error) {
 	instance, err := verifier.InstanceByHost(ctx, domain, "")
 	if err != nil {
-		notFoundErr := new(zerrors.NotFoundError)
-		if errors.As(err, &notFoundErr) {
+		notFoundErr := new(zerrors.ZitadelError)
+		if errors.As(err, &notFoundErr) && notFoundErr.Kind == zerrors.KindNotFound {
 			notFoundErr.Message = translator.LocalizeFromCtx(ctx, notFoundErr.GetMessage(), nil)
 		}
 		return nil, status.Error(codes.NotFound, fmt.Errorf("unable to set instance using domain %s: %w", domain, notFoundErr).Error())

internal/api/grpc/server/middleware/log_interceptor.go

@@ -0,0 +1,11 @@
+package middleware
+
+import (
+	"google.golang.org/grpc"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
+	return logging.NewGrpcInterceptor(ignoredMethodSuffixes...)
+}

internal/api/grpc/server/middleware/metrics_interceptor.go

@@ -9,8 +9,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/status"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	_ "github.com/zitadel/zitadel/internal/statik"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (

internal/api/grpc/server/server.go

@@ -11,6 +11,7 @@ import (
 	healthpb "google.golang.org/grpc/health/grpc_health_v1"
 	"google.golang.org/protobuf/reflect/protoreflect"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	grpc_api "github.com/zitadel/zitadel/internal/api/grpc"
 	"github.com/zitadel/zitadel/internal/api/grpc/server/middleware"
@@ -19,7 +20,6 @@ import (
 	"github.com/zitadel/zitadel/internal/logstore"
 	"github.com/zitadel/zitadel/internal/logstore/record"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	system_pb "github.com/zitadel/zitadel/pkg/grpc/system"
 )
 
@@ -71,6 +71,7 @@ func CreateServer(
 			grpc_middleware.ChainUnaryServer(
 				middleware.CallDurationHandler(),
 				middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
+				middleware.LogHandler(grpc_api.Probes...),
 				middleware.NoCacheInterceptor(),
 				middleware.InstanceInterceptor(queries, externalDomain, translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName),
 				middleware.AccessStorageInterceptor(accessSvc),

internal/api/grpc/user/v2/integration_test/intent_test.go

@@ -12,13 +12,13 @@ import (
 	"github.com/muhlemmer/gu"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/timestamppb"
 
 	"github.com/zitadel/zitadel/internal/integration"
 	"github.com/zitadel/zitadel/internal/integration/sink"
 	"github.com/zitadel/zitadel/pkg/grpc/object/v2"
 	"github.com/zitadel/zitadel/pkg/grpc/user/v2"
-	"google.golang.org/protobuf/types/known/structpb"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
 func TestServer_StartIdentityProviderIntent(t *testing.T) {
@@ -631,7 +631,7 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 						DisplayName:       gu.Ptr("displayname"),
 					},
 					Email: &user.SetHumanEmail{
-						Email: "[email protected]",
+						Email:        "[email protected]",
 						Verification: &user.SetHumanEmail_SendCode{SendCode: &user.SendEmailVerificationCode{}},
 					},
 				},
@@ -1037,4 +1037,4 @@ func TestServer_RetrieveIdentityProviderIntent(t *testing.T) {
 			assert.EqualExportedValues(t, tt.want, got)
 		})
 	}
-}
+}

internal/api/http/error.go

@@ -1,49 +1,77 @@
 package http
 
 import (
-	"errors"
+	"context"
+	"log/slog"
 	"net/http"
 
+	slogctx "github.com/veqryn/slog-context"
+
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func ZitadelErrorToHTTPStatusCode(err error) (statusCode int, ok bool) {
+func ZitadelErrorToHTTPStatusCode(ctx context.Context, err error) (statusCode int, ok bool) {
 	if err == nil {
 		return http.StatusOK, true
 	}
-	//nolint:errorlint
-	switch err.(type) {
-	case *zerrors.AlreadyExistsError:
-		return http.StatusConflict, true
-	case *zerrors.DeadlineExceededError:
-		return http.StatusGatewayTimeout, true
-	case *zerrors.InternalError:
-		return http.StatusInternalServerError, true
-	case *zerrors.InvalidArgumentError:
-		return http.StatusBadRequest, true
-	case *zerrors.NotFoundError:
-		return http.StatusNotFound, true
-	case *zerrors.PermissionDeniedError:
-		return http.StatusForbidden, true
-	case *zerrors.PreconditionFailedError:
-		// use the same code as grpc-gateway:
-		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
-		return http.StatusBadRequest, true
-	case *zerrors.UnauthenticatedError:
-		return http.StatusUnauthorized, true
-	case *zerrors.UnavailableError:
-		return http.StatusServiceUnavailable, true
-	case *zerrors.UnimplementedError:
-		return http.StatusNotImplemented, true
-	case *zerrors.ResourceExhaustedError:
-		return http.StatusTooManyRequests, true
-	default:
-		c := new(zerrors.ZitadelError)
-		if errors.As(err, &c) {
-			return ZitadelErrorToHTTPStatusCode(errors.Unwrap(err))
-		}
+	statusCode, key, id, lvl := extractError(err)
+	msg := key
+	msg += " (" + id + ")"
+	slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
+	if statusCode == statusUnknown {
 		return http.StatusInternalServerError, false
 	}
+	return statusCode, true
+}
+
+const statusUnknown = 0
+
+func extractError(err error) (statusCode int, msg, id string, lvl slog.Level) {
+	zitadelErr, ok := zerrors.AsZitadelError(err)
+	if !ok {
+		return statusUnknown, err.Error(), "", slog.LevelError
+	}
+	msg, id = zitadelErr.GetMessage(), zitadelErr.GetID()
+
+	switch zitadelErr.Kind {
+	case zerrors.KindAlreadyExists:
+		statusCode, lvl = http.StatusConflict, slog.LevelError
+	case zerrors.KindDeadlineExceeded:
+		statusCode, lvl = http.StatusGatewayTimeout, slog.LevelError
+	case zerrors.KindInternal:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindInvalidArgument:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindNotFound:
+		statusCode, lvl = http.StatusNotFound, slog.LevelWarn
+	case zerrors.KindPermissionDenied:
+		statusCode, lvl = http.StatusForbidden, slog.LevelWarn
+	case zerrors.KindPreconditionFailed:
+		// use the same code as grpc-gateway:
+		// https://github.com/grpc-ecosystem/grpc-gateway/blob/9e33e38f15cb7d2f11096366e62ea391a3459ba9/runtime/errors.go#L59
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindUnauthenticated:
+		statusCode, lvl = http.StatusUnauthorized, slog.LevelWarn
+	case zerrors.KindUnavailable:
+		statusCode, lvl = http.StatusServiceUnavailable, slog.LevelError
+	case zerrors.KindUnimplemented:
+		statusCode, lvl = http.StatusNotImplemented, slog.LevelInfo
+	case zerrors.KindResourceExhausted:
+		statusCode, lvl = http.StatusTooManyRequests, slog.LevelError
+	case zerrors.KindCanceled:
+		statusCode, lvl = 499, slog.LevelWarn
+	case zerrors.KindDataLoss:
+		statusCode, lvl = http.StatusInternalServerError, slog.LevelError
+	case zerrors.KindOutOfRange:
+		statusCode, lvl = http.StatusBadRequest, slog.LevelWarn
+	case zerrors.KindAborted:
+		statusCode, lvl = http.StatusConflict, slog.LevelWarn
+	case zerrors.KindUnknown:
+		fallthrough
+	default:
+		statusCode, lvl = statusUnknown, slog.LevelError
+	}
+	return statusCode, msg, id, lvl
 }
 
 func HTTPStatusCodeToZitadelError(parent error, statusCode int, id string, message string) error {

internal/api/http/error_test.go

@@ -3,6 +3,7 @@ package http
 import (
 	"errors"
 	"fmt"
+	"io"
 	"net/http"
 	"testing"
 
@@ -118,7 +119,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 			wantOk:         true,
 		},
 		{
-			name: "no caos/zitadel error",
+			name: "no zitadel error",
 			args: args{
 				err: errors.New("error"),
 			},
@@ -128,7 +129,7 @@ func TestZitadelErrorToHTTPStatusCode(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(tt.args.err)
+			gotStatusCode, gotOk := ZitadelErrorToHTTPStatusCode(t.Context(), tt.args.err)
 			if gotStatusCode != tt.wantStatusCode {
 				t.Errorf("ZitadelErrorToHTTPStatusCode() gotStatusCode = %v, want %v", gotStatusCode, tt.wantStatusCode)
 			}
@@ -272,9 +273,9 @@ func TestHTTPStatusCodeToZitadelError(t *testing.T) {
 				statusCode: 1000,
 				id:         "id",
 				message:    "message",
-				parent:     errors.New("parent error"),
+				parent:     io.EOF,
 			},
-			wantErr: zerrors.ThrowUnknown(nil, "id", "message"),
+			wantErr: zerrors.ThrowUnknown(io.EOF, "id", "message"),
 		},
 	}
 	for _, tt := range tests {

internal/api/http/middleware/log_interceptor.go

@@ -0,0 +1,13 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
+)
+
+func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(h http.Handler) http.Handler {
+		return logging.NewHandler(h, service, ignoredPrefix...)
+	}
+}

internal/api/http/middleware/metrics_interceptor.go

@@ -3,11 +3,11 @@ package middleware
 import (
 	"net/http"
 
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethods ...string) func(http.Handler) http.Handler {
 	return func(handler http.Handler) http.Handler {
-		return metrics.NewMetricsHandler(handler, metricTypes, ignoredMethods...)
+		return metrics.NewHandler(handler, metricTypes, ignoredMethods...)
 	}
 }

internal/api/http/middleware/telemetry_interceptor.go

@@ -1,18 +0,0 @@
-package middleware
-
-import (
-	"net/http"
-
-	http_utils "github.com/zitadel/zitadel/internal/api/http"
-	"github.com/zitadel/zitadel/internal/telemetry"
-)
-
-func DefaultTelemetryHandler(handler http.Handler) http.Handler {
-	return TelemetryHandler(http_utils.Probes...)(handler)
-}
-
-func TelemetryHandler(ignoredMethods ...string) func(http.Handler) http.Handler {
-	return func(handler http.Handler) http.Handler {
-		return telemetry.TelemetryHandler(handler, ignoredMethods...)
-	}
-}

internal/api/http/middleware/trace_interceptor.go

@@ -0,0 +1,18 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
+	http_utils "github.com/zitadel/zitadel/internal/api/http"
+)
+
+func DefaultTraceHandler(handler http.Handler) http.Handler {
+	return TraceHandler(http_utils.Probes...)(handler)
+}
+
+func TraceHandler(ignoredPrefix ...string) func(http.Handler) http.Handler {
+	return func(handler http.Handler) http.Handler {
+		return tracing.NewHandler(handler, ignoredPrefix...)
+	}
+}

internal/api/http/request_context.go

@@ -3,6 +3,7 @@ package http
 import (
 	"context"
 	"fmt"
+	"log/slog"
 	"net"
 	"net/url"
 )
@@ -67,6 +68,22 @@ func (r *DomainCtx) Origin() string {
 	return fmt.Sprintf("%s://%s", r.Protocol, host)
 }
 
+var _ slog.LogValuer = (*DomainCtx)(nil)
+
+func (r *DomainCtx) LogValue() slog.Value {
+	values := make([]slog.Attr, 0, 3)
+	if r.InstanceHost != "" {
+		values = append(values, slog.String("instance_host", r.InstanceHost))
+	}
+	if r.PublicHost != "" {
+		values = append(values, slog.String("public_host", r.PublicHost))
+	}
+	if r.Protocol != "" {
+		values = append(values, slog.String("protocol", r.Protocol))
+	}
+	return slog.GroupValue(values...)
+}
+
 func DomainContext(ctx context.Context) *DomainCtx {
 	o, ok := ctx.Value(domainCtx).(*DomainCtx)
 	if !ok {

internal/api/oidc/auth_request.go

@@ -44,7 +44,7 @@ const (
 func (o *OPStorage) CreateAuthRequest(ctx context.Context, req *oidc.AuthRequest, userID string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -167,7 +167,7 @@ func (o *OPStorage) audienceFromProjectID(ctx context.Context, projectID string)
 func (o *OPStorage) AuthRequestByID(ctx context.Context, id string) (_ op.AuthRequest, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -206,7 +206,7 @@ func (o *OPStorage) decryptGrant(grant string) (string, error) {
 func (o *OPStorage) SaveAuthCode(ctx context.Context, id, code string) (err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -249,7 +249,7 @@ func (o *OPStorage) TerminateSession(ctx context.Context, userID, clientID strin
 func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessions []command.HumanSignOutSession, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	userAgentID, ok := middleware.UserAgentIDFromCtx(ctx)
@@ -276,7 +276,7 @@ func (o *OPStorage) terminateSession(ctx context.Context, userID string) (sessio
 func (o *OPStorage) TerminateSessionFromRequest(ctx context.Context, endSessionRequest *op.EndSessionRequest) (redirectURI string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 
@@ -505,7 +505,7 @@ func (o *OPStorage) revokeTokenV1(ctx context.Context, token, userID, clientID s
 func (o *OPStorage) GetRefreshTokenInfo(ctx context.Context, clientID string, token string) (userID string, tokenID string, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/client.go

@@ -36,8 +36,6 @@ const (
 	ClaimUserGroups                 = ScopeUserGroups
 	ScopeCustomUserGroups           = "urn:zitadel:iam:user:groups"
 	ClaimCustomUserGroups           = ScopeCustomUserGroups
-
-	oidcCtx = "oidc"
 )
 
 // GetClientByClientID implements the op.Storage interface to retrieve an OIDC client by its ID.
@@ -46,7 +44,7 @@ const (
 func (o *OPStorage) GetClientByClientID(ctx context.Context, id string) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	client, err := o.query.ActiveOIDCClientByID(ctx, id, false)
@@ -188,7 +186,7 @@ func userinfoClaims(userInfo *oidc.UserInfo) func(c *actions.FieldConfig) interf
 func (s *Server) VerifyClient(ctx context.Context, r *op.Request[op.ClientCredentials]) (_ op.Client, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/error.go

@@ -1,6 +1,7 @@
 package oidc
 
 import (
+	"context"
 	"errors"
 
 	"github.com/zitadel/oidc/v3/pkg/oidc"
@@ -15,7 +16,7 @@ import (
 // When err is already of the correct type is passed as-is.
 // If the err is a Zitadel error, it is transformed with a proper HTTP status code.
 // Unknown errors are treated as internal server errors.
-func oidcError(err error) error {
+func oidcError(ctx context.Context, err error) error {
 	if err == nil {
 		return nil
 	}
@@ -37,7 +38,7 @@ func oidcError(err error) error {
 		errors.As(err, &zError)
 	}
 
-	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, _ := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	newOidcErr := oidc.ErrServerError
 	if statusCode < 500 {
 		newOidcErr = oidc.ErrInvalidRequest

internal/api/oidc/error_test.go

@@ -56,7 +56,7 @@ func Test_oidcError(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			err := oidcError(tt.err)
+			err := oidcError(t.Context(), tt.err)
 			require.ErrorIs(t, err, tt.wantErr)
 		})
 	}

internal/api/oidc/introspect.go

@@ -20,7 +20,7 @@ import (
 func (s *Server) Introspect(ctx context.Context, r *op.Request[op.IntrospectionRequest]) (resp *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	ctx, cancel := context.WithCancel(ctx)

internal/api/oidc/key.go

@@ -150,7 +150,7 @@ func (k *publicKeyCache) getKey(ctx context.Context, keyID string) (_ *cachedPub
 func (k *publicKeyCache) verifySignature(ctx context.Context, jws *jose.JSONWebSignature, checkKeyExpiry bool) (_ []byte, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 

internal/api/oidc/op.go

@@ -10,6 +10,7 @@ import (
 	"github.com/zitadel/oidc/v3/pkg/oidc"
 	"github.com/zitadel/oidc/v3/pkg/op"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/api/assets"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -21,7 +22,6 @@ import (
 	"github.com/zitadel/zitadel/internal/domain/federatedlogout"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
@@ -173,7 +173,8 @@ func NewServer(
 		op.WithFallbackLogger(fallbackLogger),
 		op.WithHTTPMiddleware(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("oidc"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/oidc/server.go

@@ -117,7 +117,7 @@ func (s *Server) Ready(ctx context.Context, r *op.Request[struct{}]) (_ *op.Resp
 func (s *Server) Discovery(ctx context.Context, r *op.Request[struct{}]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	restrictions, err := s.query.GetInstanceRestrictions(ctx)

internal/api/oidc/token_client_credentials.go

@@ -16,7 +16,7 @@ func (s *Server) ClientCredentialsExchange(ctx context.Context, r *op.ClientRequ
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 	client, ok := r.Client.(*clientCredentialsClient)
 	if !ok {

internal/api/oidc/token_code.go

@@ -19,7 +19,7 @@ func (s *Server) CodeExchange(ctx context.Context, r *op.ClientRequest[oidc.Acce
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_device.go

@@ -18,7 +18,7 @@ func (s *Server) DeviceToken(ctx context.Context, r *op.ClientRequest[oidc.Devic
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/token_exchange.go

@@ -32,7 +32,7 @@ func init() {
 func (s *Server) TokenExchange(ctx context.Context, r *op.ClientRequest[oidc.TokenExchangeRequest]) (_ *op.Response, err error) {
 	resp, err := s.tokenExchange(ctx, r)
 	if err != nil {
-		return nil, oidcError(err)
+		return nil, oidcError(ctx, err)
 	}
 	return resp, nil
 }

internal/api/oidc/token_jwt_profile.go

@@ -18,7 +18,7 @@ func (s *Server) JWTProfile(ctx context.Context, r *op.Request[oidc.JWTProfileGr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	user, err := s.verifyJWTProfile(ctx, r.Data)

internal/api/oidc/token_refresh.go

@@ -18,7 +18,7 @@ func (s *Server) RefreshToken(ctx context.Context, r *op.ClientRequest[oidc.Refr
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
 		span.EndWithError(err)
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 	}()
 
 	client, ok := r.Client.(*Client)

internal/api/oidc/userinfo.go

@@ -30,7 +30,7 @@ import (
 func (s *Server) UserInfo(ctx context.Context, r *op.Request[oidc.UserInfoRequest]) (_ *op.Response, err error) {
 	ctx, span := tracing.NewSpan(ctx)
 	defer func() {
-		err = oidcError(err)
+		err = oidcError(ctx, err)
 		span.EndWithError(err)
 	}()
 	token, err := s.verifyAccessToken(ctx, r.Data.AccessToken)

internal/api/saml/provider.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/zitadel/saml/pkg/provider"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	http_utils "github.com/zitadel/zitadel/internal/api/http"
 	"github.com/zitadel/zitadel/internal/api/http/middleware"
 	"github.com/zitadel/zitadel/internal/api/ui/login"
@@ -17,7 +18,6 @@ import (
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/eventstore/handler/crdb"
 	"github.com/zitadel/zitadel/internal/query"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 const (
@@ -71,7 +71,8 @@ func NewProvider(
 	options := []provider.Option{
 		provider.WithHttpInterceptors(
 			middleware.MetricsHandler(metricTypes),
-			middleware.TelemetryHandler(),
+			middleware.TraceHandler(),
+			middleware.LogHandler("saml"),
 			middleware.NoCacheInterceptor().Handler,
 			instanceHandler,
 			userAgentCookie,

internal/api/scim/serrors/errors.go

@@ -127,7 +127,8 @@ func ThrowPayloadTooLarge(parent error) error {
 }
 
 func IsScimOrZitadelError(err error) bool {
-	return IsScimError(err) || zerrors.IsZitadelError(err)
+	_, zok := zerrors.AsZitadelError(err)
+	return IsScimError(err) || zok
 }
 
 func IsScimError(err error) bool {
@@ -174,7 +175,7 @@ func MapToScimError(ctx context.Context, translator *i18n.Translator, err error)
 		}
 	}
 
-	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(err)
+	statusCode, ok := http_util.ZitadelErrorToHTTPStatusCode(ctx, err)
 	if !ok {
 		statusCode = http.StatusInternalServerError
 	}

internal/api/ui/console/console.go

@@ -114,7 +114,12 @@ func Start(config Config, externalSecure bool, issuer op.IssuerFromRequest, call
 	handler.Use(security, limitingAccessInterceptor.WithoutLimiting().Handle)
 
 	env := handler.NewRoute().Path(envRequestPath).Subrouter()
-	env.Use(callDurationInterceptor, middleware.TelemetryHandler(), instanceHandler)
+	env.Use(
+		callDurationInterceptor,
+		middleware.TraceHandler(),
+		middleware.LogHandler("console"),
+		instanceHandler,
+	)
 	env.HandleFunc("", func(w http.ResponseWriter, r *http.Request) {
 		url := http_util.BuildOrigin(r.Host, externalSecure)
 		ctx := r.Context()

internal/api/ui/login/external_provider_handler.go

@@ -1611,5 +1611,5 @@ func WrapIdPError(err error) *IdPError {
 	if errors.As(err, &zErr) {
 		id = zErr.ID
 	}
-	return &IdPError{err: zerrors.CreateZitadelError(err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
+	return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
 }

internal/api/ui/login/login.go

@@ -128,7 +128,18 @@ func CreateLogin(
 	cacheInterceptor := createCacheInterceptor(config.Cache.MaxAge, config.Cache.SharedMaxAge, assetCache)
 	security := middleware.SecurityHeaders(csp(), login.cspErrorHandler)
 
-	login.router = CreateRouter(login, middleware.TelemetryHandler(IgnoreInstanceEndpoints...), oidcInstanceHandler, samlInstanceHandler, csrfInterceptor, cacheInterceptor, security, userAgentCookie, issuerInterceptor, accessHandler)
+	login.router = CreateRouter(login,
+		middleware.TraceHandler(IgnoreInstanceEndpoints...),
+		middleware.LogHandler("login_v1", IgnoreInstanceEndpoints...),
+		oidcInstanceHandler,
+		samlInstanceHandler,
+		csrfInterceptor,
+		cacheInterceptor,
+		security,
+		userAgentCookie,
+		issuerInterceptor,
+		accessHandler,
+	)
 	login.renderer = CreateRenderer(HandlerPrefix, staticStorage, config.LanguageCookieName)
 	login.parser = form.NewParser()
 

internal/domain/refresh_token_test.go

@@ -122,8 +122,7 @@ func FuzzFromRefreshToken(f *testing.F) {
 
 	f.Fuzz(func(t *testing.T, refreshToken string) {
 		gotUserID, gotTokenID, gotToken, err := FromRefreshToken(refreshToken, algorithm)
-		target := zerrors.InvalidArgumentError{ZitadelError: new(zerrors.ZitadelError)}
 		t.Log(gotUserID, gotTokenID, gotToken)
-		require.ErrorAs(t, err, &target)
+		require.True(t, zerrors.IsErrorInvalidArgument(err))
 	})
 }

internal/eventstore/handler/v2/metrics.go

@@ -6,7 +6,7 @@ import (
 	"github.com/zitadel/logging"
 	"go.opentelemetry.io/otel/attribute"
 
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 const (
@@ -23,7 +23,11 @@ type ProjectionMetrics struct {
 }
 
 func NewProjectionMetrics() *ProjectionMetrics {
-	projectionMetrics := &ProjectionMetrics{provider: metrics.M}
+	return newProjectionMetrics(metrics.GlobalMeter())
+}
+
+func newProjectionMetrics(m metrics.Metrics) *ProjectionMetrics {
+	projectionMetrics := &ProjectionMetrics{provider: m}
 
 	err := projectionMetrics.provider.RegisterCounter(
 		ProjectionEventsProcessed,

internal/eventstore/handler/v2/metrics_test.go

@@ -8,23 +8,19 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 )
 
 func TestNewProjectionMetrics(t *testing.T) {
 	mockMetrics := metrics.NewMockMetrics()
-	metrics.M = mockMetrics
-
-	metrics := NewProjectionMetrics()
+	metrics := newProjectionMetrics(mockMetrics)
 	require.NotNil(t, metrics)
 	assert.NotNil(t, metrics.provider)
 }
 
 func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
-
 	mockMetrics := metrics.NewMockMetrics()
-	metrics.M = mockMetrics
-	projectionMetrics := NewProjectionMetrics()
+	projectionMetrics := newProjectionMetrics(mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -42,10 +38,8 @@ func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
 }
 
 func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
-
 	mockMetrics := metrics.NewMockMetrics()
-	metrics.M = mockMetrics
-	projectionMetrics := NewProjectionMetrics()
+	projectionMetrics := newProjectionMetrics(mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -64,10 +58,8 @@ func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
 }
 
 func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
-
 	mockMetrics := metrics.NewMockMetrics()
-	metrics.M = mockMetrics
-	projectionMetrics := NewProjectionMetrics()
+	projectionMetrics := newProjectionMetrics(mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"
@@ -85,10 +77,8 @@ func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
 }
 
 func TestProjectionMetrics_Integration(t *testing.T) {
-
 	mockMetrics := metrics.NewMockMetrics()
-	metrics.M = mockMetrics
-	projectionMetrics := NewProjectionMetrics()
+	projectionMetrics := newProjectionMetrics(mockMetrics)
 
 	ctx := context.Background()
 	projection := "test_projection"

internal/notification/channels.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/notification/channels/email"
 	"github.com/zitadel/zitadel/internal/notification/channels/set"
 	"github.com/zitadel/zitadel/internal/notification/channels/sms"
@@ -12,7 +13,6 @@ import (
 	"github.com/zitadel/zitadel/internal/notification/handlers"
 	"github.com/zitadel/zitadel/internal/notification/senders"
 	"github.com/zitadel/zitadel/internal/notification/types"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 var _ types.ChannelChains = (*channels)(nil)

internal/notification/channels/instrumenting/metrics.go

@@ -6,8 +6,8 @@ import (
 	"github.com/zitadel/logging"
 	"go.opentelemetry.io/otel/attribute"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
 	"github.com/zitadel/zitadel/internal/notification/channels"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 func countMessages(ctx context.Context, channel channels.NotificationChannel, successMetricName, errorMetricName string) channels.NotificationChannel {

internal/queue/queue.go

@@ -11,9 +11,9 @@ import (
 	"github.com/riverqueue/rivercontrib/otelriver"
 	"github.com/robfig/cron/v3"
 	"github.com/zitadel/logging"
+	"go.opentelemetry.io/otel"
 
 	"github.com/zitadel/zitadel/internal/database"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
 )
 
 // Queue abstracts the underlying queuing library
@@ -32,7 +32,7 @@ type Config struct {
 
 func NewQueue(config *Config) (_ *Queue, err error) {
 	middleware := []rivertype.Middleware{otelriver.NewMiddleware(&otelriver.MiddlewareConfig{
-		MeterProvider: metrics.GetMetricsProvider(),
+		MeterProvider: otel.GetMeterProvider(),
 		DurationUnit:  "ms",
 	})}
 	return &Queue{

internal/telemetry/http_handler.go

@@ -1,34 +0,0 @@
-package telemetry
-
-import (
-	"net/http"
-	"strings"
-
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
-
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
-)
-
-func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
-	return func(r *http.Request) bool {
-		for _, endpoint := range endpoints {
-			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
-				return false
-			}
-		}
-		return true
-	}
-}
-
-func TelemetryHandler(handler http.Handler, ignoredEndpoints ...string) http.Handler {
-	return otelhttp.NewHandler(handler,
-		"zitadel",
-		otelhttp.WithFilter(shouldNotIgnore(ignoredEndpoints...)),
-		otelhttp.WithPublicEndpoint(),
-		otelhttp.WithSpanNameFormatter(spanNameFormatter),
-		otelhttp.WithMeterProvider(metrics.GetMetricsProvider()))
-}
-
-func spanNameFormatter(_ string, r *http.Request) string {
-	return strings.Split(r.RequestURI, "?")[0]
-}

internal/telemetry/metrics/config/config.go

@@ -1,37 +0,0 @@
-package config
-
-import (
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
-	"github.com/zitadel/zitadel/internal/telemetry/metrics/otel"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-type Config struct {
-	Type   string
-	Config map[string]interface{} `mapstructure:",remain"`
-}
-
-var meter = map[string]func(map[string]interface{}) error{
-	"otel": otel.NewTracerFromConfig,
-	"none": registerNoopMetrics,
-	"":     registerNoopMetrics,
-}
-
-func (c *Config) NewMeter() error {
-	// When using start-from-init or start-from-setup the metric provider
-	// was already set in the setup phase and the start phase must not overwrite it.
-	if metrics.M != nil {
-		return nil
-	}
-	t, ok := meter[c.Type]
-	if !ok {
-		return zerrors.ThrowInternalf(nil, "METER-Dfqsx", "config type %s not supported", c.Type)
-	}
-
-	return t(c.Config)
-}
-
-func registerNoopMetrics(rawConfig map[string]interface{}) (err error) {
-	metrics.M = &metrics.NoopMetrics{}
-	return nil
-}

internal/telemetry/metrics/http_handler.go

@@ -1,147 +0,0 @@
-package metrics
-
-import (
-	"context"
-	"net/http"
-	"strings"
-
-	"go.opentelemetry.io/otel/attribute"
-)
-
-const (
-	RequestCounter               = "http.server.request_count"
-	RequestCountDescription      = "Request counter"
-	TotalRequestCounter          = "http.server.total_request_count"
-	TotalRequestDescription      = "Total return code counter"
-	ReturnCodeCounter            = "http.server.return_code_counter"
-	ReturnCodeCounterDescription = "Return code counter"
-	Method                       = "method"
-	URI                          = "uri"
-	ReturnCode                   = "return_code"
-)
-
-type Handler struct {
-	handler http.Handler
-	methods []MetricType
-	filters []Filter
-}
-
-type MetricType int32
-
-const (
-	MetricTypeTotalCount MetricType = iota
-	MetricTypeStatusCode
-	MetricTypeRequestCount
-)
-
-type StatusRecorder struct {
-	http.ResponseWriter
-	RequestURI *string
-	Status     int
-}
-
-func (r *StatusRecorder) WriteHeader(status int) {
-	r.Status = status
-	r.ResponseWriter.WriteHeader(status)
-}
-
-type Filter func(*http.Request) bool
-
-func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
-	h := Handler{
-		handler: handler,
-		methods: metricMethods,
-	}
-	if len(ignoredEndpoints) > 0 {
-		h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
-	}
-	return &h
-}
-
-type key int
-
-const requestURI key = iota
-
-func SetRequestURIPattern(ctx context.Context, pattern string) {
-	uri, ok := ctx.Value(requestURI).(*string)
-	if !ok {
-		return
-	}
-	*uri = pattern
-}
-
-// ServeHTTP serves HTTP requests (http.Handler)
-func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	if len(h.methods) == 0 {
-		h.handler.ServeHTTP(w, r)
-		return
-	}
-	for _, f := range h.filters {
-		if !f(r) {
-			// Simply pass through to the handler if a filter rejects the request
-			h.handler.ServeHTTP(w, r)
-			return
-		}
-	}
-	uri := strings.Split(r.RequestURI, "?")[0]
-	recorder := &StatusRecorder{
-		ResponseWriter: w,
-		RequestURI:     &uri,
-		Status:         200,
-	}
-	r = r.WithContext(context.WithValue(r.Context(), requestURI, &uri))
-	h.handler.ServeHTTP(recorder, r)
-	if h.containsMetricsMethod(MetricTypeRequestCount) {
-		RegisterRequestCounter(recorder, r)
-	}
-	if h.containsMetricsMethod(MetricTypeTotalCount) {
-		RegisterTotalRequestCounter(r)
-	}
-	if h.containsMetricsMethod(MetricTypeStatusCode) {
-		RegisterRequestCodeCounter(recorder, r)
-	}
-}
-
-func (h *Handler) containsMetricsMethod(method MetricType) bool {
-	for _, m := range h.methods {
-		if m == method {
-			return true
-		}
-	}
-	return false
-}
-
-func RegisterRequestCounter(recorder *StatusRecorder, r *http.Request) {
-	var labels = map[string]attribute.Value{
-		URI:    attribute.StringValue(*recorder.RequestURI),
-		Method: attribute.StringValue(r.Method),
-	}
-	RegisterCounter(RequestCounter, RequestCountDescription)
-	AddCount(r.Context(), RequestCounter, 1, labels)
-}
-
-func RegisterTotalRequestCounter(r *http.Request) {
-	RegisterCounter(TotalRequestCounter, TotalRequestDescription)
-	AddCount(r.Context(), TotalRequestCounter, 1, nil)
-}
-
-func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
-	var labels = map[string]attribute.Value{
-		URI:        attribute.StringValue(*recorder.RequestURI),
-		Method:     attribute.StringValue(r.Method),
-		ReturnCode: attribute.IntValue(recorder.Status),
-	}
-	RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
-	AddCount(r.Context(), ReturnCodeCounter, 1, labels)
-}
-
-func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
-	return func(r *http.Request) bool {
-		for _, endpoint := range endpoints {
-			if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
-				return false
-			}
-		}
-		return true
-	}
-}

internal/telemetry/metrics/metrics.go

@@ -1,87 +0,0 @@
-package metrics
-
-import (
-	"context"
-	"net/http"
-
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-const (
-	ActiveSessionCounter            = "zitadel.active_session_counter"
-	ActiveSessionCounterDescription = "Active session counter"
-	SpoolerDivCounter               = "zitadel.spooler_div_milliseconds"
-	SpoolerDivCounterDescription    = "Spooler div from last successful run to now in milliseconds"
-	Database                        = "database"
-	ViewName                        = "view_name"
-)
-
-type Metrics interface {
-	GetExporter() http.Handler
-	GetMetricsProvider() metric.MeterProvider
-	RegisterCounter(name, description string) error
-	AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error
-	AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error
-	RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error
-	RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error
-	RegisterHistogram(name, description, unit string, buckets []float64) error
-}
-
-var M Metrics
-
-func GetExporter() http.Handler {
-	if M == nil {
-		return nil
-	}
-	return M.GetExporter()
-}
-
-func GetMetricsProvider() metric.MeterProvider {
-	if M == nil {
-		return nil
-	}
-	return M.GetMetricsProvider()
-}
-
-func RegisterCounter(name, description string) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterCounter(name, description)
-}
-
-func AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
-	if M == nil {
-		return nil
-	}
-	return M.AddCount(ctx, name, value, labels)
-}
-
-func AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
-	if M == nil {
-		return nil
-	}
-	return M.AddHistogramMeasurement(ctx, name, value, labels)
-}
-
-func RegisterHistogram(name, description, unit string, buckets []float64) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterHistogram(name, description, unit, buckets)
-}
-
-func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterUpDownSumObserver(name, description, callbackFunc)
-}
-
-func RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if M == nil {
-		return nil
-	}
-	return M.RegisterValueObserver(name, description, callbackFunc)
-}

internal/telemetry/metrics/mock.go

@@ -1,95 +0,0 @@
-package metrics
-
-import (
-	"context"
-	"net/http"
-	"sync"
-
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-// MockMetrics implements the metrics.Metrics interface for testing
-type MockMetrics struct {
-	mu              sync.RWMutex
-	histogramValues map[string][]float64
-	counterValues   map[string]int64
-	histogramLabels map[string][]map[string]attribute.Value
-	counterLabels   map[string][]map[string]attribute.Value
-}
-
-var _ Metrics = new(MockMetrics)
-
-// NewMockMetrics creates a new Metrics instance for testing
-func NewMockMetrics() *MockMetrics {
-	return &MockMetrics{
-		histogramValues: make(map[string][]float64),
-		counterValues:   make(map[string]int64),
-		histogramLabels: make(map[string][]map[string]attribute.Value),
-		counterLabels:   make(map[string][]map[string]attribute.Value),
-	}
-}
-
-func (m *MockMetrics) GetExporter() http.Handler {
-	return nil
-}
-
-func (m *MockMetrics) GetMetricsProvider() metric.MeterProvider {
-	return nil
-}
-
-func (m *MockMetrics) RegisterCounter(name, description string) error {
-	return nil
-}
-
-func (m *MockMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.counterValues[name] += value
-	m.counterLabels[name] = append(m.counterLabels[name], labels)
-	return nil
-}
-
-func (m *MockMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
-	m.mu.Lock()
-	defer m.mu.Unlock()
-	m.histogramValues[name] = append(m.histogramValues[name], value)
-	m.histogramLabels[name] = append(m.histogramLabels[name], labels)
-	return nil
-}
-
-func (m *MockMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	return nil
-}
-
-func (m *MockMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	return nil
-}
-
-func (m *MockMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
-	return nil
-}
-
-func (m *MockMetrics) GetHistogramValues(name string) []float64 {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.histogramValues[name]
-}
-
-func (m *MockMetrics) GetHistogramLabels(name string) []map[string]attribute.Value {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.histogramLabels[name]
-}
-
-func (m *MockMetrics) GetCounterValue(name string) int64 {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.counterValues[name]
-}
-
-func (m *MockMetrics) GetCounterLabels(name string) []map[string]attribute.Value {
-	m.mu.RLock()
-	defer m.mu.RUnlock()
-	return m.counterLabels[name]
-}

internal/telemetry/metrics/noop.go

@@ -1,45 +0,0 @@
-package metrics
-
-import (
-	"context"
-	"net/http"
-
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/metric"
-)
-
-type NoopMetrics struct{}
-
-var _ Metrics = new(NoopMetrics)
-
-func (n *NoopMetrics) GetExporter() http.Handler {
-	return nil
-}
-
-func (n *NoopMetrics) GetMetricsProvider() metric.MeterProvider {
-	return nil
-}
-
-func (n *NoopMetrics) RegisterCounter(name, description string) error {
-	return nil
-}
-
-func (n *NoopMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
-	return nil
-}
-
-func (n *NoopMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
-	return nil
-}
-
-func (n *NoopMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	return nil
-}
-
-func (n *NoopMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	return nil
-}
-
-func (n *NoopMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
-	return nil
-}

internal/telemetry/metrics/otel/config.go

@@ -1,20 +0,0 @@
-package otel
-
-import (
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
-)
-
-type Config struct {
-	MeterName string
-}
-
-func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
-	c := new(Config)
-	c.MeterName, _ = rawConfig["metername"].(string)
-	return c.NewMetrics()
-}
-
-func (c *Config) NewMetrics() (err error) {
-	metrics.M, err = NewMetrics(c.MeterName)
-	return err
-}

internal/telemetry/metrics/otel/open_telemetry.go

@@ -1,163 +0,0 @@
-package otel
-
-import (
-	"context"
-	"net/http"
-	"sync"
-
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/exporters/prometheus"
-	"go.opentelemetry.io/otel/metric"
-	"go.opentelemetry.io/otel/sdk/instrumentation"
-	sdk_metric "go.opentelemetry.io/otel/sdk/metric"
-
-	"github.com/zitadel/zitadel/internal/telemetry/metrics"
-	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-type Metrics struct {
-	Provider          metric.MeterProvider
-	Meter             metric.Meter
-	Counters          sync.Map
-	UpDownSumObserver sync.Map
-	ValueObservers    sync.Map
-	Histograms        sync.Map
-}
-
-func NewMetrics(meterName string) (metrics.Metrics, error) {
-	resource, err := otel_resource.ResourceWithService("ZITADEL")
-	if err != nil {
-		return nil, err
-	}
-	exporter, err := prometheus.New(prometheus.WithoutScopeInfo())
-	if err != nil {
-		return &Metrics{}, err
-	}
-	// create a view to filter out unwanted attributes
-	view := sdk_metric.NewView(
-		sdk_metric.Instrument{
-			Scope: instrumentation.Scope{Name: otelhttp.ScopeName},
-		},
-		sdk_metric.Stream{
-			AttributeFilter: attribute.NewAllowKeysFilter("http.method", "http.status_code", "http.target"),
-		},
-	)
-	meterProvider := sdk_metric.NewMeterProvider(
-		sdk_metric.WithReader(exporter),
-		sdk_metric.WithResource(resource),
-		sdk_metric.WithView(view),
-	)
-	return &Metrics{
-		Provider: meterProvider,
-		Meter:    meterProvider.Meter(meterName),
-	}, nil
-}
-
-func (m *Metrics) GetExporter() http.Handler {
-	return promhttp.Handler()
-}
-
-func (m *Metrics) GetMetricsProvider() metric.MeterProvider {
-	return m.Provider
-}
-
-func (m *Metrics) RegisterCounter(name, description string) error {
-	if _, exists := m.Counters.Load(name); exists {
-		return nil
-	}
-	counter, err := m.Meter.Int64Counter(name, metric.WithDescription(description))
-	if err != nil {
-		return err
-	}
-	m.Counters.Store(name, counter)
-	return nil
-}
-
-func (m *Metrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
-	counter, exists := m.Counters.Load(name)
-	if !exists {
-		return zerrors.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
-	}
-	counter.(metric.Int64Counter).Add(ctx, value, MapToAddOption(labels)...)
-	return nil
-}
-
-func (m *Metrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
-	histogram, exists := m.Histograms.Load(name)
-	if !exists {
-		return zerrors.ThrowNotFound(nil, "METER-5wwb1", "Errors.Metrics.Histogram.NotFound")
-	}
-	histogram.(metric.Float64Histogram).Record(ctx, value, MapToRecordOption(labels)...)
-	return nil
-}
-
-func (m *Metrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
-	if _, exists := m.Histograms.Load(name); exists {
-		return nil
-	}
-
-	histogram, err := m.Meter.Float64Histogram(name,
-		metric.WithDescription(description),
-		metric.WithUnit(unit),
-		metric.WithExplicitBucketBoundaries(buckets...),
-	)
-	if err != nil {
-		return err
-	}
-
-	m.Histograms.Store(name, histogram)
-	return nil
-}
-
-func (m *Metrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if _, exists := m.UpDownSumObserver.Load(name); exists {
-		return nil
-	}
-
-	counter, err := m.Meter.Int64ObservableUpDownCounter(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
-	if err != nil {
-		return err
-	}
-
-	m.UpDownSumObserver.Store(name, counter)
-	return nil
-}
-
-func (m *Metrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
-	if _, exists := m.UpDownSumObserver.Load(name); exists {
-		return nil
-	}
-
-	gauge, err := m.Meter.Int64ObservableGauge(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
-	if err != nil {
-		return err
-	}
-
-	m.UpDownSumObserver.Store(name, gauge)
-	return nil
-}
-
-func MapToAddOption(labels map[string]attribute.Value) []metric.AddOption {
-	return []metric.AddOption{metric.WithAttributes(labelsToAttributes(labels)...)}
-}
-
-func MapToRecordOption(labels map[string]attribute.Value) []metric.RecordOption {
-	return []metric.RecordOption{metric.WithAttributes(labelsToAttributes(labels)...)}
-}
-
-func labelsToAttributes(labels map[string]attribute.Value) []attribute.KeyValue {
-	if labels == nil {
-		return nil
-	}
-	attributes := make([]attribute.KeyValue, 0, len(labels))
-	for key, value := range labels {
-		attributes = append(attributes, attribute.KeyValue{
-			Key:   attribute.Key(key),
-			Value: value,
-		})
-	}
-	return attributes
-}

internal/telemetry/otel/resource.go

@@ -1,22 +0,0 @@
-package otel
-
-import (
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/sdk/resource"
-	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
-
-	"github.com/zitadel/zitadel/cmd/build"
-)
-
-func ResourceWithService(serviceName string) (*resource.Resource, error) {
-	attributes := []attribute.KeyValue{
-		semconv.ServiceNameKey.String(serviceName),
-	}
-	if build.Version() != "" {
-		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
-	}
-	return resource.Merge(
-		resource.Default(),
-		resource.NewWithAttributes("", attributes...),
-	)
-}

internal/telemetry/profiler/config/config.go

@@ -1,30 +0,0 @@
-package config
-
-import (
-	"github.com/zitadel/zitadel/internal/telemetry/profiler/google"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-type Config struct {
-	Type   string
-	Config map[string]interface{} `mapstructure:",remain"`
-}
-
-var profiler = map[string]func(map[string]interface{}) error{
-	"google": google.NewProfiler,
-	"none":   NoProfiler,
-	"":       NoProfiler,
-}
-
-func (c *Config) NewProfiler() error {
-	t, ok := profiler[c.Type]
-	if !ok {
-		return zerrors.ThrowInternalf(nil, "PROFI-Dfqsx", "config type %s not supported", c.Type)
-	}
-
-	return t(c.Config)
-}
-
-func NoProfiler(_ map[string]interface{}) error {
-	return nil
-}

internal/telemetry/profiler/google/profiler.go

@@ -1,26 +0,0 @@
-package google
-
-import (
-	"cloud.google.com/go/profiler"
-
-	"github.com/zitadel/zitadel/cmd/build"
-)
-
-type Config struct {
-	ProjectID string
-}
-
-func NewProfiler(rawConfig map[string]interface{}) (err error) {
-	c := new(Config)
-	c.ProjectID, _ = rawConfig["projectid"].(string)
-	return c.NewProfiler()
-}
-
-func (c *Config) NewProfiler() (err error) {
-	cfg := profiler.Config{
-		Service:        "zitadel",
-		ServiceVersion: build.Version(),
-		ProjectID:      c.ProjectID,
-	}
-	return profiler.Start(cfg)
-}

internal/telemetry/tracing/caller.go

@@ -1,22 +0,0 @@
-package tracing
-
-import (
-	"runtime"
-
-	"github.com/zitadel/logging"
-)
-
-func GetCaller() string {
-	fpcs := make([]uintptr, 1)
-	n := runtime.Callers(3, fpcs)
-	if n == 0 {
-		logging.WithFields("logID", "TRACE-rWjfC").Debug("no caller")
-		return ""
-	}
-	caller := runtime.FuncForPC(fpcs[0] - 1)
-	if caller == nil {
-		logging.WithFields("logID", "TRACE-25POw").Debug("caller was nil")
-		return ""
-	}
-	return caller.Name()
-}

internal/telemetry/tracing/config/config.go

@@ -1,34 +0,0 @@
-package config
-
-import (
-	"github.com/zitadel/zitadel/internal/telemetry/tracing/google"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing/log"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-type Config struct {
-	Type   string
-	Config map[string]interface{} `mapstructure:",remain"`
-}
-
-func (c *Config) NewTracer() error {
-	t, ok := tracer[c.Type]
-	if !ok {
-		return zerrors.ThrowInternalf(nil, "TRACE-dsbjh", "config type %s not supported", c.Type)
-	}
-
-	return t(c.Config)
-}
-
-var tracer = map[string]func(map[string]interface{}) error{
-	"otel":   otel.NewTracerFromConfig,
-	"google": google.NewTracer,
-	"log":    log.NewTracer,
-	"none":   NoTracer,
-	"":       NoTracer,
-}
-
-func NoTracer(_ map[string]interface{}) error {
-	return nil
-}

internal/telemetry/tracing/google/google_tracer.go

@@ -1,41 +0,0 @@
-package google
-
-import (
-	texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
-)
-
-type Config struct {
-	ProjectID   string
-	Fraction    float64
-	ServiceName string
-}
-
-func NewTracer(rawConfig map[string]interface{}) (err error) {
-	c := new(Config)
-	c.ProjectID, _ = rawConfig["projectid"].(string)
-	c.ServiceName, _ = rawConfig["servicename"].(string)
-	c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
-	if err != nil {
-		return err
-	}
-	return c.NewTracer()
-}
-
-type Tracer struct {
-	otel.Tracer
-}
-
-func (c *Config) NewTracer() error {
-	sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
-	exporter, err := texporter.New(texporter.WithProjectID(c.ProjectID))
-	if err != nil {
-		return err
-	}
-
-	tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
-	return err
-}

internal/telemetry/tracing/log/config.go

@@ -1,39 +0,0 @@
-package log
-
-import (
-	stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
-)
-
-type Config struct {
-	Fraction    float64
-	ServiceName string
-}
-
-func NewTracer(rawConfig map[string]interface{}) (err error) {
-	c := new(Config)
-	c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
-	c.ServiceName, _ = rawConfig["servicename"].(string)
-	if err != nil {
-		return err
-	}
-	return c.NewTracer()
-}
-
-type Tracer struct {
-	otel.Tracer
-}
-
-func (c *Config) NewTracer() error {
-	sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
-	exporter, err := stdout.New(stdout.WithPrettyPrint())
-	if err != nil {
-		return err
-	}
-
-	tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
-	return err
-}

internal/telemetry/tracing/otel/config.go

@@ -1,77 +0,0 @@
-package otel
-
-import (
-	"context"
-	"strconv"
-
-	otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	api_trace "go.opentelemetry.io/otel/trace"
-
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-type Config struct {
-	Fraction    float64
-	Endpoint    string
-	ServiceName string
-}
-
-func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
-	c := new(Config)
-	c.Endpoint, _ = rawConfig["endpoint"].(string)
-	c.ServiceName, _ = rawConfig["servicename"].(string)
-	c.Fraction, err = FractionFromConfig(rawConfig["fraction"])
-	if err != nil {
-		return err
-	}
-	return c.NewTracer()
-}
-
-func FractionFromConfig(i interface{}) (float64, error) {
-	if i == nil {
-		return 0, nil
-	}
-	switch fraction := i.(type) {
-	case float64:
-		return fraction, nil
-	case int:
-		return float64(fraction), nil
-	case string:
-		f, err := strconv.ParseFloat(fraction, 64)
-		if err != nil {
-			return 0, zerrors.ThrowInternal(err, "OTEL-SAfe1", "could not map fraction")
-		}
-		return f, nil
-	default:
-		return 0, zerrors.ThrowInternal(nil, "OTEL-Dd2s", "could not map fraction, unknown type")
-	}
-}
-
-func (c *Config) NewTracer() error {
-	sampler := NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
-	exporter, err := otlpgrpc.New(context.Background(), otlpgrpc.WithEndpoint(c.Endpoint), otlpgrpc.WithInsecure())
-	if err != nil {
-		return err
-	}
-
-	tracing.T, err = NewTracer(sampler, exporter, c.ServiceName)
-	return err
-}
-
-// NewSampler returns a sampler decorator which behaves differently,
-// based on the parent of the span. If the span has no parent and is of kind server,
-// the decorated sampler is used to make sampling decision.
-// If the span has a parent, depending on whether the parent is remote and whether it
-// is sampled, one of the following samplers will apply:
-//   - remote parent sampled -> always sample
-//   - remote parent not sampled -> sample based on the decorated sampler (fraction based)
-//   - local parent sampled -> always sample
-//   - local parent not sampled -> never sample
-func NewSampler(sampler sdk_trace.Sampler) sdk_trace.Sampler {
-	return sdk_trace.ParentBased(
-		tracing.SpanKindBased(sampler, api_trace.SpanKindServer),
-		sdk_trace.WithRemoteParentNotSampled(sampler),
-	)
-}

internal/telemetry/tracing/otel/open_telemetry.go

@@ -1,77 +0,0 @@
-package otel
-
-import (
-	"context"
-	"net/http"
-
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/propagation"
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	api_trace "go.opentelemetry.io/otel/trace"
-
-	otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
-	"github.com/zitadel/zitadel/internal/telemetry/tracing"
-)
-
-type Tracer struct {
-	Exporter api_trace.Tracer
-	sampler  sdk_trace.Sampler
-}
-
-func NewTracer(sampler sdk_trace.Sampler, exporter sdk_trace.SpanExporter, serviceName string) (*Tracer, error) {
-	resource, err := otel_resource.ResourceWithService(serviceName)
-	if err != nil {
-		return nil, err
-	}
-
-	tp := sdk_trace.NewTracerProvider(
-		sdk_trace.WithSampler(sampler),
-		sdk_trace.WithBatcher(exporter),
-		sdk_trace.WithResource(resource),
-	)
-
-	otel.SetTracerProvider(tp)
-	tc := propagation.TraceContext{}
-	otel.SetTextMapPropagator(tc)
-
-	return &Tracer{Exporter: tp.Tracer(""), sampler: sampler}, nil
-}
-
-func (t *Tracer) Sampler() sdk_trace.Sampler {
-	return t.sampler
-}
-
-func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
-	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
-}
-
-func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
-	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
-}
-
-func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
-	return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
-}
-
-func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
-	return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
-}
-
-func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
-	return t.newSpan(ctx, caller)
-}
-
-func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
-	return t.newSpanFromName(ctx, caller, options...)
-}
-
-func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
-	ctx, span := t.Exporter.Start(ctx, name, options...)
-	return ctx, tracing.CreateSpan(span)
-}
-
-func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
-	ctx, span := t.NewSpan(r.Context(), caller)
-	r = r.WithContext(ctx)
-	return r, span
-}

internal/telemetry/tracing/sampler.go

@@ -1,46 +0,0 @@
-package tracing
-
-import (
-	"fmt"
-	"slices"
-
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	"go.opentelemetry.io/otel/trace"
-)
-
-type spanKindSampler struct {
-	sampler sdk_trace.Sampler
-	kinds   []trace.SpanKind
-}
-
-// ShouldSample implements the [sdk_trace.Sampler] interface.
-// It will not sample any spans which do not match the configured span kinds.
-// For spans which do match, the decorated sampler is used to make the sampling decision.
-func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
-	psc := trace.SpanContextFromContext(p.ParentContext)
-	if !slices.Contains(sk.kinds, p.Kind) {
-		return sdk_trace.SamplingResult{
-			Decision:   sdk_trace.Drop,
-			Tracestate: psc.TraceState(),
-		}
-	}
-	s := sk.sampler.ShouldSample(p)
-	return s
-}
-
-func (sk spanKindSampler) Description() string {
-	return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
-		sk.sampler.Description(),
-		sk.kinds,
-	)
-}
-
-// SpanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
-// If the span kind does not match one of the configured kinds, it will not be sampled.
-// If the span kind matches, the decorated sampler is used to make sampling decision.
-func SpanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
-	return spanKindSampler{
-		sampler: sampler,
-		kinds:   kinds,
-	}
-}

internal/telemetry/tracing/sampler_test.go

@@ -1,80 +0,0 @@
-package tracing
-
-import (
-	"context"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	"go.opentelemetry.io/otel/trace"
-)
-
-func TestSpanKindBased(t *testing.T) {
-	type args struct {
-		sampler sdk_trace.Sampler
-		kinds   []trace.SpanKind
-	}
-	type want struct {
-		description string
-		sampled     int
-	}
-	tests := []struct {
-		name string
-		args args
-		want want
-	}{
-		{
-			"never sample, no sample",
-			args{
-				sampler: sdk_trace.NeverSample(),
-				kinds:   []trace.SpanKind{trace.SpanKindServer},
-			},
-			want{
-				description: "SpanKindBased{sampler:AlwaysOffSampler,kinds:[server]}",
-				sampled:     0,
-			},
-		},
-		{
-			"always sample, no kind, no sample",
-			args{
-				sampler: sdk_trace.AlwaysSample(),
-				kinds:   nil,
-			},
-			want{
-				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[]}",
-				sampled:     0,
-			},
-		},
-		{
-			"always sample, 2 kinds, 2 samples",
-			args{
-				sampler: sdk_trace.AlwaysSample(),
-				kinds:   []trace.SpanKind{trace.SpanKindServer, trace.SpanKindClient},
-			},
-			want{
-				description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[server client]}",
-				sampled:     2,
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			sampler := SpanKindBased(tt.args.sampler, tt.args.kinds...)
-			assert.Equal(t, tt.want.description, sampler.Description())
-
-			p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))
-			tr := p.Tracer("test")
-
-			var sampled int
-			for i := trace.SpanKindUnspecified; i <= trace.SpanKindConsumer; i++ {
-				ctx := context.Background()
-				_, span := tr.Start(ctx, "test", trace.WithSpanKind(i))
-				if span.SpanContext().IsSampled() {
-					sampled++
-				}
-			}
-
-			assert.Equal(t, tt.want.sampled, sampled)
-		})
-	}
-}

internal/telemetry/tracing/span.go

@@ -1,46 +0,0 @@
-package tracing
-
-import (
-	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/trace"
-
-	"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
-)
-
-type Span struct {
-	span trace.Span
-	opts []trace.SpanEndOption
-}
-
-func CreateSpan(span trace.Span) *Span {
-	return &Span{span: span, opts: []trace.SpanEndOption{}}
-}
-
-func (s *Span) End() {
-	if s.span == nil {
-		return
-	}
-
-	s.span.End(s.opts...)
-}
-
-func (s *Span) EndWithError(err error) {
-	s.SetStatusByError(err)
-	s.End()
-}
-
-func (s *Span) SetStatusByError(err error) {
-	if s.span == nil {
-		return
-	}
-	if err != nil {
-		// trace.WithErrorStatus(codes.Error)
-		s.span.RecordError(err)
-		s.span.SetAttributes(
-			attribute.KeyValue{},
-		)
-	}
-
-	code, msg, id, _ := gerrors.ExtractZITADELError(err)
-	s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
-}

internal/telemetry/tracing/tracing.go

@@ -1,81 +1,43 @@
+// Package tracing acts as a shim to [tracing],
+// so we can migrate the tracing package without changing all imports at once.
 package tracing
 
 import (
 	"context"
 	"net/http"
 
-	sdk_trace "go.opentelemetry.io/otel/sdk/trace"
-	api_trace "go.opentelemetry.io/otel/trace"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
 )
 
-type Tracer interface {
-	NewSpan(ctx context.Context, caller string) (context.Context, *Span)
-	NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
-	NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
-	NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
-	NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
-	NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
-	Sampler() sdk_trace.Sampler
+func NewSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewSpan(ctx, instrumentation.GetCallingFunc(1))
 }
 
-var T Tracer
-
-func Sampler() sdk_trace.Sampler {
-	if T == nil {
-		return sdk_trace.NeverSample()
-	}
-	return T.Sampler()
+func NewNamedSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span) {
+	return tracing.NewNamedSpan(ctx, name)
 }
 
-func NewSpan(ctx context.Context) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewSpan(ctx, GetCaller())
+func NewClientSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewClientSpan(ctx, instrumentation.GetCallingFunc(1))
 }
 
-func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewSpan(ctx, name)
+func NewServerSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewServerSpan(ctx, instrumentation.GetCallingFunc(1))
 }
 
-func NewClientSpan(ctx context.Context) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewClientSpan(ctx, GetCaller())
+func NewClientInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewClientInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
 }
 
-func NewServerSpan(ctx context.Context) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewServerSpan(ctx, GetCaller())
+func NewServerInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewServerInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
 }
 
-func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewClientInterceptorSpan(ctx, GetCaller())
-}
-
-func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
-	if T == nil {
-		return ctx, CreateSpan(nil)
-	}
-	return T.NewServerInterceptorSpan(ctx, GetCaller())
-}
-
-func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
-	if T == nil {
-		return r, CreateSpan(nil)
-	}
-	return T.NewSpanHTTP(r, GetCaller())
+func NewSpanHTTP(r *http.Request) (*http.Request, *instrumentation.Span) {
+	return tracing.GlobalTracer().NewSpanHTTP(r, instrumentation.GetCallingFunc(1))
 }
 
 func TraceIDFromCtx(ctx context.Context) string {
-	return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
+	return tracing.TraceIDFromCtx(ctx)
 }

internal/zerrors/already_exists.go

@@ -2,43 +2,15 @@ package zerrors
 
 import "fmt"
 
-var (
-	_ AlreadyExists = (*AlreadyExistsError)(nil)
-	_ Error         = (*AlreadyExistsError)(nil)
-)
-
-type AlreadyExists interface {
-	error
-	IsAlreadyExists()
-}
-
-type AlreadyExistsError struct {
-	*ZitadelError
-}
-
 func ThrowAlreadyExists(parent error, id, message string) error {
-	return &AlreadyExistsError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindAlreadyExists, parent, id, message)
 }
 
-func ThrowAlreadyExistsf(parent error, id, format string, a ...interface{}) error {
-	return &AlreadyExistsError{CreateZitadelError(parent, id, fmt.Sprintf(format, a...))}
-}
-
-func (err *AlreadyExistsError) IsAlreadyExists() {}
-
-func (err *AlreadyExistsError) Is(target error) bool {
-	t, ok := target.(*AlreadyExistsError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
+func ThrowAlreadyExistsf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindAlreadyExists, parent, id, fmt.Sprintf(format, a...))
 }
 
 func IsErrorAlreadyExists(err error) bool {
-	_, ok := err.(AlreadyExists)
-	return ok
-}
-
-func (err *AlreadyExistsError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindAlreadyExists
 }

internal/zerrors/already_exists_test.go

@@ -9,25 +9,58 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestAlreadyExistsError(t *testing.T) {
-	var alreadyExistsError interface{} = new(zerrors.AlreadyExistsError)
-	_, ok := alreadyExistsError.(zerrors.AlreadyExists)
-	assert.True(t, ok)
-}
+func TestAlreadyExists(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowAlreadyExistsf(t *testing.T) {
-	err := zerrors.ThrowAlreadyExistsf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.AlreadyExistsError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowAlreadyExists", func(t *testing.T) {
+		err := zerrors.ThrowAlreadyExists(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsErrorAlreadyExists(t *testing.T) {
-	err := zerrors.ThrowAlreadyExists(nil, "id", "msg")
-	ok := zerrors.IsErrorAlreadyExists(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindAlreadyExists, zitadelErr.Kind)
 
-	err = errors.New("Already Exists!")
-	ok = zerrors.IsErrorAlreadyExists(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowAlreadyExistsf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowAlreadyExistsf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindAlreadyExists, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsErrorAlreadyExists", func(t *testing.T) {
+		err := zerrors.ThrowAlreadyExists(parentErr, id, message)
+		isAlreadyExists := zerrors.IsErrorAlreadyExists(err)
+		assert.True(t, isAlreadyExists)
+
+		otherErr := errors.New("some other error")
+		isAlreadyExists = zerrors.IsErrorAlreadyExists(otherErr)
+		assert.False(t, isAlreadyExists)
+	})
 }

internal/zerrors/deadline_exceeded.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ DeadlineExceeded = (*DeadlineExceededError)(nil)
-	_ Error            = (*DeadlineExceededError)(nil)
-)
-
-type DeadlineExceeded interface {
-	error
-	IsDeadlineExceeded()
-}
-
-type DeadlineExceededError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowDeadlineExceeded(parent error, id, message string) error {
-	return &DeadlineExceededError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindDeadlineExceeded, parent, id, message)
 }
 
-func ThrowDeadlineExceededf(parent error, id, format string, a ...interface{}) error {
-	return ThrowDeadlineExceeded(parent, id, fmt.Sprintf(format, a...))
+func ThrowDeadlineExceededf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindDeadlineExceeded, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *DeadlineExceededError) IsDeadlineExceeded() {}
-
 func IsDeadlineExceeded(err error) bool {
-	_, ok := err.(DeadlineExceeded)
-	return ok
-}
-
-func (err *DeadlineExceededError) Is(target error) bool {
-	t, ok := target.(*DeadlineExceededError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *DeadlineExceededError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindDeadlineExceeded
 }

internal/zerrors/deadline_exceeded_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestDeadlineExceededError(t *testing.T) {
-	var err interface{} = new(zerrors.DeadlineExceededError)
-	_, ok := err.(zerrors.DeadlineExceeded)
-	assert.True(t, ok)
-}
+func TestDeadlineExceeded(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowDeadlineExceededf(t *testing.T) {
-	err := zerrors.ThrowDeadlineExceededf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.DeadlineExceededError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowDeadlineExceeded", func(t *testing.T) {
+		err := zerrors.ThrowDeadlineExceeded(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsDeadlineExceeded(t *testing.T) {
-	err := zerrors.ThrowDeadlineExceeded(nil, "id", "msg")
-	ok := zerrors.IsDeadlineExceeded(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindDeadlineExceeded, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsDeadlineExceeded(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowDeadlineExceededf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowDeadlineExceededf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindDeadlineExceeded, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsDeadlineExceeded", func(t *testing.T) {
+		err := zerrors.ThrowDeadlineExceeded(parentErr, id, message)
+		isDeadlineExceeded := zerrors.IsDeadlineExceeded(err)
+		assert.True(t, isDeadlineExceeded)
+	})
 }

internal/zerrors/error.go

@@ -1,19 +0,0 @@
-package zerrors
-
-import (
-	"strings"
-)
-
-// Error is a stdlib error extension.
-// It contains parameters to identify errors through all application layers
-type Error interface {
-	GetParent() error
-	GetMessage() string
-	SetMessage(string)
-	GetID() string
-}
-
-// Contains compares the error message with needle
-func Contains(err error, needle string) bool {
-	return err != nil && strings.Contains(err.Error(), needle)
-}

internal/zerrors/error_test.go

@@ -1,19 +0,0 @@
-package zerrors_test
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func TestContains(t *testing.T) {
-	err := errors.New("hello world")
-	world := zerrors.Contains(err, "hello")
-	assert.True(t, world)
-
-	mars := zerrors.Contains(err, "mars")
-	assert.False(t, mars)
-}

internal/zerrors/generate/error.go.tmpl

@@ -1,34 +0,0 @@
-package errors
-
-import (
-	"fmt"
-)
-
-var (
-	_ {{.ErrorName}} = (*{{.ErrorName}}Error)(nil)
-	_ Error    = (*{{.ErrorName}}Error)(nil)
-)
-
-type {{.ErrorName}} interface {
-	error
-	Is{{.ErrorName}}()
-}
-
-type {{.ErrorName}}Error struct {
-	*CaosError
-}
-
-func Throw{{.ErrorName}}(parent error, id, message string) error {
-	return &{{.ErrorName}}Error{CreateCaosError(parent, id, message)}
-}
-
-func Throw{{.ErrorName}}f(parent error, id, format string, a ...interface{}) error {
-	return Throw{{.ErrorName}}(parent, id, fmt.Sprintf(format, a...))
-}
-
-func (err *{{.ErrorName}}Error) Is{{.ErrorName}}() {}
-
-func Is{{.ErrorName}}(err error) bool {
-	_, ok := err.({{.ErrorName}})
-	return ok
-}

internal/zerrors/generate/error_creator.go

@@ -1,75 +0,0 @@
-// go generate
-package main
-
-import (
-	"flag"
-	"fmt"
-	"html/template"
-	"log"
-	"os"
-	"strings"
-)
-
-func main() {
-	errorName := readErrorName()
-	errorName = validateErrorName(errorName)
-
-	data := &Data{
-		ErrorName: errorName,
-	}
-
-	errorFile := data.createFile("error.go.tmpl")
-	data.createTemplate("error.go.tmpl", errorFile)
-	if err := errorFile.Close(); err != nil {
-		log.Fatal(err)
-	}
-
-	testFile := data.createFile("error_test.go.tmpl")
-	data.createTemplate("error_test.go.tmpl", testFile)
-	if err := testFile.Close(); err != nil {
-		log.Fatal(err)
-	}
-
-	fmt.Print(`
-!!!!!
-  Add status mapping in internal/api/grpc/zerrors.go	
-!!!!!`)
-}
-
-type Data struct {
-	ErrorName string
-}
-
-func (data *Data) createFile(tmplName string) *os.File {
-	filename := strings.Replace(tmplName, "error", strings.ToLower(data.ErrorName), 1)
-	filename = filename[:len(filename)-5]
-	filePath := fmt.Sprintf("../%s", filename)
-	file, err := os.Create(filePath)
-	if err != nil {
-		log.Fatalf("unable to create file (%s): %v", filePath, err)
-	}
-	return file
-}
-
-func (data *Data) createTemplate(templateName string, file *os.File) {
-	tmpl := template.Must(template.New(templateName).ParseFiles(templateName))
-	if err := tmpl.Execute(file, data); err != nil {
-		log.Fatal("unable to execute tmpl: ", err)
-	}
-}
-
-func readErrorName() (errorName string) {
-	flag.StringVar(&errorName, "Name", "", "KeyType of the error (e.g. Internal)")
-	flag.Parse()
-	return errorName
-}
-
-func validateErrorName(errorName string) string {
-	if errorName == "" {
-		log.Fatal("pass argument name")
-	}
-	if strings.Contains(errorName, " ") || strings.Contains(errorName, ".") {
-		log.Fatal("name cannot contain spaces or points")
-	}
-	return strings.Title(errorName)
-}

internal/zerrors/generate/error_interface.go.tmpl

@@ -1,5 +0,0 @@
-
-type {{.ErrorName}} interface {
-	error
-	Is{{.ErrorName}}()
-}

internal/zerrors/generate/error_test.go.tmpl

@@ -1,33 +0,0 @@
-package errors_test
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-
-	"github.com/zitadel/zitadel/internal/zerrors"
-)
-
-func Test{{.ErrorName}}Error(t *testing.T) {
-	var err interface{}
-	err = new(zerrors.{{.ErrorName}}Error)
-	_, ok := err.(*zerrors.{{.ErrorName}})
-	assert.True(t, ok)
-}
-
-func TestThrow{{.ErrorName}}f(t *testing.T) {
-	err := zerrors.Throw{{.ErrorName}}f(nil, "id", "msg")
-	_, ok := err.(*zerrors.{{.ErrorName}}Error)
-	assert.True(t, ok)
-}
-
-func TestIs{{.ErrorName}}(t *testing.T) {
-	err := zerrors.Throw{{.ErrorName}}(nil, "id", "msg")
-	ok := zerrors.Is{{.ErrorName}}(err)
-	assert.True(t, ok)
-
-	err = errors.New("I am found!")
-	ok = zerrors.Is{{.ErrorName}}(err)
-	assert.False(t, ok)
-}

internal/zerrors/internal.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ Internal = (*InternalError)(nil)
-	_ Error    = (*InternalError)(nil)
-)
-
-type Internal interface {
-	error
-	IsInternal()
-}
-
-type InternalError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowInternal(parent error, id, message string) error {
-	return &InternalError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindInternal, parent, id, message)
 }
 
-func ThrowInternalf(parent error, id, format string, a ...interface{}) error {
-	return ThrowInternal(parent, id, fmt.Sprintf(format, a...))
+func ThrowInternalf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindInternal, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *InternalError) IsInternal() {}
-
 func IsInternal(err error) bool {
-	_, ok := err.(Internal)
-	return ok
-}
-
-func (err *InternalError) Is(target error) bool {
-	t, ok := target.(*InternalError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *InternalError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindInternal
 }

internal/zerrors/internal_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestInternalError(t *testing.T) {
-	var err interface{} = new(zerrors.InternalError)
-	_, ok := err.(zerrors.Internal)
-	assert.True(t, ok)
-}
+func TestInternal(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowInternalf(t *testing.T) {
-	err := zerrors.ThrowInternalf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.InternalError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowInternal", func(t *testing.T) {
+		err := zerrors.ThrowInternal(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsInternal(t *testing.T) {
-	err := zerrors.ThrowInternal(nil, "id", "msg")
-	ok := zerrors.IsInternal(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindInternal, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsInternal(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowInternalf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowInternalf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindInternal, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsInternal", func(t *testing.T) {
+		err := zerrors.ThrowInternal(parentErr, id, message)
+		isInternal := zerrors.IsInternal(err)
+		assert.True(t, isInternal)
+	})
 }

internal/zerrors/invalid_argument.go

@@ -1,55 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ InvalidArgument = (*InvalidArgumentError)(nil)
-	_ Error           = (*InvalidArgumentError)(nil)
-)
-
-type InvalidArgument interface {
-	error
-	IsInvalidArgument()
-}
-
-type InvalidArgumentError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowInvalidArgument(parent error, id, message string) error {
-	return &InvalidArgumentError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindInvalidArgument, parent, id, message)
 }
 
-func ThrowInvalidArgumentf(parent error, id, format string, a ...interface{}) error {
-	return ThrowInvalidArgument(parent, id, fmt.Sprintf(format, a...))
+func ThrowInvalidArgumentf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindInvalidArgument, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *InvalidArgumentError) IsInvalidArgument() {}
-
 func IsErrorInvalidArgument(err error) bool {
-	_, ok := err.(InvalidArgument)
-	return ok
-}
-
-func (err *InvalidArgumentError) Is(target error) bool {
-	t, ok := target.(*InvalidArgumentError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *InvalidArgumentError) As(target any) bool {
-	targetErr, ok := target.(*InvalidArgumentError)
-	if !ok {
-		return false
-	}
-	*targetErr = *err
-	return true
-}
-
-func (err *InvalidArgumentError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindInvalidArgument
 }

internal/zerrors/invalid_argument_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestInvalidArgumentError(t *testing.T) {
-	var invalidArgumentError interface{} = new(zerrors.InvalidArgumentError)
-	_, ok := invalidArgumentError.(zerrors.InvalidArgument)
-	assert.True(t, ok)
-}
+func TestInvalidArgument(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowInvalidArgumentf(t *testing.T) {
-	err := zerrors.ThrowInvalidArgumentf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.InvalidArgumentError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowInvalidArgument", func(t *testing.T) {
+		err := zerrors.ThrowInvalidArgument(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsErrorInvalidArgument(t *testing.T) {
-	err := zerrors.ThrowInvalidArgument(nil, "id", "msg")
-	ok := zerrors.IsErrorInvalidArgument(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindInvalidArgument, zitadelErr.Kind)
 
-	err = errors.New("I am invalid!")
-	ok = zerrors.IsErrorInvalidArgument(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowInvalidArgumentf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowInvalidArgumentf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindInvalidArgument, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsInvalidArgument", func(t *testing.T) {
+		err := zerrors.ThrowInvalidArgument(parentErr, id, message)
+		isInvalidArgument := zerrors.IsErrorInvalidArgument(err)
+		assert.True(t, isInvalidArgument)
+	})
 }

internal/zerrors/kind_enumer.go

@@ -0,0 +1,153 @@
+// Code generated by "enumer -type=Kind -trimprefix=Kind -json"; DO NOT EDIT.
+
+package zerrors
+
+import (
+	"encoding/json"
+	"fmt"
+	"strings"
+)
+
+const _KindName = "CanceledUnknownInvalidArgumentDeadlineExceededNotFoundAlreadyExistsPermissionDeniedResourceExhaustedPreconditionFailedAbortedOutOfRangeUnimplementedInternalUnavailableDataLossUnauthenticated"
+
+var _KindIndex = [...]uint8{0, 8, 15, 30, 46, 54, 67, 83, 100, 118, 125, 135, 148, 156, 167, 175, 190}
+
+const _KindLowerName = "canceledunknowninvalidargumentdeadlineexceedednotfoundalreadyexistspermissiondeniedresourceexhaustedpreconditionfailedabortedoutofrangeunimplementedinternalunavailabledatalossunauthenticated"
+
+func (i Kind) String() string {
+	i -= 1
+	if i < 0 || i >= Kind(len(_KindIndex)-1) {
+		return fmt.Sprintf("Kind(%d)", i+1)
+	}
+	return _KindName[_KindIndex[i]:_KindIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _KindNoOp() {
+	var x [1]struct{}
+	_ = x[KindCanceled-(1)]
+	_ = x[KindUnknown-(2)]
+	_ = x[KindInvalidArgument-(3)]
+	_ = x[KindDeadlineExceeded-(4)]
+	_ = x[KindNotFound-(5)]
+	_ = x[KindAlreadyExists-(6)]
+	_ = x[KindPermissionDenied-(7)]
+	_ = x[KindResourceExhausted-(8)]
+	_ = x[KindPreconditionFailed-(9)]
+	_ = x[KindAborted-(10)]
+	_ = x[KindOutOfRange-(11)]
+	_ = x[KindUnimplemented-(12)]
+	_ = x[KindInternal-(13)]
+	_ = x[KindUnavailable-(14)]
+	_ = x[KindDataLoss-(15)]
+	_ = x[KindUnauthenticated-(16)]
+}
+
+var _KindValues = []Kind{KindCanceled, KindUnknown, KindInvalidArgument, KindDeadlineExceeded, KindNotFound, KindAlreadyExists, KindPermissionDenied, KindResourceExhausted, KindPreconditionFailed, KindAborted, KindOutOfRange, KindUnimplemented, KindInternal, KindUnavailable, KindDataLoss, KindUnauthenticated}
+
+var _KindNameToValueMap = map[string]Kind{
+	_KindName[0:8]:          KindCanceled,
+	_KindLowerName[0:8]:     KindCanceled,
+	_KindName[8:15]:         KindUnknown,
+	_KindLowerName[8:15]:    KindUnknown,
+	_KindName[15:30]:        KindInvalidArgument,
+	_KindLowerName[15:30]:   KindInvalidArgument,
+	_KindName[30:46]:        KindDeadlineExceeded,
+	_KindLowerName[30:46]:   KindDeadlineExceeded,
+	_KindName[46:54]:        KindNotFound,
+	_KindLowerName[46:54]:   KindNotFound,
+	_KindName[54:67]:        KindAlreadyExists,
+	_KindLowerName[54:67]:   KindAlreadyExists,
+	_KindName[67:83]:        KindPermissionDenied,
+	_KindLowerName[67:83]:   KindPermissionDenied,
+	_KindName[83:100]:       KindResourceExhausted,
+	_KindLowerName[83:100]:  KindResourceExhausted,
+	_KindName[100:118]:      KindPreconditionFailed,
+	_KindLowerName[100:118]: KindPreconditionFailed,
+	_KindName[118:125]:      KindAborted,
+	_KindLowerName[118:125]: KindAborted,
+	_KindName[125:135]:      KindOutOfRange,
+	_KindLowerName[125:135]: KindOutOfRange,
+	_KindName[135:148]:      KindUnimplemented,
+	_KindLowerName[135:148]: KindUnimplemented,
+	_KindName[148:156]:      KindInternal,
+	_KindLowerName[148:156]: KindInternal,
+	_KindName[156:167]:      KindUnavailable,
+	_KindLowerName[156:167]: KindUnavailable,
+	_KindName[167:175]:      KindDataLoss,
+	_KindLowerName[167:175]: KindDataLoss,
+	_KindName[175:190]:      KindUnauthenticated,
+	_KindLowerName[175:190]: KindUnauthenticated,
+}
+
+var _KindNames = []string{
+	_KindName[0:8],
+	_KindName[8:15],
+	_KindName[15:30],
+	_KindName[30:46],
+	_KindName[46:54],
+	_KindName[54:67],
+	_KindName[67:83],
+	_KindName[83:100],
+	_KindName[100:118],
+	_KindName[118:125],
+	_KindName[125:135],
+	_KindName[135:148],
+	_KindName[148:156],
+	_KindName[156:167],
+	_KindName[167:175],
+	_KindName[175:190],
+}
+
+// KindString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func KindString(s string) (Kind, error) {
+	if val, ok := _KindNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _KindNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to Kind values", s)
+}
+
+// KindValues returns all values of the enum
+func KindValues() []Kind {
+	return _KindValues
+}
+
+// KindStrings returns a slice of all String values of the enum
+func KindStrings() []string {
+	strs := make([]string, len(_KindNames))
+	copy(strs, _KindNames)
+	return strs
+}
+
+// IsAKind returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i Kind) IsAKind() bool {
+	for _, v := range _KindValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalJSON implements the json.Marshaler interface for Kind
+func (i Kind) MarshalJSON() ([]byte, error) {
+	return json.Marshal(i.String())
+}
+
+// UnmarshalJSON implements the json.Unmarshaler interface for Kind
+func (i *Kind) UnmarshalJSON(data []byte) error {
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return fmt.Errorf("Kind should be a string, got %s", data)
+	}
+
+	var err error
+	*i, err = KindString(s)
+	return err
+}

internal/zerrors/not_found.go

@@ -2,38 +2,15 @@ package zerrors
 
 import "fmt"
 
-type NotFound interface {
-	error
-	IsNotFound()
-}
-
-type NotFoundError struct {
-	*ZitadelError
-}
-
 func ThrowNotFound(parent error, id, message string) error {
-	return &NotFoundError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindNotFound, parent, id, message)
 }
 
-func ThrowNotFoundf(parent error, id, format string, a ...interface{}) error {
-	return ThrowNotFound(parent, id, fmt.Sprintf(format, a...))
+func ThrowNotFoundf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindNotFound, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *NotFoundError) IsNotFound() {}
-
 func IsNotFound(err error) bool {
-	_, ok := err.(NotFound)
-	return ok
-}
-
-func (err *NotFoundError) Is(target error) bool {
-	t, ok := target.(*NotFoundError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *NotFoundError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindNotFound
 }

internal/zerrors/not_found_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestNotFoundError(t *testing.T) {
-	var notFoundError interface{} = new(zerrors.NotFoundError)
-	_, ok := notFoundError.(zerrors.NotFound)
-	assert.True(t, ok)
-}
+func TestNotFound(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowNotFoundf(t *testing.T) {
-	err := zerrors.ThrowNotFoundf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.NotFoundError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowNotFound", func(t *testing.T) {
+		err := zerrors.ThrowNotFound(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsNotFound(t *testing.T) {
-	err := zerrors.ThrowNotFound(nil, "id", "msg")
-	ok := zerrors.IsNotFound(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindNotFound, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsNotFound(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowNotFoundf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowNotFoundf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindNotFound, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsNotFound", func(t *testing.T) {
+		err := zerrors.ThrowNotFound(parentErr, id, message)
+		isNotFound := zerrors.IsNotFound(err)
+		assert.True(t, isNotFound)
+	})
 }

internal/zerrors/permission_denied.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ PermissionDenied = (*PermissionDeniedError)(nil)
-	_ Error            = (*PermissionDeniedError)(nil)
-)
-
-type PermissionDenied interface {
-	error
-	IsPermissionDenied()
-}
-
-type PermissionDeniedError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowPermissionDenied(parent error, id, message string) error {
-	return &PermissionDeniedError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindPermissionDenied, parent, id, message)
 }
 
-func ThrowPermissionDeniedf(parent error, id, format string, a ...interface{}) error {
-	return ThrowPermissionDenied(parent, id, fmt.Sprintf(format, a...))
+func ThrowPermissionDeniedf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindPermissionDenied, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *PermissionDeniedError) IsPermissionDenied() {}
-
 func IsPermissionDenied(err error) bool {
-	_, ok := err.(PermissionDenied)
-	return ok
-}
-
-func (err *PermissionDeniedError) Is(target error) bool {
-	t, ok := target.(*PermissionDeniedError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *PermissionDeniedError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindPermissionDenied
 }

internal/zerrors/permission_denied_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestPermissionDeniedError(t *testing.T) {
-	var err interface{} = new(zerrors.PermissionDeniedError)
-	_, ok := err.(zerrors.PermissionDenied)
-	assert.True(t, ok)
-}
+func TestPermissionDenied(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "PERMISSION_DENIED"
+	message := "you do not have permission"
 
-func TestThrowPermissionDeniedf(t *testing.T) {
-	err := zerrors.ThrowPermissionDeniedf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.PermissionDeniedError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowPermissionDenied", func(t *testing.T) {
+		err := zerrors.ThrowPermissionDenied(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsPermissionDenied(t *testing.T) {
-	err := zerrors.ThrowPermissionDenied(nil, "id", "msg")
-	ok := zerrors.IsPermissionDenied(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindPermissionDenied, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsPermissionDenied(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowPermissionDeniedf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowPermissionDeniedf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindPermissionDenied, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsPermissionDenied", func(t *testing.T) {
+		err := zerrors.ThrowPermissionDenied(parentErr, id, message)
+		isPermissionDenied := zerrors.IsPermissionDenied(err)
+		assert.True(t, isPermissionDenied)
+	})
 }

internal/zerrors/precondition_failed.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ PreconditionFailed = (*PreconditionFailedError)(nil)
-	_ Error              = (*PreconditionFailedError)(nil)
-)
-
-type PreconditionFailed interface {
-	error
-	IsPreconditionFailed()
-}
-
-type PreconditionFailedError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowPreconditionFailed(parent error, id, message string) error {
-	return &PreconditionFailedError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindPreconditionFailed, parent, id, message)
 }
 
-func ThrowPreconditionFailedf(parent error, id, format string, a ...interface{}) error {
-	return ThrowPreconditionFailed(parent, id, fmt.Sprintf(format, a...))
+func ThrowPreconditionFailedf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindPreconditionFailed, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *PreconditionFailedError) IsPreconditionFailed() {}
-
 func IsPreconditionFailed(err error) bool {
-	_, ok := err.(PreconditionFailed)
-	return ok
-}
-
-func (err *PreconditionFailedError) Is(target error) bool {
-	t, ok := target.(*PreconditionFailedError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *PreconditionFailedError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindPreconditionFailed
 }

internal/zerrors/precondition_failed_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestPreconditionFailedError(t *testing.T) {
-	var err interface{} = new(zerrors.PreconditionFailedError)
-	_, ok := err.(zerrors.PreconditionFailed)
-	assert.True(t, ok)
-}
+func TestPreconditionFailed(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowPreconditionFailedf(t *testing.T) {
-	err := zerrors.ThrowPreconditionFailedf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.PreconditionFailedError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowPreconditionFailed", func(t *testing.T) {
+		err := zerrors.ThrowPreconditionFailed(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsPreconditionFailed(t *testing.T) {
-	err := zerrors.ThrowPreconditionFailed(nil, "id", "msg")
-	ok := zerrors.IsPreconditionFailed(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindPreconditionFailed, zitadelErr.Kind)
 
-	err = errors.New("Precondition failed!")
-	ok = zerrors.IsPreconditionFailed(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowPreconditionFailedf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowPreconditionFailedf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindPreconditionFailed, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsPreconditionFailed", func(t *testing.T) {
+		err := zerrors.ThrowPreconditionFailed(parentErr, id, message)
+		isPreconditionFailed := zerrors.IsPreconditionFailed(err)
+		assert.True(t, isPreconditionFailed)
+	})
 }

internal/zerrors/resource_exhausted.go

@@ -1,48 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ ResourceExhausted = (*ResourceExhaustedError)(nil)
-	_ Error             = (*ResourceExhaustedError)(nil)
-)
-
-type ResourceExhausted interface {
-	error
-	IsResourceExhausted()
-}
-
-type ResourceExhaustedError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowResourceExhausted(parent error, id, message string) error {
-	return &ResourceExhaustedError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindResourceExhausted, parent, id, message)
 }
 
-func ThrowResourceExhaustedf(parent error, id, format string, a ...interface{}) error {
-	return ThrowResourceExhausted(parent, id, fmt.Sprintf(format, a...))
+func ThrowResourceExhaustedf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindResourceExhausted, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *ResourceExhaustedError) IsResourceExhausted() {}
-
 func IsResourceExhausted(err error) bool {
-	//nolint:errorlint
-	_, ok := err.(ResourceExhausted)
-	return ok
-}
-
-func (err *ResourceExhaustedError) Is(target error) bool {
-	//nolint:errorlint
-	t, ok := target.(*ResourceExhaustedError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *ResourceExhaustedError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindResourceExhausted
 }

internal/zerrors/resource_exhausted_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestResourceExhaustedError(t *testing.T) {
-	var err interface{} = new(zerrors.ResourceExhaustedError)
-	_, ok := err.(zerrors.ResourceExhausted)
-	assert.True(t, ok)
-}
+func TestResourceExhausted(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowResourceExhaustedf(t *testing.T) {
-	err := zerrors.ThrowResourceExhaustedf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.ResourceExhaustedError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowResourceExhausted", func(t *testing.T) {
+		err := zerrors.ThrowResourceExhausted(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsResourceExhausted(t *testing.T) {
-	err := zerrors.ThrowResourceExhausted(nil, "id", "msg")
-	ok := zerrors.IsResourceExhausted(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindResourceExhausted, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsResourceExhausted(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowResourceExhaustedf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowResourceExhaustedf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindResourceExhausted, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsResourceExhausted", func(t *testing.T) {
+		err := zerrors.ThrowResourceExhausted(parentErr, id, message)
+		isResourceExhausted := zerrors.IsResourceExhausted(err)
+		assert.True(t, isResourceExhausted)
+	})
 }

internal/zerrors/unauthenticated.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ Unauthenticated = (*UnauthenticatedError)(nil)
-	_ Error           = (*UnauthenticatedError)(nil)
-)
-
-type Unauthenticated interface {
-	error
-	IsUnauthenticated()
-}
-
-type UnauthenticatedError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowUnauthenticated(parent error, id, message string) error {
-	return &UnauthenticatedError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindUnauthenticated, parent, id, message)
 }
 
-func ThrowUnauthenticatedf(parent error, id, format string, a ...interface{}) error {
-	return ThrowUnauthenticated(parent, id, fmt.Sprintf(format, a...))
+func ThrowUnauthenticatedf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindUnauthenticated, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *UnauthenticatedError) IsUnauthenticated() {}
-
 func IsUnauthenticated(err error) bool {
-	_, ok := err.(Unauthenticated)
-	return ok
-}
-
-func (err *UnauthenticatedError) Is(target error) bool {
-	t, ok := target.(*UnauthenticatedError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *UnauthenticatedError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindUnauthenticated
 }

internal/zerrors/unauthenticated_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestUnauthenticatedError(t *testing.T) {
-	var err interface{} = new(zerrors.UnauthenticatedError)
-	_, ok := err.(zerrors.Unauthenticated)
-	assert.True(t, ok)
-}
+func TestUnauthenticated(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowUnauthenticatedf(t *testing.T) {
-	err := zerrors.ThrowUnauthenticatedf(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.UnauthenticatedError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowUnauthenticated", func(t *testing.T) {
+		err := zerrors.ThrowUnauthenticated(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsUnauthenticated(t *testing.T) {
-	err := zerrors.ThrowUnauthenticated(nil, "id", "msg")
-	ok := zerrors.IsUnauthenticated(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindUnauthenticated, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsUnauthenticated(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowUnauthenticatedf", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowUnauthenticatedf(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindUnauthenticated, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsUnauthenticated", func(t *testing.T) {
+		err := zerrors.ThrowUnauthenticated(parentErr, id, message)
+		isUnauthenticated := zerrors.IsUnauthenticated(err)
+		assert.True(t, isUnauthenticated)
+	})
 }

internal/zerrors/unavailable.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ Unavailable = (*UnavailableError)(nil)
-	_ Error       = (*UnavailableError)(nil)
-)
-
-type Unavailable interface {
-	error
-	IsUnavailable()
-}
-
-type UnavailableError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowUnavailable(parent error, id, message string) error {
-	return &UnavailableError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindUnavailable, parent, id, message)
 }
 
-func ThrowUnavailablef(parent error, id, format string, a ...interface{}) error {
-	return ThrowUnavailable(parent, id, fmt.Sprintf(format, a...))
+func ThrowUnavailablef(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindUnavailable, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *UnavailableError) IsUnavailable() {}
-
 func IsUnavailable(err error) bool {
-	_, ok := err.(Unavailable)
-	return ok
-}
-
-func (err *UnavailableError) Is(target error) bool {
-	t, ok := target.(*UnavailableError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *UnavailableError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindUnavailable
 }

internal/zerrors/unavailable_test.go

@@ -9,25 +9,54 @@ import (
 	"github.com/zitadel/zitadel/internal/zerrors"
 )
 
-func TestUnavailableError(t *testing.T) {
-	var err interface{} = new(zerrors.UnavailableError)
-	_, ok := err.(zerrors.Unavailable)
-	assert.True(t, ok)
-}
+func TestUnavailable(t *testing.T) {
+	parentErr := errors.New("parent error")
+	id := "test_id"
+	message := "test message"
 
-func TestThrowUnavailablef(t *testing.T) {
-	err := zerrors.ThrowUnavailablef(nil, "id", "msg")
-	//nolint:errorlint
-	_, ok := err.(*zerrors.UnavailableError)
-	assert.True(t, ok)
-}
+	t.Run("ThrowUnavailable", func(t *testing.T) {
+		err := zerrors.ThrowUnavailable(parentErr, id, message)
+		assert.NotNil(t, err)
 
-func TestIsUnavailable(t *testing.T) {
-	err := zerrors.ThrowUnavailable(nil, "id", "msg")
-	ok := zerrors.IsUnavailable(err)
-	assert.True(t, ok)
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindUnavailable, zitadelErr.Kind)
 
-	err = errors.New("I am found!")
-	ok = zerrors.IsUnavailable(err)
-	assert.False(t, ok)
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, message, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("ThrowUnavailablef", func(t *testing.T) {
+		format := "formatted %s"
+		arg := "message"
+		expectedMessage := "formatted message"
+
+		err := zerrors.ThrowUnavailablef(parentErr, id, format, arg)
+		assert.NotNil(t, err)
+
+		zitadelErr, ok := zerrors.AsZitadelError(err)
+		assert.True(t, ok)
+		assert.Equal(t, zerrors.KindUnavailable, zitadelErr.Kind)
+
+		zitadelError := new(zerrors.ZitadelError)
+		if errors.As(err, &zitadelError) {
+			assert.Equal(t, parentErr, zitadelError.Unwrap())
+			assert.Equal(t, id, zitadelError.ID)
+			assert.Equal(t, expectedMessage, zitadelError.Message)
+		} else {
+			t.Errorf("error is not of type ZitadelError")
+		}
+	})
+
+	t.Run("IsUnavailable", func(t *testing.T) {
+		err := zerrors.ThrowUnavailable(parentErr, id, message)
+		isUnavailable := zerrors.IsUnavailable(err)
+		assert.True(t, isUnavailable)
+	})
 }

internal/zerrors/unimplemented.go

@@ -1,46 +1,16 @@
 package zerrors
 
-import (
-	"fmt"
-)
-
-var (
-	_ Unimplemented = (*UnimplementedError)(nil)
-	_ Error         = (*UnimplementedError)(nil)
-)
-
-type Unimplemented interface {
-	error
-	IsUnimplemented()
-}
-
-type UnimplementedError struct {
-	*ZitadelError
-}
+import "fmt"
 
 func ThrowUnimplemented(parent error, id, message string) error {
-	return &UnimplementedError{CreateZitadelError(parent, id, message)}
+	return CreateZitadelError(KindUnimplemented, parent, id, message)
 }
 
-func ThrowUnimplementedf(parent error, id, format string, a ...interface{}) error {
-	return ThrowUnimplemented(parent, id, fmt.Sprintf(format, a...))
+func ThrowUnimplementedf(parent error, id, format string, a ...any) error {
+	return CreateZitadelError(KindUnimplemented, parent, id, fmt.Sprintf(format, a...))
 }
 
-func (err *UnimplementedError) IsUnimplemented() {}
-
 func IsUnimplemented(err error) bool {
-	_, ok := err.(Unimplemented)
-	return ok
-}
-
-func (err *UnimplementedError) Is(target error) bool {
-	t, ok := target.(*UnimplementedError)
-	if !ok {
-		return false
-	}
-	return err.ZitadelError.Is(t.ZitadelError)
-}
-
-func (err *UnimplementedError) Unwrap() error {
-	return err.ZitadelError
+	zitadelErr, ok := AsZitadelError(err)
+	return ok && zitadelErr.Kind == KindUnimplemented
 }