fix(api): add tracing interceptor on connectRPC services (#11147)

# Which Problems Are Solved

ConnectRPC based services (v2 API) was missing the root span of all
requests. Rest calls through grpc-gateway more or less worked properly.
This was due to a missing server interceptor for connectRPC services
itself. Rest call would work as the trace would start on the gateway
already.

# How the Problems Are Solved

Initiate an connectRPC OTEL interceptor and pass it to all services.

# Additional Changes

None

# Additional Context

- noticed internally
- requires backport to v4.x

internal/api/api.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"connectrpc.com/grpcreflect"
+	"connectrpc.com/otelconnect"
 	"github.com/gorilla/mux"
 	"github.com/improbable-eng/grpc-web/go/grpcweb"
 	"github.com/zitadel/logging"
@@ -54,6 +55,7 @@ type API struct {
 
 	targetEncryptionAlgorithm crypto.EncryptionAlgorithm
 	translator                *i18n.Translator
+	connectOTELInterceptor    *otelconnect.Interceptor
 }
 
 func (a *API) ListGrpcServices() []string {
@@ -129,6 +131,10 @@ func New(
 	if err != nil {
 		return nil, err
 	}
+	api.connectOTELInterceptor, err = otelconnect.NewInterceptor()
+	if err != nil {
+		return nil, err
+	}
 	api.registerHealthServer()
 
 	api.RegisterHandlerOnPrefix("/debug", api.healthHandler())
@@ -191,6 +197,7 @@ func (a *API) RegisterService(ctx context.Context, srv server.Server) error {
 func (a *API) registerConnectServer(service server.ConnectServer) {
 	prefix, handler := service.RegisterConnectServer(
 		connect_middleware.CallDurationHandler(),
+		a.connectOTELInterceptor,
 		connect_middleware.MetricsHandler(metricTypes, grpc_api.Probes...),
 		connect_middleware.NoCacheInterceptor(),
 		connect_middleware.InstanceInterceptor(a.queries, a.externalDomain, a.translator, system_pb.SystemService_ServiceDesc.ServiceName, healthpb.Health_ServiceDesc.ServiceName, instance_pb.InstanceService_ServiceDesc.ServiceName),