fix: recover from request panics (#11713)

# Which Problems Are Solved

Panics may cause a service discruption by unexpectedly closing an
request's connection. Or in the case of gRPC completely killing the
service.

Allthough panics are still individual bugs that need to be solved, this
PR makes sure a panic is gracefully handled and an understandable error
is returned to the client.

# How the Problems Are Solved

- Recover in the middleware interceptors for the 3 API protocols (HTTP,
gRPC, connect).
- HTTP middleware uses formatted responses for:
  - OIDC errors (JSON formatted response)
  - UI (error page rendering)
  - SCIM
- Upon recovery an alert level log is printed (ERROR+4 for stdlib log
handlers)

# Additional Context

- internal observation

---------

Co-authored-by: Livio Spring <[email protected]>
Co-authored-by: Livio Spring <[email protected]>

backend/v3/instrumentation/slog.go

@@ -2,6 +2,7 @@ package instrumentation
 
 import (
 	"context"
+	"errors"
 	"log/slog"
 	"os"
 	"slices"
@@ -159,6 +160,7 @@ func setLogger(provider *log.LoggerProvider, cfg LogConfig) {
 			Prependers: []slogctx.AttrExtractor{
 				instanceExtractor,
 				requestIDExtractor,
+				causeExtractor,
 				slogotel.ExtractTraceSpanID,
 				slogctx.ExtractPrepended,
 			},
@@ -224,6 +226,17 @@ func requestIDExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string
 	return nil
 }
 
+// causeExtractor sets the cause of a canceled context to a log entry.
+func causeExtractor(ctx context.Context, _ time.Time, _ slog.Level, _ string) []slog.Attr {
+	err := context.Cause(ctx)
+	if err != nil && !errors.Is(err, context.Canceled) && !errors.Is(err, context.DeadlineExceeded) {
+		return []slog.Attr{
+			slog.String("cause", err.Error()),
+		}
+	}
+	return nil
+}
+
 type replacer func(groups []string, a slog.Attr) slog.Attr
 
 func chainReplacers(replacers ...replacer) replacer {

backend/v3/instrumentation/slog_test.go

@@ -1,8 +1,11 @@
 package instrumentation
 
 import (
+	"context"
+	"errors"
 	"log/slog"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	old_logging "github.com/zitadel/logging" //nolint:staticcheck
@@ -164,3 +167,48 @@ func TestLogConfig_replacer(t *testing.T) {
 		})
 	}
 }
+
+func Test_causeExtractor(t *testing.T) {
+	background := context.Background()
+	canceled, cancel := context.WithCancel(background)
+	cancel()
+	timedOut, cancel := context.WithTimeout(background, -1)
+	defer cancel()
+	canceledCause, cause := context.WithCancelCause(background)
+	cause(errors.New("oops"))
+
+	tests := []struct {
+		name string // description of this test case
+		ctx  context.Context
+		want []slog.Attr
+	}{
+		{
+			name: "valid background, no attributes",
+			ctx:  background,
+			want: nil,
+		},
+		{
+			name: "regular canceled context",
+			ctx:  canceled,
+			want: nil,
+		},
+		{
+			name: "regular timed out context",
+			ctx:  timedOut,
+			want: nil,
+		},
+		{
+			name: "canceled context with cause",
+			ctx:  canceledCause,
+			want: []slog.Attr{
+				slog.String("cause", "oops"),
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := causeExtractor(tt.ctx, time.Time{}, 0, "")
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}