fix: recover from request panics (#11713)

# Which Problems Are Solved

Panics may cause a service discruption by unexpectedly closing an
request's connection. Or in the case of gRPC completely killing the
service.

Allthough panics are still individual bugs that need to be solved, this
PR makes sure a panic is gracefully handled and an understandable error
is returned to the client.

# How the Problems Are Solved

- Recover in the middleware interceptors for the 3 API protocols (HTTP,
gRPC, connect).
- HTTP middleware uses formatted responses for:
  - OIDC errors (JSON formatted response)
  - UI (error page rendering)
  - SCIM
- Upon recovery an alert level log is printed (ERROR+4 for stdlib log
handlers)

# Additional Context

- internal observation

---------

Co-authored-by: Livio Spring <[email protected]>
Co-authored-by: Livio Spring <[email protected]>

internal/api/ui/login/renderer.go

@@ -11,8 +11,9 @@ import (
 	"time"
 
 	"github.com/gorilla/csrf"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	http_mw "github.com/zitadel/zitadel/internal/api/http/middleware"
@@ -249,7 +250,7 @@ func CreateRenderer(pathPrefix string, staticStorage static.Storage, cookieName
 		tmplMapping, funcs,
 		cookieName,
 	)
-	logging.New().OnError(err).WithError(err).Panic("error creating renderer")
+	old_logging.New().OnError(err).WithError(err).Panic("error creating renderer")
 	return r
 }
 
@@ -342,18 +343,20 @@ func (l *Login) chooseNextStep(w http.ResponseWriter, r *http.Request, authReq *
 }
 
 func (l *Login) renderInternalError(w http.ResponseWriter, r *http.Request, authReq *domain.AuthRequest, err error) {
+	ctx := r.Context()
 	if err != nil {
-		log := logging.WithError(err)
+		log := logging.FromCtx(ctx).With("err", err)
 		if authReq != nil {
-			log = log.WithField("auth_req_id", authReq.ID)
+			log = log.With("auth_req_id", authReq.ID)
 		}
+		const msg = "render internal error"
 		if zerrors.IsInternal(err) {
-			log.Error()
+			log.ErrorContext(ctx, msg)
 		} else {
-			log.Info()
+			log.InfoContext(ctx, msg)
 		}
 	}
-	translator := l.getTranslator(r.Context(), authReq)
+	translator := l.getTranslator(ctx, authReq)
 	data := l.getBaseData(r, authReq, translator, "Errors.Internal", "", err)
 	l.renderer.RenderTemplate(w, r, translator, l.renderer.Templates[tmplError], data, nil)
 }
@@ -568,7 +571,7 @@ func (l *Login) getOrgPrimaryDomain(r *http.Request, authReq *domain.AuthRequest
 	}
 	org, err := l.query.OrgByID(r.Context(), orgID)
 	if err != nil {
-		logging.New().WithError(err).Error("cannot get default org")
+		old_logging.New().WithError(err).Error("cannot get default org")
 		return ""
 	}
 	return org.Domain
@@ -591,7 +594,7 @@ func (l *Login) addLoginTranslations(translator *i18n.Translator, customTexts []
 			Text: text.Text,
 		}
 		err := l.renderer.AddMessages(translator, text.Language, msg)
-		logging.OnError(err).Warn("could no add message to translator")
+		old_logging.OnError(err).Warn("could no add message to translator")
 	}
 }
 
@@ -599,7 +602,7 @@ func (l *Login) customTexts(ctx context.Context, translator *i18n.Translator, or
 	instanceID := authz.GetInstance(ctx).InstanceID()
 	instanceTexts, err := l.query.CustomTextListByTemplate(ctx, instanceID, domain.LoginCustomText, false)
 	if err != nil {
-		logging.WithFields("instanceID", instanceID).Warn("unable to load custom texts for instance")
+		old_logging.WithFields("instanceID", instanceID).Warn("unable to load custom texts for instance")
 		return
 	}
 	l.addLoginTranslations(translator, query.CustomTextsToDomain(instanceTexts))
@@ -608,7 +611,7 @@ func (l *Login) customTexts(ctx context.Context, translator *i18n.Translator, or
 	}
 	orgTexts, err := l.query.CustomTextListByTemplate(ctx, orgID, domain.LoginCustomText, false)
 	if err != nil {
-		logging.WithFields("instanceID", instanceID, "org", orgID).Warn("unable to load custom texts for org")
+		old_logging.WithFields("instanceID", instanceID, "org", orgID).Warn("unable to load custom texts for org")
 		return
 	}
 	l.addLoginTranslations(translator, query.CustomTextsToDomain(orgTexts))

internal/api/ui/login/router.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/zitadel/zitadel/internal/api/http/middleware"
 )
 
 const (
@@ -71,6 +72,7 @@ var (
 func CreateRouter(login *Login, interceptors ...mux.MiddlewareFunc) *mux.Router {
 	router := mux.NewRouter()
 	router.Use(interceptors...)
+	router.Use(middleware.RecoverHandler(login.writeRecoverError))
 	router.HandleFunc(EndpointRoot, login.handleLogin).Methods(http.MethodGet)
 	router.HandleFunc(EndpointHealthz, login.handleHealthz).Methods(http.MethodGet)
 	router.HandleFunc(EndpointReadiness, login.handleReadiness).Methods(http.MethodGet)
@@ -131,3 +133,7 @@ func CreateRouter(login *Login, interceptors ...mux.MiddlewareFunc) *mux.Router
 	router.HandleFunc(EndpointDeviceAuthAction, login.handleDeviceAuthAction).Methods(http.MethodGet, http.MethodPost)
 	return router
 }
+
+func (l *Login) writeRecoverError(w http.ResponseWriter, r *http.Request, err error) {
+	l.renderError(w, r, nil, err)
+}