andwati/zitadel
1
0
Fork 0
mirror of https://github.com/andwati/zitadel.git synced 2026-11-07 06:42:15 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(logging): add streams (#11435)

Browse Source 
# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <[email protected]>
This commit is contained in:
Tim Möhlmann
2026-02-04 11:51:43 +01:00
committed by GitHub
co-authored by Silvan
parent b99271755b
commit 11dbb1b277
112 changed files with 2679 additions and 981 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/authz/instance.go
+3 -1
View File
@@ -6,6 +6,7 @@ import (
"golang.org/x/text/language"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/internal/execution/target"
"github.com/zitadel/zitadel/internal/feature"
)
@@ -107,11 +108,12 @@ func GetFeatures(ctx context.Context) feature.Features {
}
func WithInstance(ctx context.Context, instance Instance) context.Context {
ctx = instrumentation.SetInstance(ctx, instance)
return context.WithValue(ctx, instanceKey, instance)
}
func WithInstanceID(ctx context.Context, id string) context.Context {
return context.WithValue(ctx, instanceKey, &instance{id: id})
return WithInstance(ctx, &instance{id: id})
}
func WithDefaultLanguage(ctx context.Context, defaultLanguage language.Tag) context.Context {
internal/api/grpc/gerrors/zitadel_errors.go
+5 -6
View File
@@ -7,13 +7,12 @@ import (
"connectrpc.com/connect"
"github.com/jackc/pgx/v5/pgconn"
slogctx "github.com/veqryn/slog-context"
"github.com/zitadel/logging"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/protoadapt"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
commandErrors "github.com/zitadel/zitadel/internal/command/errors"
"github.com/zitadel/zitadel/internal/zerrors"
"github.com/zitadel/zitadel/pkg/grpc/message"
@@ -27,13 +26,13 @@ func ZITADELToGRPCError(ctx context.Context, err error) error {
code, key, id, lvl := extractError(err)
msg := key
msg += " (" + id + ")"
slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err, "code", code)
logging.Log(ctx, lvl, msg, "err", err, "code", code)
errorInfo := getErrorInfo(id, key, err)
s, err := status.New(code, msg).WithDetails(errorInfo)
if err != nil {
logging.WithError(err).WithField("logID", "GRPC-gIeRw").Debug("unable to add detail")
logging.WithError(ctx, err).Debug("unable to add error detail")
return status.New(code, msg).Err()
}
@@ -48,13 +47,13 @@ func ZITADELToConnectError(ctx context.Context, err error) error {
if errors.As(err, &connectError) {
// Connect error may be returned by other middlewares,
// so we assume it's a client error and log as warning.
slogctx.FromCtx(ctx).WarnContext(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
logging.Warn(ctx, connectError.Message(), "err", connectError.Unwrap(), "code", connectError.Code())
return err
}
code, key, id, lvl := extractError(err)
msg := key
msg += " (" + id + ")"
slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
logging.Log(ctx, lvl, msg, "err", err)
errorInfo := getErrorInfo(id, key, err)
internal/api/grpc/server/connect_middleware/log_interceptor.go
+44 -2
View File
@@ -1,13 +1,55 @@
package connect_middleware
import (
"context"
"log/slog"
"slices"
"strings"
"time"
"connectrpc.com/connect"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
http_util "github.com/zitadel/zitadel/internal/api/http"
)
func LogHandler(ignoredPrefix ...string) connect.UnaryInterceptorFunc {
func LogHandler(ignoredMethodSuffixes ...string) connect.UnaryInterceptorFunc {
return func(next connect.UnaryFunc) connect.UnaryFunc {
return logging.NewConnectInterceptor(next, ignoredPrefix...)
return func(ctx context.Context, req connect.AnyRequest) (connect.AnyResponse, error) {
if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
return strings.HasSuffix(req.Spec().Procedure, s)
}) {
return next(ctx, req)
}
start := time.Now()
ctx = logging.NewCtx(ctx, logging.StreamRequest)
ctx = instrumentation.SetRequestID(ctx, start)
resp, err := next(ctx, req)
var code connect.Code
if err != nil {
code = connect.CodeOf(err)
}
spec := req.Spec()
logging.Info(ctx, "request served",
slog.String("protocol", "connect"),
slog.Any("domain", http_util.DomainContext(ctx)),
slog.String("service", serviceFromRPCMethod(spec.Procedure)),
slog.String("http_method", req.HTTPMethod()),
slog.String("path", spec.Procedure),
slog.Any("code", code),
slog.Duration("duration", time.Since(start)),
)
return resp, err
}
}
}
func serviceFromRPCMethod(fullMethod string) string {
parts := strings.Split(fullMethod, "/")
if len(parts) >= 2 {
return parts[1]
}
return "unknown"
}
internal/api/grpc/server/gateway.go
-1
View File
@@ -284,5 +284,4 @@ func setRequestURIPattern(ctx context.Context) {
}
span := trace.SpanFromContext(ctx)
span.SetName(pattern)
metrics.SetRequestURIPattern(ctx, pattern)
}
internal/api/grpc/server/middleware/log_interceptor.go
+46 -2
View File
@@ -1,11 +1,55 @@
package middleware
import (
"google.golang.org/grpc"
"context"
"log/slog"
"net/http"
"slices"
"strings"
"time"
"google.golang.org/grpc"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
http_util "github.com/zitadel/zitadel/internal/api/http"
)
func LogHandler(ignoredMethodSuffixes ...string) grpc.UnaryServerInterceptor {
return logging.NewGrpcInterceptor(ignoredMethodSuffixes...)
return func(ctx context.Context, req any, info *grpc.UnaryServerInfo, next grpc.UnaryHandler) (any, error) {
if slices.ContainsFunc(ignoredMethodSuffixes, func(s string) bool {
return strings.HasSuffix(info.FullMethod, s)
}) {
return next(ctx, req)
}
start := time.Now()
ctx = logging.NewCtx(ctx, logging.StreamRequest)
ctx = instrumentation.SetRequestID(ctx, start)
resp, err := next(ctx, req)
var code codes.Code
if err != nil {
code = status.Code(err)
}
logging.Info(ctx, "request served",
slog.String("protocol", "grpc"),
slog.Any("domain", http_util.DomainContext(ctx)),
slog.String("service", serviceFromRPCMethod(info.FullMethod)),
slog.String("http_method", http.MethodPost), // gRPC always uses POST
slog.String("path", info.FullMethod),
slog.Any("code", code),
slog.Duration("duration", time.Since(start)),
)
return resp, err
}
}
func serviceFromRPCMethod(fullMethod string) string {
parts := strings.Split(fullMethod, "/")
if len(parts) >= 2 {
return parts[1]
}
return "unknown"
}
internal/api/http/error.go
+2 -3
View File
@@ -5,8 +5,7 @@ import (
"log/slog"
"net/http"
slogctx "github.com/veqryn/slog-context"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -17,7 +16,7 @@ func ZitadelErrorToHTTPStatusCode(ctx context.Context, err error) (statusCode in
statusCode, key, id, lvl := extractError(err)
msg := key
msg += " (" + id + ")"
slogctx.FromCtx(ctx).Log(ctx, lvl, msg, "err", err)
logging.Log(ctx, lvl, msg, "err", err)
if statusCode == statusUnknown {
return http.StatusInternalServerError, false
}
internal/api/http/middleware/access_interceptor.go
+2 -16
View File
@@ -133,7 +133,7 @@ func (a *AccessInterceptor) handle(publicAuthPathPrefixes ...string) func(http.H
return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
ctx := request.Context()
tracingCtx, checkSpan := tracing.NewNamedSpan(ctx, "checkAccessQuota")
wrappedWriter := &statusRecorder{ResponseWriter: writer, status: 0}
wrappedWriter := newStatusWriter(writer)
limited := a.Limit(wrappedWriter, request.WithContext(tracingCtx), publicAuthPathPrefixes...)
checkSpan.End()
if limited {
@@ -151,7 +151,7 @@ func (a *AccessInterceptor) handle(publicAuthPathPrefixes ...string) func(http.H
}
}
func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusRecorder, writer http.ResponseWriter, request *http.Request, notCountable bool) {
func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusWriter, writer http.ResponseWriter, request *http.Request, notCountable bool) {
if !a.logstoreSvc.Enabled() {
return
}
@@ -178,17 +178,3 @@ func (a *AccessInterceptor) writeLog(ctx context.Context, wrappedWriter *statusR
NotCountable: notCountable,
})
}
type statusRecorder struct {
http.ResponseWriter
status int
ignoreWrites bool
}
func (r *statusRecorder) WriteHeader(status int) {
if r.ignoreWrites {
return
}
r.status = status
r.ResponseWriter.WriteHeader(status)
}
internal/api/http/middleware/log_interceptor.go
+28 -2
View File
@@ -1,13 +1,39 @@
package middleware
import (
"log/slog"
"net/http"
"time"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
http_util "github.com/zitadel/zitadel/internal/api/http"
)
func LogHandler(service string, ignoredPrefix ...string) func(http.Handler) http.Handler {
return func(h http.Handler) http.Handler {
return logging.NewHandler(h, service, ignoredPrefix...)
return func(next http.Handler) http.Handler {
filter := instrumentation.RequestFilter(ignoredPrefix...)
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !filter(r) {
next.ServeHTTP(w, r)
return
}
start := time.Now()
ctx := logging.NewCtx(r.Context(), logging.StreamRequest)
ctx = instrumentation.SetRequestID(ctx, start)
sw := newStatusWriter(w)
next.ServeHTTP(sw, r.WithContext(ctx))
logging.Info(ctx, "request served",
slog.String("protocol", "http"),
slog.Any("domain", http_util.DomainContext(ctx)),
slog.String("service", service),
slog.String("http_method", r.Method), // gRPC always uses POST
slog.String("path", r.URL.Path),
slog.Int("status", sw.status),
slog.Duration("duration", time.Since(start)),
)
})
}
}
internal/api/http/middleware/metrics_interceptor.go
+47 -1
View File
@@ -3,11 +3,57 @@ package middleware
import (
"net/http"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
)
func MetricsHandler(metricTypes []metrics.MetricType, ignoredMethods ...string) func(http.Handler) http.Handler {
return func(handler http.Handler) http.Handler {
return metrics.NewHandler(handler, metricTypes, ignoredMethods...)
return &Handler{
handler: handler,
methods: metricTypes,
filter: instrumentation.RequestFilter(ignoredMethods...),
}
}
}
type Handler struct {
handler http.Handler
methods []metrics.MetricType
filter otelhttp.Filter
}
// ServeHTTP implements [http.Handler]
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(h.methods) == 0 {
h.handler.ServeHTTP(w, r)
return
}
if !h.filter(r) {
// Simply pass through to the handler if a filter rejects the request
h.handler.ServeHTTP(w, r)
return
}
recorder := newStatusWriter(w)
h.handler.ServeHTTP(recorder, r)
if h.containsMetricsMethod(metrics.MetricTypeRequestCount) {
metrics.RegisterRequestCounter(recorder, r)
}
if h.containsMetricsMethod(metrics.MetricTypeTotalCount) {
metrics.RegisterTotalRequestCounter(r)
}
if h.containsMetricsMethod(metrics.MetricTypeStatusCode) {
metrics.RegisterRequestCodeCounter(recorder, r)
}
}
func (h *Handler) containsMetricsMethod(method metrics.MetricType) bool {
for _, m := range h.methods {
if m == method {
return true
}
}
return false
}
internal/api/http/middleware/middleware.go
+29
View File
@@ -0,0 +1,29 @@
package middleware
import "net/http"
// statusWriter is a [http.ResponseWriter] that captures the status code for logging.
type statusWriter struct {
http.ResponseWriter
status int
}
func newStatusWriter(w http.ResponseWriter) *statusWriter {
return &statusWriter{ResponseWriter: w}
}
func (w *statusWriter) Write(p []byte) (int, error) {
if w.status == 0 {
w.status = http.StatusOK
}
return w.ResponseWriter.Write(p)
}
func (w *statusWriter) WriteHeader(statusCode int) {
w.status = statusCode
w.ResponseWriter.WriteHeader(statusCode)
}
func (w *statusWriter) Status() int {
return w.status
}
internal/api/http/middleware/middleware_test.go
+38
View File
@@ -1,9 +1,12 @@
package middleware
import (
"net/http/httptest"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"golang.org/x/text/language"
"github.com/zitadel/zitadel/internal/i18n"
@@ -17,3 +20,38 @@ func TestMain(m *testing.M) {
i18n.SupportLanguages(SupportedLanguages...)
os.Exit(m.Run())
}
func Test_statusWriter(t *testing.T) {
tests := []struct {
name string
writeStatus int
wantStatus int
}{
{
name: "default status is 200",
wantStatus: 200,
},
{
name: "writes status code and body correctly",
writeStatus: 201,
wantStatus: 201,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
const body = "Hello, World!"
rec := httptest.NewRecorder()
sw := newStatusWriter(rec)
if tt.writeStatus != 0 {
sw.WriteHeader(tt.writeStatus)
}
_, err := sw.Write([]byte(body))
require.NoError(t, err)
assert.Equal(t, tt.wantStatus, sw.Status())
resp := rec.Result()
assert.Equal(t, tt.wantStatus, resp.StatusCode)
assert.Equal(t, body, rec.Body.String())
})
}
}
internal/api/http/middleware/trace_interceptor.go
+15 -2
View File
@@ -3,7 +3,10 @@ package middleware
import (
"net/http"
"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
http_utils "github.com/zitadel/zitadel/internal/api/http"
)
@@ -13,6 +16,16 @@ func DefaultTraceHandler(handler http.Handler) http.Handler {
func TraceHandler(ignoredPrefix ...string) func(http.Handler) http.Handler {
return func(handler http.Handler) http.Handler {
return tracing.NewHandler(handler, ignoredPrefix...)
return otelhttp.NewHandler(handler,
"zitadel",
otelhttp.WithFilter(instrumentation.RequestFilter(ignoredPrefix...)),
otelhttp.WithPublicEndpoint(),
otelhttp.WithSpanNameFormatter(spanNameFormatter),
otelhttp.WithMeterProvider(otel.GetMeterProvider()),
)
}
}
func spanNameFormatter(_ string, r *http.Request) string {
return r.URL.Path
}
internal/api/ui/login/external_provider_handler.go
+1 -1
View File
@@ -1611,5 +1611,5 @@ func WrapIdPError(err error) *IdPError {
if errors.As(err, &zErr) {
id = zErr.ID
}
return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal")}
return &IdPError{err: zerrors.CreateZitadelError(zerrors.KindPreconditionFailed, err, id, "Errors.User.ExternalIDP.LoginFailedSwitchLocal", 1)}
}
internal/eventstore/event.go
+58
View File
@@ -2,7 +2,10 @@ package eventstore
import (
"encoding/json"
"log/slog"
"maps"
"reflect"
"slices"
"time"
"github.com/shopspring/decimal"
@@ -109,3 +112,58 @@ func isEventTypes(command Command, types ...EventType) bool {
}
return false
}
type logValue struct {
event Event
}
func eventToLogValue(event Event) slog.LogValuer {
return &logValue{
event: event,
}
}
func (lv *logValue) LogValue() slog.Value {
attributes := make([]slog.Attr, 0, 12)
aggregate := lv.event.Aggregate()
attributes = append(attributes,
slog.String("aggregate_id", aggregate.ID),
slog.String("aggregate_type", string(aggregate.Type)),
slog.String("resource_owner", aggregate.ResourceOwner),
slog.String("instance_id", aggregate.InstanceID),
slog.String("version", string(aggregate.Version)),
slog.String("creator", lv.event.Creator()),
slog.String("event_type", string(lv.event.Type())),
slog.Uint64("revision", uint64(lv.event.Revision())),
slog.Uint64("sequence", lv.event.Sequence()),
slog.Time("created_at", lv.event.CreatedAt()),
slog.String("position", lv.event.Position().String()),
)
var m map[string]any
err := lv.event.Unmarshal(&m)
if err != nil {
attributes = append(attributes,
slog.String("msg", "failed to unmarshal event for logging"),
slog.String("err", err.Error()),
)
return slog.GroupValue(attributes...)
}
attributes = append(attributes,
slog.Any("data", mapToLogValue(m)),
)
return slog.GroupValue(attributes...)
}
// mapToLogValue converts a map[string]any to a [slog.Value], handling nested maps recursively.
func mapToLogValue(m map[string]any) slog.Value {
attributes := make([]slog.Attr, 0, len(m))
for _, key := range slices.Sorted(maps.Keys(m)) {
value := m[key]
if nestedMap, ok := value.(map[string]any); ok {
value = mapToLogValue(nestedMap)
}
attributes = append(attributes, slog.Any(key, value))
}
return slog.GroupValue(attributes...)
}
internal/eventstore/event_test.go
+139
View File
@@ -0,0 +1,139 @@
package eventstore
import (
"log/slog"
"testing"
"time"
"github.com/shopspring/decimal"
"github.com/stretchr/testify/assert"
)
func Test_logValue_LogValue(t *testing.T) {
tests := []struct {
name string
event *BaseEvent
want slog.Value
}{
{
name: "output event",
event: &BaseEvent{
EventType: "test.type",
Agg: &Aggregate{
ID: "agg-id",
Type: "agg-type",
ResourceOwner: "owner",
InstanceID: "instance-1",
Version: "x.y.z",
},
Seq: 42,
Pos: decimal.NewFromInt(1001),
Creation: time.Unix(123, 456),
User: "user-123",
Data: []byte(`{"key1":"value1", "key2":"value2"}`),
},
want: slog.GroupValue(
slog.String("aggregate_id", "agg-id"),
slog.String("aggregate_type", "agg-type"),
slog.String("resource_owner", "owner"),
slog.String("instance_id", "instance-1"),
slog.String("version", "x.y.z"),
slog.String("creator", "user-123"),
slog.String("event_type", "test.type"),
slog.Uint64("revision", uint64(0)),
slog.Uint64("sequence", 42),
slog.Time("created_at", time.Unix(123, 456)),
slog.String("position", "1001"),
slog.Any("data", slog.GroupValue(
slog.String("key1", "value1"),
slog.String("key2", "value2"),
)),
),
},
{
name: "unmarshal error",
event: &BaseEvent{
EventType: "test.type",
Agg: &Aggregate{
ID: "agg-id",
Type: "agg-type",
ResourceOwner: "owner",
InstanceID: "instance-1",
Version: "x.y.z",
},
Seq: 42,
Pos: decimal.NewFromInt(1001),
Creation: time.Unix(123, 456),
User: "user-123",
Data: []byte(`invalid-json`),
},
want: slog.GroupValue(
slog.String("aggregate_id", "agg-id"),
slog.String("aggregate_type", "agg-type"),
slog.String("resource_owner", "owner"),
slog.String("instance_id", "instance-1"),
slog.String("version", "x.y.z"),
slog.String("creator", "user-123"),
slog.String("event_type", "test.type"),
slog.Uint64("revision", uint64(0)),
slog.Uint64("sequence", 42),
slog.Time("created_at", time.Unix(123, 456)),
slog.String("position", "1001"),
slog.String("msg", "failed to unmarshal event for logging"),
slog.String("err", "invalid character 'i' looking for beginning of value"),
),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
lv := eventToLogValue(tt.event)
got := lv.LogValue()
assert.Equal(t, tt.want, got)
})
}
}
func Test_mapToLogValue(t *testing.T) {
tests := []struct {
name string
m map[string]any
want slog.Value
}{
{
name: "flat map",
m: map[string]any{
"key1": "value1",
"key2": 42,
"key3": true,
},
want: slog.GroupValue(
slog.String("key1", "value1"),
slog.Int("key2", 42),
slog.Bool("key3", true),
),
},
{
name: "nested map",
m: map[string]any{
"key1": "value1",
"key2": map[string]any{
"nestedKey1": 3.14,
"nestedKey2": "nestedValue",
},
},
want: slog.GroupValue(
slog.String("key1", "value1"),
slog.Any("key2", slog.GroupValue(
slog.Float64("nestedKey1", 3.14),
slog.String("nestedKey2", "nestedValue"),
)),
),
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got := mapToLogValue(tt.m)
assert.Equal(t, tt.want, got)
})
}
}
internal/eventstore/eventstore.go
+22 -6
View File
@@ -4,13 +4,14 @@ import (
"context"
"database/sql"
"errors"
"log/slog"
"sort"
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
new_db "github.com/zitadel/zitadel/backend/v3/storage/database"
new_sql "github.com/zitadel/zitadel/backend/v3/storage/database/dialect/sql"
"github.com/zitadel/zitadel/internal/api/authz"
@@ -33,6 +34,8 @@ type Eventstore struct {
pusher Pusher
querier Querier
searcher Searcher
logger *slog.Logger
}
var (
@@ -73,6 +76,7 @@ func NewEventstore(config *Config) *Eventstore {
pusher: config.Pusher,
querier: config.Querier,
searcher: config.Searcher,
logger: logging.New(logging.StreamEventPusher),
}
}
@@ -107,21 +111,27 @@ func (es *Eventstore) PushWithClient(ctx context.Context, client database.Contex
}
func (es *Eventstore) PushWithNewClient(ctx context.Context, client new_db.QueryExecutor, cmds ...Command) ([]Event, error) {
ctx = logging.ToCtx(ctx, es.logger)
if es.PushTimeout > 0 {
var cancel func()
ctx, cancel = context.WithTimeout(ctx, es.PushTimeout)
defer cancel()
}
var (
events []Event
err error
events []Event
err error
retries int
)
defer func() {
logging.OnError(ctx, err).Error("eventstore push failed", "retries", retries)
logPushedEvents(ctx, events)
}()
// Retry when there is a collision of the sequence as part of the primary key.
// "duplicate key value violates unique constraint \"events2_pkey\" (SQLSTATE 23505)"
// https://github.com/zitadel/zitadel/issues/7202
retry:
for i := 0; i <= es.maxRetries; i++ {
for ; retries <= es.maxRetries; retries++ {
events, err = es.pusher.Push(ctx, client, cmds...)
// if there is a transaction passed the calling function needs to retry
if _, ok := client.(new_db.Transaction); ok {
@@ -132,11 +142,11 @@ retry:
break retry
}
if pgErr.ConstraintName == "events2_pkey" && pgErr.SQLState() == "23505" {
logging.WithError(err).Info("eventstore push retry")
logging.WithError(ctx, err).Info("eventstore push retry")
continue
}
if pgErr.SQLState() == "CR000" || pgErr.SQLState() == "40001" {
logging.WithError(err).Info("eventstore push retry")
logging.WithError(ctx, err).Info("eventstore push retry")
continue
}
break retry
@@ -334,3 +344,9 @@ func appendAggregateType(typ AggregateType) {
}
aggregateTypes = append(aggregateTypes[:i], append([]string{string(typ)}, aggregateTypes[i:]...)...)
}
func logPushedEvents(ctx context.Context, events []Event) {
for _, event := range events {
logging.Info(ctx, "event pushed", "event", eventToLogValue(event))
}
}
internal/eventstore/handler/v2/failed_event.go
+15 -3
View File
@@ -1,10 +1,13 @@
package handler
import (
"context"
"database/sql"
_ "embed"
"log/slog"
"time"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -25,6 +28,14 @@ type failure struct {
err error
}
func (f *failure) LogValue() slog.Value {
return slog.GroupValue(
slog.Uint64("sequence", f.sequence),
slog.String("instance", f.instance),
slog.String("aggregate_id", f.aggregateID),
)
}
func failureFromEvent(event eventstore.Event, err error) *failure {
return &failure{
sequence: event.Sequence(),
@@ -47,15 +58,16 @@ func failureFromStatement(statement *Statement, err error) *failure {
}
}
func (h *Handler) handleFailedStmt(tx *sql.Tx, f *failure) (shouldContinue bool) {
func (h *Handler) handleFailedStmt(ctx context.Context, tx *sql.Tx, f *failure) (shouldContinue bool) {
ctx = logging.With(ctx, "failure", f)
failureCount, err := h.failureCount(tx, f)
if err != nil {
h.logFailure(f).WithError(err).Warn("unable to get failure count")
logging.Warn(ctx, "unable to get failure count", "err", err)
return false
}
failureCount += 1
err = h.setFailureCount(tx, failureCount, f)
h.logFailure(f).OnError(err).Warn("unable to update failure count")
logging.OnError(ctx, err).Warn("unable to update failure count")
return failureCount >= h.maxFailureCount
}
internal/eventstore/handler/v2/field_handler.go
+7 -7
View File
@@ -10,6 +10,7 @@ import (
"github.com/jackc/pgx/v5/pgconn"
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
@@ -79,8 +80,7 @@ func (h *FieldHandler) Trigger(ctx context.Context, opts ...TriggerOpt) (err err
wg.Done()
}
wg.Wait()
h.log().OnError(err).Info("process events failed")
h.log().WithField("iteration", i).Debug("trigger iteration")
logging.Debug(ctx, "trigger iteration", "iteration", i)
if !additionalIteration || err != nil {
return err
}
@@ -93,7 +93,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
if errors.As(err, &pgErr) {
// error returned if the row is currently locked by another connection
if pgErr.Code == "55P03" {
h.log().Debug("state already locked")
logging.WithError(ctx, err).Info("another handler is already updating this projection")
err = nil
additionalIteration = false
}
@@ -117,7 +117,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
defer func() {
if err != nil && !errors.Is(err, &executionError{}) {
rollbackErr := tx.Rollback()
h.log().OnError(rollbackErr).Debug("unable to rollback tx")
logging.OnError(ctx, rollbackErr).Error("unable to rollback tx")
return
}
commitErr := tx.Commit()
@@ -156,7 +156,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
return additionalIteration, err
}
if len(events) == 0 {
err = h.setState(tx, currentState)
err = h.setState(ctx, tx, currentState)
return additionalIteration, err
}
@@ -165,7 +165,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
return false, err
}
err = h.setState(tx, currentState)
err = h.setState(ctx, tx, currentState)
return additionalIteration, err
}
@@ -173,7 +173,7 @@ func (h *FieldHandler) processEvents(ctx context.Context, config *triggerConfig)
func (h *FieldHandler) fetchEvents(ctx context.Context, tx *sql.Tx, currentState *state) (_ []eventstore.FillFieldsEvent, additionalIteration bool, err error) {
events, err := h.es.Filter(ctx, h.eventQuery(currentState).SetTx(tx))
if err != nil || len(events) == 0 {
h.log().OnError(err).Debug("filter eventstore failed")
logging.OnError(ctx, err).Debug("filter eventstore failed")
return nil, false, err
}
eventAmount := len(events)
internal/eventstore/handler/v2/handler.go
+35 -29
View File
@@ -4,15 +4,17 @@ import (
"context"
"database/sql"
"errors"
"log/slog"
"math/rand"
"slices"
"sync"
"time"
"github.com/jackc/pgx/v5/pgconn"
"github.com/rs/xid"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/api/call"
"github.com/zitadel/zitadel/internal/database"
@@ -80,11 +82,12 @@ var _ migration.Migration = (*Handler)(nil)
// Execute implements migration.Migration.
func (h *Handler) Execute(ctx context.Context, startedEvent eventstore.Event) error {
start := time.Now()
logging.WithFields("projection", h.ProjectionName()).Info("projection starts prefilling")
ctx = logging.With(ctx, "projection", h.ProjectionName())
logging.Info(ctx, "projection starts prefilling")
logTicker := time.NewTicker(30 * time.Second)
go func() {
for range logTicker.C {
logging.WithFields("projection", h.ProjectionName()).Info("projection is prefilling")
logging.Info(ctx, "projection is prefilling")
}
}()
@@ -123,7 +126,7 @@ func (h *Handler) Execute(ctx context.Context, startedEvent eventstore.Event) er
wg.Wait()
logTicker.Stop()
logging.WithFields("projection", h.ProjectionName(), "took", time.Since(start)).Info("projections ended prefilling")
logging.Info(ctx, "projections ended prefilling", "took", time.Since(start))
return nil
}
@@ -170,7 +173,7 @@ func NewHandler(
aggregates[reducer.Aggregate] = eventTypes
}
metrics := NewProjectionMetrics()
metrics := NewProjectionMetrics(ctx)
handler := &Handler{
projection: projection,
@@ -203,6 +206,7 @@ func NewHandler(
}
func (h *Handler) Start(ctx context.Context) {
ctx = logging.NewCtx(ctx, logging.StreamEventHandler, slog.String("projection", h.ProjectionName()))
go h.schedule(ctx)
if h.triggerWithoutEvents != nil {
return
@@ -267,10 +271,12 @@ func (h *Handler) schedule(ctx context.Context) {
t.Stop()
return
case <-t.C:
jobCtx := logging.With(ctx, "job_id", xid.New(), "invoker", "schedule")
instances, err := h.queryInstances()
h.log().OnError(err).Debug("unable to query instances")
h.triggerInstances(call.WithTimestamp(ctx), instances)
if err != nil {
logging.Debug(jobCtx, "unable to query instances", "err", err)
}
h.triggerInstances(call.WithTimestamp(jobCtx), instances)
t.Reset(h.requeueEvery)
}
}
@@ -286,12 +292,12 @@ func (h *Handler) triggerInstances(ctx context.Context, instances []string, trig
if err == nil {
continue
}
h.log().WithField("instance", instance).WithError(err).Debug("trigger failed")
logging.Debug(instanceCtx, "trigger failed", "err", err)
time.Sleep(h.retryFailedAfter)
// retry if trigger failed
for ; err != nil; _, err = h.Trigger(instanceCtx, triggerOpts...) {
time.Sleep(h.retryFailedAfter)
h.log().WithField("instance", instance).WithError(err).Debug("trigger failed")
logging.Debug(instanceCtx, "trigger failed", "err", err)
}
}
}
@@ -302,28 +308,29 @@ func randomizeStart(min, maxSeconds float64) time.Duration {
}
func (h *Handler) subscribe(ctx context.Context) {
defer logging.Info(ctx, "handler has shutdown")
queue := make(chan eventstore.Event, 100)
subscription := eventstore.SubscribeEventTypes(queue, h.eventTypes)
for {
select {
case <-ctx.Done():
subscription.Unsubscribe()
h.log().Debug("shutdown")
return
case event := <-queue:
events := checkAdditionalEvents(queue, event)
solvedInstances := make([]string, 0, len(events))
queueCtx := call.WithTimestamp(ctx)
queueCtx = logging.With(queueCtx, "job_id", xid.New(), "invoker", "subscribe")
for _, e := range events {
if slices.Contains(solvedInstances, e.Aggregate().InstanceID) {
continue
}
queueCtx = authz.WithInstanceID(queueCtx, e.Aggregate().InstanceID)
_, err := h.Trigger(queueCtx)
h.log().OnError(err).Debug("trigger of queued event failed")
if err == nil {
solvedInstances = append(solvedInstances, e.Aggregate().InstanceID)
if _, err := h.Trigger(queueCtx); err != nil {
logging.Warn(queueCtx, "trigger of queued event failed", "err", err)
continue
}
solvedInstances = append(solvedInstances, e.Aggregate().InstanceID)
}
}
}
@@ -468,8 +475,7 @@ func (h *Handler) Trigger(ctx context.Context, opts ...TriggerOpt) (_ context.Co
wg.Done()
}
wg.Wait()
h.log().OnError(err).Info("process events failed")
h.log().WithField("iteration", i).Debug("trigger iteration")
logging.Debug(ctx, "trigger iteration", "iteration", i)
if !additionalIteration || err != nil {
return call.ResetTimestamp(ctx), err
}
@@ -523,7 +529,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
if errors.As(err, &pgErr) {
// error returned if the row is currently locked by another connection
if pgErr.Code == "55P03" {
h.log().Debug("state already locked")
logging.WithError(ctx, err).Info("another handler is already updating this projection")
err = nil
additionalIteration = false
}
@@ -549,7 +555,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
defer func() {
if err != nil && !errors.Is(err, &executionError{}) {
rollbackErr := tx.Rollback()
h.log().OnError(rollbackErr).Debug("unable to rollback tx")
logging.OnError(ctx, rollbackErr).Error("unable to rollback tx")
return
}
commitErr := tx.Commit()
@@ -569,7 +575,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
return false, err
}
if !hasLocked {
h.log().Debug("skip execution, projection already locked")
logging.Info(ctx, "another handler is already updating this projection")
return false, nil
}
@@ -611,12 +617,12 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
}()
if len(statements) == 0 {
err = h.setState(tx, currentState)
err = h.setState(ctx, tx, currentState)
return additionalIteration, err
}
lastProcessedIndex, err := h.executeStatements(ctx, tx, statements)
h.log().OnError(err).WithField("lastProcessedIndex", lastProcessedIndex).Debug("execution of statements failed")
logging.OnError(ctx, err).Debug("execution of statements failed", "lastProcessedIndex", lastProcessedIndex)
if lastProcessedIndex < 0 {
return false, err
}
@@ -628,7 +634,7 @@ func (h *Handler) processEvents(ctx context.Context, config *triggerConfig) (add
currentState.sequence = statements[lastProcessedIndex].Sequence
currentState.eventTimestamp = statements[lastProcessedIndex].CreationDate
setStateErr := h.setState(tx, currentState)
setStateErr := h.setState(ctx, tx, currentState)
if setStateErr != nil {
err = setStateErr
}
@@ -647,12 +653,12 @@ func (h *Handler) generateStatements(ctx context.Context, tx *sql.Tx, currentSta
events, err := h.es.Filter(ctx, h.eventQuery(currentState).SetTx(tx))
if err != nil {
h.log().WithError(err).Debug("filter eventstore failed")
logging.WithError(ctx, err).Debug("filter eventstore failed")
return nil, false, err
}
eventAmount := len(events)
statements, err := h.eventsToStatements(tx, events, currentState)
statements, err := h.eventsToStatements(ctx, tx, events, currentState)
if err != nil || len(statements) == 0 {
return nil, false, err
}
@@ -716,17 +722,17 @@ func (h *Handler) executeStatement(ctx context.Context, tx *sql.Tx, statement *S
_, err = tx.ExecContext(ctx, "SAVEPOINT exec_stmt")
if err != nil {
h.log().WithError(err).Debug("create savepoint failed")
logging.WithError(ctx, err).Debug("create savepoint failed")
return err
}
if err = statement.Execute(ctx, tx, h.projection.Name()); err != nil {
h.log().WithError(err).Error("statement execution failed")
logging.WithError(ctx, err).Error("statement execution failed")
_, rollbackErr := tx.ExecContext(ctx, "ROLLBACK TO SAVEPOINT exec_stmt")
h.log().OnError(rollbackErr).Error("rollback to savepoint failed")
logging.OnError(ctx, rollbackErr).Debug("rollback to savepoint failed")
shouldContinue := h.handleFailedStmt(tx, failureFromStatement(statement, err))
shouldContinue := h.handleFailedStmt(ctx, tx, failureFromStatement(statement, err))
if shouldContinue {
return nil
}
internal/eventstore/handler/v2/init.go
+7 -4
View File
@@ -7,8 +7,8 @@ import (
"strings"
"github.com/jackc/pgx/v5/pgconn"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/eventstore/handler"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -198,15 +198,18 @@ func (h *Handler) Init(ctx context.Context) error {
if err != nil {
return zerrors.ThrowInternal(err, "CRDB-SAdf2", "begin failed")
}
ctx = logging.With(ctx, "projection", h.ProjectionName())
for i, execute := range check.Init().Executes {
logging.WithFields("projection", h.projection.Name(), "execute", i).Debug("executing check")
logging.Debug(ctx, "executing check", "execute", i)
next, err := execute(ctx, tx, h.projection.Name())
if err != nil {
logging.OnError(tx.Rollback()).Debug("unable to rollback")
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logging.Error(ctx, "unable to rollback", "err", rollbackErr)
}
return err
}
if !next {
logging.WithFields("projection", h.projection.Name(), "execute", i).Debug("projection set up")
logging.Debug(ctx, "projection set up", "execute", i)
break
}
}
internal/eventstore/handler/v2/log.go
-23
View File
@@ -1,23 +0,0 @@
package handler
import (
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/internal/eventstore"
)
func (h *Handler) log() *logging.Entry {
return logging.WithFields("projection", h.projection.Name())
}
func (h *Handler) logFailure(fail *failure) *logging.Entry {
return h.log().WithField("sequence", fail.sequence).
WithField("instance", fail.instance).
WithField("aggregate", fail.aggregateID)
}
func (h *Handler) logEvent(event eventstore.Event) *logging.Entry {
return h.log().WithField("sequence", event.Sequence()).
WithField("instance", event.Aggregate().InstanceID).
WithField("aggregate", event.Aggregate().Type)
}
internal/eventstore/handler/v2/metrics.go
+10 -10
View File
@@ -3,9 +3,9 @@ package handler
import (
"context"
"github.com/zitadel/logging"
"go.opentelemetry.io/otel/attribute"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/metrics"
)
@@ -22,32 +22,32 @@ type ProjectionMetrics struct {
provider metrics.Metrics
}
func NewProjectionMetrics() *ProjectionMetrics {
return newProjectionMetrics(metrics.GlobalMeter())
func NewProjectionMetrics(ctx context.Context) *ProjectionMetrics {
return newProjectionMetrics(ctx, metrics.GlobalMeter())
}
func newProjectionMetrics(m metrics.Metrics) *ProjectionMetrics {
func newProjectionMetrics(ctx context.Context, m metrics.Metrics) *ProjectionMetrics {
projectionMetrics := &ProjectionMetrics{provider: m}
err := projectionMetrics.provider.RegisterCounter(
ProjectionEventsProcessed,
"Number of events reduced to process projection updates",
)
logging.OnError(err).Error("failed to register projection events processed counter")
logging.OnError(ctx, err).Error("failed to register projection events processed counter")
err = projectionMetrics.provider.RegisterHistogram(
ProjectionHandleTimerMetric,
"Time taken to process a projection update",
"s",
[]float64{0.005, 0.01, 0.05, 0.1, 1, 5, 10, 30, 60, 120},
)
logging.OnError(err).Error("failed to register projection handle timer metric")
logging.OnError(ctx, err).Error("failed to register projection handle timer metric")
err = projectionMetrics.provider.RegisterHistogram(
ProjectionStateLatencyMetric,
"When finishing processing a batch of events, this track the age of the last events seen from current time",
"s",
[]float64{0.1, 0.5, 1, 5, 10, 30, 60, 300, 600, 1800},
)
logging.OnError(err).Error("failed to register projection state latency metric")
logging.OnError(ctx, err).Error("failed to register projection state latency metric")
return projectionMetrics
}
@@ -55,7 +55,7 @@ func (m *ProjectionMetrics) ProjectionUpdateTiming(ctx context.Context, projecti
err := m.provider.AddHistogramMeasurement(ctx, ProjectionHandleTimerMetric, duration, map[string]attribute.Value{
ProjectionLabel: attribute.StringValue(projection),
})
logging.OnError(err).Error("failed to add projection trigger timing")
logging.OnError(ctx, err).Error("failed to add projection trigger timing")
}
func (m *ProjectionMetrics) ProjectionEventsProcessed(ctx context.Context, projection string, count int64, success bool) {
@@ -63,12 +63,12 @@ func (m *ProjectionMetrics) ProjectionEventsProcessed(ctx context.Context, proje
ProjectionLabel: attribute.StringValue(projection),
SuccessLabel: attribute.BoolValue(success),
})
logging.OnError(err).Error("failed to add projection events processed metric")
logging.OnError(ctx, err).Error("failed to add projection events processed metric")
}
func (m *ProjectionMetrics) ProjectionStateLatency(ctx context.Context, projection string, latency float64) {
err := m.provider.AddHistogramMeasurement(ctx, ProjectionStateLatencyMetric, latency, map[string]attribute.Value{
ProjectionLabel: attribute.StringValue(projection),
})
logging.OnError(err).Error("failed to add projection state latency metric")
logging.OnError(ctx, err).Error("failed to add projection state latency metric")
}
internal/eventstore/handler/v2/metrics_test.go
+5 -5
View File
@@ -13,14 +13,14 @@ import (
func TestNewProjectionMetrics(t *testing.T) {
mockMetrics := metrics.NewMockMetrics()
metrics := newProjectionMetrics(mockMetrics)
metrics := newProjectionMetrics(t.Context(), mockMetrics)
require.NotNil(t, metrics)
assert.NotNil(t, metrics.provider)
}
func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
mockMetrics := metrics.NewMockMetrics()
projectionMetrics := newProjectionMetrics(mockMetrics)
projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
ctx := context.Background()
projection := "test_projection"
@@ -39,7 +39,7 @@ func TestProjectionMetrics_ProjectionUpdateTiming(t *testing.T) {
func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
mockMetrics := metrics.NewMockMetrics()
projectionMetrics := newProjectionMetrics(mockMetrics)
projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
ctx := context.Background()
projection := "test_projection"
@@ -59,7 +59,7 @@ func TestProjectionMetrics_ProjectionEventsProcessed(t *testing.T) {
func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
mockMetrics := metrics.NewMockMetrics()
projectionMetrics := newProjectionMetrics(mockMetrics)
projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
ctx := context.Background()
projection := "test_projection"
@@ -78,7 +78,7 @@ func TestProjectionMetrics_ProjectionStateLatency(t *testing.T) {
func TestProjectionMetrics_Integration(t *testing.T) {
mockMetrics := metrics.NewMockMetrics()
projectionMetrics := newProjectionMetrics(mockMetrics)
projectionMetrics := newProjectionMetrics(t.Context(), mockMetrics)
ctx := context.Background()
projection := "test_projection"
internal/eventstore/handler/v2/state.go
+9 -6
View File
@@ -9,6 +9,7 @@ import (
"github.com/shopspring/decimal"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/api/authz"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
@@ -55,7 +56,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx) (currentState *s
offset,
)
if err != nil && !errors.Is(err, sql.ErrNoRows) {
h.log().WithError(err).Debug("unable to query current state")
logging.WithError(ctx, err).Debug("unable to query current state")
return nil, err
}
@@ -69,7 +70,7 @@ func (h *Handler) currentState(ctx context.Context, tx *sql.Tx) (currentState *s
return currentState, nil
}
func (h *Handler) setState(tx *sql.Tx, updatedState *state) error {
func (h *Handler) setState(ctx context.Context, tx *sql.Tx, updatedState *state) error {
res, err := tx.Exec(updateStateStmt,
h.projection.Name(),
updatedState.instanceID,
@@ -81,12 +82,14 @@ func (h *Handler) setState(tx *sql.Tx, updatedState *state) error {
updatedState.offset,
)
if err != nil {
h.log().WithError(err).Warn("unable to update state")
return zerrors.ThrowInternal(err, "V2-WF23g2", "unable to update state")
err = zerrors.ThrowInternal(err, "V2-WF23g2", "unable to update state")
logging.Warn(ctx, "unable to update state", "err", err)
return err
}
if affected, err := res.RowsAffected(); affected == 0 {
h.log().OnError(err).Error("unable to check if states are updated")
return zerrors.ThrowInternal(err, "V2-FGEKi", "unable to update state")
err = zerrors.ThrowInternal(err, "V2-FGEKi", "unable to update state")
logging.Error(ctx, "unable to check if states are updated", "err", err)
return err
}
return nil
}
internal/eventstore/handler/v2/state_test.go
+1 -1
View File
@@ -137,7 +137,7 @@ func TestHandler_updateLastUpdated(t *testing.T) {
h := &Handler{
projection: tt.fields.projection,
}
err = h.setState(tx, tt.args.updatedState)
err = h.setState(t.Context(), tx, tt.args.updatedState)
tt.isErr(t, err)
tt.fields.mock.Assert(t)
internal/eventstore/handler/v2/statement.go
+7 -5
View File
@@ -11,9 +11,9 @@ import (
"time"
"github.com/shopspring/decimal"
"github.com/zitadel/logging"
"golang.org/x/exp/constraints"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/database"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
@@ -39,7 +39,7 @@ func (s *executionError) Unwrap() error {
return s.parent
}
func (h *Handler) eventsToStatements(tx *sql.Tx, events []eventstore.Event, currentState *state) (statements []*Statement, err error) {
func (h *Handler) eventsToStatements(ctx context.Context, tx *sql.Tx, events []eventstore.Event, currentState *state) (statements []*Statement, err error) {
statements = make([]*Statement, 0, len(events))
previousPosition := currentState.position
@@ -47,8 +47,8 @@ func (h *Handler) eventsToStatements(tx *sql.Tx, events []eventstore.Event, curr
for _, event := range events {
statement, err := h.reduce(event)
if err != nil {
h.logEvent(event).WithError(err).Error("reduce failed")
if shouldContinue := h.handleFailedStmt(tx, failureFromEvent(event, err)); shouldContinue {
logging.Error(ctx, "reduce failed", "err", err, "event", failureFromEvent(event, err))
if shouldContinue := h.handleFailedStmt(ctx, tx, failureFromEvent(event, err)); shouldContinue {
continue
}
return statements, &executionError{err}
@@ -577,7 +577,9 @@ func NewCol(name string, value interface{}) Column {
func NewJSONCol(name string, value interface{}) Column {
marshalled, err := json.Marshal(value)
if err != nil {
logging.WithFields("column", name).WithError(err).Panic("unable to marshal column")
err = zerrors.ThrowInternal(err, "V2-aez5X", "unable to marshal column")
ctx := logging.NewCtx(context.Background(), logging.StreamEventHandler)
logging.OnError(ctx, err).Panic("unable to marshal column", "column", name, "err", err)
}
return NewCol(name, marshalled)
internal/execution/projections.go
+2 -1
View File
@@ -14,13 +14,14 @@ var (
)
func Register(
ctx context.Context,
workerConfig WorkerConfig,
queue *queue.Queue,
targetEncAlg crypto.EncryptionAlgorithm,
activeSigningKey GetActiveSigningWebKey,
) {
queue.ShouldStart()
queue.AddWorkers(NewWorker(workerConfig, targetEncAlg, activeSigningKey, time.Now))
queue.AddWorkers(ctx, NewWorker(workerConfig, targetEncAlg, activeSigningKey, time.Now))
}
func Start(ctx context.Context) {
internal/migration/migration.go
+7 -8
View File
@@ -5,8 +5,7 @@ import (
"errors"
"time"
"github.com/zitadel/logging"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/eventstore"
"github.com/zitadel/zitadel/internal/zerrors"
)
@@ -43,7 +42,8 @@ type RepeatableMigration interface {
}
func Migrate(ctx context.Context, es *eventstore.Eventstore, migration Migration) (err error) {
logging.WithFields("name", migration.String()).Info("verify migration")
ctx = logging.With(ctx, "name", migration.String())
logging.Info(ctx, "verify migration")
continueOnErr := func(err error) bool {
return false
@@ -66,12 +66,12 @@ func Migrate(ctx context.Context, es *eventstore.Eventstore, migration Migration
return err
}
logging.WithFields("name", migration.String()).Info("starting migration")
logging.Info(ctx, "starting migration")
err = migration.Execute(ctx, startedEvent[0])
logging.WithFields("name", migration.String()).OnError(err).Error("migration failed")
logging.OnError(ctx, err).Error("migration failed")
_, pushErr := es.Push(ctx, setupDoneCmd(ctx, migration, err))
logging.WithFields("name", migration.String()).OnError(pushErr).Error("migration finish failed")
logging.OnError(ctx, pushErr).Error("migration finish failed")
if err != nil {
return err
}
@@ -129,8 +129,7 @@ func checkExec(ctx context.Context, es *eventstore.Eventstore, migration Migrati
if !errors.Is(err, errMigrationAlreadyStarted) {
return false, err
}
logging.WithFields("migration step", migration.String()).
Warn("migration already started, will check again in 5 seconds")
logging.Warn(ctx, "migration already started, will check again in 5 seconds")
timer.Reset(5 * time.Second)
break
}
internal/notification/projections.go
+2 -2
View File
@@ -57,12 +57,12 @@ func Register(
queue,
backChannelLogoutWorkerConfig.MaxAttempts,
))
queue.AddWorkers(handlers.NewBackChannelLogoutWorker(commands, q, es, queue, c, backChannelLogoutWorkerConfig, id.SonyFlakeGenerator()))
queue.AddWorkers(ctx, handlers.NewBackChannelLogoutWorker(commands, q, es, queue, c, backChannelLogoutWorkerConfig, id.SonyFlakeGenerator()))
if telemetryCfg.Enabled {
projections = append(projections, handlers.NewTelemetryPusher(ctx, telemetryCfg, projection.ApplyCustomConfig(telemetryHandlerCustomConfig), commands, q, c))
}
if !notificationWorkerConfig.LegacyEnabled {
queue.AddWorkers(handlers.NewNotificationWorker(notificationWorkerConfig, commands, q, c))
queue.AddWorkers(ctx, handlers.NewNotificationWorker(notificationWorkerConfig, commands, q, c))
}
}
internal/queue/log.go
+106
View File
@@ -0,0 +1,106 @@
package queue
import (
"context"
"encoding/hex"
"log/slog"
"time"
"github.com/riverqueue/river"
"github.com/riverqueue/river/rivertype"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
)
type logMiddleware struct {
river.MiddlewareDefaults
logger *slog.Logger
}
func newLogMiddleware() rivertype.Middleware {
return &logMiddleware{
logger: logging.New(logging.StreamQueue),
}
}
func (m *logMiddleware) InsertMany(
ctx context.Context,
manyParams []*rivertype.JobInsertParams,
doInner func(context.Context) ([]*rivertype.JobInsertResult, error),
) ([]*rivertype.JobInsertResult, error) {
start := time.Now()
ctx = logging.ToCtx(ctx, m.logger)
results, err := doInner(ctx)
if err != nil {
logging.WithError(ctx, err).Error("insert many error")
return results, err
}
logging.Debug(ctx, "jobs inserted",
slog.Int("count", len(results)),
slog.Duration("duration", time.Since(start)),
)
// Only do expensive operations if debug is enabled
if m.logger.Enabled(ctx, slog.LevelDebug) {
for _, result := range results {
logging.Debug(ctx, "inserted job details", attributesFromJobInsertResult(result)...)
}
}
return results, err
}
func (m *logMiddleware) Work(
ctx context.Context,
job *rivertype.JobRow,
doInner func(context.Context) error,
) error {
start := time.Now()
ctx = logging.ToCtx(ctx, m.logger)
ctx = logging.With(ctx, attributesFromJobRow(job)...)
if err := doInner(ctx); err != nil {
logging.WithError(ctx, err).Warn("job processing error")
return err
}
logging.Info(ctx, "job processed successfully",
slog.Duration("duration", time.Since(start)),
)
return nil
}
func attributesFromJobRow(j *rivertype.JobRow) []any {
attributes := make([]any, 0, 14)
attributes = append(attributes,
slog.String("queue", j.Queue),
slog.Int64("job_id", j.ID),
slog.String("kind", j.Kind),
slog.Int("priority", j.Priority),
slog.Int("max_attempts", j.MaxAttempts),
slog.String("state", string(j.State)),
slog.String("unique_key", hex.EncodeToString(j.UniqueKey)),
)
if j.AttemptedAt != nil {
attributes = append(attributes,
slog.Time("created_at", j.CreatedAt),
slog.Int("attempt", j.Attempt),
slog.Time("attempted_at", *j.AttemptedAt),
slog.Any("attempted_by", j.AttemptedBy),
)
}
if j.FinalizedAt != nil {
attributes = append(attributes, slog.Time("finalized_at", *j.FinalizedAt))
}
if !j.ScheduledAt.IsZero() {
attributes = append(attributes, slog.Time("scheduled_at", j.ScheduledAt))
}
if len(j.Tags) > 0 {
attributes = append(attributes, slog.Any("tags", j.Tags))
}
return attributes
}
func attributesFromJobInsertResult(j *rivertype.JobInsertResult) []any {
attributes := make([]any, 0, 15)
attributes = append(attributes, attributesFromJobRow(j.Job)...)
attributes = append(attributes, slog.Bool("unique_skipped_as_duplicate", j.UniqueSkippedAsDuplicate))
return attributes
}
internal/queue/queue.go
+12 -9
View File
@@ -10,9 +10,9 @@ import (
"github.com/riverqueue/river/rivertype"
"github.com/riverqueue/rivercontrib/otelriver"
"github.com/robfig/cron/v3"
"github.com/zitadel/logging"
"go.opentelemetry.io/otel"
"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
"github.com/zitadel/zitadel/internal/database"
)
@@ -31,10 +31,13 @@ type Config struct {
}
func NewQueue(config *Config) (_ *Queue, err error) {
middleware := []rivertype.Middleware{otelriver.NewMiddleware(&otelriver.MiddlewareConfig{
MeterProvider: otel.GetMeterProvider(),
DurationUnit: "ms",
})}
middleware := []rivertype.Middleware{
otelriver.NewMiddleware(&otelriver.MiddlewareConfig{
MeterProvider: otel.GetMeterProvider(),
DurationUnit: "ms",
}),
newLogMiddleware(),
}
return &Queue{
driver: riverdatabasesql.New(config.Client.DB),
config: &river.Config{
@@ -67,9 +70,9 @@ func (q *Queue) Start(ctx context.Context) (err error) {
return q.client.Start(ctx)
}
func (q *Queue) AddWorkers(w ...Worker) {
func (q *Queue) AddWorkers(ctx context.Context, w ...Worker) {
if q == nil {
logging.Info("skip adding workers because queue is not set")
logging.Info(ctx, "skip adding workers because queue is not set")
return
}
for _, worker := range w {
@@ -77,9 +80,9 @@ func (q *Queue) AddWorkers(w ...Worker) {
}
}
func (q *Queue) AddPeriodicJob(schedule cron.Schedule, jobArgs river.JobArgs, opts ...InsertOpt) (handle rivertype.PeriodicJobHandle) {
func (q *Queue) AddPeriodicJob(ctx context.Context, schedule cron.Schedule, jobArgs river.JobArgs, opts ...InsertOpt) (handle rivertype.PeriodicJobHandle) {
if q == nil {
logging.Info("skip adding periodic job because queue is not set")
logging.Info(ctx, "skip adding periodic job because queue is not set")
return
}
options := new(river.InsertOpts)
internal/serviceping/worker.go
+3 -2
View File
@@ -229,7 +229,7 @@ func Register(
if err != nil {
return err
}
q.AddWorkers(&Worker{
q.AddWorkers(ctx, &Worker{
reportClient: NewClient(config),
db: queries,
queue: q,
@@ -240,7 +240,7 @@ func Register(
return nil
}
func Start(config *Config, q *queue.Queue) error {
func Start(ctx context.Context, config *Config, q *queue.Queue) error {
if !config.Enabled {
return nil
}
@@ -249,6 +249,7 @@ func Start(config *Config, q *queue.Queue) error {
return err
}
q.AddPeriodicJob(
ctx,
schedule,
&ServicePingReport{},
queue.WithQueueName(QueueName),
internal/zerrors/already_exists.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowAlreadyExists(parent error, id, message string) error {
return newZitadelError(KindAlreadyExists, parent, id, message)
return CreateZitadelError(KindAlreadyExists, parent, id, message, 1)
}
func ThrowAlreadyExistsf(parent error, id, format string, a ...any) error {
return newZitadelError(KindAlreadyExists, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindAlreadyExists, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsErrorAlreadyExists(err error) bool {
internal/zerrors/deadline_exceeded.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowDeadlineExceeded(parent error, id, message string) error {
return newZitadelError(KindDeadlineExceeded, parent, id, message)
return CreateZitadelError(KindDeadlineExceeded, parent, id, message, 1)
}
func ThrowDeadlineExceededf(parent error, id, format string, a ...any) error {
return newZitadelError(KindDeadlineExceeded, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindDeadlineExceeded, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsDeadlineExceeded(err error) bool {
internal/zerrors/internal.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowInternal(parent error, id, message string) error {
return newZitadelError(KindInternal, parent, id, message)
return CreateZitadelError(KindInternal, parent, id, message, 1)
}
func ThrowInternalf(parent error, id, format string, a ...any) error {
return newZitadelError(KindInternal, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindInternal, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsInternal(err error) bool {
internal/zerrors/invalid_argument.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowInvalidArgument(parent error, id, message string) error {
return newZitadelError(KindInvalidArgument, parent, id, message)
return CreateZitadelError(KindInvalidArgument, parent, id, message, 1)
}
func ThrowInvalidArgumentf(parent error, id, format string, a ...any) error {
return newZitadelError(KindInvalidArgument, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindInvalidArgument, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsErrorInvalidArgument(err error) bool {
internal/zerrors/not_found.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowNotFound(parent error, id, message string) error {
return newZitadelError(KindNotFound, parent, id, message)
return CreateZitadelError(KindNotFound, parent, id, message, 1)
}
func ThrowNotFoundf(parent error, id, format string, a ...any) error {
return newZitadelError(KindNotFound, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindNotFound, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsNotFound(err error) bool {
internal/zerrors/permission_denied.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowPermissionDenied(parent error, id, message string) error {
return newZitadelError(KindPermissionDenied, parent, id, message)
return CreateZitadelError(KindPermissionDenied, parent, id, message, 1)
}
func ThrowPermissionDeniedf(parent error, id, format string, a ...any) error {
return newZitadelError(KindPermissionDenied, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindPermissionDenied, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsPermissionDenied(err error) bool {
internal/zerrors/precondition_failed.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowPreconditionFailed(parent error, id, message string) error {
return newZitadelError(KindPreconditionFailed, parent, id, message)
return CreateZitadelError(KindPreconditionFailed, parent, id, message, 1)
}
func ThrowPreconditionFailedf(parent error, id, format string, a ...any) error {
return newZitadelError(KindPreconditionFailed, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindPreconditionFailed, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsPreconditionFailed(err error) bool {
internal/zerrors/resource_exhausted.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowResourceExhausted(parent error, id, message string) error {
return newZitadelError(KindResourceExhausted, parent, id, message)
return CreateZitadelError(KindResourceExhausted, parent, id, message, 1)
}
func ThrowResourceExhaustedf(parent error, id, format string, a ...any) error {
return newZitadelError(KindResourceExhausted, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindResourceExhausted, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsResourceExhausted(err error) bool {
internal/zerrors/unauthenticated.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowUnauthenticated(parent error, id, message string) error {
return newZitadelError(KindUnauthenticated, parent, id, message)
return CreateZitadelError(KindUnauthenticated, parent, id, message, 1)
}
func ThrowUnauthenticatedf(parent error, id, format string, a ...any) error {
return newZitadelError(KindUnauthenticated, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindUnauthenticated, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsUnauthenticated(err error) bool {
internal/zerrors/unavailable.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowUnavailable(parent error, id, message string) error {
return newZitadelError(KindUnavailable, parent, id, message)
return CreateZitadelError(KindUnavailable, parent, id, message, 1)
}
func ThrowUnavailablef(parent error, id, format string, a ...any) error {
return newZitadelError(KindUnavailable, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindUnavailable, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsUnavailable(err error) bool {
internal/zerrors/unimplemented.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowUnimplemented(parent error, id, message string) error {
return newZitadelError(KindUnimplemented, parent, id, message)
return CreateZitadelError(KindUnimplemented, parent, id, message, 1)
}
func ThrowUnimplementedf(parent error, id, format string, a ...any) error {
return newZitadelError(KindUnimplemented, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindUnimplemented, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsUnimplemented(err error) bool {
internal/zerrors/unknown.go
+2 -2
View File
@@ -3,11 +3,11 @@ package zerrors
import "fmt"
func ThrowUnknown(parent error, id, message string) error {
return newZitadelError(KindUnknown, parent, id, message)
return CreateZitadelError(KindUnknown, parent, id, message, 1)
}
func ThrowUnknownf(parent error, id, format string, a ...any) error {
return newZitadelError(KindUnknown, parent, id, fmt.Sprintf(format, a...))
return CreateZitadelError(KindUnknown, parent, id, fmt.Sprintf(format, a...), 1)
}
func IsUnknown(err error) bool {
internal/zerrors/zerror.go
+19 -8
View File
@@ -117,23 +117,34 @@ type ZitadelError struct {
}
func ThrowError(parent error, id, message string) error {
return newZitadelError(KindUnknown, parent, id, message)
return CreateZitadelError(KindUnknown, parent, id, message, 1)
}
func CreateZitadelError(kind Kind, parent error, id, message string) *ZitadelError {
return newZitadelError(kind, parent, id, message)
}
func newZitadelError(kind Kind, parent error, id, message string) *ZitadelError {
// CreateZitadelError creates a ZitadelError of the given kind.
// If ReportLocation or StackTrace are enabled, they are added to the error.
// The skip parameter defines how many stack frames to skip when determining
// the report location.
// A skip value of 0 means the location where CreateZitadelError is called.
// When the parent error is also a ZitadelError, the report location and stack trace
// are inherited from the parent error.
func CreateZitadelError(kind Kind, parent error, id, message string, skip int) *ZitadelError {
err := &ZitadelError{
Kind: kind,
Parent: parent,
ID: id,
Message: message,
}
var target *ZitadelError
if errors.As(parent, &target) {
// inherit stack trace and report location from parent error
err.reportLocation = target.reportLocation
err.stackTrace = target.stackTrace
err.hasStackTrace = target.hasStackTrace
return err
}
if enableReportLocation.Load() {
// skip 2: newZitadelError + Create / Throw function.
err.reportLocation = sloggcp.NewReportLocation(2)
err.reportLocation = sloggcp.NewReportLocation(skip + 1)
}
if enableStackTrace.Load() {
err.stackTrace = debug.Stack()
internal/zerrors/zerror_test.go
+6 -6
View File
@@ -19,7 +19,7 @@ func TestErrorMethod(t *testing.T) {
func TestZitadelError_Is(t *testing.T) {
parent := errors.New("parent error")
target := newZitadelError(KindAborted, parent, "id", "message")
target := CreateZitadelError(KindAborted, parent, "id", "message", 0)
tests := []struct {
name string // description of this test case
err error
@@ -32,27 +32,27 @@ func TestZitadelError_Is(t *testing.T) {
},
{
name: "different kind",
err: newZitadelError(KindNotFound, parent, "id", "message"),
err: CreateZitadelError(KindNotFound, parent, "id", "message", 0),
want: false,
},
{
name: "different id",
err: newZitadelError(KindAborted, parent, "otherID", "message"),
err: CreateZitadelError(KindAborted, parent, "otherID", "message", 0),
want: false,
},
{
name: "different message",
err: newZitadelError(KindAborted, parent, "id", "other message"),
err: CreateZitadelError(KindAborted, parent, "id", "other message", 0),
want: false,
},
{
name: "different parent",
err: newZitadelError(KindAborted, errors.New("other parent"), "id", "message"),
err: CreateZitadelError(KindAborted, errors.New("other parent"), "id", "message", 0),
want: false,
},
{
name: "same error",
err: newZitadelError(KindAborted, parent, "id", "message"),
err: CreateZitadelError(KindAborted, parent, "id", "message", 0),
want: true,
},
}