feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4
This commit is contained in:
Tim Möhlmann
2026-01-12 05:51:39 +00:00
committed by GitHub
parent d7a9e9d48b
commit eb22b58756
160 changed files with 4320 additions and 2135 deletions
-34
View File
@@ -1,34 +0,0 @@
package telemetry
import (
"net/http"
"strings"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"github.com/zitadel/zitadel/internal/telemetry/metrics"
)
func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
return func(r *http.Request) bool {
for _, endpoint := range endpoints {
if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
return false
}
}
return true
}
}
func TelemetryHandler(handler http.Handler, ignoredEndpoints ...string) http.Handler {
return otelhttp.NewHandler(handler,
"zitadel",
otelhttp.WithFilter(shouldNotIgnore(ignoredEndpoints...)),
otelhttp.WithPublicEndpoint(),
otelhttp.WithSpanNameFormatter(spanNameFormatter),
otelhttp.WithMeterProvider(metrics.GetMetricsProvider()))
}
func spanNameFormatter(_ string, r *http.Request) string {
return strings.Split(r.RequestURI, "?")[0]
}
@@ -1,37 +0,0 @@
package config
import (
"github.com/zitadel/zitadel/internal/telemetry/metrics"
"github.com/zitadel/zitadel/internal/telemetry/metrics/otel"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Type string
Config map[string]interface{} `mapstructure:",remain"`
}
var meter = map[string]func(map[string]interface{}) error{
"otel": otel.NewTracerFromConfig,
"none": registerNoopMetrics,
"": registerNoopMetrics,
}
func (c *Config) NewMeter() error {
// When using start-from-init or start-from-setup the metric provider
// was already set in the setup phase and the start phase must not overwrite it.
if metrics.M != nil {
return nil
}
t, ok := meter[c.Type]
if !ok {
return zerrors.ThrowInternalf(nil, "METER-Dfqsx", "config type %s not supported", c.Type)
}
return t(c.Config)
}
func registerNoopMetrics(rawConfig map[string]interface{}) (err error) {
metrics.M = &metrics.NoopMetrics{}
return nil
}
-147
View File
@@ -1,147 +0,0 @@
package metrics
import (
"context"
"net/http"
"strings"
"go.opentelemetry.io/otel/attribute"
)
const (
RequestCounter = "http.server.request_count"
RequestCountDescription = "Request counter"
TotalRequestCounter = "http.server.total_request_count"
TotalRequestDescription = "Total return code counter"
ReturnCodeCounter = "http.server.return_code_counter"
ReturnCodeCounterDescription = "Return code counter"
Method = "method"
URI = "uri"
ReturnCode = "return_code"
)
type Handler struct {
handler http.Handler
methods []MetricType
filters []Filter
}
type MetricType int32
const (
MetricTypeTotalCount MetricType = iota
MetricTypeStatusCode
MetricTypeRequestCount
)
type StatusRecorder struct {
http.ResponseWriter
RequestURI *string
Status int
}
func (r *StatusRecorder) WriteHeader(status int) {
r.Status = status
r.ResponseWriter.WriteHeader(status)
}
type Filter func(*http.Request) bool
func NewMetricsHandler(handler http.Handler, metricMethods []MetricType, ignoredEndpoints ...string) http.Handler {
h := Handler{
handler: handler,
methods: metricMethods,
}
if len(ignoredEndpoints) > 0 {
h.filters = append(h.filters, shouldNotIgnore(ignoredEndpoints...))
}
return &h
}
type key int
const requestURI key = iota
func SetRequestURIPattern(ctx context.Context, pattern string) {
uri, ok := ctx.Value(requestURI).(*string)
if !ok {
return
}
*uri = pattern
}
// ServeHTTP serves HTTP requests (http.Handler)
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if len(h.methods) == 0 {
h.handler.ServeHTTP(w, r)
return
}
for _, f := range h.filters {
if !f(r) {
// Simply pass through to the handler if a filter rejects the request
h.handler.ServeHTTP(w, r)
return
}
}
uri := strings.Split(r.RequestURI, "?")[0]
recorder := &StatusRecorder{
ResponseWriter: w,
RequestURI: &uri,
Status: 200,
}
r = r.WithContext(context.WithValue(r.Context(), requestURI, &uri))
h.handler.ServeHTTP(recorder, r)
if h.containsMetricsMethod(MetricTypeRequestCount) {
RegisterRequestCounter(recorder, r)
}
if h.containsMetricsMethod(MetricTypeTotalCount) {
RegisterTotalRequestCounter(r)
}
if h.containsMetricsMethod(MetricTypeStatusCode) {
RegisterRequestCodeCounter(recorder, r)
}
}
func (h *Handler) containsMetricsMethod(method MetricType) bool {
for _, m := range h.methods {
if m == method {
return true
}
}
return false
}
func RegisterRequestCounter(recorder *StatusRecorder, r *http.Request) {
var labels = map[string]attribute.Value{
URI: attribute.StringValue(*recorder.RequestURI),
Method: attribute.StringValue(r.Method),
}
RegisterCounter(RequestCounter, RequestCountDescription)
AddCount(r.Context(), RequestCounter, 1, labels)
}
func RegisterTotalRequestCounter(r *http.Request) {
RegisterCounter(TotalRequestCounter, TotalRequestDescription)
AddCount(r.Context(), TotalRequestCounter, 1, nil)
}
func RegisterRequestCodeCounter(recorder *StatusRecorder, r *http.Request) {
var labels = map[string]attribute.Value{
URI: attribute.StringValue(*recorder.RequestURI),
Method: attribute.StringValue(r.Method),
ReturnCode: attribute.IntValue(recorder.Status),
}
RegisterCounter(ReturnCodeCounter, ReturnCodeCounterDescription)
AddCount(r.Context(), ReturnCodeCounter, 1, labels)
}
func shouldNotIgnore(endpoints ...string) func(r *http.Request) bool {
return func(r *http.Request) bool {
for _, endpoint := range endpoints {
if strings.HasPrefix(r.URL.RequestURI(), endpoint) {
return false
}
}
return true
}
}
-87
View File
@@ -1,87 +0,0 @@
package metrics
import (
"context"
"net/http"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
const (
ActiveSessionCounter = "zitadel.active_session_counter"
ActiveSessionCounterDescription = "Active session counter"
SpoolerDivCounter = "zitadel.spooler_div_milliseconds"
SpoolerDivCounterDescription = "Spooler div from last successful run to now in milliseconds"
Database = "database"
ViewName = "view_name"
)
type Metrics interface {
GetExporter() http.Handler
GetMetricsProvider() metric.MeterProvider
RegisterCounter(name, description string) error
AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error
AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error
RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error
RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error
RegisterHistogram(name, description, unit string, buckets []float64) error
}
var M Metrics
func GetExporter() http.Handler {
if M == nil {
return nil
}
return M.GetExporter()
}
func GetMetricsProvider() metric.MeterProvider {
if M == nil {
return nil
}
return M.GetMetricsProvider()
}
func RegisterCounter(name, description string) error {
if M == nil {
return nil
}
return M.RegisterCounter(name, description)
}
func AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
if M == nil {
return nil
}
return M.AddCount(ctx, name, value, labels)
}
func AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
if M == nil {
return nil
}
return M.AddHistogramMeasurement(ctx, name, value, labels)
}
func RegisterHistogram(name, description, unit string, buckets []float64) error {
if M == nil {
return nil
}
return M.RegisterHistogram(name, description, unit, buckets)
}
func RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
if M == nil {
return nil
}
return M.RegisterUpDownSumObserver(name, description, callbackFunc)
}
func RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
if M == nil {
return nil
}
return M.RegisterValueObserver(name, description, callbackFunc)
}
-95
View File
@@ -1,95 +0,0 @@
package metrics
import (
"context"
"net/http"
"sync"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
// MockMetrics implements the metrics.Metrics interface for testing
type MockMetrics struct {
mu sync.RWMutex
histogramValues map[string][]float64
counterValues map[string]int64
histogramLabels map[string][]map[string]attribute.Value
counterLabels map[string][]map[string]attribute.Value
}
var _ Metrics = new(MockMetrics)
// NewMockMetrics creates a new Metrics instance for testing
func NewMockMetrics() *MockMetrics {
return &MockMetrics{
histogramValues: make(map[string][]float64),
counterValues: make(map[string]int64),
histogramLabels: make(map[string][]map[string]attribute.Value),
counterLabels: make(map[string][]map[string]attribute.Value),
}
}
func (m *MockMetrics) GetExporter() http.Handler {
return nil
}
func (m *MockMetrics) GetMetricsProvider() metric.MeterProvider {
return nil
}
func (m *MockMetrics) RegisterCounter(name, description string) error {
return nil
}
func (m *MockMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
m.mu.Lock()
defer m.mu.Unlock()
m.counterValues[name] += value
m.counterLabels[name] = append(m.counterLabels[name], labels)
return nil
}
func (m *MockMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
m.mu.Lock()
defer m.mu.Unlock()
m.histogramValues[name] = append(m.histogramValues[name], value)
m.histogramLabels[name] = append(m.histogramLabels[name], labels)
return nil
}
func (m *MockMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
return nil
}
func (m *MockMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
return nil
}
func (m *MockMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
return nil
}
func (m *MockMetrics) GetHistogramValues(name string) []float64 {
m.mu.RLock()
defer m.mu.RUnlock()
return m.histogramValues[name]
}
func (m *MockMetrics) GetHistogramLabels(name string) []map[string]attribute.Value {
m.mu.RLock()
defer m.mu.RUnlock()
return m.histogramLabels[name]
}
func (m *MockMetrics) GetCounterValue(name string) int64 {
m.mu.RLock()
defer m.mu.RUnlock()
return m.counterValues[name]
}
func (m *MockMetrics) GetCounterLabels(name string) []map[string]attribute.Value {
m.mu.RLock()
defer m.mu.RUnlock()
return m.counterLabels[name]
}
-45
View File
@@ -1,45 +0,0 @@
package metrics
import (
"context"
"net/http"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/metric"
)
type NoopMetrics struct{}
var _ Metrics = new(NoopMetrics)
func (n *NoopMetrics) GetExporter() http.Handler {
return nil
}
func (n *NoopMetrics) GetMetricsProvider() metric.MeterProvider {
return nil
}
func (n *NoopMetrics) RegisterCounter(name, description string) error {
return nil
}
func (n *NoopMetrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
return nil
}
func (n *NoopMetrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
return nil
}
func (n *NoopMetrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
return nil
}
func (n *NoopMetrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
return nil
}
func (n *NoopMetrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
return nil
}
-20
View File
@@ -1,20 +0,0 @@
package otel
import (
"github.com/zitadel/zitadel/internal/telemetry/metrics"
)
type Config struct {
MeterName string
}
func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.MeterName, _ = rawConfig["metername"].(string)
return c.NewMetrics()
}
func (c *Config) NewMetrics() (err error) {
metrics.M, err = NewMetrics(c.MeterName)
return err
}
@@ -1,163 +0,0 @@
package otel
import (
"context"
"net/http"
"sync"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/prometheus"
"go.opentelemetry.io/otel/metric"
"go.opentelemetry.io/otel/sdk/instrumentation"
sdk_metric "go.opentelemetry.io/otel/sdk/metric"
"github.com/zitadel/zitadel/internal/telemetry/metrics"
otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Metrics struct {
Provider metric.MeterProvider
Meter metric.Meter
Counters sync.Map
UpDownSumObserver sync.Map
ValueObservers sync.Map
Histograms sync.Map
}
func NewMetrics(meterName string) (metrics.Metrics, error) {
resource, err := otel_resource.ResourceWithService("ZITADEL")
if err != nil {
return nil, err
}
exporter, err := prometheus.New(prometheus.WithoutScopeInfo())
if err != nil {
return &Metrics{}, err
}
// create a view to filter out unwanted attributes
view := sdk_metric.NewView(
sdk_metric.Instrument{
Scope: instrumentation.Scope{Name: otelhttp.ScopeName},
},
sdk_metric.Stream{
AttributeFilter: attribute.NewAllowKeysFilter("http.method", "http.status_code", "http.target"),
},
)
meterProvider := sdk_metric.NewMeterProvider(
sdk_metric.WithReader(exporter),
sdk_metric.WithResource(resource),
sdk_metric.WithView(view),
)
return &Metrics{
Provider: meterProvider,
Meter: meterProvider.Meter(meterName),
}, nil
}
func (m *Metrics) GetExporter() http.Handler {
return promhttp.Handler()
}
func (m *Metrics) GetMetricsProvider() metric.MeterProvider {
return m.Provider
}
func (m *Metrics) RegisterCounter(name, description string) error {
if _, exists := m.Counters.Load(name); exists {
return nil
}
counter, err := m.Meter.Int64Counter(name, metric.WithDescription(description))
if err != nil {
return err
}
m.Counters.Store(name, counter)
return nil
}
func (m *Metrics) AddCount(ctx context.Context, name string, value int64, labels map[string]attribute.Value) error {
counter, exists := m.Counters.Load(name)
if !exists {
return zerrors.ThrowNotFound(nil, "METER-4u8fs", "Errors.Metrics.Counter.NotFound")
}
counter.(metric.Int64Counter).Add(ctx, value, MapToAddOption(labels)...)
return nil
}
func (m *Metrics) AddHistogramMeasurement(ctx context.Context, name string, value float64, labels map[string]attribute.Value) error {
histogram, exists := m.Histograms.Load(name)
if !exists {
return zerrors.ThrowNotFound(nil, "METER-5wwb1", "Errors.Metrics.Histogram.NotFound")
}
histogram.(metric.Float64Histogram).Record(ctx, value, MapToRecordOption(labels)...)
return nil
}
func (m *Metrics) RegisterHistogram(name, description, unit string, buckets []float64) error {
if _, exists := m.Histograms.Load(name); exists {
return nil
}
histogram, err := m.Meter.Float64Histogram(name,
metric.WithDescription(description),
metric.WithUnit(unit),
metric.WithExplicitBucketBoundaries(buckets...),
)
if err != nil {
return err
}
m.Histograms.Store(name, histogram)
return nil
}
func (m *Metrics) RegisterUpDownSumObserver(name, description string, callbackFunc metric.Int64Callback) error {
if _, exists := m.UpDownSumObserver.Load(name); exists {
return nil
}
counter, err := m.Meter.Int64ObservableUpDownCounter(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
if err != nil {
return err
}
m.UpDownSumObserver.Store(name, counter)
return nil
}
func (m *Metrics) RegisterValueObserver(name, description string, callbackFunc metric.Int64Callback) error {
if _, exists := m.UpDownSumObserver.Load(name); exists {
return nil
}
gauge, err := m.Meter.Int64ObservableGauge(name, metric.WithInt64Callback(callbackFunc), metric.WithDescription(description))
if err != nil {
return err
}
m.UpDownSumObserver.Store(name, gauge)
return nil
}
func MapToAddOption(labels map[string]attribute.Value) []metric.AddOption {
return []metric.AddOption{metric.WithAttributes(labelsToAttributes(labels)...)}
}
func MapToRecordOption(labels map[string]attribute.Value) []metric.RecordOption {
return []metric.RecordOption{metric.WithAttributes(labelsToAttributes(labels)...)}
}
func labelsToAttributes(labels map[string]attribute.Value) []attribute.KeyValue {
if labels == nil {
return nil
}
attributes := make([]attribute.KeyValue, 0, len(labels))
for key, value := range labels {
attributes = append(attributes, attribute.KeyValue{
Key: attribute.Key(key),
Value: value,
})
}
return attributes
}
-22
View File
@@ -1,22 +0,0 @@
package otel
import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"github.com/zitadel/zitadel/cmd/build"
)
func ResourceWithService(serviceName string) (*resource.Resource, error) {
attributes := []attribute.KeyValue{
semconv.ServiceNameKey.String(serviceName),
}
if build.Version() != "" {
attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
}
return resource.Merge(
resource.Default(),
resource.NewWithAttributes("", attributes...),
)
}
@@ -1,30 +0,0 @@
package config
import (
"github.com/zitadel/zitadel/internal/telemetry/profiler/google"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Type string
Config map[string]interface{} `mapstructure:",remain"`
}
var profiler = map[string]func(map[string]interface{}) error{
"google": google.NewProfiler,
"none": NoProfiler,
"": NoProfiler,
}
func (c *Config) NewProfiler() error {
t, ok := profiler[c.Type]
if !ok {
return zerrors.ThrowInternalf(nil, "PROFI-Dfqsx", "config type %s not supported", c.Type)
}
return t(c.Config)
}
func NoProfiler(_ map[string]interface{}) error {
return nil
}
@@ -1,26 +0,0 @@
package google
import (
"cloud.google.com/go/profiler"
"github.com/zitadel/zitadel/cmd/build"
)
type Config struct {
ProjectID string
}
func NewProfiler(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.ProjectID, _ = rawConfig["projectid"].(string)
return c.NewProfiler()
}
func (c *Config) NewProfiler() (err error) {
cfg := profiler.Config{
Service: "zitadel",
ServiceVersion: build.Version(),
ProjectID: c.ProjectID,
}
return profiler.Start(cfg)
}
-22
View File
@@ -1,22 +0,0 @@
package tracing
import (
"runtime"
"github.com/zitadel/logging"
)
func GetCaller() string {
fpcs := make([]uintptr, 1)
n := runtime.Callers(3, fpcs)
if n == 0 {
logging.WithFields("logID", "TRACE-rWjfC").Debug("no caller")
return ""
}
caller := runtime.FuncForPC(fpcs[0] - 1)
if caller == nil {
logging.WithFields("logID", "TRACE-25POw").Debug("caller was nil")
return ""
}
return caller.Name()
}
@@ -1,34 +0,0 @@
package config
import (
"github.com/zitadel/zitadel/internal/telemetry/tracing/google"
"github.com/zitadel/zitadel/internal/telemetry/tracing/log"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Type string
Config map[string]interface{} `mapstructure:",remain"`
}
func (c *Config) NewTracer() error {
t, ok := tracer[c.Type]
if !ok {
return zerrors.ThrowInternalf(nil, "TRACE-dsbjh", "config type %s not supported", c.Type)
}
return t(c.Config)
}
var tracer = map[string]func(map[string]interface{}) error{
"otel": otel.NewTracerFromConfig,
"google": google.NewTracer,
"log": log.NewTracer,
"none": NoTracer,
"": NoTracer,
}
func NoTracer(_ map[string]interface{}) error {
return nil
}
@@ -1,41 +0,0 @@
package google
import (
texporter "github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/trace"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
)
type Config struct {
ProjectID string
Fraction float64
ServiceName string
}
func NewTracer(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.ProjectID, _ = rawConfig["projectid"].(string)
c.ServiceName, _ = rawConfig["servicename"].(string)
c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
if err != nil {
return err
}
return c.NewTracer()
}
type Tracer struct {
otel.Tracer
}
func (c *Config) NewTracer() error {
sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := texporter.New(texporter.WithProjectID(c.ProjectID))
if err != nil {
return err
}
tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
return err
}
-39
View File
@@ -1,39 +0,0 @@
package log
import (
stdout "go.opentelemetry.io/otel/exporters/stdout/stdouttrace"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/telemetry/tracing/otel"
)
type Config struct {
Fraction float64
ServiceName string
}
func NewTracer(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.Fraction, err = otel.FractionFromConfig(rawConfig["fraction"])
c.ServiceName, _ = rawConfig["servicename"].(string)
if err != nil {
return err
}
return c.NewTracer()
}
type Tracer struct {
otel.Tracer
}
func (c *Config) NewTracer() error {
sampler := otel.NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := stdout.New(stdout.WithPrettyPrint())
if err != nil {
return err
}
tracing.T, err = otel.NewTracer(sampler, exporter, c.ServiceName)
return err
}
-77
View File
@@ -1,77 +0,0 @@
package otel
import (
"context"
"strconv"
otlpgrpc "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
"github.com/zitadel/zitadel/internal/zerrors"
)
type Config struct {
Fraction float64
Endpoint string
ServiceName string
}
func NewTracerFromConfig(rawConfig map[string]interface{}) (err error) {
c := new(Config)
c.Endpoint, _ = rawConfig["endpoint"].(string)
c.ServiceName, _ = rawConfig["servicename"].(string)
c.Fraction, err = FractionFromConfig(rawConfig["fraction"])
if err != nil {
return err
}
return c.NewTracer()
}
func FractionFromConfig(i interface{}) (float64, error) {
if i == nil {
return 0, nil
}
switch fraction := i.(type) {
case float64:
return fraction, nil
case int:
return float64(fraction), nil
case string:
f, err := strconv.ParseFloat(fraction, 64)
if err != nil {
return 0, zerrors.ThrowInternal(err, "OTEL-SAfe1", "could not map fraction")
}
return f, nil
default:
return 0, zerrors.ThrowInternal(nil, "OTEL-Dd2s", "could not map fraction, unknown type")
}
}
func (c *Config) NewTracer() error {
sampler := NewSampler(sdk_trace.TraceIDRatioBased(c.Fraction))
exporter, err := otlpgrpc.New(context.Background(), otlpgrpc.WithEndpoint(c.Endpoint), otlpgrpc.WithInsecure())
if err != nil {
return err
}
tracing.T, err = NewTracer(sampler, exporter, c.ServiceName)
return err
}
// NewSampler returns a sampler decorator which behaves differently,
// based on the parent of the span. If the span has no parent and is of kind server,
// the decorated sampler is used to make sampling decision.
// If the span has a parent, depending on whether the parent is remote and whether it
// is sampled, one of the following samplers will apply:
// - remote parent sampled -> always sample
// - remote parent not sampled -> sample based on the decorated sampler (fraction based)
// - local parent sampled -> always sample
// - local parent not sampled -> never sample
func NewSampler(sampler sdk_trace.Sampler) sdk_trace.Sampler {
return sdk_trace.ParentBased(
tracing.SpanKindBased(sampler, api_trace.SpanKindServer),
sdk_trace.WithRemoteParentNotSampled(sampler),
)
}
@@ -1,77 +0,0 @@
package otel
import (
"context"
"net/http"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
otel_resource "github.com/zitadel/zitadel/internal/telemetry/otel"
"github.com/zitadel/zitadel/internal/telemetry/tracing"
)
type Tracer struct {
Exporter api_trace.Tracer
sampler sdk_trace.Sampler
}
func NewTracer(sampler sdk_trace.Sampler, exporter sdk_trace.SpanExporter, serviceName string) (*Tracer, error) {
resource, err := otel_resource.ResourceWithService(serviceName)
if err != nil {
return nil, err
}
tp := sdk_trace.NewTracerProvider(
sdk_trace.WithSampler(sampler),
sdk_trace.WithBatcher(exporter),
sdk_trace.WithResource(resource),
)
otel.SetTracerProvider(tp)
tc := propagation.TraceContext{}
otel.SetTextMapPropagator(tc)
return &Tracer{Exporter: tp.Tracer(""), sampler: sampler}, nil
}
func (t *Tracer) Sampler() sdk_trace.Sampler {
return t.sampler
}
func (t *Tracer) NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindServer))
}
func (t *Tracer) NewServerSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindServer))
}
func (t *Tracer) NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, name, api_trace.WithSpanKind(api_trace.SpanKindClient))
}
func (t *Tracer) NewClientSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller, api_trace.WithSpanKind(api_trace.SpanKindClient))
}
func (t *Tracer) NewSpan(ctx context.Context, caller string) (context.Context, *tracing.Span) {
return t.newSpan(ctx, caller)
}
func (t *Tracer) newSpan(ctx context.Context, caller string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
return t.newSpanFromName(ctx, caller, options...)
}
func (t *Tracer) newSpanFromName(ctx context.Context, name string, options ...api_trace.SpanStartOption) (context.Context, *tracing.Span) {
ctx, span := t.Exporter.Start(ctx, name, options...)
return ctx, tracing.CreateSpan(span)
}
func (t *Tracer) NewSpanHTTP(r *http.Request, caller string) (*http.Request, *tracing.Span) {
ctx, span := t.NewSpan(r.Context(), caller)
r = r.WithContext(ctx)
return r, span
}
-46
View File
@@ -1,46 +0,0 @@
package tracing
import (
"fmt"
"slices"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
type spanKindSampler struct {
sampler sdk_trace.Sampler
kinds []trace.SpanKind
}
// ShouldSample implements the [sdk_trace.Sampler] interface.
// It will not sample any spans which do not match the configured span kinds.
// For spans which do match, the decorated sampler is used to make the sampling decision.
func (sk spanKindSampler) ShouldSample(p sdk_trace.SamplingParameters) sdk_trace.SamplingResult {
psc := trace.SpanContextFromContext(p.ParentContext)
if !slices.Contains(sk.kinds, p.Kind) {
return sdk_trace.SamplingResult{
Decision: sdk_trace.Drop,
Tracestate: psc.TraceState(),
}
}
s := sk.sampler.ShouldSample(p)
return s
}
func (sk spanKindSampler) Description() string {
return fmt.Sprintf("SpanKindBased{sampler:%s,kinds:%v}",
sk.sampler.Description(),
sk.kinds,
)
}
// SpanKindBased returns a sampler decorator which behaves differently, based on the kind of the span.
// If the span kind does not match one of the configured kinds, it will not be sampled.
// If the span kind matches, the decorated sampler is used to make sampling decision.
func SpanKindBased(sampler sdk_trace.Sampler, kinds ...trace.SpanKind) sdk_trace.Sampler {
return spanKindSampler{
sampler: sampler,
kinds: kinds,
}
}
@@ -1,80 +0,0 @@
package tracing
import (
"context"
"testing"
"github.com/stretchr/testify/assert"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
func TestSpanKindBased(t *testing.T) {
type args struct {
sampler sdk_trace.Sampler
kinds []trace.SpanKind
}
type want struct {
description string
sampled int
}
tests := []struct {
name string
args args
want want
}{
{
"never sample, no sample",
args{
sampler: sdk_trace.NeverSample(),
kinds: []trace.SpanKind{trace.SpanKindServer},
},
want{
description: "SpanKindBased{sampler:AlwaysOffSampler,kinds:[server]}",
sampled: 0,
},
},
{
"always sample, no kind, no sample",
args{
sampler: sdk_trace.AlwaysSample(),
kinds: nil,
},
want{
description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[]}",
sampled: 0,
},
},
{
"always sample, 2 kinds, 2 samples",
args{
sampler: sdk_trace.AlwaysSample(),
kinds: []trace.SpanKind{trace.SpanKindServer, trace.SpanKindClient},
},
want{
description: "SpanKindBased{sampler:AlwaysOnSampler,kinds:[server client]}",
sampled: 2,
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sampler := SpanKindBased(tt.args.sampler, tt.args.kinds...)
assert.Equal(t, tt.want.description, sampler.Description())
p := sdk_trace.NewTracerProvider(sdk_trace.WithSampler(sampler))
tr := p.Tracer("test")
var sampled int
for i := trace.SpanKindUnspecified; i <= trace.SpanKindConsumer; i++ {
ctx := context.Background()
_, span := tr.Start(ctx, "test", trace.WithSpanKind(i))
if span.SpanContext().IsSampled() {
sampled++
}
}
assert.Equal(t, tt.want.sampled, sampled)
})
}
}
-46
View File
@@ -1,46 +0,0 @@
package tracing
import (
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/internal/api/grpc/gerrors"
)
type Span struct {
span trace.Span
opts []trace.SpanEndOption
}
func CreateSpan(span trace.Span) *Span {
return &Span{span: span, opts: []trace.SpanEndOption{}}
}
func (s *Span) End() {
if s.span == nil {
return
}
s.span.End(s.opts...)
}
func (s *Span) EndWithError(err error) {
s.SetStatusByError(err)
s.End()
}
func (s *Span) SetStatusByError(err error) {
if s.span == nil {
return
}
if err != nil {
// trace.WithErrorStatus(codes.Error)
s.span.RecordError(err)
s.span.SetAttributes(
attribute.KeyValue{},
)
}
code, msg, id, _ := gerrors.ExtractZITADELError(err)
s.span.SetAttributes(attribute.Int("grpc_code", int(code)), attribute.String("grpc_msg", msg), attribute.String("error_id", id))
}
+19 -57
View File
@@ -1,81 +1,43 @@
// Package tracing acts as a shim to [tracing],
// so we can migrate the tracing package without changing all imports at once.
package tracing
import (
"context"
"net/http"
sdk_trace "go.opentelemetry.io/otel/sdk/trace"
api_trace "go.opentelemetry.io/otel/trace"
"github.com/zitadel/zitadel/backend/v3/instrumentation"
"github.com/zitadel/zitadel/backend/v3/instrumentation/tracing"
)
type Tracer interface {
NewSpan(ctx context.Context, caller string) (context.Context, *Span)
NewClientSpan(ctx context.Context, caller string) (context.Context, *Span)
NewServerSpan(ctx context.Context, caller string) (context.Context, *Span)
NewClientInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
NewServerInterceptorSpan(ctx context.Context, name string) (context.Context, *Span)
NewSpanHTTP(r *http.Request, caller string) (*http.Request, *Span)
Sampler() sdk_trace.Sampler
func NewSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewSpan(ctx, instrumentation.GetCallingFunc(1))
}
var T Tracer
func Sampler() sdk_trace.Sampler {
if T == nil {
return sdk_trace.NeverSample()
}
return T.Sampler()
func NewNamedSpan(ctx context.Context, name string) (context.Context, *instrumentation.Span) {
return tracing.NewNamedSpan(ctx, name)
}
func NewSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewSpan(ctx, GetCaller())
func NewClientSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewClientSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewNamedSpan(ctx context.Context, name string) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewSpan(ctx, name)
func NewServerSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewServerSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewClientSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewClientSpan(ctx, GetCaller())
func NewClientInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewClientInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewServerSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewServerSpan(ctx, GetCaller())
func NewServerInterceptorSpan(ctx context.Context) (context.Context, *instrumentation.Span) {
return tracing.GlobalTracer().NewServerInterceptorSpan(ctx, instrumentation.GetCallingFunc(1))
}
func NewClientInterceptorSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewClientInterceptorSpan(ctx, GetCaller())
}
func NewServerInterceptorSpan(ctx context.Context) (context.Context, *Span) {
if T == nil {
return ctx, CreateSpan(nil)
}
return T.NewServerInterceptorSpan(ctx, GetCaller())
}
func NewSpanHTTP(r *http.Request) (*http.Request, *Span) {
if T == nil {
return r, CreateSpan(nil)
}
return T.NewSpanHTTP(r, GetCaller())
func NewSpanHTTP(r *http.Request) (*http.Request, *instrumentation.Span) {
return tracing.GlobalTracer().NewSpanHTTP(r, instrumentation.GetCallingFunc(1))
}
func TraceIDFromCtx(ctx context.Context) string {
return api_trace.SpanFromContext(ctx).SpanContext().TraceID().String()
return tracing.TraceIDFromCtx(ctx)
}