diff --git a/backend/v3/instrumentation/detectortype_enumer.go b/backend/v3/instrumentation/detectortype_enumer.go new file mode 100644 index 000000000..e61fc6cdd --- /dev/null +++ b/backend/v3/instrumentation/detectortype_enumer.go @@ -0,0 +1,90 @@ +// Code generated by "enumer -type=DetectorType -trimprefix=DetectorType -text -linecomment"; DO NOT EDIT. + +package instrumentation + +import ( + "fmt" + "strings" +) + +const _DetectorTypeName = "Google" + +var _DetectorTypeIndex = [...]uint8{0, 0, 6} + +const _DetectorTypeLowerName = "google" + +func (i DetectorType) String() string { + if i < 0 || i >= DetectorType(len(_DetectorTypeIndex)-1) { + return fmt.Sprintf("DetectorType(%d)", i) + } + return _DetectorTypeName[_DetectorTypeIndex[i]:_DetectorTypeIndex[i+1]] +} + +// An "invalid array index" compiler error signifies that the constant values have changed. +// Re-run the stringer command to generate them again. +func _DetectorTypeNoOp() { + var x [1]struct{} + _ = x[DetectorTypeUnspecified-(0)] + _ = x[DetectorTypeGoogle-(1)] +} + +var _DetectorTypeValues = []DetectorType{DetectorTypeUnspecified, DetectorTypeGoogle} + +var _DetectorTypeNameToValueMap = map[string]DetectorType{ + _DetectorTypeName[0:0]: DetectorTypeUnspecified, + _DetectorTypeLowerName[0:0]: DetectorTypeUnspecified, + _DetectorTypeName[0:6]: DetectorTypeGoogle, + _DetectorTypeLowerName[0:6]: DetectorTypeGoogle, +} + +var _DetectorTypeNames = []string{ + _DetectorTypeName[0:0], + _DetectorTypeName[0:6], +} + +// DetectorTypeString retrieves an enum value from the enum constants string name. +// Throws an error if the param is not part of the enum. +func DetectorTypeString(s string) (DetectorType, error) { + if val, ok := _DetectorTypeNameToValueMap[s]; ok { + return val, nil + } + + if val, ok := _DetectorTypeNameToValueMap[strings.ToLower(s)]; ok { + return val, nil + } + return 0, fmt.Errorf("%s does not belong to DetectorType values", s) +} + +// DetectorTypeValues returns all values of the enum +func DetectorTypeValues() []DetectorType { + return _DetectorTypeValues +} + +// DetectorTypeStrings returns a slice of all String values of the enum +func DetectorTypeStrings() []string { + strs := make([]string, len(_DetectorTypeNames)) + copy(strs, _DetectorTypeNames) + return strs +} + +// IsADetectorType returns "true" if the value is listed in the enum definition. "false" otherwise +func (i DetectorType) IsADetectorType() bool { + for _, v := range _DetectorTypeValues { + if i == v { + return true + } + } + return false +} + +// MarshalText implements the encoding.TextMarshaler interface for DetectorType +func (i DetectorType) MarshalText() ([]byte, error) { + return []byte(i.String()), nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface for DetectorType +func (i *DetectorType) UnmarshalText(text []byte) error { + var err error + *i, err = DetectorTypeString(string(text)) + return err +} diff --git a/backend/v3/instrumentation/instrumentation.go b/backend/v3/instrumentation/instrumentation.go index d83dbfbfb..95aea0998 100644 --- a/backend/v3/instrumentation/instrumentation.go +++ b/backend/v3/instrumentation/instrumentation.go @@ -20,10 +20,13 @@ const Name = "zitadel/backend/v3/instrumentation/tracing" type Config struct { ServiceName string - Trace TraceConfig - Metric MetricConfig - Log LogConfig - Profile ProfileConfig + // Detectors enrich the resource of every exported signal with attributes + // describing the platform we run on. Empty by default. + Detectors []DetectorType + Trace TraceConfig + Metric MetricConfig + Log LogConfig + Profile ProfileConfig } type ShutdownFunc func(context.Context) error diff --git a/backend/v3/instrumentation/otel.go b/backend/v3/instrumentation/otel.go index 9e85ba63a..ecd9d1716 100644 --- a/backend/v3/instrumentation/otel.go +++ b/backend/v3/instrumentation/otel.go @@ -7,6 +7,7 @@ import ( "os" "time" + google_detector "go.opentelemetry.io/contrib/detectors/gcp" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/log/global" @@ -52,6 +53,20 @@ type ExporterConfig struct { GoogleProjectID string // only for tracing, metrics and profiler } +//go:generate enumer -type=DetectorType -trimprefix=DetectorType -text -linecomment +type DetectorType int + +// DetectorType defines a resource detector that describes the platform ZITADEL +// runs on. Detected attributes are attached to every trace, metric and log we +// export. Detectors are opt-in: each one probes its platform on startup, and the +// attributes it finds identify the account and host we run on to whichever +// backend we export to. +const ( + // Empty line comment sets empty string of unspecified value + DetectorTypeUnspecified DetectorType = iota // + DetectorTypeGoogle +) + // setupOTelSDK bootstraps the OpenTelemetry pipeline. // If it does not return an error, make sure to call shutdown for proper cleanup. func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, ShutdownFunc, error) { @@ -81,7 +96,7 @@ func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, Shutdow prop := newPropagator() otel.SetTextMapPropagator(prop) - resource, err := resourceWithService(cfg.ServiceName) + resource, err := newResource(ctx, cfg.ServiceName, cfg.Detectors) if err != nil { return nil, shutdown, err } @@ -122,7 +137,11 @@ func newPropagator() propagation.TextMapPropagator { ) } -func resourceWithService(serviceName string) (*resource.Resource, error) { +// newResource builds the resource that identifies this process on every signal +// it emits. Detectors are applied in order and the later ones win, so the +// service attributes we are explicitly configured with always take precedence +// over anything a platform detector found. +func newResource(ctx context.Context, serviceName string, detectorTypes []DetectorType) (*resource.Resource, error) { // OTEL_SERVICE_NAME takes priority over programmatic config per the // OpenTelemetry specification. if envName := os.Getenv("OTEL_SERVICE_NAME"); envName != "" { @@ -134,12 +153,49 @@ func resourceWithService(serviceName string) (*resource.Resource, error) { if build.Version() != "" { attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version())) } - return resource.Merge( - resource.Default(), - resource.NewWithAttributes("", attributes...), + detectors, err := newDetectors(detectorTypes) + if err != nil { + return nil, err + } + return resource.New(ctx, + resource.WithDetectors(detectors...), + resource.WithTelemetrySDK(), + resource.WithFromEnv(), + resource.WithAttributes(attributes...), ) } +// newDetectors resolves the configured detector types, rejecting any value +// that does not name a detector. An empty list yields no detectors, which is +// the default: detection is opt-in because it costs a platform probe on startup +// and adds attributes that identify the account and host we run on to +// everything we export. +func newDetectors(types []DetectorType) ([]resource.Detector, error) { + detectors := make([]resource.Detector, 0, len(types)) + for _, typ := range types { + switch typ { + case DetectorTypeGoogle: + // Fills in faas.*, cloud.* and host.* attributes when we run on + // Cloud Run, GKE, GCE, App Engine or Cloud Functions. Google Cloud + // Monitoring derives the monitored resource a metric is written to + // from exactly those attributes, so without them every process + // reports as the same empty generic_node and the series overlap. + detectors = append(detectors, google_detector.NewDetector()) + case DetectorTypeUnspecified: + // The zero value is not a valid entry: an empty string in the list + // is a typo, not a way to switch detection off. An empty list is. + return nil, errDetectorType(typ) + default: + return nil, errDetectorType(typ) + } + } + return detectors, nil +} + func errExporterType(typ ExporterType, instrument string) error { return fmt.Errorf("exporter type \"%v\" unsupported for %s", typ, instrument) } + +func errDetectorType(typ DetectorType) error { + return fmt.Errorf("detector type \"%v\" unsupported", typ) +} diff --git a/backend/v3/instrumentation/otel_test.go b/backend/v3/instrumentation/otel_test.go index e433fd416..4a5f5b368 100644 --- a/backend/v3/instrumentation/otel_test.go +++ b/backend/v3/instrumentation/otel_test.go @@ -4,6 +4,9 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/otel/attribute" + semconv "go.opentelemetry.io/otel/semconv/v1.17.0" ) func TestExporterType_isNone(t *testing.T) { @@ -35,3 +38,164 @@ func TestExporterType_isNone(t *testing.T) { }) } } + +func Test_newResource(t *testing.T) { + tests := []struct { + name string + serviceName string + env map[string]string + want map[attribute.Key]string + }{ + { + name: "service name from config", + serviceName: "zitadel", + want: map[attribute.Key]string{ + semconv.ServiceNameKey: "zitadel", + }, + }, + { + name: "OTEL_SERVICE_NAME overrides config", + serviceName: "zitadel", + env: map[string]string{"OTEL_SERVICE_NAME": "from-env"}, + want: map[attribute.Key]string{ + semconv.ServiceNameKey: "from-env", + }, + }, + { + name: "config service name wins over OTEL_RESOURCE_ATTRIBUTES", + serviceName: "zitadel", + env: map[string]string{"OTEL_RESOURCE_ATTRIBUTES": "service.name=from-attrs,deployment.environment=qa"}, + want: map[attribute.Key]string{ + semconv.ServiceNameKey: "zitadel", + semconv.DeploymentEnvironmentKey: "qa", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.env { + t.Setenv(k, v) + } + got, err := newResource(t.Context(), tt.serviceName, nil) + require.NoError(t, err) + + attributes := make(map[attribute.Key]string) + for _, kv := range got.Attributes() { + attributes[kv.Key] = kv.Value.String() + } + for key, value := range tt.want { + assert.Equal(t, value, attributes[key], "attribute %q", key) + } + // The telemetry SDK detector always contributes, proving the + // detector chain ran even when nothing else is detectable. + assert.NotEmpty(t, attributes[semconv.TelemetrySDKNameKey]) + }) + } +} + +func Test_newResource_errors(t *testing.T) { + // Resource construction is part of startup: anything that cannot be + // resolved is returned rather than dropped, so a mistyped environment + // fails loudly instead of silently losing attributes. + tests := []struct { + name string + resourceAttrs string + detectors []DetectorType + }{ + { + name: "attribute pair without a value", + resourceAttrs: "deployment.environment", + }, + { + name: "trailing comma", + resourceAttrs: "deployment.environment=prod,", + }, + { + name: "unsupported detector", + detectors: []DetectorType{DetectorType(99)}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.resourceAttrs != "" { + t.Setenv("OTEL_RESOURCE_ATTRIBUTES", tt.resourceAttrs) + } + _, err := newResource(t.Context(), "zitadel", tt.detectors) + assert.Error(t, err) + }) + } +} + +func Test_newDetectors(t *testing.T) { + tests := []struct { + name string + types []DetectorType + wantLen int + wantErr bool + }{ + { + name: "nil is off", + types: nil, + wantLen: 0, + }, + { + name: "empty is off", + types: []DetectorType{}, + wantLen: 0, + }, + { + name: "unspecified is rejected", + types: []DetectorType{DetectorTypeUnspecified}, + wantErr: true, + }, + { + name: "a valid entry does not excuse an unspecified one", + types: []DetectorType{DetectorTypeGoogle, DetectorTypeUnspecified}, + wantErr: true, + }, + { + name: "google", + types: []DetectorType{DetectorTypeGoogle}, + wantLen: 1, + }, + { + name: "unknown type errors", + types: []DetectorType{DetectorType(99)}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newDetectors(tt.types) + if tt.wantErr { + require.Error(t, err) + return + } + require.NoError(t, err) + assert.Len(t, got, tt.wantLen) + }) + } +} + +func TestDetectorType_text(t *testing.T) { + // The config decoder resolves detectors through encoding.TextUnmarshaler, + // so the YAML and env var spellings have to round-trip. + for _, spelling := range []string{"google", "Google"} { + var typ DetectorType + require.NoError(t, typ.UnmarshalText([]byte(spelling))) + assert.Equal(t, DetectorTypeGoogle, typ) + } + + // The empty string still parses, because the zero value is a member of the + // enum by project convention. It is newDetectors that rejects it as a list + // entry, so an empty string in the config fails startup rather than being + // silently skipped. + var unspecified DetectorType + require.NoError(t, unspecified.UnmarshalText([]byte(""))) + assert.Equal(t, DetectorTypeUnspecified, unspecified) + _, err := newDetectors([]DetectorType{unspecified}) + assert.Error(t, err) + + var unknown DetectorType + assert.Error(t, unknown.UnmarshalText([]byte("aws"))) +} diff --git a/cmd/defaults.yaml b/cmd/defaults.yaml index 294e63d2a..0032b9061 100644 --- a/cmd/defaults.yaml +++ b/cmd/defaults.yaml @@ -1,5 +1,18 @@ Instrumentation: ServiceName: "zitadel" # ZITADEL_INSTRUMENTATION_SERVICENAME + # Resource detectors describe the platform ZITADEL runs on. Their attributes + # are attached to every exported trace, metric and log, and telemetry backends + # use them to tell instances apart instead of merging them into one series. + # Detection is off by default: each detector probes its platform on startup and + # the attributes it finds identify the account and host to whichever backend + # you export to. Enable one or more of: + # "google": Detects Cloud Run, GKE, GCE, App Engine and Cloud Functions by + # querying the Google Cloud metadata server. Required when + # exporting to Google Cloud: without it every instance reports as + # the same empty generic_node and the series overlap. + # Leave the list empty to disable detection. An empty or unknown entry is a + # config error and fails startup. + Detectors: [] # ZITADEL_INSTRUMENTATION_DETECTORS (comma separated list) Trace: Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION # Trust incoming trace context from remote services for distributed tracing. diff --git a/go.mod b/go.mod index 86111c140..4fc738529 100644 --- a/go.mod +++ b/go.mod @@ -106,6 +106,7 @@ require ( github.com/zitadel/schema v1.3.2 github.com/zitadel/sloggcp v0.6.0 go.opentelemetry.io/contrib/bridges/otelslog v0.18.0 + go.opentelemetry.io/contrib/detectors/gcp v1.44.0 go.opentelemetry.io/contrib/exporters/autoexport v0.68.0 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0 @@ -259,7 +260,6 @@ require ( github.com/yuin/gopher-lua v1.1.2 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 // indirect - go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect go.opentelemetry.io/proto/otlp v1.10.0 // indirect go.uber.org/atomic v1.11.0 // indirect