feat(telemetry): auto detect google attributes (#12694)

# Which Problems Are Solved

We run multiple deployments (regions) under a single GCP project. Within
these deployments we scale containers. Currently all metrics target the
same time series in GCP, regardless or region or container. All metics
ended up in `generic_node` and Cloud Monitoring rejects the duplicate
and out-of-order points.

This can be fixed by setting vendor specific attributes such as
`faas.name` (K_SERVICE), `faas.version` (K_REVISION), `faas.instance`
(instance id from the metadata server), and `cloud.region` (metadata
instance/region).


# How the Problems Are Solved

Use `go.opentelemetry.io/contrib/detectors/gcp` to auto-detect values as
mentioned above. A new `Instrumentation.Detectors` config entry is
added, which accepts and array of enum for detecotrs. At the moment only
Google is accepted, other entries from
[contrib](https://github.com/open-telemetry/opentelemetry-go-contrib/tree/main/detectors)
can be added if deemed usefull.

I've considered using `autodetect` contrib package, identifiers and
constructor, but this pulls in the full AWS SDK, increasing zitadel's
binary size by 50MiB.

By default detection is switched off, as it relies on unauthenticated
link-local calls that are insecure anywhere outside of the
infrastructure it targets. It also prevents breaking existing
deployments that rely on detection being off already. (May break
existing trace and logging pipelines).

# Additional Changes

- none

# Additional Context

- (internal only): https://github.com/caos/infra/pull/1813

---------

Co-authored-by: Copilot Autofix powered by AI <[email protected]>
Co-authored-by: Claude Opus 5 <[email protected]>

backend/v3/instrumentation/detectortype_enumer.go

@@ -0,0 +1,90 @@
+// Code generated by "enumer -type=DetectorType -trimprefix=DetectorType -text -linecomment"; DO NOT EDIT.
+
+package instrumentation
+
+import (
+	"fmt"
+	"strings"
+)
+
+const _DetectorTypeName = "Google"
+
+var _DetectorTypeIndex = [...]uint8{0, 0, 6}
+
+const _DetectorTypeLowerName = "google"
+
+func (i DetectorType) String() string {
+	if i < 0 || i >= DetectorType(len(_DetectorTypeIndex)-1) {
+		return fmt.Sprintf("DetectorType(%d)", i)
+	}
+	return _DetectorTypeName[_DetectorTypeIndex[i]:_DetectorTypeIndex[i+1]]
+}
+
+// An "invalid array index" compiler error signifies that the constant values have changed.
+// Re-run the stringer command to generate them again.
+func _DetectorTypeNoOp() {
+	var x [1]struct{}
+	_ = x[DetectorTypeUnspecified-(0)]
+	_ = x[DetectorTypeGoogle-(1)]
+}
+
+var _DetectorTypeValues = []DetectorType{DetectorTypeUnspecified, DetectorTypeGoogle}
+
+var _DetectorTypeNameToValueMap = map[string]DetectorType{
+	_DetectorTypeName[0:0]:      DetectorTypeUnspecified,
+	_DetectorTypeLowerName[0:0]: DetectorTypeUnspecified,
+	_DetectorTypeName[0:6]:      DetectorTypeGoogle,
+	_DetectorTypeLowerName[0:6]: DetectorTypeGoogle,
+}
+
+var _DetectorTypeNames = []string{
+	_DetectorTypeName[0:0],
+	_DetectorTypeName[0:6],
+}
+
+// DetectorTypeString retrieves an enum value from the enum constants string name.
+// Throws an error if the param is not part of the enum.
+func DetectorTypeString(s string) (DetectorType, error) {
+	if val, ok := _DetectorTypeNameToValueMap[s]; ok {
+		return val, nil
+	}
+
+	if val, ok := _DetectorTypeNameToValueMap[strings.ToLower(s)]; ok {
+		return val, nil
+	}
+	return 0, fmt.Errorf("%s does not belong to DetectorType values", s)
+}
+
+// DetectorTypeValues returns all values of the enum
+func DetectorTypeValues() []DetectorType {
+	return _DetectorTypeValues
+}
+
+// DetectorTypeStrings returns a slice of all String values of the enum
+func DetectorTypeStrings() []string {
+	strs := make([]string, len(_DetectorTypeNames))
+	copy(strs, _DetectorTypeNames)
+	return strs
+}
+
+// IsADetectorType returns "true" if the value is listed in the enum definition. "false" otherwise
+func (i DetectorType) IsADetectorType() bool {
+	for _, v := range _DetectorTypeValues {
+		if i == v {
+			return true
+		}
+	}
+	return false
+}
+
+// MarshalText implements the encoding.TextMarshaler interface for DetectorType
+func (i DetectorType) MarshalText() ([]byte, error) {
+	return []byte(i.String()), nil
+}
+
+// UnmarshalText implements the encoding.TextUnmarshaler interface for DetectorType
+func (i *DetectorType) UnmarshalText(text []byte) error {
+	var err error
+	*i, err = DetectorTypeString(string(text))
+	return err
+}

backend/v3/instrumentation/instrumentation.go

@@ -20,6 +20,9 @@ const Name = "zitadel/backend/v3/instrumentation/tracing"
 
 type Config struct {
 	ServiceName string
+	// Detectors enrich the resource of every exported signal with attributes
+	// describing the platform we run on. Empty by default.
+	Detectors []DetectorType
 	Trace     TraceConfig
 	Metric    MetricConfig
 	Log       LogConfig

backend/v3/instrumentation/otel.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"time"
 
+	google_detector "go.opentelemetry.io/contrib/detectors/gcp"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/log/global"
@@ -52,6 +53,20 @@ type ExporterConfig struct {
 	GoogleProjectID string // only for tracing, metrics and profiler
 }
 
+//go:generate enumer -type=DetectorType -trimprefix=DetectorType -text -linecomment
+type DetectorType int
+
+// DetectorType defines a resource detector that describes the platform ZITADEL
+// runs on. Detected attributes are attached to every trace, metric and log we
+// export. Detectors are opt-in: each one probes its platform on startup, and the
+// attributes it finds identify the account and host we run on to whichever
+// backend we export to.
+const (
+	// Empty line comment sets empty string of unspecified value
+	DetectorTypeUnspecified DetectorType = iota //
+	DetectorTypeGoogle
+)
+
 // setupOTelSDK bootstraps the OpenTelemetry pipeline.
 // If it does not return an error, make sure to call shutdown for proper cleanup.
 func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, ShutdownFunc, error) {
@@ -81,7 +96,7 @@ func setupOTelSDK(ctx context.Context, cfg Config) (*log.LoggerProvider, Shutdow
 	prop := newPropagator()
 	otel.SetTextMapPropagator(prop)
 
-	resource, err := resourceWithService(cfg.ServiceName)
+	resource, err := newResource(ctx, cfg.ServiceName, cfg.Detectors)
 	if err != nil {
 		return nil, shutdown, err
 	}
@@ -122,7 +137,11 @@ func newPropagator() propagation.TextMapPropagator {
 	)
 }
 
-func resourceWithService(serviceName string) (*resource.Resource, error) {
+// newResource builds the resource that identifies this process on every signal
+// it emits. Detectors are applied in order and the later ones win, so the
+// service attributes we are explicitly configured with always take precedence
+// over anything a platform detector found.
+func newResource(ctx context.Context, serviceName string, detectorTypes []DetectorType) (*resource.Resource, error) {
 	// OTEL_SERVICE_NAME takes priority over programmatic config per the
 	// OpenTelemetry specification.
 	if envName := os.Getenv("OTEL_SERVICE_NAME"); envName != "" {
@@ -134,12 +153,49 @@ func resourceWithService(serviceName string) (*resource.Resource, error) {
 	if build.Version() != "" {
 		attributes = append(attributes, semconv.ServiceVersionKey.String(build.Version()))
 	}
-	return resource.Merge(
+	detectors, err := newDetectors(detectorTypes)
-		resource.Default(),
+	if err != nil {
-		resource.NewWithAttributes("", attributes...),
+		return nil, err
+	}
+	return resource.New(ctx,
+		resource.WithDetectors(detectors...),
+		resource.WithTelemetrySDK(),
+		resource.WithFromEnv(),
+		resource.WithAttributes(attributes...),
 	)
 }
 
+// newDetectors resolves the configured detector types, rejecting any value
+// that does not name a detector. An empty list yields no detectors, which is
+// the default: detection is opt-in because it costs a platform probe on startup
+// and adds attributes that identify the account and host we run on to
+// everything we export.
+func newDetectors(types []DetectorType) ([]resource.Detector, error) {
+	detectors := make([]resource.Detector, 0, len(types))
+	for _, typ := range types {
+		switch typ {
+		case DetectorTypeGoogle:
+			// Fills in faas.*, cloud.* and host.* attributes when we run on
+			// Cloud Run, GKE, GCE, App Engine or Cloud Functions. Google Cloud
+			// Monitoring derives the monitored resource a metric is written to
+			// from exactly those attributes, so without them every process
+			// reports as the same empty generic_node and the series overlap.
+			detectors = append(detectors, google_detector.NewDetector())
+		case DetectorTypeUnspecified:
+			// The zero value is not a valid entry: an empty string in the list
+			// is a typo, not a way to switch detection off. An empty list is.
+			return nil, errDetectorType(typ)
+		default:
+			return nil, errDetectorType(typ)
+		}
+	}
+	return detectors, nil
+}
+
 func errExporterType(typ ExporterType, instrument string) error {
 	return fmt.Errorf("exporter type \"%v\" unsupported for %s", typ, instrument)
 }
+
+func errDetectorType(typ DetectorType) error {
+	return fmt.Errorf("detector type \"%v\" unsupported", typ)
+}

backend/v3/instrumentation/otel_test.go

@@ -4,6 +4,9 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel/attribute"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 )
 
 func TestExporterType_isNone(t *testing.T) {
@@ -35,3 +38,164 @@ func TestExporterType_isNone(t *testing.T) {
 		})
 	}
 }
+
+func Test_newResource(t *testing.T) {
+	tests := []struct {
+		name        string
+		serviceName string
+		env         map[string]string
+		want        map[attribute.Key]string
+	}{
+		{
+			name:        "service name from config",
+			serviceName: "zitadel",
+			want: map[attribute.Key]string{
+				semconv.ServiceNameKey: "zitadel",
+			},
+		},
+		{
+			name:        "OTEL_SERVICE_NAME overrides config",
+			serviceName: "zitadel",
+			env:         map[string]string{"OTEL_SERVICE_NAME": "from-env"},
+			want: map[attribute.Key]string{
+				semconv.ServiceNameKey: "from-env",
+			},
+		},
+		{
+			name:        "config service name wins over OTEL_RESOURCE_ATTRIBUTES",
+			serviceName: "zitadel",
+			env:         map[string]string{"OTEL_RESOURCE_ATTRIBUTES": "service.name=from-attrs,deployment.environment=qa"},
+			want: map[attribute.Key]string{
+				semconv.ServiceNameKey:           "zitadel",
+				semconv.DeploymentEnvironmentKey: "qa",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			for k, v := range tt.env {
+				t.Setenv(k, v)
+			}
+			got, err := newResource(t.Context(), tt.serviceName, nil)
+			require.NoError(t, err)
+
+			attributes := make(map[attribute.Key]string)
+			for _, kv := range got.Attributes() {
+				attributes[kv.Key] = kv.Value.String()
+			}
+			for key, value := range tt.want {
+				assert.Equal(t, value, attributes[key], "attribute %q", key)
+			}
+			// The telemetry SDK detector always contributes, proving the
+			// detector chain ran even when nothing else is detectable.
+			assert.NotEmpty(t, attributes[semconv.TelemetrySDKNameKey])
+		})
+	}
+}
+
+func Test_newResource_errors(t *testing.T) {
+	// Resource construction is part of startup: anything that cannot be
+	// resolved is returned rather than dropped, so a mistyped environment
+	// fails loudly instead of silently losing attributes.
+	tests := []struct {
+		name          string
+		resourceAttrs string
+		detectors     []DetectorType
+	}{
+		{
+			name:          "attribute pair without a value",
+			resourceAttrs: "deployment.environment",
+		},
+		{
+			name:          "trailing comma",
+			resourceAttrs: "deployment.environment=prod,",
+		},
+		{
+			name:      "unsupported detector",
+			detectors: []DetectorType{DetectorType(99)},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if tt.resourceAttrs != "" {
+				t.Setenv("OTEL_RESOURCE_ATTRIBUTES", tt.resourceAttrs)
+			}
+			_, err := newResource(t.Context(), "zitadel", tt.detectors)
+			assert.Error(t, err)
+		})
+	}
+}
+
+func Test_newDetectors(t *testing.T) {
+	tests := []struct {
+		name    string
+		types   []DetectorType
+		wantLen int
+		wantErr bool
+	}{
+		{
+			name:    "nil is off",
+			types:   nil,
+			wantLen: 0,
+		},
+		{
+			name:    "empty is off",
+			types:   []DetectorType{},
+			wantLen: 0,
+		},
+		{
+			name:    "unspecified is rejected",
+			types:   []DetectorType{DetectorTypeUnspecified},
+			wantErr: true,
+		},
+		{
+			name:    "a valid entry does not excuse an unspecified one",
+			types:   []DetectorType{DetectorTypeGoogle, DetectorTypeUnspecified},
+			wantErr: true,
+		},
+		{
+			name:    "google",
+			types:   []DetectorType{DetectorTypeGoogle},
+			wantLen: 1,
+		},
+		{
+			name:    "unknown type errors",
+			types:   []DetectorType{DetectorType(99)},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := newDetectors(tt.types)
+			if tt.wantErr {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			assert.Len(t, got, tt.wantLen)
+		})
+	}
+}
+
+func TestDetectorType_text(t *testing.T) {
+	// The config decoder resolves detectors through encoding.TextUnmarshaler,
+	// so the YAML and env var spellings have to round-trip.
+	for _, spelling := range []string{"google", "Google"} {
+		var typ DetectorType
+		require.NoError(t, typ.UnmarshalText([]byte(spelling)))
+		assert.Equal(t, DetectorTypeGoogle, typ)
+	}
+
+	// The empty string still parses, because the zero value is a member of the
+	// enum by project convention. It is newDetectors that rejects it as a list
+	// entry, so an empty string in the config fails startup rather than being
+	// silently skipped.
+	var unspecified DetectorType
+	require.NoError(t, unspecified.UnmarshalText([]byte("")))
+	assert.Equal(t, DetectorTypeUnspecified, unspecified)
+	_, err := newDetectors([]DetectorType{unspecified})
+	assert.Error(t, err)
+
+	var unknown DetectorType
+	assert.Error(t, unknown.UnmarshalText([]byte("aws")))
+}

cmd/defaults.yaml

@@ -1,5 +1,18 @@
 Instrumentation:
   ServiceName: "zitadel" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  # Resource detectors describe the platform ZITADEL runs on. Their attributes
+  # are attached to every exported trace, metric and log, and telemetry backends
+  # use them to tell instances apart instead of merging them into one series.
+  # Detection is off by default: each detector probes its platform on startup and
+  # the attributes it finds identify the account and host to whichever backend
+  # you export to. Enable one or more of:
+  #   "google": Detects Cloud Run, GKE, GCE, App Engine and Cloud Functions by
+  #             querying the Google Cloud metadata server. Required when
+  #             exporting to Google Cloud: without it every instance reports as
+  #             the same empty generic_node and the series overlap.
+  # Leave the list empty to disable detection. An empty or unknown entry is a
+  # config error and fails startup.
+  Detectors: [] # ZITADEL_INSTRUMENTATION_DETECTORS (comma separated list)
   Trace:
     Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
     # Trust incoming trace context from remote services for distributed tracing.

go.mod

@@ -106,6 +106,7 @@ require (
 	github.com/zitadel/schema v1.3.2
 	github.com/zitadel/sloggcp v0.6.0
 	go.opentelemetry.io/contrib/bridges/otelslog v0.18.0
+	go.opentelemetry.io/contrib/detectors/gcp v1.44.0
 	go.opentelemetry.io/contrib/exporters/autoexport v0.68.0
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.68.0
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.68.0
@@ -259,7 +260,6 @@ require (
 	github.com/yuin/gopher-lua v1.1.2 // indirect
 	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
 	go.opentelemetry.io/contrib/bridges/prometheus v0.68.0 // indirect
-	go.opentelemetry.io/contrib/detectors/gcp v1.44.0 // indirect
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.43.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.10.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect