diff --git a/cmd/ateapi/internal/store/atepg/atepg.go b/cmd/ateapi/internal/store/atepg/atepg.go index 914c8f00d0..7ea5e88250 100644 --- a/cmd/ateapi/internal/store/atepg/atepg.go +++ b/cmd/ateapi/internal/store/atepg/atepg.go @@ -35,6 +35,7 @@ import ( "github.com/jackc/pgx/v5" "github.com/jackc/pgx/v5/pgconn" "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" ) @@ -144,6 +145,8 @@ func poolConfig(dsn string) (*pgxpool.Config, error) { if err != nil { return nil, fmt.Errorf("parsing PostgreSQL connection string: %w", err) } + // Per-statement trace spans; the watch pool inherits this through Copy(). + cfg.ConnConfig.Tracer = newQueryTracer(otel.GetTracerProvider(), cfg.ConnConfig) usesTLS := cfg.ConnConfig.TLSConfig != nil for _, fallback := range cfg.ConnConfig.Fallbacks { usesTLS = usesTLS || fallback.TLSConfig != nil diff --git a/cmd/ateapi/internal/store/atepg/tracing.go b/cmd/ateapi/internal/store/atepg/tracing.go new file mode 100644 index 0000000000..666c95fe1f --- /dev/null +++ b/cmd/ateapi/internal/store/atepg/tracing.go @@ -0,0 +1,180 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atepg + +import ( + "context" + "fmt" + "strings" + + "github.com/jackc/pgx/v5" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/trace" +) + +// queryTracer is a pgx QueryTracer that opens one client span per statement, +// so an RPC trace shows where its time went inside PostgreSQL. Statements are +// parameterized ($1, $2, ...), so db.query.text carries no argument values. +// +// Spans follow the OpenTelemetry database semantic conventions: the span name +// is the query summary ("SELECT actors", "UPDATE workers", "commit"), and the +// connection-level attributes are computed once from the pool configuration. +type queryTracer struct { + tracer trace.Tracer + // attrs holds db.system.name, server.address, server.port and + // db.namespace, which are the same for every statement on the pool. + attrs []attribute.KeyValue +} + +var _ pgx.QueryTracer = (*queryTracer)(nil) + +// newQueryTracer builds the tracer for pools opened from cc. The tracer is +// resolved once here rather than per statement. +func newQueryTracer(tp trace.TracerProvider, cc *pgx.ConnConfig) *queryTracer { + return &queryTracer{ + tracer: tp.Tracer("atepg"), + attrs: []attribute.KeyValue{ + semconv.DBSystemNamePostgreSQL, + semconv.ServerAddress(cc.Host), + semconv.ServerPort(int(cc.Port)), + semconv.DBNamespace(cc.Database), + }, + } +} + +// querySpanKey marks a context whose statement span was opened by +// TraceQueryStart, so TraceQueryEnd never ends a span it did not start. +type querySpanKey struct{} + +func (t *queryTracer) TraceQueryStart(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryStartData) context.Context { + // Join sampled traces only. Statements issued from background work + // (outbox polling, lease maintenance) carry no span, and opening a root + // span for each would flood the backend with single-span traces. An + // unsampled parent is skipped too: every sampler serverboot installs is + // ParentBased, so its children could never be sampled, and skipping + // them saves the non-recording span and context wrap per statement. + if !trace.SpanContextFromContext(ctx).IsSampled() { + return ctx + } + operation, collection := querySummary(data.SQL) + attrs := make([]attribute.KeyValue, 0, len(t.attrs)+4) + attrs = append(attrs, t.attrs...) + attrs = append(attrs, semconv.DBQueryText(data.SQL)) + name := "postgresql" + if operation != "" { + name = operation + attrs = append(attrs, semconv.DBOperationName(operation)) + if collection != "" { + name += " " + collection + attrs = append(attrs, semconv.DBCollectionName(collection)) + } + attrs = append(attrs, semconv.DBQuerySummary(name)) + } + ctx, span := t.tracer.Start(ctx, name, + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes(attrs...)) + return context.WithValue(ctx, querySpanKey{}, span) +} + +func (*queryTracer) TraceQueryEnd(ctx context.Context, _ *pgx.Conn, data pgx.TraceQueryEndData) { + span, ok := ctx.Value(querySpanKey{}).(trace.Span) + if !ok { + return + } + // pgx.ErrNoRows never arrives here: pgx synthesizes it for the caller + // after the rows are closed, so a lookup miss ends the span cleanly. + if data.Err != nil { + span.RecordError(data.Err) + span.SetStatus(codes.Error, data.Err.Error()) + if code := pgErrCode(data.Err); code != "" { + // SQLSTATE is both the server's status code and the most + // useful low-cardinality error class. + span.SetAttributes(semconv.DBResponseStatusCode(code), semconv.ErrorTypeKey.String(code)) + } else { + span.SetAttributes(semconv.ErrorTypeKey.String(fmt.Sprintf("%T", data.Err))) + } + } + span.End() +} + +// querySummary extracts the db.operation.name and db.collection.name of a +// statement: the leading keyword, and the single table it acts on. Both are +// returned as written, without case normalization, as the conventions ask. +// +// The store issues one hand-written statement per call, so a keyword scan is +// enough: the table follows INTO for INSERT, UPDATE for UPDATE, TABLE for +// LOCK, and the first FROM for SELECT and DELETE. A statement that reads +// several tables (any JOIN), none (SELECT clock_timestamp()) or only a +// subquery's (SELECT EXISTS(SELECT 1 FROM t)) has no collection, and DDL and +// transaction control report only their keyword. +func querySummary(sql string) (operation, collection string) { + fields := strings.Fields(sql) + if len(fields) == 0 { + return "", "" + } + operation = fields[0] + var marker string + switch strings.ToUpper(operation) { + case "SELECT", "DELETE": + marker = "FROM" + case "INSERT": + marker = "INTO" + case "LOCK": + marker = "TABLE" + case "UPDATE": + return operation, collectionToken(fields, 1) + default: + return operation, "" + } + at := -1 + for i, f := range fields[1:] { + switch { + case strings.EqualFold(f, "JOIN"): + return operation, "" + case at < 0 && opensSubquery(f): + // A subquery opened before the marker owns the first FROM, + // as in SELECT EXISTS(SELECT 1 FROM t): the outer statement + // reads no table of its own. + return operation, "" + case at < 0 && strings.EqualFold(f, marker): + at = i + 2 + } + } + if at < 0 { + return operation, "" + } + return operation, collectionToken(fields, at) +} + +// opensSubquery reports whether a whitespace-delimited token starts a nested +// SELECT: "SELECT", "(SELECT" or "EXISTS(SELECT". +func opensSubquery(tok string) bool { + return strings.EqualFold(tok[strings.LastIndexByte(tok, '(')+1:], "SELECT") +} + +// collectionToken returns fields[i] stripped of surrounding punctuation, or +// "" when it is absent or is not an identifier (a subquery, for instance). +func collectionToken(fields []string, i int) string { + if i >= len(fields) { + return "" + } + tok := strings.Trim(fields[i], "(),;") + if tok == "" || strings.EqualFold(tok, "SELECT") { + return "" + } + return tok +} diff --git a/cmd/ateapi/internal/store/atepg/tracing_test.go b/cmd/ateapi/internal/store/atepg/tracing_test.go new file mode 100644 index 0000000000..b2177677a9 --- /dev/null +++ b/cmd/ateapi/internal/store/atepg/tracing_test.go @@ -0,0 +1,340 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package atepg + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/jackc/pgx/v5" + "github.com/jackc/pgx/v5/pgconn" + "github.com/jackc/pgx/v5/pgxpool" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "go.opentelemetry.io/otel/sdk/trace/tracetest" + semconv "go.opentelemetry.io/otel/semconv/v1.40.0" + "go.opentelemetry.io/otel/trace" +) + +// newTestQueryTracer builds a queryTracer against a local recording provider, +// never touching the global one, so these tests stay parallel-safe. +func newTestQueryTracer(t *testing.T) (*queryTracer, *sdktrace.TracerProvider, *tracetest.SpanRecorder) { + t.Helper() + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr)) + t.Cleanup(func() { _ = tp.Shutdown(context.Background()) }) + cc, err := pgx.ParseConfig("postgres://ate@db.example.internal:5433/atedb?sslmode=disable") + if err != nil { + t.Fatalf("ParseConfig: %v", err) + } + return newQueryTracer(tp, cc), tp, sr +} + +func spanAttrs(s sdktrace.ReadOnlySpan) map[attribute.Key]attribute.Value { + m := make(map[attribute.Key]attribute.Value, len(s.Attributes())) + for _, kv := range s.Attributes() { + m[kv.Key] = kv.Value + } + return m +} + +func findSpan(t *testing.T, sr *tracetest.SpanRecorder, name string) sdktrace.ReadOnlySpan { + t.Helper() + for _, s := range sr.Ended() { + if s.Name() == name { + return s + } + } + var names []string + for _, s := range sr.Ended() { + names = append(names, s.Name()) + } + t.Fatalf("no %q span recorded; got %q", name, names) + return nil +} + +func TestQueryTracerJoinsParentTrace(t *testing.T) { + t.Parallel() + qt, tp, sr := newTestQueryTracer(t) + ctx, parent := tp.Tracer("test").Start(context.Background(), "parent") + + const sql = "SELECT id FROM actors WHERE name = $1" + qctx := qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: sql}) + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{}) + parent.End() + + dbSpan := findSpan(t, sr, "SELECT actors") + if got, want := dbSpan.Parent().SpanID(), parent.SpanContext().SpanID(); got != want { + t.Errorf("db span parent = %s, want the RPC span %s", got, want) + } + if dbSpan.SpanKind() != trace.SpanKindClient { + t.Errorf("db span kind = %v, want client", dbSpan.SpanKind()) + } + if dbSpan.Status().Code != codes.Unset { + t.Errorf("status = %v, want Unset", dbSpan.Status().Code) + } + attrs := spanAttrs(dbSpan) + for key, want := range map[attribute.Key]attribute.Value{ + semconv.DBSystemNameKey: semconv.DBSystemNamePostgreSQL.Value, + semconv.ServerAddressKey: attribute.StringValue("db.example.internal"), + semconv.ServerPortKey: attribute.IntValue(5433), + semconv.DBNamespaceKey: attribute.StringValue("atedb"), + semconv.DBQueryTextKey: attribute.StringValue(sql), + semconv.DBOperationNameKey: attribute.StringValue("SELECT"), + semconv.DBCollectionNameKey: attribute.StringValue("actors"), + semconv.DBQuerySummaryKey: attribute.StringValue("SELECT actors"), + } { + if got, ok := attrs[key]; !ok { + t.Errorf("missing %s", key) + } else if got != want { + t.Errorf("%s = %v, want %v", key, got.String(), want.String()) + } + } + for _, key := range []attribute.Key{semconv.ErrorTypeKey, semconv.DBResponseStatusCodeKey} { + if _, ok := attrs[key]; ok { + t.Errorf("successful statement must not carry %s", key) + } + } +} + +func TestQueryTracerSkipsWithoutSampledParent(t *testing.T) { + t.Parallel() + qt, _, sr := newTestQueryTracer(t) + + // No span at all: background work. + ctx := context.Background() + qctx := qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "SELECT 1"}) + if qctx != ctx { + t.Error("TraceQueryStart without a parent span must return the context unchanged") + } + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{}) + + // A parent that was not sampled: its children could never be sampled + // either, and ending the statement must leave the parent untouched. + unsampled := sdktrace.NewTracerProvider(sdktrace.WithSampler(sdktrace.NeverSample()), sdktrace.WithSpanProcessor(sr)) + ctx, parent := unsampled.Tracer("test").Start(context.Background(), "parent") + if parent.SpanContext().IsSampled() { + t.Fatal("test setup: parent must not be sampled") + } + qctx = qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "SELECT 1"}) + if qctx != ctx { + t.Error("TraceQueryStart under an unsampled parent must return the context unchanged") + } + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{}) + if n := len(sr.Ended()); n != 0 { + t.Errorf("unsampled statements recorded %d spans, want 0", n) + } +} + +// recordOnlySampler records every span without sampling it, the one +// decision that yields a parent that is recording yet not sampled. +type recordOnlySampler struct{} + +func (recordOnlySampler) ShouldSample(sdktrace.SamplingParameters) sdktrace.SamplingResult { + return sdktrace.SamplingResult{Decision: sdktrace.RecordOnly} +} + +func (recordOnlySampler) Description() string { return "RecordOnly" } + +func TestQueryTracerLeavesRecordOnlyParentOpen(t *testing.T) { + t.Parallel() + qt, _, sr := newTestQueryTracer(t) + recordOnly := sdktrace.NewTracerProvider(sdktrace.WithSampler(recordOnlySampler{}), sdktrace.WithSpanProcessor(sr)) + ctx, parent := recordOnly.Tracer("test").Start(context.Background(), "parent") + if parent.SpanContext().IsSampled() || !parent.IsRecording() { + t.Fatal("test setup: parent must be recording but not sampled") + } + + // The statement is skipped because the parent is not sampled. Ending it + // must not touch the parent, which is the span the context now carries. + qctx := qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "SELECT 1"}) + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{}) + + if !parent.IsRecording() { + t.Error("TraceQueryEnd ended the parent span it did not start") + } + if n := len(sr.Ended()); n != 0 { + t.Errorf("recorded %d ended spans before the parent finished, want 0", n) + } +} + +func TestQueryTracerErrorStatus(t *testing.T) { + t.Parallel() + qt, tp, sr := newTestQueryTracer(t) + ctx, parent := tp.Tracer("test").Start(context.Background(), "parent") + + // A PostgreSQL error carries its SQLSTATE as status code and error type. + pgErr := &pgconn.PgError{Code: "40P01", Message: "deadlock detected"} + qctx := qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "UPDATE actors SET state = $1"}) + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{Err: pgErr}) + + // Any other failure is typed by its Go type. + qctx = qt.TraceQueryStart(ctx, nil, pgx.TraceQueryStartData{SQL: "DELETE FROM workers WHERE name = $1"}) + qt.TraceQueryEnd(qctx, nil, pgx.TraceQueryEndData{Err: context.DeadlineExceeded}) + parent.End() + + update := findSpan(t, sr, "UPDATE actors") + if update.Status().Code != codes.Error { + t.Errorf("failed statement status = %v, want Error", update.Status().Code) + } + attrs := spanAttrs(update) + if got := attrs[semconv.DBResponseStatusCodeKey].AsString(); got != "40P01" { + t.Errorf("db.response.status_code = %q, want 40P01", got) + } + if got := attrs[semconv.ErrorTypeKey].AsString(); got != "40P01" { + t.Errorf("error.type = %q, want 40P01", got) + } + if len(update.Events()) == 0 { + t.Error("failed statement recorded no exception event") + } + + del := findSpan(t, sr, "DELETE workers") + if del.Status().Code != codes.Error { + t.Errorf("failed statement status = %v, want Error", del.Status().Code) + } + attrs = spanAttrs(del) + if _, ok := attrs[semconv.DBResponseStatusCodeKey]; ok { + t.Error("non-PostgreSQL error must not carry db.response.status_code") + } + if got := attrs[semconv.ErrorTypeKey].AsString(); got == "" || got == "40P01" { + t.Errorf("error.type = %q, want the Go error type", got) + } +} + +func TestQuerySummary(t *testing.T) { + t.Parallel() + for _, tc := range []struct { + sql, operation, collection string + }{ + // Statements as the store writes them. + {"SELECT proto FROM actors WHERE atespace = $1 AND name = $2", "SELECT", "actors"}, + {"SELECT proto FROM workers WHERE name = $1 FOR UPDATE", "SELECT", "workers"}, + {"\n\t\tSELECT atespace, name, proto\n\t\tFROM actor_templates\n\t\tWHERE atespace = $1\n", "SELECT", "actor_templates"}, + {"INSERT INTO actors (atespace, name, uid, version, proto) VALUES ($1, $2, $3, $4, $5)", "INSERT", "actors"}, + {"UPDATE actor_templates SET version = $1, proto = $2 WHERE atespace = $3", "UPDATE", "actor_templates"}, + {"DELETE FROM actor_templates AS t WHERE t.atespace = $1 RETURNING t.proto", "DELETE", "actor_templates"}, + {"DELETE FROM leases WHERE expires_at <= clock_timestamp()", "DELETE", "leases"}, + {"INSERT INTO leases (key, token, expires_at) VALUES ($1, $2, clock_timestamp()) ON CONFLICT (key) DO UPDATE SET token = $2", "INSERT", "leases"}, + {"INSERT INTO worker_outbox_trim (xid) SELECT xid FROM worker_outbox_default ORDER BY xid DESC LIMIT 1", "INSERT", "worker_outbox_trim"}, + {"SELECT proto FROM actors WHERE name IN (SELECT name FROM actor_templates WHERE atespace = $1)", "SELECT", "actors"}, + {"LOCK TABLE worker_outbox IN ACCESS EXCLUSIVE MODE", "LOCK", "worker_outbox"}, + // No single table. + {"SELECT clock_timestamp()", "SELECT", ""}, + {"SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", "SELECT", ""}, + {"SELECT child.relname FROM pg_inherits i JOIN pg_class child ON child.oid = i.inhrelid", "SELECT", ""}, + {"SELECT count(*) FROM (SELECT 1) AS sub", "SELECT", ""}, + {"SELECT EXISTS(SELECT 1 FROM worker_outbox_default)", "SELECT", ""}, + {"SELECT (SELECT xid FROM worker_outbox_trim) >= $1::xid8", "SELECT", ""}, + {"CREATE SCHEMA IF NOT EXISTS \"ate\"", "CREATE", ""}, + {"DROP TABLE worker_outbox_p1", "DROP", ""}, + // Transaction control as pgx issues it. + {"begin", "begin", ""}, + {"begin isolation level serializable", "begin", ""}, + {"commit", "commit", ""}, + {"rollback", "rollback", ""}, + {"", "", ""}, + } { + op, coll := querySummary(tc.sql) + if op != tc.operation || coll != tc.collection { + t.Errorf("querySummary(%q) = (%q, %q), want (%q, %q)", tc.sql, op, coll, tc.operation, tc.collection) + } + } +} + +func TestPoolConfigTracerSharedWithWatchPool(t *testing.T) { + t.Parallel() + cfg, err := poolConfig("postgres://postgres@localhost:5432/atepg?sslmode=disable") + if err != nil { + t.Fatalf("poolConfig: %v", err) + } + qt, ok := cfg.ConnConfig.Tracer.(*queryTracer) + if !ok { + t.Fatalf("pool tracer = %T, want *queryTracer", cfg.ConnConfig.Tracer) + } + watch := cfg.Copy() + if watch.ConnConfig.Tracer != qt { + t.Errorf("watch pool tracer = %v, want the same *queryTracer as the main pool", watch.ConnConfig.Tracer) + } +} + +// TestQueryTracerAgainstPostgreSQL drives real statements through a traced +// pool, so the span names, transaction control spans, lookup misses and +// server errors are what pgx actually delivers rather than what the unit +// tests feed the tracer by hand. +func TestQueryTracerAgainstPostgreSQL(t *testing.T) { + requirePool(t) + ctx := context.Background() + sr := tracetest.NewSpanRecorder() + tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(sr)) + t.Cleanup(func() { _ = tp.Shutdown(ctx) }) + + cfg, err := poolConfig(containerDSN) + if err != nil { + t.Fatalf("poolConfig: %v", err) + } + cfg.ConnConfig.Tracer = newQueryTracer(tp, cfg.ConnConfig) + pool, err := pgxpool.NewWithConfig(ctx, cfg) + if err != nil { + t.Fatalf("opening traced pool: %v", err) + } + t.Cleanup(pool.Close) + + ctx, parent := tp.Tracer("test").Start(ctx, "rpc") + tx, err := pool.Begin(ctx) + if err != nil { + t.Fatalf("Begin: %v", err) + } + var relname string + if err := tx.QueryRow(ctx, "SELECT relname FROM pg_class WHERE false").Scan(&relname); !errors.Is(err, pgx.ErrNoRows) { + t.Fatalf("lookup miss err = %v, want ErrNoRows", err) + } + if err := tx.Commit(ctx); err != nil { + t.Fatalf("Commit: %v", err) + } + if _, err := pool.Exec(ctx, "SELECT 1 FROM no_such_table"); err == nil { + t.Fatal("querying a missing table succeeded") + } + parent.End() + + var names []string + for _, s := range sr.Ended() { + if s.Name() == "rpc" { + continue + } + names = append(names, s.Name()) + if got, want := s.Parent().SpanID(), parent.SpanContext().SpanID(); got != want { + t.Errorf("%s parent = %s, want the rpc span %s", s.Name(), got, want) + } + } + want := []string{"begin", "SELECT pg_class", "commit", "SELECT no_such_table"} + if !slices.Equal(names, want) { + t.Fatalf("span names = %q, want %q", names, want) + } + + miss := findSpan(t, sr, "SELECT pg_class") + if miss.Status().Code != codes.Unset { + t.Errorf("lookup miss status = %v, want Unset: pgx must not hand ErrNoRows to the tracer", miss.Status().Code) + } + failed := findSpan(t, sr, "SELECT no_such_table") + if failed.Status().Code != codes.Error { + t.Errorf("missing table status = %v, want Error", failed.Status().Code) + } + if got := spanAttrs(failed)[semconv.DBResponseStatusCodeKey].AsString(); got != "42P01" { + t.Errorf("db.response.status_code = %q, want 42P01 (undefined_table)", got) + } +} diff --git a/docs/dev/best-practices/tracing.md b/docs/dev/best-practices/tracing.md index 18f0c4d5c9..83eba390ca 100644 --- a/docs/dev/best-practices/tracing.md +++ b/docs/dev/best-practices/tracing.md @@ -64,6 +64,8 @@ Samplers are resolved with `serverboot.ResolveTraceSampling`, which applies the All defaults are `ParentBased`, so a request that arrives already sampled stays sampled on every hop, and one that arrives explicitly unsampled stays unsampled. Only parentless requests are subject to the ratio, at whichever component roots the trace. +The `ateapi` store follows the same rule and has no switch of its own. Its pgx tracer opens one client span per PostgreSQL statement, `begin` and `commit` included, only when the request is already sampled. `always_off` on `ateapi` therefore silences store spans together with everything else, and statements from background work never produce any. + An invalid sampler name, or a missing or unparsable ratio arg, keeps the component default and logs a warning. This deliberately diverges from the OTel SDK's own env handling, which falls back to 100% sampling on invalid input and reads a missing arg as ratio 1.0. In agentgateway mode the data plane root fraction lives in the agentgateway ConfigMap (`randomSampling`, same 0.01 default). Unlike Envoy's `RandomSampling`, it is static config that env overrides on the router do not reach, so adjust both together. diff --git a/docs/observability.md b/docs/observability.md index 91b121eff4..61728def60 100644 --- a/docs/observability.md +++ b/docs/observability.md @@ -298,6 +298,8 @@ To visualize traces locally: > ateom carries no manual spans — its only instrumentation is the `otelgrpc` interceptor on the gRPC surface `atelet` calls. So it produces a span for an actor lifecycle operation (`suspend`, `resume`) and nothing at all for a read like `kubectl ate get actor`. Its sampler is parent based, so a lifecycle command is traced end to end into ateom whenever `ateapi` roots a sampled trace, which the kind overlay makes unconditional; the per-component ratio never enters into it. To check whether ateom exported its spans through the [OTLP relay](#the-ateom-otlp-relay) rather than falling back to direct network egress, inspect the span's resource attributes: `ate.otlp.relay` will be set to `"relay"` (instead of `"direct"`). +> `ateapi` traces its PostgreSQL store. Every statement issued while serving a sampled request becomes a client span under the RPC span, named by its query summary (`SELECT actors`, `UPDATE workers`), with the parameterized statement in `db.query.text` and never the argument values. Transaction control appears as well, as `begin` and `commit` spans, so commit latency is visible. Store spans only join a trace that is already sampled: they follow `ateapi`'s sampler and have no switch of their own, and background work such as outbox polling and lease upkeep produces none. + > **Developer Guide:** For detailed instructions on configuring OpenTelemetry tracer providers, middleware, and exporters in your servers or clients, please refer to the [Tracing Best Practices](dev/best-practices/tracing.md) guide. ---