Files
Krisztian F 15eecd02c9 Define a production-ready default tracing policy (#711)
Today everything outside ateapi hardcodes `ParentBased(NeverSample)`,
and setting `OTEL_TRACES_SAMPLER` does nothing because an explicit
sampler silences the SDK's env handling. This makes troubleshooting
production issues impossible via traces.

This PR gives every component a sane default and makes the standard OTel
env vars work:

- Control plane (ateapi, atelet, ateom) defaults to
`parentbased_traceidratio` 0.1, the router (data plane root) to 0.01,
per the discussion on #584.
- `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` override any of this
without a rebuild. Invalid values keep the component default and log a
warning instead of inheriting the SDK's fall-open-to-100% behavior.
- Envoy's `RandomSampling` is derived from the router's resolved policy,
so the two root decisions cannot drift.
- kubectl-ate without `--trace` no longer installs a tracer provider at
all: the old `NeverSample` provider injected `sampled=0`, which pinned
every parent based sampler downstream and would have defeated the server
side ratios. `--trace` still forces a full end to end trace.
- ate-controller propagates the two env vars to the ateom worker pods it
creates, same as the metric export vars.
- kind pins ateapi to `parentbased_always_on`, so the local Jaeger flow
keeps showing every API call.
- The agentgateway integration follows what we have above. its
`randomSampling` changes from `true` to `0.01` to match the data plane
default, though as static config it does not follow
`OTEL_TRACES_SAMPLER` overrides, so the two need adjusting together.


Verified on a kind cluster e2e manually. It resolved samplers logged at
startup, a `--trace` resume produced one trace across ateapi, atelet,
and ateom, an unsampled CLI call got picked up server side, Envoy
continued a sampled traceparent while sampling 0 of 30 parentless
requests at the 1% default, and an invalid env value fell back to the
component default.

Gating who may use `--trace` stays a separate follow-up (and a
discussion), and the more sophisticaed tracing policies belongs in a
collector, not in substrate.

Fixes #584
cc. @git286 

- [x] Tests pass
- [x] Appropriate changes to documentation are included in the PR
2026-08-04 11:31:04 -04:00

249 lines
7.2 KiB
Go

// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package serverboot
import (
"context"
"testing"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
)
const testDefaultRatio = 0.25
func TestResolveTraceSampling(t *testing.T) {
def := ParentRatioSampling(testDefaultRatio)
tests := []struct {
name string
sampler string
samplerSet bool
arg string
argSet bool
def TraceSampling
want sdktrace.Sampler
wantPercent float64
wantErr bool
}{
{
name: "unset keeps ratio default",
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
},
{
name: "unset keeps never default",
def: ParentNeverSampling(),
want: ParentNeverSampling().Sampler(),
wantPercent: 0,
},
{
name: "set but empty keeps default",
samplerSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
},
{
name: "always_on",
sampler: "always_on",
samplerSet: true,
def: def,
want: sdktrace.AlwaysSample(),
wantPercent: 100,
},
{
name: "always_off",
sampler: "always_off",
samplerSet: true,
def: def,
want: sdktrace.NeverSample(),
wantPercent: 0,
},
{
name: "traceidratio with arg",
sampler: "traceidratio",
samplerSet: true,
arg: "0.5",
argSet: true,
def: def,
want: sdktrace.TraceIDRatioBased(0.5),
wantPercent: 50,
},
{
name: "parentbased_always_on",
sampler: "parentbased_always_on",
samplerSet: true,
def: def,
want: sdktrace.ParentBased(sdktrace.AlwaysSample()),
wantPercent: 100,
},
{
name: "parentbased_always_off",
sampler: "parentbased_always_off",
samplerSet: true,
def: def,
want: sdktrace.ParentBased(sdktrace.NeverSample()),
wantPercent: 0,
},
{
name: "parentbased_traceidratio with arg",
sampler: "parentbased_traceidratio",
samplerSet: true,
arg: "0.5",
argSet: true,
def: def,
want: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5)),
wantPercent: 50,
},
{
name: "mixed case and whitespace normalize",
sampler: " ParentBased_TraceIDRatio ",
samplerSet: true,
arg: " 0.5 ",
argSet: true,
def: def,
want: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5)),
wantPercent: 50,
},
{
name: "ratio without arg keeps default",
sampler: "parentbased_traceidratio",
samplerSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
{
name: "ratio with empty arg keeps default",
sampler: "traceidratio",
samplerSet: true,
arg: " ",
argSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
{
name: "unknown sampler keeps default",
sampler: "jaeger_remote",
samplerSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
{
name: "unparsable arg keeps default",
sampler: "parentbased_traceidratio",
samplerSet: true,
arg: "abc",
argSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
{
name: "arg above 1 keeps default",
sampler: "parentbased_traceidratio",
samplerSet: true,
arg: "1.5",
argSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
{
name: "negative arg keeps default",
sampler: "traceidratio",
samplerSet: true,
arg: "-0.1",
argSet: true,
def: def,
want: def.Sampler(),
wantPercent: testDefaultRatio * 100,
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := resolveTraceSampling(tt.sampler, tt.samplerSet, tt.arg, tt.argSet, tt.def)
if (err != nil) != tt.wantErr {
t.Fatalf("resolveTraceSampling() error = %v, wantErr %v", err, tt.wantErr)
}
if got.Sampler().Description() != tt.want.Description() {
t.Errorf("sampler = %s, want %s", got.Sampler().Description(), tt.want.Description())
}
if got.RootSamplingPercent() != tt.wantPercent {
t.Errorf("RootSamplingPercent() = %v, want %v", got.RootSamplingPercent(), tt.wantPercent)
}
})
}
}
func TestResolveTraceSamplingReadsEnv(t *testing.T) {
def := ParentNeverSampling()
t.Setenv("OTEL_TRACES_SAMPLER", "parentbased_traceidratio")
t.Setenv("OTEL_TRACES_SAMPLER_ARG", "0.5")
got := ResolveTraceSampling(context.Background(), def)
want := sdktrace.ParentBased(sdktrace.TraceIDRatioBased(0.5))
if got.Sampler().Description() != want.Description() {
t.Errorf("sampler = %s, want %s", got.Sampler().Description(), want.Description())
}
t.Setenv("OTEL_TRACES_SAMPLER", "not_a_sampler")
got = ResolveTraceSampling(context.Background(), def)
if got.Sampler().Description() != def.Sampler().Description() {
t.Errorf("sampler = %s, want the default %s", got.Sampler().Description(), def.Sampler().Description())
}
}
func TestResolveTraceSamplingZeroValueDefault(t *testing.T) {
t.Setenv("OTEL_TRACES_SAMPLER", "not_a_sampler")
got := ResolveTraceSampling(context.Background(), TraceSampling{})
if got.Sampler() != nil {
t.Errorf("Sampler() = %v, want nil from a zero-value default", got.Sampler())
}
if got.RootSamplingPercent() != 0 {
t.Errorf("RootSamplingPercent() = %v, want 0", got.RootSamplingPercent())
}
}
func TestParentSamplingConstructors(t *testing.T) {
if got := ParentRatioSampling(0.01).RootSamplingPercent(); got != 1 {
t.Errorf("ParentRatioSampling(0.01).RootSamplingPercent() = %v, want 1", got)
}
if got := ParentNeverSampling().RootSamplingPercent(); got != 0 {
t.Errorf("ParentNeverSampling().RootSamplingPercent() = %v, want 0", got)
}
if got := ParentRatioSampling(1.5).RootSamplingPercent(); got != 100 {
t.Errorf("ParentRatioSampling(1.5).RootSamplingPercent() = %v, want 100 (clamped)", got)
}
if got := ParentRatioSampling(-1).RootSamplingPercent(); got != 0 {
t.Errorf("ParentRatioSampling(-1).RootSamplingPercent() = %v, want 0 (clamped)", got)
}
}
func TestInitTracingRequiresSampling(t *testing.T) {
if _, err := InitTracing(context.Background(), TracingOptions{ServiceName: "x"}); err == nil {
t.Error("InitTracing() with zero Sampling: expected error, got nil")
}
}