mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-11-10 10:13:50 +08:00
Today everything outside ateapi hardcodes `ParentBased(NeverSample)`, and setting `OTEL_TRACES_SAMPLER` does nothing because an explicit sampler silences the SDK's env handling. This makes troubleshooting production issues impossible via traces. This PR gives every component a sane default and makes the standard OTel env vars work: - Control plane (ateapi, atelet, ateom) defaults to `parentbased_traceidratio` 0.1, the router (data plane root) to 0.01, per the discussion on #584. - `OTEL_TRACES_SAMPLER` / `OTEL_TRACES_SAMPLER_ARG` override any of this without a rebuild. Invalid values keep the component default and log a warning instead of inheriting the SDK's fall-open-to-100% behavior. - Envoy's `RandomSampling` is derived from the router's resolved policy, so the two root decisions cannot drift. - kubectl-ate without `--trace` no longer installs a tracer provider at all: the old `NeverSample` provider injected `sampled=0`, which pinned every parent based sampler downstream and would have defeated the server side ratios. `--trace` still forces a full end to end trace. - ate-controller propagates the two env vars to the ateom worker pods it creates, same as the metric export vars. - kind pins ateapi to `parentbased_always_on`, so the local Jaeger flow keeps showing every API call. - The agentgateway integration follows what we have above. its `randomSampling` changes from `true` to `0.01` to match the data plane default, though as static config it does not follow `OTEL_TRACES_SAMPLER` overrides, so the two need adjusting together. Verified on a kind cluster e2e manually. It resolved samplers logged at startup, a `--trace` resume produced one trace across ateapi, atelet, and ateom, an unsampled CLI call got picked up server side, Envoy continued a sampled traceparent while sampling 0 of 30 parentless requests at the 1% default, and an invalid env value fell back to the component default. Gating who may use `--trace` stays a separate follow-up (and a discussion), and the more sophisticaed tracing policies belongs in a collector, not in substrate. Fixes #584 cc. @git286 - [x] Tests pass - [x] Appropriate changes to documentation are included in the PR
145 lines
4.8 KiB
Go
145 lines
4.8 KiB
Go
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package serverboot
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"os"
|
|
"strconv"
|
|
"strings"
|
|
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
)
|
|
|
|
const (
|
|
tracesSamplerEnv = "OTEL_TRACES_SAMPLER"
|
|
tracesSamplerArgEnv = "OTEL_TRACES_SAMPLER_ARG"
|
|
)
|
|
|
|
// ControlPlaneTraceRatio is the default root sampling ratio for the control
|
|
// plane binaries: low volume, and their traces are what lifecycle debugging
|
|
// needs, so the default leans generous.
|
|
const ControlPlaneTraceRatio = 0.1
|
|
|
|
// TraceSampling is a resolved sampling policy. The root fraction is kept
|
|
// alongside the sampler because the atenet router must mirror it into Envoy's
|
|
// RandomSampling percent, which roots the data plane traces.
|
|
type TraceSampling struct {
|
|
sampler sdktrace.Sampler
|
|
rootRatio float64
|
|
}
|
|
|
|
func (s TraceSampling) Sampler() sdktrace.Sampler { return s.sampler }
|
|
|
|
// RootSamplingPercent is the percentage (0 to 100) of parentless requests
|
|
// sampled at the root.
|
|
func (s TraceSampling) RootSamplingPercent() float64 { return s.rootRatio * 100 }
|
|
|
|
// ParentRatioSampling is ParentBased(TraceIDRatioBased(ratio)), clamped to [0, 1].
|
|
func ParentRatioSampling(ratio float64) TraceSampling {
|
|
ratio = min(max(ratio, 0), 1)
|
|
return TraceSampling{
|
|
sampler: sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio)),
|
|
rootRatio: ratio,
|
|
}
|
|
}
|
|
|
|
// ParentNeverSampling is ParentBased(NeverSample()).
|
|
func ParentNeverSampling() TraceSampling {
|
|
return TraceSampling{sampler: sdktrace.ParentBased(sdktrace.NeverSample())}
|
|
}
|
|
|
|
// ResolveTraceSampling applies OTEL_TRACES_SAMPLER / OTEL_TRACES_SAMPLER_ARG on
|
|
// top of the component default. serverboot resolves the env itself because an
|
|
// explicit WithSampler silences the SDK's env handling, and the SDK falls open
|
|
// to 100% sampling on invalid values where this keeps the default and logs.
|
|
// That includes a ratio sampler without an arg, which the spec reads as 1.0.
|
|
func ResolveTraceSampling(ctx context.Context, def TraceSampling) TraceSampling {
|
|
name, nameSet := os.LookupEnv(tracesSamplerEnv)
|
|
arg, argSet := os.LookupEnv(tracesSamplerArgEnv)
|
|
resolved, err := resolveTraceSampling(name, nameSet, arg, argSet, def)
|
|
if err != nil {
|
|
slog.WarnContext(ctx, "Invalid trace sampler environment, keeping the component default",
|
|
slog.String("sampler", name),
|
|
slog.String("arg", arg),
|
|
slog.String("default", def.description()),
|
|
slog.Any("err", err))
|
|
}
|
|
return resolved
|
|
}
|
|
|
|
func (s TraceSampling) description() string {
|
|
if s.sampler == nil {
|
|
return "<unset>"
|
|
}
|
|
return s.sampler.Description()
|
|
}
|
|
|
|
func isRatioSampler(name string) bool {
|
|
switch strings.ToLower(strings.TrimSpace(name)) {
|
|
case "traceidratio", "parentbased_traceidratio":
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
// resolveTraceSampling accepts the sampler names the OTel SDK understands,
|
|
// minus the remote ones substrate does not vendor. Any error means def was kept.
|
|
func resolveTraceSampling(name string, nameSet bool, arg string, argSet bool, def TraceSampling) (TraceSampling, error) {
|
|
if !nameSet {
|
|
return def, nil
|
|
}
|
|
name = strings.ToLower(strings.TrimSpace(name))
|
|
// Unlike the SDK, treat set-but-empty as unset: templated manifests can
|
|
// render empty env vars.
|
|
if name == "" {
|
|
return def, nil
|
|
}
|
|
|
|
var ratio float64
|
|
if isRatioSampler(name) {
|
|
trimmed := strings.TrimSpace(arg)
|
|
if !argSet || trimmed == "" {
|
|
return def, fmt.Errorf("%s %q requires %s to be a ratio in [0, 1]", tracesSamplerEnv, name, tracesSamplerArgEnv)
|
|
}
|
|
parsed, err := strconv.ParseFloat(trimmed, 64)
|
|
if err != nil {
|
|
return def, fmt.Errorf("parse %s %q: %w", tracesSamplerArgEnv, arg, err)
|
|
}
|
|
if parsed < 0 || parsed > 1 {
|
|
return def, fmt.Errorf("%s %q outside [0, 1]", tracesSamplerArgEnv, arg)
|
|
}
|
|
ratio = parsed
|
|
}
|
|
|
|
switch name {
|
|
case "always_on":
|
|
return TraceSampling{sampler: sdktrace.AlwaysSample(), rootRatio: 1}, nil
|
|
case "always_off":
|
|
return TraceSampling{sampler: sdktrace.NeverSample()}, nil
|
|
case "traceidratio":
|
|
return TraceSampling{sampler: sdktrace.TraceIDRatioBased(ratio), rootRatio: ratio}, nil
|
|
case "parentbased_always_on":
|
|
return TraceSampling{sampler: sdktrace.ParentBased(sdktrace.AlwaysSample()), rootRatio: 1}, nil
|
|
case "parentbased_always_off":
|
|
return ParentNeverSampling(), nil
|
|
case "parentbased_traceidratio":
|
|
return ParentRatioSampling(ratio), nil
|
|
}
|
|
return def, fmt.Errorf("unsupported %s %q", tracesSamplerEnv, name)
|
|
}
|