trending/substrate
1
0
Fork 0
mirror of https://github.com/agent-substrate/substrate.git synced 2026-10-02 03:24:42 +08:00
Code Issues Packages Projects Releases Wiki Activity

add otel export timeout

Browse Source
This commit is contained in:
Lior Lieberman
2026-07-31 08:22:50 -07:00
committed by Dmitry Berkovich
parent 543fd3c974
commit 1b329c9872
5 changed files with 80 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/atecontroller/internal/controllers/workerpool_apply.go
+8
View File
@@ -43,6 +43,9 @@ type ateomOTelSettings struct {
// first export tick fires. Shortening the interval is what keeps that
// startup gap inside an e2e budget; production leaves it unset.
MetricExportInterval string
// MetricExportTimeout overrides the SDK's per-export timeout, in the same
// whole-millisecond form as MetricExportInterval. Empty keeps the default.
MetricExportTimeout string
}
// buildDeploymentApplyConfig constructs the SSA apply configuration for the
@@ -118,6 +121,11 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat
WithName("OTEL_METRIC_EXPORT_INTERVAL").
WithValue(otel.MetricExportInterval))
}
if otel.MetricExportTimeout != "" {
envs = append(envs, corev1ac.EnvVar().
WithName("OTEL_METRIC_EXPORT_TIMEOUT").
WithValue(otel.MetricExportTimeout))
}
return envs
}
cmd/atecontroller/internal/controllers/workerpool_apply_test.go
+47 -19
View File
@@ -360,7 +360,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) {
}
if !tt.wantTelemetry {
for _, k := range []string{"OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", "OTEL_METRIC_EXPORT_INTERVAL", "POD_NAME", "POD_NAMESPACE"} {
for _, k := range []string{"OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", "OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT", "POD_NAME", "POD_NAMESPACE"} {
if _, ok := env[k]; ok {
t.Errorf("%s must be absent without an OTLP endpoint", k)
}
@@ -388,32 +388,60 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) {
}
}
// TestBuildDeploymentApplyConfigMetricExportInterval asserts the export interval
// reaches the ateom container only when both it and an endpoint are set. ateom is
// invisible to the collector until its first export tick, so the kind stack
// shortens the SDK's 60s default to keep that gap inside the e2e budget.
func TestBuildDeploymentApplyConfigMetricExportInterval(t *testing.T) {
// TestBuildDeploymentApplyConfigMetricExportTuning asserts the export interval and
// per-export timeout reach the ateom container only when each is set alongside an
// endpoint. ateom is invisible to the collector until its first successful export
// tick, so the kind stack shortens the SDK's 60s interval to keep that gap inside
// the e2e budget, and its 30s timeout so a failing tick cannot swallow three
// shortened intervals.
func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) {
const endpoint = "http://collector.otel-system.svc:4317"
tests := []struct {
name string
otel ateomOTelSettings
want string
wantSet bool
name string
otel ateomOTelSettings
want map[string]string // env name -> value; absent key means must not be set
}{
{"unset keeps SDK default", ateomOTelSettings{Endpoint: endpoint}, "", false},
{"set with endpoint", ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000"}, "10000", true},
{"ignored without endpoint", ateomOTelSettings{MetricExportInterval: "10000"}, "", false},
{
name: "unset keeps SDK defaults",
otel: ateomOTelSettings{Endpoint: endpoint},
want: nil,
},
{
name: "both set with endpoint",
otel: ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000", MetricExportTimeout: "10000"},
want: map[string]string{"OTEL_METRIC_EXPORT_INTERVAL": "10000", "OTEL_METRIC_EXPORT_TIMEOUT": "10000"},
},
{
name: "interval alone",
otel: ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000"},
want: map[string]string{"OTEL_METRIC_EXPORT_INTERVAL": "10000"},
},
{
name: "timeout alone",
otel: ateomOTelSettings{Endpoint: endpoint, MetricExportTimeout: "10000"},
want: map[string]string{"OTEL_METRIC_EXPORT_TIMEOUT": "10000"},
},
{
name: "ignored without endpoint",
otel: ateomOTelSettings{MetricExportInterval: "10000", MetricExportTimeout: "10000"},
want: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel).
Spec.Template.Spec.Containers[0]
got, ok := envByName(c.Env)["OTEL_METRIC_EXPORT_INTERVAL"]
if ok != tt.wantSet {
t.Fatalf("OTEL_METRIC_EXPORT_INTERVAL present = %v, want %v", ok, tt.wantSet)
}
if ok && got.value != tt.want {
t.Errorf("OTEL_METRIC_EXPORT_INTERVAL = %q, want %q", got.value, tt.want)
env := envByName(c.Env)
for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} {
got, ok := env[k]
want, wantSet := tt.want[k]
if ok != wantSet {
t.Errorf("%s present = %v, want %v", k, ok, wantSet)
continue
}
if ok && got.value != want {
t.Errorf("%s = %q, want %q", k, got.value, want)
}
}
})
}
cmd/atecontroller/internal/controllers/workerpool_controller.go
+5 -1
View File
@@ -38,8 +38,11 @@ type WorkerPoolReconciler struct {
Scheme *runtime.Scheme
OTelEndpoint string
// OTelMetricExportInterval is the OTEL_METRIC_EXPORT_INTERVAL propagated to
// ateom pods. Empty keeps the SDK's 60s default.
// ateom pods. Empty keeps the SDK's default.
OTelMetricExportInterval string
// OTelMetricExportTimeout is the OTEL_METRIC_EXPORT_TIMEOUT propagated to
// ateom pods. Empty keeps the SDK's default.
OTelMetricExportTimeout string
}
//+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete
@@ -98,6 +101,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp
depAC := buildDeploymentApplyConfig(wp, ateomOTelSettings{
Endpoint: r.OTelEndpoint,
MetricExportInterval: r.OTelMetricExportInterval,
MetricExportTimeout: r.OTelMetricExportTimeout,
})
if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil {
return fmt.Errorf("failed to apply Deployment: %w", err)
cmd/atecontroller/main.go
+4
View File
@@ -45,6 +45,9 @@ var (
otelMetricExportInterval = pflag.String("otel-metric-export-interval", os.Getenv("OTEL_METRIC_EXPORT_INTERVAL"),
"Metric export interval in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 60s default. Defaults to the controller's own OTEL_METRIC_EXPORT_INTERVAL.")
otelMetricExportTimeout = pflag.String("otel-metric-export-timeout", os.Getenv("OTEL_METRIC_EXPORT_TIMEOUT"),
"Per-export timeout in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 30s default. Defaults to the controller's own OTEL_METRIC_EXPORT_TIMEOUT.")
ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.")
ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.")
ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.")
@@ -94,6 +97,7 @@ func main() {
Scheme: mgr.GetScheme(),
OTelEndpoint: *otelEndpoint,
OTelMetricExportInterval: *otelMetricExportInterval,
OTelMetricExportTimeout: *otelMetricExportTimeout,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "WorkerPool")
os.Exit(1)
manifests/ate-install/kind/kustomization.yaml
+16 -4
View File
@@ -30,10 +30,16 @@ resources:
# OTEL_METRIC_EXPORT_INTERVAL shortens the OTel SDK's 60s metric export tick to
# 10s. A component is invisible to the collector until its first tick, and the
# metrics e2e suite asserts against the collector's scrape on a bounded deadline,
# so the default leaves too little headroom on a loaded runner. On ate-controller
# it does double duty: the controller also propagates it to the ateom worker pods
# it creates, which have no other way to receive it. Kind only; production keeps
# the SDK default.
# so the default leaves too little headroom on a loaded runner.
#
# OTEL_METRIC_EXPORT_TIMEOUT bounds a tick that fails rather than one that
# succeeds, and its 30s default swallows three shortened intervals: a collector
# blip or slow DNS drops a component back to roughly one attempt per 30s, undoing
# the shorter interval. Matching it to the interval keeps every tick an attempt.
#
# On ate-controller both do double duty: the controller also propagates them to
# the ateom worker pods it creates, which have no other way to receive them.
# Kind only; production keeps the SDK defaults.
patches:
- patch: |-
apiVersion: apps/v1
@@ -51,6 +57,8 @@ patches:
value: http://opentelemetry-collector.otel-system.svc:4317
- name: OTEL_METRIC_EXPORT_INTERVAL
value: "10000"
- name: OTEL_METRIC_EXPORT_TIMEOUT
value: "10000"
- patch: |-
apiVersion: apps/v1
kind: DaemonSet
@@ -67,6 +75,8 @@ patches:
value: http://opentelemetry-collector.otel-system.svc:4317
- name: OTEL_METRIC_EXPORT_INTERVAL
value: "10000"
- name: OTEL_METRIC_EXPORT_TIMEOUT
value: "10000"
- patch: |-
apiVersion: apps/v1
kind: Deployment
@@ -83,3 +93,5 @@ patches:
value: http://opentelemetry-collector.otel-system.svc:4317
- name: OTEL_METRIC_EXPORT_INTERVAL
value: "10000"
- name: OTEL_METRIC_EXPORT_TIMEOUT
value: "10000"