From 1b329c9872c0a3fdb32ceb5f1109f0650fd17154 Mon Sep 17 00:00:00 2001 From: Lior Lieberman Date: Fri, 31 Jul 2026 08:10:03 -0700 Subject: [PATCH] add otel export timeout --- .../internal/controllers/workerpool_apply.go | 8 +++ .../controllers/workerpool_apply_test.go | 66 +++++++++++++------ .../controllers/workerpool_controller.go | 6 +- cmd/atecontroller/main.go | 4 ++ manifests/ate-install/kind/kustomization.yaml | 20 ++++-- 5 files changed, 80 insertions(+), 24 deletions(-) diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply.go b/cmd/atecontroller/internal/controllers/workerpool_apply.go index 11e7f552f..10ee3dc41 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply.go @@ -43,6 +43,9 @@ type ateomOTelSettings struct { // first export tick fires. Shortening the interval is what keeps that // startup gap inside an e2e budget; production leaves it unset. MetricExportInterval string + // MetricExportTimeout overrides the SDK's per-export timeout, in the same + // whole-millisecond form as MetricExportInterval. Empty keeps the default. + MetricExportTimeout string } // buildDeploymentApplyConfig constructs the SSA apply configuration for the @@ -118,6 +121,11 @@ func ateomContainerEnv(otel ateomOTelSettings) []*corev1ac.EnvVarApplyConfigurat WithName("OTEL_METRIC_EXPORT_INTERVAL"). WithValue(otel.MetricExportInterval)) } + if otel.MetricExportTimeout != "" { + envs = append(envs, corev1ac.EnvVar(). + WithName("OTEL_METRIC_EXPORT_TIMEOUT"). + WithValue(otel.MetricExportTimeout)) + } return envs } diff --git a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go index 8032675f0..bc05760a5 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_apply_test.go +++ b/cmd/atecontroller/internal/controllers/workerpool_apply_test.go @@ -360,7 +360,7 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } if !tt.wantTelemetry { - for _, k := range []string{"OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", "OTEL_METRIC_EXPORT_INTERVAL", "POD_NAME", "POD_NAMESPACE"} { + for _, k := range []string{"OTEL_EXPORTER_OTLP_ENDPOINT", "OTEL_RESOURCE_ATTRIBUTES", "OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT", "POD_NAME", "POD_NAMESPACE"} { if _, ok := env[k]; ok { t.Errorf("%s must be absent without an OTLP endpoint", k) } @@ -388,32 +388,60 @@ func TestBuildDeploymentApplyConfigOTelEndpoint(t *testing.T) { } } -// TestBuildDeploymentApplyConfigMetricExportInterval asserts the export interval -// reaches the ateom container only when both it and an endpoint are set. ateom is -// invisible to the collector until its first export tick, so the kind stack -// shortens the SDK's 60s default to keep that gap inside the e2e budget. -func TestBuildDeploymentApplyConfigMetricExportInterval(t *testing.T) { +// TestBuildDeploymentApplyConfigMetricExportTuning asserts the export interval and +// per-export timeout reach the ateom container only when each is set alongside an +// endpoint. ateom is invisible to the collector until its first successful export +// tick, so the kind stack shortens the SDK's 60s interval to keep that gap inside +// the e2e budget, and its 30s timeout so a failing tick cannot swallow three +// shortened intervals. +func TestBuildDeploymentApplyConfigMetricExportTuning(t *testing.T) { const endpoint = "http://collector.otel-system.svc:4317" tests := []struct { - name string - otel ateomOTelSettings - want string - wantSet bool + name string + otel ateomOTelSettings + want map[string]string // env name -> value; absent key means must not be set }{ - {"unset keeps SDK default", ateomOTelSettings{Endpoint: endpoint}, "", false}, - {"set with endpoint", ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000"}, "10000", true}, - {"ignored without endpoint", ateomOTelSettings{MetricExportInterval: "10000"}, "", false}, + { + name: "unset keeps SDK defaults", + otel: ateomOTelSettings{Endpoint: endpoint}, + want: nil, + }, + { + name: "both set with endpoint", + otel: ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000", MetricExportTimeout: "10000"}, + want: map[string]string{"OTEL_METRIC_EXPORT_INTERVAL": "10000", "OTEL_METRIC_EXPORT_TIMEOUT": "10000"}, + }, + { + name: "interval alone", + otel: ateomOTelSettings{Endpoint: endpoint, MetricExportInterval: "10000"}, + want: map[string]string{"OTEL_METRIC_EXPORT_INTERVAL": "10000"}, + }, + { + name: "timeout alone", + otel: ateomOTelSettings{Endpoint: endpoint, MetricExportTimeout: "10000"}, + want: map[string]string{"OTEL_METRIC_EXPORT_TIMEOUT": "10000"}, + }, + { + name: "ignored without endpoint", + otel: ateomOTelSettings{MetricExportInterval: "10000", MetricExportTimeout: "10000"}, + want: nil, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := buildDeploymentApplyConfig(testWorkerPoolApplyConfig(nil), tt.otel). Spec.Template.Spec.Containers[0] - got, ok := envByName(c.Env)["OTEL_METRIC_EXPORT_INTERVAL"] - if ok != tt.wantSet { - t.Fatalf("OTEL_METRIC_EXPORT_INTERVAL present = %v, want %v", ok, tt.wantSet) - } - if ok && got.value != tt.want { - t.Errorf("OTEL_METRIC_EXPORT_INTERVAL = %q, want %q", got.value, tt.want) + env := envByName(c.Env) + for _, k := range []string{"OTEL_METRIC_EXPORT_INTERVAL", "OTEL_METRIC_EXPORT_TIMEOUT"} { + got, ok := env[k] + want, wantSet := tt.want[k] + if ok != wantSet { + t.Errorf("%s present = %v, want %v", k, ok, wantSet) + continue + } + if ok && got.value != want { + t.Errorf("%s = %q, want %q", k, got.value, want) + } } }) } diff --git a/cmd/atecontroller/internal/controllers/workerpool_controller.go b/cmd/atecontroller/internal/controllers/workerpool_controller.go index 9f3ced203..27318c81c 100644 --- a/cmd/atecontroller/internal/controllers/workerpool_controller.go +++ b/cmd/atecontroller/internal/controllers/workerpool_controller.go @@ -38,8 +38,11 @@ type WorkerPoolReconciler struct { Scheme *runtime.Scheme OTelEndpoint string // OTelMetricExportInterval is the OTEL_METRIC_EXPORT_INTERVAL propagated to - // ateom pods. Empty keeps the SDK's 60s default. + // ateom pods. Empty keeps the SDK's default. OTelMetricExportInterval string + // OTelMetricExportTimeout is the OTEL_METRIC_EXPORT_TIMEOUT propagated to + // ateom pods. Empty keeps the SDK's default. + OTelMetricExportTimeout string } //+kubebuilder:rbac:groups=ate.dev,resources=workerpools,verbs=get;list;watch;create;update;patch;delete @@ -98,6 +101,7 @@ func (r *WorkerPoolReconciler) applyDeployment(ctx context.Context, wp *atev1alp depAC := buildDeploymentApplyConfig(wp, ateomOTelSettings{ Endpoint: r.OTelEndpoint, MetricExportInterval: r.OTelMetricExportInterval, + MetricExportTimeout: r.OTelMetricExportTimeout, }) if err := r.Apply(ctx, depAC, client.FieldOwner(workerPoolFieldOwner), client.ForceOwnership); err != nil { return fmt.Errorf("failed to apply Deployment: %w", err) diff --git a/cmd/atecontroller/main.go b/cmd/atecontroller/main.go index 7c5418b77..d7d49324b 100644 --- a/cmd/atecontroller/main.go +++ b/cmd/atecontroller/main.go @@ -45,6 +45,9 @@ var ( otelMetricExportInterval = pflag.String("otel-metric-export-interval", os.Getenv("OTEL_METRIC_EXPORT_INTERVAL"), "Metric export interval in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 60s default. Defaults to the controller's own OTEL_METRIC_EXPORT_INTERVAL.") + otelMetricExportTimeout = pflag.String("otel-metric-export-timeout", os.Getenv("OTEL_METRIC_EXPORT_TIMEOUT"), + "Per-export timeout in milliseconds set on ateom worker pods. Empty keeps the OTel SDK's 30s default. Defaults to the controller's own OTEL_METRIC_EXPORT_TIMEOUT.") + ateapiCAFile = pflag.String("ateapi-ca-file", ateapiauth.DefaultServiceAccountCAFile, "PEM file with CAs trusted to verify the ateapi server cert.") ateapiServerName = pflag.String("ateapi-server-name", "", "SNI / hostname expected on the ateapi server cert. Optional.") ateapiTokenAuth = pflag.Bool("ateapi-use-token-auth", false, "Authenticate to ateapi with the Bearer token from --ateapi-token-file instead of the client certificate from --ateapi-client-cert.") @@ -94,6 +97,7 @@ func main() { Scheme: mgr.GetScheme(), OTelEndpoint: *otelEndpoint, OTelMetricExportInterval: *otelMetricExportInterval, + OTelMetricExportTimeout: *otelMetricExportTimeout, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WorkerPool") os.Exit(1) diff --git a/manifests/ate-install/kind/kustomization.yaml b/manifests/ate-install/kind/kustomization.yaml index 43c65cfa0..16b30fbf0 100644 --- a/manifests/ate-install/kind/kustomization.yaml +++ b/manifests/ate-install/kind/kustomization.yaml @@ -30,10 +30,16 @@ resources: # OTEL_METRIC_EXPORT_INTERVAL shortens the OTel SDK's 60s metric export tick to # 10s. A component is invisible to the collector until its first tick, and the # metrics e2e suite asserts against the collector's scrape on a bounded deadline, -# so the default leaves too little headroom on a loaded runner. On ate-controller -# it does double duty: the controller also propagates it to the ateom worker pods -# it creates, which have no other way to receive it. Kind only; production keeps -# the SDK default. +# so the default leaves too little headroom on a loaded runner. +# +# OTEL_METRIC_EXPORT_TIMEOUT bounds a tick that fails rather than one that +# succeeds, and its 30s default swallows three shortened intervals: a collector +# blip or slow DNS drops a component back to roughly one attempt per 30s, undoing +# the shorter interval. Matching it to the interval keeps every tick an attempt. +# +# On ate-controller both do double duty: the controller also propagates them to +# the ateom worker pods it creates, which have no other way to receive them. +# Kind only; production keeps the SDK defaults. patches: - patch: |- apiVersion: apps/v1 @@ -51,6 +57,8 @@ patches: value: http://opentelemetry-collector.otel-system.svc:4317 - name: OTEL_METRIC_EXPORT_INTERVAL value: "10000" + - name: OTEL_METRIC_EXPORT_TIMEOUT + value: "10000" - patch: |- apiVersion: apps/v1 kind: DaemonSet @@ -67,6 +75,8 @@ patches: value: http://opentelemetry-collector.otel-system.svc:4317 - name: OTEL_METRIC_EXPORT_INTERVAL value: "10000" + - name: OTEL_METRIC_EXPORT_TIMEOUT + value: "10000" - patch: |- apiVersion: apps/v1 kind: Deployment @@ -83,3 +93,5 @@ patches: value: http://opentelemetry-collector.otel-system.svc:4317 - name: OTEL_METRIC_EXPORT_INTERVAL value: "10000" + - name: OTEL_METRIC_EXPORT_TIMEOUT + value: "10000"