(otel): onboard ateom to the OTLP metrics path (#562)

This PR wires `ateom` into the OTLP push path so it can emit metrics,
and unblock unblocking
https://github.com/agent-substrate/substrate/issues/550

- serverboot: new InitMetricsPushOnly (OTLP periodic reader, no
Prometheus pull surface) for binaries that run no metrics HTTP server
- ateom-gvisor + ateom-microvm call it, so the existing otelgrpc server
handler now feeds `rpc.server.*` to the collector
- atecontroller propagates its `OTEL_EXPORTER_OTLP_ENDPOINT` +
`OTEL_RESOURCE_ATTRIBUTES` into the ateom worker pods
- fixes ateom telemetry not reaching the collector 
- e2e: metrics suite asserts an `ateom` service reaches the collector

cc. @git286 

- [x] Tests pass
- [ ] Appropriate changes to documentation are included in the PR
This commit is contained in:
Krisztian F
2026-07-29 10:25:37 -04:00
committed by GitHub
parent 6fc3467ac5
commit acf5c12df4
12 changed files with 236 additions and 26 deletions
+24 -7
View File
@@ -13,9 +13,9 @@
// limitations under the License.
// Package serverboot collects the startup boilerplate shared by the
// long-running substrate server binaries (ateapi, atelet, ateom-gvisor):
// slog wiring, OTel tracer + meter providers, a Prometheus + /readyz
// HTTP surface, and a couple of small helpers for startup fail-fast.
// long-running substrate server binaries (ateapi, atelet, ateom-gvisor,
// ateom-microvm): slog wiring, OTel tracer + meter providers, a Prometheus +
// /readyz HTTP surface, and a couple of small helpers for startup fail-fast.
package serverboot
import (
@@ -130,6 +130,20 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi
if err != nil {
return nil, fmt.Errorf("create Prometheus metric exporter: %w", err)
}
return newMeterProvider(ctx, serviceName, promExporter)
}
// InitMetricsPushOnly is InitMetrics without the Prometheus reader, for binaries
// that run no metrics HTTP server (ateom): a pull reader would collect into a
// registry nothing serves.
func InitMetricsPushOnly(ctx context.Context, serviceName string) (*sdkmetric.MeterProvider, error) {
return newMeterProvider(ctx, serviceName)
}
func newMeterProvider(ctx context.Context, serviceName string, extraReaders ...sdkmetric.Reader) (*sdkmetric.MeterProvider, error) {
if serviceName == "" {
return nil, fmt.Errorf("serviceName is required")
}
otlpExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithInsecure())
if err != nil {
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
@@ -138,11 +152,14 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi
if err != nil {
return nil, fmt.Errorf("create metric resource: %w", err)
}
mp := sdkmetric.NewMeterProvider(
sdkmetric.WithReader(promExporter),
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(otlpExporter)),
opts := []sdkmetric.Option{
sdkmetric.WithResource(res),
)
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(otlpExporter)),
}
for _, r := range extraReaders {
opts = append(opts, sdkmetric.WithReader(r))
}
mp := sdkmetric.NewMeterProvider(opts...)
otel.SetMeterProvider(mp)
return mp, nil
}
+43
View File
@@ -18,8 +18,11 @@ import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.opentelemetry.io/otel/sdk/resource"
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
)
@@ -107,6 +110,46 @@ func TestHealthzAbsentUnlessEnabled(t *testing.T) {
}
}
func TestInitMetricsPushOnlyHasNoPrometheusSurface(t *testing.T) {
mp, err := InitMetricsPushOnly(context.Background(), "test-pushonly")
if err != nil {
t.Fatalf("InitMetricsPushOnly: %v", err)
}
// Bound shutdown: the periodic reader would otherwise block flushing to the
// unreachable default OTLP endpoint until the export timeout.
t.Cleanup(func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
_ = mp.Shutdown(ctx)
})
ctr, err := mp.Meter("test").Int64Counter("ate.test.pushonly.count")
if err != nil {
t.Fatalf("create counter: %v", err)
}
ctr.Add(context.Background(), 1)
// A push-only provider registers no Prometheus reader, so what it records must
// not surface on the default registry StartMetricsServer's /metrics serves.
rec := httptest.NewRecorder()
promhttp.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
if strings.Contains(rec.Body.String(), "ate_test_pushonly") {
t.Error("push-only MeterProvider must not expose a Prometheus pull surface")
}
}
func TestInitMetricsPushOnlyRequiresServiceName(t *testing.T) {
if _, err := InitMetricsPushOnly(context.Background(), ""); err == nil {
t.Error("InitMetricsPushOnly(\"\") must return an error")
}
}
func TestInitMetricsRequiresServiceName(t *testing.T) {
if _, err := InitMetrics(context.Background(), ""); err == nil {
t.Error("InitMetrics(\"\") must return an error")
}
}
func getCode(t *testing.T, mux *http.ServeMux, path string) int {
t.Helper()
rec := httptest.NewRecorder()