mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
(otel): onboard ateom to the OTLP metrics path (#562)
This PR wires `ateom` into the OTLP push path so it can emit metrics, and unblock unblocking https://github.com/agent-substrate/substrate/issues/550 - serverboot: new InitMetricsPushOnly (OTLP periodic reader, no Prometheus pull surface) for binaries that run no metrics HTTP server - ateom-gvisor + ateom-microvm call it, so the existing otelgrpc server handler now feeds `rpc.server.*` to the collector - atecontroller propagates its `OTEL_EXPORTER_OTLP_ENDPOINT` + `OTEL_RESOURCE_ATTRIBUTES` into the ateom worker pods - fixes ateom telemetry not reaching the collector - e2e: metrics suite asserts an `ateom` service reaches the collector cc. @git286 - [x] Tests pass - [ ] Appropriate changes to documentation are included in the PR
This commit is contained in:
@@ -13,9 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
// Package serverboot collects the startup boilerplate shared by the
|
||||
// long-running substrate server binaries (ateapi, atelet, ateom-gvisor):
|
||||
// slog wiring, OTel tracer + meter providers, a Prometheus + /readyz
|
||||
// HTTP surface, and a couple of small helpers for startup fail-fast.
|
||||
// long-running substrate server binaries (ateapi, atelet, ateom-gvisor,
|
||||
// ateom-microvm): slog wiring, OTel tracer + meter providers, a Prometheus +
|
||||
// /readyz HTTP surface, and a couple of small helpers for startup fail-fast.
|
||||
package serverboot
|
||||
|
||||
import (
|
||||
@@ -130,6 +130,20 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create Prometheus metric exporter: %w", err)
|
||||
}
|
||||
return newMeterProvider(ctx, serviceName, promExporter)
|
||||
}
|
||||
|
||||
// InitMetricsPushOnly is InitMetrics without the Prometheus reader, for binaries
|
||||
// that run no metrics HTTP server (ateom): a pull reader would collect into a
|
||||
// registry nothing serves.
|
||||
func InitMetricsPushOnly(ctx context.Context, serviceName string) (*sdkmetric.MeterProvider, error) {
|
||||
return newMeterProvider(ctx, serviceName)
|
||||
}
|
||||
|
||||
func newMeterProvider(ctx context.Context, serviceName string, extraReaders ...sdkmetric.Reader) (*sdkmetric.MeterProvider, error) {
|
||||
if serviceName == "" {
|
||||
return nil, fmt.Errorf("serviceName is required")
|
||||
}
|
||||
otlpExporter, err := otlpmetricgrpc.New(ctx, otlpmetricgrpc.WithInsecure())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create OTLP metric exporter: %w", err)
|
||||
@@ -138,11 +152,14 @@ func InitMetrics(ctx context.Context, serviceName string) (*sdkmetric.MeterProvi
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("create metric resource: %w", err)
|
||||
}
|
||||
mp := sdkmetric.NewMeterProvider(
|
||||
sdkmetric.WithReader(promExporter),
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(otlpExporter)),
|
||||
opts := []sdkmetric.Option{
|
||||
sdkmetric.WithResource(res),
|
||||
)
|
||||
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(otlpExporter)),
|
||||
}
|
||||
for _, r := range extraReaders {
|
||||
opts = append(opts, sdkmetric.WithReader(r))
|
||||
}
|
||||
mp := sdkmetric.NewMeterProvider(opts...)
|
||||
otel.SetMeterProvider(mp)
|
||||
return mp, nil
|
||||
}
|
||||
|
||||
@@ -18,8 +18,11 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.opentelemetry.io/otel/sdk/resource"
|
||||
semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
|
||||
)
|
||||
@@ -107,6 +110,46 @@ func TestHealthzAbsentUnlessEnabled(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitMetricsPushOnlyHasNoPrometheusSurface(t *testing.T) {
|
||||
mp, err := InitMetricsPushOnly(context.Background(), "test-pushonly")
|
||||
if err != nil {
|
||||
t.Fatalf("InitMetricsPushOnly: %v", err)
|
||||
}
|
||||
// Bound shutdown: the periodic reader would otherwise block flushing to the
|
||||
// unreachable default OTLP endpoint until the export timeout.
|
||||
t.Cleanup(func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
_ = mp.Shutdown(ctx)
|
||||
})
|
||||
|
||||
ctr, err := mp.Meter("test").Int64Counter("ate.test.pushonly.count")
|
||||
if err != nil {
|
||||
t.Fatalf("create counter: %v", err)
|
||||
}
|
||||
ctr.Add(context.Background(), 1)
|
||||
|
||||
// A push-only provider registers no Prometheus reader, so what it records must
|
||||
// not surface on the default registry StartMetricsServer's /metrics serves.
|
||||
rec := httptest.NewRecorder()
|
||||
promhttp.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/metrics", nil))
|
||||
if strings.Contains(rec.Body.String(), "ate_test_pushonly") {
|
||||
t.Error("push-only MeterProvider must not expose a Prometheus pull surface")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitMetricsPushOnlyRequiresServiceName(t *testing.T) {
|
||||
if _, err := InitMetricsPushOnly(context.Background(), ""); err == nil {
|
||||
t.Error("InitMetricsPushOnly(\"\") must return an error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitMetricsRequiresServiceName(t *testing.T) {
|
||||
if _, err := InitMetrics(context.Background(), ""); err == nil {
|
||||
t.Error("InitMetrics(\"\") must return an error")
|
||||
}
|
||||
}
|
||||
|
||||
func getCode(t *testing.T, mux *http.ServeMux, path string) int {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
Reference in New Issue
Block a user