mirror of
https://github.com/agent-substrate/substrate.git
synced 2026-10-02 03:24:42 +08:00
## Benchamark(atelet, ateom-microvm): log per-actor checkpoint and
restore phase breakdowns
### Why
`SuspendActor` and `ResumeActor` latency is only attributable down to
the `ate.actor.{checkpoint,restore}.duration` histogram phases, and the
biggest buckets — `ateom_checkpoint`, `ateom_restore` — are opaque. When
a large memory benchmark suspend takes 6 s we want to able to tell where
the time is been spent during the suspend.
### What
One joinable, developer-facing log record per operation per layer, with
the full actor identity (allowed in logs, barred from metric labels),
the snapshot scope, and one float-seconds field per phase that ran.
**atelet**
- `Checkpoint` now writes a `Checkpoint timing breakdown` record, the
same way `Restore` has written `Restore timing breakdown` since #1364.
Phases: `sandbox_assets`, `ateom_checkpoint`, `persist`, `total`. One
slice feeds both the histogram and the record, so they cannot disagree.
- Both records carry `error.type` when the operation failed (the gRPC
code; context errors map to `DeadlineExceeded` / `Canceled`). The record
is written on the way out of a failure too, so its completed phases are
kept, and the marker lets a reader exclude a timed-out restore from a
latency distribution. This restores what the record lost when the
`ateerrors` taxonomy was deleted (#1817), using the `error.type`
convention the ateapi instruments already follow.
**ateom-microvm**
- New `phaselog.go`. `CheckpointWorkload` and `RestoreWorkload` emit
records with the same two messages, under
`ateom.actor.checkpoint.duration.<phase>` and
`ateom.actor.restore.duration.<phase>`, decomposing atelet's
`ateom_checkpoint` / `ateom_restore` buckets:
- checkpoint: `pause`, `snapshot`, `durable_dir`, `rootfs_upper`,
`teardown`, `total` — the three captures run concurrently on the paused
guest, so the paused window costs their max, not their sum.
- restore: `prep`, `bundles`, `upper_join`, `lowers`, `tap`,
`vmm_launch`, `vm_restore`, `resume`, `wakeup_probe`, `total` —
sequential; they partition the total. A Data-scope cold boot records
`total` only.
- These timings already existed as ad-hoc `slog.Duration` fields on the
`Actor checkpointed` / `Actor restore phases` lines; those lines are
kept. The record adds stable keys, identity, and seconds (the
histograms' unit).
- The phase names are deliberately private to the binary rather than
added to `internal/ateattr`, so they cannot be mistaken for
`ate.snapshot.phase` metric values. They are micro-VM specific;
ateom-gvisor is unchanged.
**docs/observability.md** is updated: the Restore record is no longer
the only per-actor latency record, and the ateom records are described.
No new instruments, no registry changes, no behavior change.
### Example
```json
{"msg":"Checkpoint timing breakdown","ate.actor.uid":"8f2a…","ate.template.name":"glutton",
"ate.snapshot.scope":"full",
"ateom.actor.checkpoint.duration.pause":0.003,
"ateom.actor.checkpoint.duration.snapshot":0.846,
"ateom.actor.checkpoint.duration.rootfs_upper":0.022,
"ateom.actor.checkpoint.duration.teardown":0.232,
"ateom.actor.checkpoint.duration.total":1.081}
```
Joined with atelet's record for the same actor, a run of the glutton
workload (1 GiB resident, microVM) attributes a 6.0 s p50 suspend as 75%
`persist`, 19% `ateom_checkpoint` (of which the CH `snapshot` is 0.85 s
and `teardown` 0.23 s), and a 5.3 s p50 resume as 79% `download`, 18%
`ateom_restore` (of which `vm_restore` is 0.73 s). The consumer that
produces those tables from pod logs is a separate
`benchmarking/analysis` PR.
### Testing
- `go test ./cmd/atelet/...` and `./cmd/ateom-microvm/...` pass; new
unit tests cover the record shape (seconds, identity keys, zero phases
absent, no duplicate keys), the scope mapping, and `error.type` for
gRPC, context and plain errors.
- `GOOS=linux go vet` clean for both binaries; boilerplate and gofmt
clean.
183 lines
6.2 KiB
Go
183 lines
6.2 KiB
Go
//go:build linux
|
|
|
|
// Copyright 2026 Google LLC
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"log/slog"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/agent-substrate/substrate/internal/ateattr"
|
|
"github.com/agent-substrate/substrate/internal/proto/ateompb"
|
|
"github.com/agent-substrate/substrate/internal/resources"
|
|
)
|
|
|
|
func phaseLogAttribution() resources.ActorAttribution {
|
|
return resources.ActorAttribution{
|
|
Ref: resources.ActorRef{Atespace: "team-a", Name: "support-agent-42"},
|
|
UID: "uid-abc",
|
|
TemplateAtespace: "templates",
|
|
TemplateName: "support-agent",
|
|
}
|
|
}
|
|
|
|
// renderPhaseRecord logs attrs through a local JSON handler, so the test sees
|
|
// the record a collector would parse rather than the slice that built it.
|
|
func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any {
|
|
t.Helper()
|
|
var buf bytes.Buffer
|
|
slog.New(slog.NewJSONHandler(&buf, nil)).LogAttrs(context.Background(), slog.LevelInfo, "Checkpoint timing breakdown", attrs...)
|
|
var rec map[string]any
|
|
if err := json.Unmarshal(buf.Bytes(), &rec); err != nil {
|
|
t.Fatalf("unmarshal record %q: %v", buf.String(), err)
|
|
}
|
|
return rec
|
|
}
|
|
|
|
func TestSnapshotPhaseAttrs(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
attrs := snapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
|
|
checkpointDurationKey, nil, []phase{
|
|
{phasePrep, 40 * time.Millisecond},
|
|
{phasePause, 3 * time.Millisecond},
|
|
{phaseSnapshot, 850 * time.Millisecond},
|
|
// A capture that did not run stays off the record.
|
|
{phaseDurableDir, 0},
|
|
{phaseTeardown, 230 * time.Millisecond},
|
|
{phaseTotal, 1250 * time.Millisecond},
|
|
})
|
|
|
|
// json.Unmarshal keeps the last of a repeated key, so a duplicate is
|
|
// invisible in the rendered record and has to be caught on the slice.
|
|
seen := make(map[string]bool, len(attrs))
|
|
for _, attr := range attrs {
|
|
if seen[attr.Key] {
|
|
t.Errorf("key %s emitted twice", attr.Key)
|
|
}
|
|
seen[attr.Key] = true
|
|
}
|
|
|
|
rec := renderPhaseRecord(t, attrs)
|
|
for k, want := range map[string]string{
|
|
"ate.atespace": "team-a",
|
|
"ate.actor.name": "support-agent-42",
|
|
"ate.actor.uid": "uid-abc",
|
|
"ate.template.atespace": "templates",
|
|
"ate.template.name": "support-agent",
|
|
"ate.snapshot.scope": ateattr.SnapshotScopeFull,
|
|
} {
|
|
if got, ok := rec[k]; !ok {
|
|
t.Errorf("missing %s", k)
|
|
} else if got != want {
|
|
t.Errorf("%s = %v, want %q", k, got, want)
|
|
}
|
|
}
|
|
// Seconds, not slog.Duration's nanoseconds: the keys extend the atelet
|
|
// histograms' names, and those declare unit s.
|
|
for k, want := range map[string]float64{
|
|
"ateom.actor.checkpoint.duration.prep": 0.04,
|
|
"ateom.actor.checkpoint.duration.pause": 0.003,
|
|
"ateom.actor.checkpoint.duration.snapshot": 0.85,
|
|
"ateom.actor.checkpoint.duration.teardown": 0.23,
|
|
"ateom.actor.checkpoint.duration.total": 1.25,
|
|
} {
|
|
got, ok := rec[k].(float64)
|
|
if !ok {
|
|
t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k])
|
|
} else if got != want {
|
|
t.Errorf("%s = %v, want %v", k, got, want)
|
|
}
|
|
}
|
|
for _, k := range []string{
|
|
"ateom.actor.checkpoint.duration.durable_dir",
|
|
// The phase key names the one step a datapoint timed; this record
|
|
// carries them all.
|
|
"ate.snapshot.phase",
|
|
// Absent error.type is success, as on the instruments.
|
|
"error.type",
|
|
} {
|
|
if v, ok := rec[k]; ok {
|
|
t.Errorf("%s is present with %v, want absent", k, v)
|
|
}
|
|
}
|
|
}
|
|
|
|
// TestSnapshotPhaseAttrsFailure: a checkpoint that died keeps the phases it
|
|
// completed and is marked, so a reader can exclude it from percentiles.
|
|
func TestSnapshotPhaseAttrsFailure(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
name string
|
|
err error
|
|
want string
|
|
}{
|
|
{"a wrapped context deadline reports DeadlineExceeded",
|
|
fmt.Errorf("while snapshotting guest: %w", context.DeadlineExceeded), "DeadlineExceeded"},
|
|
{"a wrapped context cancellation reports Canceled",
|
|
fmt.Errorf("while pausing guest: %w", context.Canceled), "Canceled"},
|
|
{"a plain error is a bounded Unknown, not its message",
|
|
fmt.Errorf("while clearing checkpoint dir: disk on fire"), "Unknown"},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
rec := renderPhaseRecord(t, snapshotPhaseAttrs(phaseLogAttribution(),
|
|
ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, checkpointDurationKey, tt.err,
|
|
[]phase{{phasePrep, 40 * time.Millisecond}, {phasePause, 3 * time.Millisecond}, {phaseTotal, 30 * time.Second}}))
|
|
if got := rec["error.type"]; got != tt.want {
|
|
t.Errorf("error.type = %v, want %q", got, tt.want)
|
|
}
|
|
// The phases that ran before the failure are still on the record;
|
|
// the ones that never started are not.
|
|
if _, ok := rec["ateom.actor.checkpoint.duration.pause"]; !ok {
|
|
t.Error("missing ateom.actor.checkpoint.duration.pause")
|
|
}
|
|
if v, ok := rec["ateom.actor.checkpoint.duration.snapshot"]; ok {
|
|
t.Errorf("snapshot phase present with %v, want absent", v)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
// TestScopeLogValue pins the mapping onto the shared scope values, so the
|
|
// ateom and atelet records of one operation agree on the scope they carry.
|
|
func TestScopeLogValue(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
tests := []struct {
|
|
scope ateompb.SnapshotScope
|
|
want string
|
|
}{
|
|
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateattr.SnapshotScopeFull},
|
|
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateattr.SnapshotScopeData},
|
|
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, ateattr.SnapshotScopeDataOnGolden},
|
|
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, ateattr.SnapshotScopeUnknown},
|
|
{ateompb.SnapshotScope(99), ateattr.SnapshotScopeUnknown},
|
|
}
|
|
for _, tt := range tests {
|
|
if got := scopeLogValue(tt.scope); got != tt.want {
|
|
t.Errorf("scopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want)
|
|
}
|
|
}
|
|
}
|