Files
substrate/cmd/ateom-microvm/phaselog_test.go
T
Lucky Abolorunke 9c1f0ba042 benchmark(atelet, ateom-microvm): log per-actor checkpoint and restor… (#1941)
## Benchamark(atelet, ateom-microvm): log per-actor checkpoint and
restore phase breakdowns

### Why

`SuspendActor` and `ResumeActor` latency is only attributable down to
the `ate.actor.{checkpoint,restore}.duration` histogram phases, and the
biggest buckets — `ateom_checkpoint`, `ateom_restore` — are opaque. When
a large memory benchmark suspend takes 6 s we want to able to tell where
the time is been spent during the suspend.

### What

One joinable, developer-facing log record per operation per layer, with
the full actor identity (allowed in logs, barred from metric labels),
the snapshot scope, and one float-seconds field per phase that ran.

**atelet**
- `Checkpoint` now writes a `Checkpoint timing breakdown` record, the
same way `Restore` has written `Restore timing breakdown` since #1364.
Phases: `sandbox_assets`, `ateom_checkpoint`, `persist`, `total`. One
slice feeds both the histogram and the record, so they cannot disagree.
- Both records carry `error.type` when the operation failed (the gRPC
code; context errors map to `DeadlineExceeded` / `Canceled`). The record
is written on the way out of a failure too, so its completed phases are
kept, and the marker lets a reader exclude a timed-out restore from a
latency distribution. This restores what the record lost when the
`ateerrors` taxonomy was deleted (#1817), using the `error.type`
convention the ateapi instruments already follow.

**ateom-microvm**
- New `phaselog.go`. `CheckpointWorkload` and `RestoreWorkload` emit
records with the same two messages, under
`ateom.actor.checkpoint.duration.<phase>` and
`ateom.actor.restore.duration.<phase>`, decomposing atelet's
`ateom_checkpoint` / `ateom_restore` buckets:
- checkpoint: `pause`, `snapshot`, `durable_dir`, `rootfs_upper`,
`teardown`, `total` — the three captures run concurrently on the paused
guest, so the paused window costs their max, not their sum.
- restore: `prep`, `bundles`, `upper_join`, `lowers`, `tap`,
`vmm_launch`, `vm_restore`, `resume`, `wakeup_probe`, `total` —
sequential; they partition the total. A Data-scope cold boot records
`total` only.
- These timings already existed as ad-hoc `slog.Duration` fields on the
`Actor checkpointed` / `Actor restore phases` lines; those lines are
kept. The record adds stable keys, identity, and seconds (the
histograms' unit).
- The phase names are deliberately private to the binary rather than
added to `internal/ateattr`, so they cannot be mistaken for
`ate.snapshot.phase` metric values. They are micro-VM specific;
ateom-gvisor is unchanged.

**docs/observability.md** is updated: the Restore record is no longer
the only per-actor latency record, and the ateom records are described.

No new instruments, no registry changes, no behavior change.

### Example

```json
{"msg":"Checkpoint timing breakdown","ate.actor.uid":"8f2a…","ate.template.name":"glutton",
 "ate.snapshot.scope":"full",
 "ateom.actor.checkpoint.duration.pause":0.003,
 "ateom.actor.checkpoint.duration.snapshot":0.846,
 "ateom.actor.checkpoint.duration.rootfs_upper":0.022,
 "ateom.actor.checkpoint.duration.teardown":0.232,
 "ateom.actor.checkpoint.duration.total":1.081}
```

Joined with atelet's record for the same actor, a run of the glutton
workload (1 GiB resident, microVM) attributes a 6.0 s p50 suspend as 75%
`persist`, 19% `ateom_checkpoint` (of which the CH `snapshot` is 0.85 s
and `teardown` 0.23 s), and a 5.3 s p50 resume as 79% `download`, 18%
`ateom_restore` (of which `vm_restore` is 0.73 s). The consumer that
produces those tables from pod logs is a separate
`benchmarking/analysis` PR.

### Testing

- `go test ./cmd/atelet/...` and `./cmd/ateom-microvm/...` pass; new
unit tests cover the record shape (seconds, identity keys, zero phases
absent, no duplicate keys), the scope mapping, and `error.type` for
gRPC, context and plain errors.
- `GOOS=linux go vet` clean for both binaries; boilerplate and gofmt
clean.
2026-09-30 10:44:20 +00:00

183 lines
6.2 KiB
Go

//go:build linux
// Copyright 2026 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"log/slog"
"testing"
"time"
"github.com/agent-substrate/substrate/internal/ateattr"
"github.com/agent-substrate/substrate/internal/proto/ateompb"
"github.com/agent-substrate/substrate/internal/resources"
)
func phaseLogAttribution() resources.ActorAttribution {
return resources.ActorAttribution{
Ref: resources.ActorRef{Atespace: "team-a", Name: "support-agent-42"},
UID: "uid-abc",
TemplateAtespace: "templates",
TemplateName: "support-agent",
}
}
// renderPhaseRecord logs attrs through a local JSON handler, so the test sees
// the record a collector would parse rather than the slice that built it.
func renderPhaseRecord(t *testing.T, attrs []slog.Attr) map[string]any {
t.Helper()
var buf bytes.Buffer
slog.New(slog.NewJSONHandler(&buf, nil)).LogAttrs(context.Background(), slog.LevelInfo, "Checkpoint timing breakdown", attrs...)
var rec map[string]any
if err := json.Unmarshal(buf.Bytes(), &rec); err != nil {
t.Fatalf("unmarshal record %q: %v", buf.String(), err)
}
return rec
}
func TestSnapshotPhaseAttrs(t *testing.T) {
t.Parallel()
attrs := snapshotPhaseAttrs(phaseLogAttribution(), ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL,
checkpointDurationKey, nil, []phase{
{phasePrep, 40 * time.Millisecond},
{phasePause, 3 * time.Millisecond},
{phaseSnapshot, 850 * time.Millisecond},
// A capture that did not run stays off the record.
{phaseDurableDir, 0},
{phaseTeardown, 230 * time.Millisecond},
{phaseTotal, 1250 * time.Millisecond},
})
// json.Unmarshal keeps the last of a repeated key, so a duplicate is
// invisible in the rendered record and has to be caught on the slice.
seen := make(map[string]bool, len(attrs))
for _, attr := range attrs {
if seen[attr.Key] {
t.Errorf("key %s emitted twice", attr.Key)
}
seen[attr.Key] = true
}
rec := renderPhaseRecord(t, attrs)
for k, want := range map[string]string{
"ate.atespace": "team-a",
"ate.actor.name": "support-agent-42",
"ate.actor.uid": "uid-abc",
"ate.template.atespace": "templates",
"ate.template.name": "support-agent",
"ate.snapshot.scope": ateattr.SnapshotScopeFull,
} {
if got, ok := rec[k]; !ok {
t.Errorf("missing %s", k)
} else if got != want {
t.Errorf("%s = %v, want %q", k, got, want)
}
}
// Seconds, not slog.Duration's nanoseconds: the keys extend the atelet
// histograms' names, and those declare unit s.
for k, want := range map[string]float64{
"ateom.actor.checkpoint.duration.prep": 0.04,
"ateom.actor.checkpoint.duration.pause": 0.003,
"ateom.actor.checkpoint.duration.snapshot": 0.85,
"ateom.actor.checkpoint.duration.teardown": 0.23,
"ateom.actor.checkpoint.duration.total": 1.25,
} {
got, ok := rec[k].(float64)
if !ok {
t.Errorf("%s = %v (%T), want a number", k, rec[k], rec[k])
} else if got != want {
t.Errorf("%s = %v, want %v", k, got, want)
}
}
for _, k := range []string{
"ateom.actor.checkpoint.duration.durable_dir",
// The phase key names the one step a datapoint timed; this record
// carries them all.
"ate.snapshot.phase",
// Absent error.type is success, as on the instruments.
"error.type",
} {
if v, ok := rec[k]; ok {
t.Errorf("%s is present with %v, want absent", k, v)
}
}
}
// TestSnapshotPhaseAttrsFailure: a checkpoint that died keeps the phases it
// completed and is marked, so a reader can exclude it from percentiles.
func TestSnapshotPhaseAttrsFailure(t *testing.T) {
t.Parallel()
tests := []struct {
name string
err error
want string
}{
{"a wrapped context deadline reports DeadlineExceeded",
fmt.Errorf("while snapshotting guest: %w", context.DeadlineExceeded), "DeadlineExceeded"},
{"a wrapped context cancellation reports Canceled",
fmt.Errorf("while pausing guest: %w", context.Canceled), "Canceled"},
{"a plain error is a bounded Unknown, not its message",
fmt.Errorf("while clearing checkpoint dir: disk on fire"), "Unknown"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
rec := renderPhaseRecord(t, snapshotPhaseAttrs(phaseLogAttribution(),
ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, checkpointDurationKey, tt.err,
[]phase{{phasePrep, 40 * time.Millisecond}, {phasePause, 3 * time.Millisecond}, {phaseTotal, 30 * time.Second}}))
if got := rec["error.type"]; got != tt.want {
t.Errorf("error.type = %v, want %q", got, tt.want)
}
// The phases that ran before the failure are still on the record;
// the ones that never started are not.
if _, ok := rec["ateom.actor.checkpoint.duration.pause"]; !ok {
t.Error("missing ateom.actor.checkpoint.duration.pause")
}
if v, ok := rec["ateom.actor.checkpoint.duration.snapshot"]; ok {
t.Errorf("snapshot phase present with %v, want absent", v)
}
})
}
}
// TestScopeLogValue pins the mapping onto the shared scope values, so the
// ateom and atelet records of one operation agree on the scope they carry.
func TestScopeLogValue(t *testing.T) {
t.Parallel()
tests := []struct {
scope ateompb.SnapshotScope
want string
}{
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_FULL, ateattr.SnapshotScopeFull},
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA, ateattr.SnapshotScopeData},
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_DATA_ON_GOLDEN, ateattr.SnapshotScopeDataOnGolden},
{ateompb.SnapshotScope_SNAPSHOT_SCOPE_UNSPECIFIED, ateattr.SnapshotScopeUnknown},
{ateompb.SnapshotScope(99), ateattr.SnapshotScopeUnknown},
}
for _, tt := range tests {
if got := scopeLogValue(tt.scope); got != tt.want {
t.Errorf("scopeLogValue(%v) = %q, want %q", tt.scope, got, tt.want)
}
}
}