diff --git a/benchmarking/README.md b/benchmarking/README.md index b285861f3..d01542e6c 100644 --- a/benchmarking/README.md +++ b/benchmarking/README.md @@ -160,12 +160,13 @@ step's think time, and the next step's first request **wakes it through the atenet router** (request parking). Mostly-idle sessions plus fast wake is exactly the oversubscription story this measures. -The entire workload is the declarative script in -[`internal/benchmarking/boomer/agentsession/script.go`](../internal/benchmarking/boomer/agentsession/script.go) -— one table entry per step, naming what the agent is doing and the resource -ops that act it out. To change the workload, edit the table. Each session is -an actor from the stock `glutton` template; steps are sequences of glutton -RPCs: +The entire workload is a YAML script. The default, +[`internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml`](../internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml), +has one entry per step naming what the agent is doing and the resource ops +that act it out. To change the workload, edit it or add a sibling file and +select it with `--agentsession-script` (see [Writing a +script](#writing-an-agent-session-script)). Each session is an actor from +the stock `glutton` template; steps are sequences of glutton RPCs: | Step | The agent is… | Sandbox effect | |---|---|---| @@ -193,9 +194,10 @@ RPCs: The script needs bigger actors than the 256Mi default: it holds ~96Mi of RAM arrays + ~110Mi of tmpfs files, and the observed guest peak with allocator transients is ~320Mi (512Mi OOMs). Deploy the workloads with -`--actor-memory 1Gi`. `TestSessionBudgets` bounds the script-declared bytes -only, so script edits that grow the working set fail the test and force this -guidance to be revisited. +`--actor-memory 1Gi`. Every script declares that floor as +`min_actor_memory`; the worker reads the `glutton` template's memory limit +at start and refuses to run a script against a smaller actor, so a too-small +deployment fails loudly instead of showing up as OOM-flaky steps. ```sh ./benchmarking/deploy_locust.sh --deploy --sandbox-class gvisor --actor-memory 1Gi @@ -204,6 +206,10 @@ guidance to be revisited. #### Agent-Session Configuration Knobs +* `--agentsession-script` — built-in script variant to run, by file name + under `internal/benchmarking/boomer/agentsession/scripts/` (default + `coding-session`). Read once, on the worker's first iteration after the + swarm starts, then fixed for the worker's lifetime. * `--agentsession-think-scale` — multiplier on every think gap; 0.5 makes the fleet twice as chatty, 4.0 models slow reasoning models (default 1.0). Each gap gets ±20% jitter so sessions don't move in lockstep. @@ -212,6 +218,50 @@ guidance to be revisited. * `--lifecycle-mode suspend|pause` — durable suspend (default) or node-local pause between steps. +#### Writing an agent-session script + +A script is a named step list with a memory floor: + +```yaml +name: coding-session +min_actor_memory: 1Gi +steps: + - name: 01_read_task + agent: Boots, reads the task prompt, loads its context window + think: 2s + ops: + - fill_ram: {key: agent_context, size: 32Mi} + - ping: {} + - name: 02_clone_repo + agent: git clone of the target repository + think: 3s + ops: + - ingest: {key: repo_tarball, size: 16Mi} + - burn_cpu: {millis: 500, parallel: 1} +``` + +`think` is the suspended gap before the step (a Go duration, scaled by +`--agentsession-think-scale`); `name` keys the `Step_` stats row. +Sizes are Kubernetes quantities. Each op is one glutton RPC: + +| Op | Arguments | Sandbox effect | +|---|---|---| +| `ingest` | `key`, `size` | bytes cross the network from the driver and land in a file: a download | +| `burn_cpu` | `millis`, `parallel` (default 1) | spins `parallel` goroutines for `millis` of wall clock | +| `write_disk` | `key`, `size` | writes locally generated bytes to a file | +| `read_disk_digest` | `key` | reads and hashes a file; nothing ships back | +| `read_disk_data` | `key` | reads a file and returns its bytes: disk read plus egress | +| `fill_ram` | `key`, `size` | allocates a resident RAM array | +| `churn_ram` | `key`, `size` | re-randomizes part of an array in place, dirtying pages | +| `walk_ram` | `key` | touches one byte per page: demand-paging cost after a resume | +| `ping` | none | a minimal round trip through the router | + +Loading is strict: unknown op kinds or fields, an argument a kind does not +take, a read of a file nothing wrote, a walk of an array nothing filled, a +duplicate step name, or a `min_actor_memory` below the declared RAM plus +disk all fail before any actor is created. Built-in variants are checked by +`TestEmbeddedScriptsAreValid`, so a broken file cannot merge. + #### Agent-Session Reported Metrics * `WakeFirstTouch`: latency of a dedicated ping sent before each step's ops — diff --git a/benchmarking/locust/common/agentsession_config.py b/benchmarking/locust/common/agentsession_config.py index a2b30d112..10fd03052 100644 --- a/benchmarking/locust/common/agentsession_config.py +++ b/benchmarking/locust/common/agentsession_config.py @@ -20,6 +20,18 @@ from locust.argument_parser import LocustArgumentParser @events.init_command_line_parser.add_listener def add_agentsession_arguments(parser: LocustArgumentParser) -> None: + parser.add_argument( + "--agentsession-script", + type=str, + default="coding-session", + env_var="LOCUST_AGENTSESSION_SCRIPT", + help=( + "Built-in agent-session script variant to run; one of the YAML " + "files under internal/benchmarking/boomer/agentsession/scripts/ " + "(default: coding-session)" + ), + include_in_web_ui=True, + ) parser.add_argument( "--agentsession-think-scale", type=float, diff --git a/benchmarking/locust/common/boomer_config.py b/benchmarking/locust/common/boomer_config.py index 15a4e77bb..3937a6734 100644 --- a/benchmarking/locust/common/boomer_config.py +++ b/benchmarking/locust/common/boomer_config.py @@ -67,6 +67,7 @@ _FLAGS = { "--sweperf-total-steps": int, "--sweperf-num-cycles": int, "--sweperf-poll-interval-ms": int, + "--agentsession-script": str, "--agentsession-think-scale": float, } diff --git a/internal/benchmarking/boomer/agentsession/agentsession.go b/internal/benchmarking/boomer/agentsession/agentsession.go index c4636e056..9e786c93d 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession.go +++ b/internal/benchmarking/boomer/agentsession/agentsession.go @@ -13,8 +13,9 @@ // limitations under the License. // Package agentsession implements the AgentSessionUser locust test: each VU -// is one coding-agent session replaying the scripted steps in script.go -// against a glutton actor. After every step the actor is suspended for the +// is one coding-agent session replaying a scripted step sequence (a YAML +// file under scripts/, selected by --agentsession-script) against a glutton +// actor. After every step the actor is suspended for the // step's "LLM thinking" gap; by default the next step's first request wakes // it through the atenet router (request parking), so the benchmark measures // the user-visible wake latency Substrate's oversubscription story rests on. @@ -55,6 +56,7 @@ import ( "google.golang.org/grpc/metadata" "google.golang.org/grpc/status" "google.golang.org/protobuf/proto" + "k8s.io/apimachinery/pkg/api/resource" ) const ( @@ -100,8 +102,7 @@ func initAgentSession(cfg *userclass.Config) (taskFn func(), shutdown func(conte if cfg.Tracer == nil { cfg.Tracer = otel.Tracer("substrate-boomer/agentsession") } - rt := &runtime{cfg: cfg, steps: Session()} - rt.ingestBuf = makeIngestBuf(rt.steps) + rt := &runtime{cfg: cfg} return rt.iterate, rt.shutdown } @@ -132,10 +133,101 @@ func makeIngestBuf(steps []Step) []byte { // goroutine keeps its own session in users, keyed by goroutine ID, because // boomer offers no per-VU context. type runtime struct { - cfg *userclass.Config - steps []Step - ingestBuf []byte // shared read-only ingest payload, sized to the largest ingest op - users sync.Map // goroutineID -> *sessionUser + cfg *userclass.Config + users sync.Map // goroutineID -> *sessionUser + + // script is resolved by loadScript on the first iteration and fixed for + // the worker's lifetime; ingestBuf is sized to it. + scriptMu sync.Mutex + script *Script + ingestBuf []byte // shared read-only ingest payload, sized to the largest ingest op + + // The template memory check runs once per worker. A fetch failure + // leaves it unanswered so the next startUser retries; a verdict is + // final until the workloads are redeployed. + templateMu sync.Mutex + templateOK bool + templateErr error +} + +// loadScript resolves the run's script on first use. It cannot happen at +// Init: the operator's knob values arrive with the first spawn message, +// after Init, so a load there would only ever see --config-json. A failed +// load is retried on the next iteration, so correcting the knob in the web +// UI and starting a new swarm recovers without a pod restart. +func (r *runtime) loadScript() (*Script, error) { + r.scriptMu.Lock() + defer r.scriptMu.Unlock() + if r.script != nil { + return r.script, nil + } + name := r.cfg.Dyn.Load().AgentSessionScript + if name == "" { + name = DefaultScript + } + s, err := Load(name) + if err != nil { + return nil, err + } + b := Budgets(s.Steps) + slog.Info("agentsession: loaded script", + slog.String("script", s.Name), + slog.Int("steps", len(s.Steps)), + slog.String("declared_ram", formatSize(b.RAM)), + slog.String("declared_disk", formatSize(b.Disk)), + slog.String("min_actor_memory", formatSize(s.MinActorMemory))) + r.script = s + r.ingestBuf = makeIngestBuf(s.Steps) + return s, nil +} + +// checkTemplateMemory refuses to start sessions against a template whose +// memory limit is below the script's min_actor_memory. A too-small actor +// OOMs partway through the script and shows up as flaky steps, which is far +// harder to read than a refusal at start. A template with no memory limit +// is allowed through with a warning. +func (r *runtime) checkTemplateMemory(ctx context.Context) error { + r.templateMu.Lock() + defer r.templateMu.Unlock() + if r.templateOK { + return nil + } + if r.templateErr != nil { + return r.templateErr + } + ref := &ateapipb.ObjectRef{Atespace: templateNS, Name: templateName} + tmpl, err := r.cfg.APIStub.GetActorTemplate(ctx, &ateapipb.GetActorTemplateRequest{ActorTemplate: ref}) + if err != nil { + return fmt.Errorf("GetActorTemplate %s/%s: %w", templateNS, templateName, err) + } + limit, ok := memoryLimit(tmpl) + switch { + case !ok: + slog.Warn("agentsession: template sets no memory limit; cannot check it against the script", + slog.String("template", templateName), + slog.String("min_actor_memory", formatSize(r.script.MinActorMemory))) + case limit < r.script.MinActorMemory: + r.templateErr = fmt.Errorf("template %s/%s memory limit %s is below script %q min_actor_memory %s; redeploy the workloads with --actor-memory %s", + templateNS, templateName, formatSize(limit), r.script.Name, formatSize(r.script.MinActorMemory), formatSize(r.script.MinActorMemory)) + return r.templateErr + } + r.templateOK = true + return nil +} + +// memoryLimit reads the template's memory limit in bytes, if it has one. +func memoryLimit(tmpl *ateapipb.ActorTemplate) (int64, bool) { + for _, l := range tmpl.GetResources().GetLimits() { + if l.GetName() != "memory" { + continue + } + q, err := resource.ParseQuantity(l.GetQuantity()) + if err != nil { + return 0, false + } + return q.AsInt64() + } + return 0, false } // think returns the suspended "LLM thinking" gap before step s, which is the @@ -155,6 +247,15 @@ func (r *runtime) think(s Step) time.Duration { // script completes, so one actor keeps producing suspend/resume cycles for // the whole run. func (r *runtime) iterate() { + script, err := r.loadScript() + if err != nil { + slog.Error("agentsession: cannot load script; goroutine will retry next iter", + slog.String("err", err.Error())) + time.Sleep(2 * time.Second) + return + } + steps := script.Steps + gid := boomerutil.GoroutineID() val, loaded := r.users.Load(gid) if !loaded { @@ -169,13 +270,13 @@ func (r *runtime) iterate() { } u := val.(*sessionUser) - step := r.steps[u.stepIndex] + step := steps[u.stepIndex] // The LLM is "thinking": the actor stays suspended for the gap. time.Sleep(r.think(step)) if u.runStep(context.Background(), step) { - u.stepIndex = (u.stepIndex + 1) % len(r.steps) + u.stepIndex = (u.stepIndex + 1) % len(steps) if u.stepIndex == 0 { slog.Info("agent session completed the full script; starting over", slog.String("actor", u.actorName)) @@ -195,6 +296,9 @@ func (r *runtime) iterate() { // then suspends it so the first scripted step begins — like every later // step — with a wake from suspension. func (r *runtime) startUser(ctx context.Context) (*sessionUser, error) { + if err := r.checkTemplateMemory(ctx); err != nil { + return nil, err + } u := &sessionUser{ cfg: r.cfg, actorName: "agent-" + uuid.NewString(), diff --git a/internal/benchmarking/boomer/agentsession/agentsession_test.go b/internal/benchmarking/boomer/agentsession/agentsession_test.go index 210a066af..8447b9d25 100644 --- a/internal/benchmarking/boomer/agentsession/agentsession_test.go +++ b/internal/benchmarking/boomer/agentsession/agentsession_test.go @@ -17,6 +17,9 @@ package agentsession import ( "context" "net/http" + "reflect" + "slices" + "strings" "sync" "testing" "time" @@ -31,108 +34,169 @@ import ( "google.golang.org/grpc/status" ) -// TestSessionScriptIsWellFormed pins the script's invariants: unique step -// names, positive think times, and no op that consumes a sandbox object -// before an earlier step created it. A broken ordering would fail at run -// time with NotFound from glutton; this catches it at test time. -func TestSessionScriptIsWellFormed(t *testing.T) { - steps := Session() - if len(steps) != 20 { - t.Fatalf("Session has %d steps, want 20", len(steps)) +// TestEmbeddedScriptsAreValid keeps every built-in variant loadable: a file +// under scripts/ that fails Validate cannot merge. +func TestEmbeddedScriptsAreValid(t *testing.T) { + names := Names() + if !slices.Contains(names, DefaultScript) { + t.Fatalf("Names() = %v, missing DefaultScript %q", names, DefaultScript) } + for _, n := range names { + if _, err := Load(n); err != nil { + t.Errorf("Load(%q): %v", n, err) + } + } + if _, err := Load("no-such-script"); err == nil { + t.Error("Load of an unknown name must fail") + } +} - seen := map[string]bool{} - ramFilled := map[string]bool{} - diskWritten := map[string]bool{} +// TestDefaultScriptShape pins what the README promises about the default +// variant: 20 steps, a 1Gi floor, and declared bytes that stay well under +// it. The declared budget is far below the floor on purpose: the guest's +// real peak (kernel, kata-agent, allocator transients) sits on top of it. +func TestDefaultScriptShape(t *testing.T) { + s, err := Load(DefaultScript) + if err != nil { + t.Fatal(err) + } + if len(s.Steps) != 20 { + t.Errorf("default script has %d steps, want 20", len(s.Steps)) + } + if s.MinActorMemory != 1<<30 { + t.Errorf("min_actor_memory = %d, want 1Gi", s.MinActorMemory) + } + b := Budgets(s.Steps) + if b.RAM > 128<<20 || b.Disk > 256<<20 { + t.Errorf("declared RAM %d / disk %d outgrew the 128Mi / 256Mi budgets; revisit the memory guidance", b.RAM, b.Disk) + } +} - for i, s := range steps { - if s.Name == "" || s.Agent == "" { - t.Errorf("step %d: Name and Agent must be set", i) - } - if seen[s.Name] { - t.Errorf("step %q: duplicate name", s.Name) - } - seen[s.Name] = true - if s.Think <= 0 { - t.Errorf("step %q: think time must be positive", s.Name) - } - if len(s.Ops) == 0 { - t.Errorf("step %q: has no ops", s.Name) - } - for _, o := range s.Ops { - switch o.kind { - case opFillRAM: - ramFilled[o.key] = true - case opChurnRAM, opWalkRAM: - if !ramFilled[o.key] { - t.Errorf("step %q: %s RAM op before any fill of %q", s.Name, opName(o.kind), o.key) - } - case opIngest, opWriteDisk: - diskWritten[o.key] = true - case opReadDiskDigest, opReadDiskData: - if !diskWritten[o.key] { - t.Errorf("step %q: read of %q before any write", s.Name, o.key) - } +// TestEncodeDecodeRoundTrip proves Encode writes exactly what Decode reads, +// so a script dumped from Go and one authored by hand are the same thing. +func TestEncodeDecodeRoundTrip(t *testing.T) { + s, err := Load(DefaultScript) + if err != nil { + t.Fatal(err) + } + out, err := Encode(s) + if err != nil { + t.Fatal(err) + } + back, err := Decode(out) + if err != nil { + t.Fatalf("Decode(Encode(s)): %v\n%s", err, out) + } + if !reflect.DeepEqual(s, back) { + t.Errorf("round trip changed the script:\n%s", out) + } +} + +const validScript = ` +name: tiny +min_actor_memory: 64Mi +steps: +- name: 01_fill + agent: fills + think: 1s + ops: + - fill_ram: {key: ctx, size: 4Mi} + - ingest: {key: repo, size: 1Mi} +- name: 02_use + agent: uses + think: 500ms + ops: + - walk_ram: {key: ctx} + - read_disk_data: {key: repo} + - burn_cpu: {millis: 10} + - ping: {} +` + +// TestDecodeAcceptsValidScript covers the argument defaults: parallel is +// optional and reads as 1. +func TestDecodeAcceptsValidScript(t *testing.T) { + s, err := Decode([]byte(validScript)) + if err != nil { + t.Fatal(err) + } + if got, want := s.Steps[1].Ops[2], (op{kind: opBurnCPU, millis: 10, parallel: 1}); got != want { + t.Errorf("burn_cpu decoded as %+v, want %+v", got, want) + } + if got, want := s.Steps[0].Ops[0], (op{kind: opFillRAM, key: "ctx", bytes: 4 << 20}); got != want { + t.Errorf("fill_ram decoded as %+v, want %+v", got, want) + } +} + +// TestDecodeRejects lists the mistakes a hand-written script can make; each +// must fail at load with a message that names the problem. +func TestDecodeRejects(t *testing.T) { + for _, tc := range []struct { + name, edit, want string + }{ + {"unknown op kind", "- ping: {}", "unknown op kind"}, + {"unknown field", "think: 1s", "field typo not found"}, + {"two-key op", "- ping: {}", "single-key map"}, + {"ping with key", "- ping: {}", "takes no key"}, + {"missing size", "- ingest: {key: repo, size: 1Mi}", "size is required"}, + {"bad key", "- ingest: {key: repo, size: 1Mi}", "must match"}, + {"zero millis", "- burn_cpu: {millis: 10}", "millis must be positive"}, + {"walk before fill", "- fill_ram: {key: ctx, size: 4Mi}", "before any fill_ram"}, + {"read before write", "- ingest: {key: repo, size: 1Mi}", "before any ingest"}, + {"duplicate step", "name: 02_use", "duplicate step name"}, + {"zero think", "think: 500ms", "think time must be positive"}, + {"no min memory", "min_actor_memory: 64Mi", "min_actor_memory is required"}, + {"min below declared", "min_actor_memory: 64Mi", "below the declared"}, + {"bad script name", "name: tiny", "script name"}, + } { + t.Run(tc.name, func(t *testing.T) { + doc := validScript + switch tc.name { + case "unknown op kind": + doc = strings.Replace(doc, tc.edit, "- sleep: {}", 1) + case "unknown field": + doc = strings.Replace(doc, tc.edit, "think: 1s\n typo: x", 1) + case "two-key op": + doc = strings.Replace(doc, tc.edit, "- {ping: {}, walk_ram: {key: ctx}}", 1) + case "ping with key": + doc = strings.Replace(doc, tc.edit, "- ping: {key: ctx}", 1) + case "missing size": + doc = strings.Replace(doc, tc.edit, "- ingest: {key: repo}", 1) + case "bad key": + doc = strings.Replace(doc, tc.edit, "- ingest: {key: ../repo, size: 1Mi}", 1) + case "zero millis": + doc = strings.Replace(doc, tc.edit, "- burn_cpu: {millis: 0}", 1) + case "walk before fill": + doc = strings.Replace(doc, tc.edit, "- ping: {}", 1) + case "read before write": + doc = strings.Replace(doc, tc.edit, "- ping: {}", 1) + case "duplicate step": + doc = strings.Replace(doc, tc.edit, "name: 01_fill", 1) + case "zero think": + doc = strings.Replace(doc, tc.edit, "think: 0s", 1) + case "no min memory": + doc = strings.Replace(doc, tc.edit, "", 1) + case "min below declared": + doc = strings.Replace(doc, tc.edit, "min_actor_memory: 4Mi", 1) + case "bad script name": + doc = strings.Replace(doc, tc.edit, "name: Tiny Script", 1) } - } - } -} - -func opName(k opKind) string { - switch k { - case opChurnRAM: - return "churn" - case opWalkRAM: - return "walk" - default: - return "op" - } -} - -// TestSessionBudgets bounds the bytes the script itself declares: resident -// RAM (largest fill per key) and disk (largest object per key). It does NOT -// model the guest's real peak — kernel, kata-agent, and allocator transients -// sit on top — so the budgets are deliberately far below the 1Gi the -// template calls for. A script edit that outgrows them must come with a -// fresh look at the actor memory guidance. -func TestSessionBudgets(t *testing.T) { - const ( - ramBudget = 128 << 20 // bytes - diskBudget = 256 << 20 - ) - ramMax := map[string]int64{} - diskMax := map[string]int64{} - for _, s := range Session() { - for _, o := range s.Ops { - switch o.kind { - case opFillRAM: - if o.bytes > ramMax[o.key] { - ramMax[o.key] = o.bytes - } - case opIngest, opWriteDisk: - if o.bytes > diskMax[o.key] { - diskMax[o.key] = o.bytes - } + if doc == validScript { + t.Fatal("test edit did not change the script") } - } - } - var ramTotal, diskTotal int64 - for _, v := range ramMax { - ramTotal += v - } - for _, v := range diskMax { - diskTotal += v - } - if ramTotal > ramBudget { - t.Errorf("script fills %d bytes of RAM, budget %d", ramTotal, ramBudget) - } - if diskTotal > diskBudget { - t.Errorf("script writes %d bytes of disk, budget %d", diskTotal, diskBudget) + _, err := Decode([]byte(doc)) + if err == nil { + t.Fatalf("Decode accepted the script:\n%s", doc) + } + if !strings.Contains(err.Error(), tc.want) { + t.Errorf("error = %q, want it to mention %q", err, tc.want) + } + }) } } -// TestExecOpAgainstFake replays every scripted op against the fake glutton -// server, proving each op marshals a request the actor-side routes accept. +// TestExecOpAgainstFake replays every op of the default script against the +// fake glutton server, proving each op marshals a request the actor-side +// routes accept. func TestExecOpAgainstFake(t *testing.T) { fakeSrv := &fake.Server{Data: []byte("filecontents")} ts := fakeSrv.Start(t) @@ -146,9 +210,13 @@ func TestExecOpAgainstFake(t *testing.T) { }, actorName: "agent-test", } + script, err := Load(DefaultScript) + if err != nil { + t.Fatal(err) + } var opCount int - for _, s := range Session() { + for _, s := range script.Steps { for i, o := range s.Ops { // Cap ingest payloads in the unit test: transport shape is what // matters here, not moving tens of MiB through httptest. @@ -179,6 +247,74 @@ func TestExecOpAgainstFake(t *testing.T) { } } +// TestLoadScriptFollowsTheKnob: the script is resolved from dynconfig on +// first use (an empty knob means the default), an unknown name is an error +// that leaves nothing cached, and a later call returns the same script. +func TestLoadScriptFollowsTheKnob(t *testing.T) { + rt := &runtime{cfg: &userclass.Config{Dyn: dynconfig.NewHolder(dynconfig.Config{AgentSessionScript: "no-such-script"})}} + if _, err := rt.loadScript(); err == nil { + t.Fatal("loadScript accepted an unknown script name") + } + if rt.script != nil { + t.Fatal("a failed load must not cache a script") + } + + rt.cfg.Dyn.Store(dynconfig.Config{}) + s, err := rt.loadScript() + if err != nil { + t.Fatal(err) + } + if s.Name != DefaultScript { + t.Errorf("empty knob loaded %q, want %q", s.Name, DefaultScript) + } + if int64(len(rt.ingestBuf)) == 0 { + t.Error("ingestBuf not sized after load") + } + + // Once loaded, the script is fixed for the worker even if the knob moves. + rt.cfg.Dyn.Store(dynconfig.Config{AgentSessionScript: "no-such-script"}) + if again, err := rt.loadScript(); err != nil || again != s { + t.Errorf("second loadScript = (%v, %v), want the cached script", again, err) + } +} + +// TestStartUserChecksTemplateMemory: a template smaller than the script's +// floor is refused before any actor is created; a big enough one proceeds. +func TestStartUserChecksTemplateMemory(t *testing.T) { + script, err := Load(DefaultScript) + if err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name, memory string + wantStart bool + }{ + {"too small", "512Mi", false}, + {"exact", "1Gi", true}, + {"no limit", "", true}, + } { + t.Run(tc.name, func(t *testing.T) { + ctl := &fakeControlClient{templateMemory: tc.memory} + u := newTestUser(t, &fake.Server{}, ctl, dynconfig.Config{}) + rt := &runtime{cfg: u.cfg, script: script, ingestBuf: makeIngestBuf(script.Steps)} + + _, err := rt.startUser(context.Background()) + created := countCalls(ctl.recordedCalls(), "CreateActor") > 0 + if tc.wantStart && (err != nil || !created) { + t.Fatalf("startUser = %v, CreateActor called = %v; want a started session", err, created) + } + if !tc.wantStart { + if err == nil || !strings.Contains(err.Error(), "min_actor_memory") { + t.Fatalf("startUser = %v, want a min_actor_memory refusal", err) + } + if created { + t.Error("CreateActor was called despite the refusal") + } + } + }) + } +} + // TestThinkScaling checks the think-time multiplier and its jitter bounds. func TestThinkScaling(t *testing.T) { r := &runtime{ @@ -214,6 +350,9 @@ type fakeControlClient struct { suspendErrs []error // sawDeadline is set when a call arrived with a context deadline. sawDeadline bool + // templateMemory is the memory limit GetActorTemplate reports; "" means + // the template sets none. + templateMemory string } func nextErr(errs *[]error) error { @@ -234,6 +373,25 @@ func (f *fakeControlClient) record(ctx context.Context, name string) { } } +func (f *fakeControlClient) GetActorTemplate(ctx context.Context, in *ateapipb.GetActorTemplateRequest, opts ...grpc.CallOption) (*ateapipb.ActorTemplate, error) { + f.record(ctx, "GetActorTemplate") + tmpl := &ateapipb.ActorTemplate{} + if f.templateMemory != "" { + tmpl.Resources = &ateapipb.Resources{Limits: []*ateapipb.Limits{{Name: "memory", Quantity: f.templateMemory}}} + } + return tmpl, nil +} + +func (f *fakeControlClient) CreateAtespace(ctx context.Context, in *ateapipb.CreateAtespaceRequest, opts ...grpc.CallOption) (*ateapipb.Atespace, error) { + f.record(ctx, "CreateAtespace") + return &ateapipb.Atespace{}, nil +} + +func (f *fakeControlClient) CreateActor(ctx context.Context, in *ateapipb.CreateActorRequest, opts ...grpc.CallOption) (*ateapipb.Actor, error) { + f.record(ctx, "CreateActor") + return &ateapipb.Actor{}, nil +} + func (f *fakeControlClient) ResumeActor(ctx context.Context, in *ateapipb.ResumeActorRequest, opts ...grpc.CallOption) (*ateapipb.ResumeActorResponse, error) { f.record(ctx, "ResumeActor") f.mu.Lock() @@ -416,11 +574,11 @@ func TestControlRPCsCarryADeadline(t *testing.T) { } func TestBurnRatePerGoroutine(t *testing.T) { - rate, ok := burnRatePerGoroutine(burn(500, 2), 10_000) + rate, ok := burnRatePerGoroutine(op{kind: opBurnCPU, millis: 500, parallel: 2}, 10_000) if !ok || rate != 10_000 { t.Errorf("burnRatePerGoroutine(500ms x2, 10k) = %v, %v; want 10000, true", rate, ok) } - if _, ok := burnRatePerGoroutine(burn(0, 1), 5); ok { + if _, ok := burnRatePerGoroutine(op{kind: opBurnCPU, parallel: 1}, 5); ok { t.Error("a zero-duration burn must not report a rate") } } diff --git a/internal/benchmarking/boomer/agentsession/script.go b/internal/benchmarking/boomer/agentsession/script.go index 9e2629f7f..6cb7fe7fb 100644 --- a/internal/benchmarking/boomer/agentsession/script.go +++ b/internal/benchmarking/boomer/agentsession/script.go @@ -16,11 +16,12 @@ package agentsession import "time" -// This file is the whole benchmark, spelled out: one coding-agent session as -// a script of steps. Each step names what the agent is doing in plain +// A script is the whole benchmark, spelled out: one coding-agent session as +// a sequence of steps. Each step names what the agent is doing in plain // English and lists the resource operations the real action would cost the // sandbox. Nothing is hidden in the driver — to understand or change the -// workload, edit this table. +// workload, read or edit the YAML under scripts/ (scriptfile.go describes +// the format). This file holds the types a decoded script becomes. // // Between steps the agent is "waiting for the LLM to think": the driver // suspends the actor, sleeps the step's think time, and lets the NEXT step's @@ -29,8 +30,8 @@ import "time" // first-class script data, not driver noise. // op is one resource effect inside a step, executed as a single glutton RPC -// through the router. Build them with the constructors below so the script -// stays declarative and greppable. +// through the router. Ops come from a decoded script (scriptfile.go); ping +// is also built directly, as the wake probe ahead of every step. type op struct { kind opKind key string // file or RAM-array name inside the sandbox @@ -68,17 +69,7 @@ const ( opPing ) -func ingest(key string, bytes int64) op { return op{kind: opIngest, key: key, bytes: bytes} } -func burn(millis int64, parallel int32) op { - return op{kind: opBurnCPU, millis: millis, parallel: parallel} -} -func writeDisk(key string, bytes int64) op { return op{kind: opWriteDisk, key: key, bytes: bytes} } -func readDigest(key string) op { return op{kind: opReadDiskDigest, key: key} } -func readData(key string) op { return op{kind: opReadDiskData, key: key} } -func fillRAM(key string, bytes int64) op { return op{kind: opFillRAM, key: key, bytes: bytes} } -func churnRAM(key string, bytes int64) op { return op{kind: opChurnRAM, key: key, bytes: bytes} } -func walkRAM(key string) op { return op{kind: opWalkRAM, key: key} } -func ping() op { return op{kind: opPing} } +func ping() op { return op{kind: opPing} } // Step is one agent action: what a coding agent would be doing, the think // time that precedes it (the LLM producing this step), and the resource @@ -94,221 +85,3 @@ type Step struct { // Ops are the resource effects, executed in order. Ops []op } - -const ( - _ = iota - kib = int64(1) << (10 * iota) - mib -) - -// Sandbox object names, so the script reads like a filesystem. -const ( - repoBlob = "repo_tarball" // the cloned repository - depsBlob = "deps_cache" // downloaded dependency archives - buildOut = "build_artifacts" // compiler output - patchFile = "patch" // edits arriving from the LLM - packageBlob = "release_package" // the final packaged artifact - contextRAM = "agent_context" // the agent process's resident working set - compilerRAM = "compiler_ws" // build-time working set - testDataBlob = "test_fixtures" // fixtures written by the test steps -) - -// Session is the default 20-step coding-agent session. The script itself -// declares ≈96Mi of resident RAM (contextRAM 32Mi + compilerRAM 64Mi) and -// ≈110Mi of files; the guest peak on top of that (kernel, kata-agent, Go -// allocator transients) has been observed around 320Mi, which is why the -// template calls for 1Gi actors. TestSessionBudgets bounds only the -// script-declared bytes, so edits that grow the working set fail the test -// and force the memory guidance to be revisited. -// -// The narrative: the agent is told to fetch a repository, get it building, -// fix a failing test, extend the test suite, refactor, and package the -// result — with an LLM round-trip (seconds of suspension) before every step. -func Session() []Step { - return []Step{ - { - Name: "01_read_task", - Agent: "Boots, reads the task prompt, loads its context window", - Think: 2 * time.Second, - Ops: []op{ - fillRAM(contextRAM, 32*mib), // the agent's resident working set for the whole session - ping(), - }, - }, - { - Name: "02_clone_repo", - Agent: "git clone of the target repository", - Think: 3 * time.Second, - Ops: []op{ - ingest(repoBlob, 16*mib), // tarball arrives over the network, lands on disk - burn(500, 1), // checkout: decompress + write tree - }, - }, - { - Name: "03_explore_tree", - Agent: "Lists directories, greps for entry points", - Think: 4 * time.Second, - Ops: []op{ - readDigest(repoBlob), // walk the repo bytes on disk - burn(200, 1), // grep-ish scanning - }, - }, - { - Name: "04_read_key_files", - Agent: "Opens the files it plans to change; they enter the LLM context", - Think: 3 * time.Second, - Ops: []op{ - readData(repoBlob), // file contents also travel back out to the 'LLM' - churnRAM(contextRAM, 8*mib), // context window grows/changes - }, - }, - { - Name: "05_install_deps", - Agent: "Installs dependencies (pip install / go mod download)", - Think: 2 * time.Second, - Ops: []op{ - ingest(depsBlob, 32*mib), // packages arrive over the network - burn(1500, 2), // unpack, byte-compile, link - }, - }, - { - Name: "06_first_build", - Agent: "First full build of the project", - Think: 3 * time.Second, - Ops: []op{ - fillRAM(compilerRAM, 64*mib), // compiler working set - burn(3000, 2), // the compile itself - writeDisk(buildOut, 24*mib), // object files and binaries - }, - }, - { - Name: "07_run_unit_tests", - Agent: "Runs the existing unit tests; one fails", - Think: 2 * time.Second, - Ops: []op{ - readDigest(buildOut), // load test binaries - burn(2500, 2), // the test run - }, - }, - { - Name: "08_reason_about_failure", - Agent: "Re-reads the failing test and traces the bug (mostly thinking)", - Think: 8 * time.Second, // the long LLM analysis turn - Ops: []op{ - walkRAM(contextRAM), // re-touch the whole context after the long suspend - ping(), - }, - }, - { - Name: "09_edit_source", - Agent: "Applies the LLM's fix to the source files", - Think: 5 * time.Second, - Ops: []op{ - ingest(patchFile, 64*kib), // the patch arrives from the LLM - writeDisk(repoBlob, 4*mib), // rewrite the touched sources - }, - }, - { - Name: "10_incremental_build", - Agent: "Rebuilds just the changed packages", - Think: 2 * time.Second, - Ops: []op{ - burn(1200, 2), - writeDisk(buildOut, 8*mib), - }, - }, - { - Name: "11_rerun_failed_test", - Agent: "Re-runs the previously failing test; it passes", - Think: 2 * time.Second, - Ops: []op{ - burn(800, 1), - }, - }, - { - Name: "12_write_new_tests", - Agent: "Writes regression tests for the fix", - Think: 6 * time.Second, // LLM authors the test file - Ops: []op{ - ingest(patchFile, 128*kib), - writeDisk(testDataBlob, 2*mib), // fixtures the new tests need - }, - }, - { - Name: "13_run_new_tests", - Agent: "Runs the new tests in isolation", - Think: 2 * time.Second, - Ops: []op{ - readDigest(testDataBlob), - burn(1000, 1), - }, - }, - { - Name: "14_full_test_suite", - Agent: "Runs the entire test suite to check for regressions", - Think: 3 * time.Second, - Ops: []op{ - burn(4000, 2), // the longest compute step in the session - readDigest(buildOut), - churnRAM(contextRAM, 8*mib), // test output enters the context - }, - }, - { - Name: "15_lint_format", - Agent: "Runs the linter and formatter over the tree", - Think: 2 * time.Second, - Ops: []op{ - readDigest(repoBlob), - burn(900, 1), - }, - }, - { - Name: "16_refactor", - Agent: "Applies the LLM's multi-file cleanup refactor", - Think: 8 * time.Second, // LLM plans the refactor - Ops: []op{ - ingest(patchFile, 512*kib), - writeDisk(repoBlob, 6*mib), - burn(600, 1), - }, - }, - { - Name: "17_rebuild", - Agent: "Full rebuild after the refactor", - Think: 2 * time.Second, - Ops: []op{ - churnRAM(compilerRAM, 32*mib), // compiler working set changes with the new tree - burn(2000, 2), - writeDisk(buildOut, 16*mib), - }, - }, - { - Name: "18_final_test_suite", - Agent: "Final full-suite run before shipping", - Think: 2 * time.Second, - Ops: []op{ - burn(3500, 2), - }, - }, - { - Name: "19_package_artifact", - Agent: "Builds the release package / container image", - Think: 3 * time.Second, - Ops: []op{ - readDigest(buildOut), - burn(1000, 1), - writeDisk(packageBlob, 24*mib), - }, - }, - { - Name: "20_commit_and_summarize", - Agent: "Commits, writes the summary, hands the session back", - Think: 5 * time.Second, // LLM writes the summary - Ops: []op{ - writeDisk(repoBlob, 256*kib), // the commit - readData(packageBlob), // artifact ships back out over the network - walkRAM(contextRAM), // final context read - }, - }, - } -} diff --git a/internal/benchmarking/boomer/agentsession/scriptfile.go b/internal/benchmarking/boomer/agentsession/scriptfile.go new file mode 100644 index 000000000..06f55a131 --- /dev/null +++ b/internal/benchmarking/boomer/agentsession/scriptfile.go @@ -0,0 +1,298 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agentsession + +import ( + "bytes" + "fmt" + "regexp" + "strconv" + "time" + + "gopkg.in/yaml.v3" + "k8s.io/apimachinery/pkg/api/resource" +) + +// Script is a loaded agent-session script: a named step sequence plus the +// actor memory it needs. +// +// The YAML form is a direct transcription of the Go types: +// +// name: coding-session +// min_actor_memory: 1Gi +// steps: +// - name: 01_read_task +// agent: Boots, reads the task prompt, loads its context window +// think: 2s +// ops: +// - fill_ram: {key: agent_context, size: 32Mi} +// - ping: {} +// +// Each op is a one-key map whose key is the op kind and whose value holds +// that kind's arguments. Sizes are Kubernetes quantities (16Mi, 64Ki); +// think times are Go durations (2s, 1.5s). Unknown kinds, unknown fields, +// and arguments a kind does not take are errors. +type Script struct { + Name string + // MinActorMemory is the smallest actor memory limit the script is known + // to run under, in bytes. The driver refuses to start against a smaller + // template. Required, and at least the script's declared RAM plus disk + // (the data dir is tmpfs). + MinActorMemory int64 + Steps []Step +} + +// scriptDoc is the YAML shape of a Script. +type scriptDoc struct { + Name string `yaml:"name"` + MinActorMemory string `yaml:"min_actor_memory"` + Steps []stepDoc `yaml:"steps"` +} + +type stepDoc struct { + Name string `yaml:"name"` + Agent string `yaml:"agent"` + Think string `yaml:"think"` + Ops []opDoc `yaml:"ops"` +} + +// opDoc is one op: exactly one entry, kind name to arguments. +type opDoc map[string]opArgs + +type opArgs struct { + Key string `yaml:"key,omitempty"` + Size string `yaml:"size,omitempty"` + Millis int64 `yaml:"millis,omitempty"` + Parallel int32 `yaml:"parallel,omitempty"` +} + +// opSpec says which arguments an op kind takes. +type opSpec struct { + kind opKind + key bool // takes key + size bool // takes size + millis bool // takes millis and parallel +} + +var opSpecs = map[string]opSpec{ + "ingest": {kind: opIngest, key: true, size: true}, + "burn_cpu": {kind: opBurnCPU, millis: true}, + "write_disk": {kind: opWriteDisk, key: true, size: true}, + "read_disk_digest": {kind: opReadDiskDigest, key: true}, + "read_disk_data": {kind: opReadDiskData, key: true}, + "fill_ram": {kind: opFillRAM, key: true, size: true}, + "churn_ram": {kind: opChurnRAM, key: true, size: true}, + "walk_ram": {kind: opWalkRAM, key: true}, + "ping": {kind: opPing}, +} + +// opNames is the inverse of opSpecs, for encoding and messages. +var opNames = func() map[opKind]string { + m := make(map[opKind]string, len(opSpecs)) + for name, spec := range opSpecs { + m[spec.kind] = name + } + return m +}() + +// keyRE mirrors glutton's diskKeyRE: keys name files under the actor's data +// dir, so nothing that could escape it is accepted. +var keyRE = regexp.MustCompile(`^[a-zA-Z0-9_-]+$`) + +// scriptNameRE bounds script names, which double as the --agentsession-script +// knob value and the embedded file name. +var scriptNameRE = regexp.MustCompile(`^[a-z0-9][a-z0-9-]*$`) + +// Decode parses a YAML script and validates it. +func Decode(data []byte) (*Script, error) { + var doc scriptDoc + dec := yaml.NewDecoder(bytes.NewReader(data)) + dec.KnownFields(true) + if err := dec.Decode(&doc); err != nil { + return nil, fmt.Errorf("parse script: %w", err) + } + s := &Script{Name: doc.Name} + if doc.MinActorMemory != "" { + n, err := parseSize(doc.MinActorMemory) + if err != nil { + return nil, fmt.Errorf("min_actor_memory: %w", err) + } + s.MinActorMemory = n + } + for i, sd := range doc.Steps { + step, err := decodeStep(sd) + if err != nil { + return nil, fmt.Errorf("step %d (%q): %w", i+1, sd.Name, err) + } + s.Steps = append(s.Steps, step) + } + if err := Validate(s); err != nil { + return nil, err + } + return s, nil +} + +func decodeStep(sd stepDoc) (Step, error) { + step := Step{Name: sd.Name, Agent: sd.Agent} + if sd.Think != "" { + d, err := time.ParseDuration(sd.Think) + if err != nil { + return step, fmt.Errorf("think: %w", err) + } + step.Think = d + } + for i, od := range sd.Ops { + o, err := decodeOp(od) + if err != nil { + return step, fmt.Errorf("op %d: %w", i+1, err) + } + step.Ops = append(step.Ops, o) + } + return step, nil +} + +func decodeOp(od opDoc) (op, error) { + if len(od) != 1 { + return op{}, fmt.Errorf("an op is a single-key map, got %d keys", len(od)) + } + var name string + var args opArgs + for name, args = range od { + } + spec, ok := opSpecs[name] + if !ok { + return op{}, fmt.Errorf("unknown op kind %q", name) + } + o := op{kind: spec.kind} + switch { + case spec.key && args.Key == "": + return op{}, fmt.Errorf("%s: key is required", name) + case !spec.key && args.Key != "": + return op{}, fmt.Errorf("%s: takes no key", name) + } + if args.Key != "" { + if !keyRE.MatchString(args.Key) { + return op{}, fmt.Errorf("%s: key %q must match %s", name, args.Key, keyRE) + } + o.key = args.Key + } + switch { + case spec.size && args.Size == "": + return op{}, fmt.Errorf("%s: size is required", name) + case !spec.size && args.Size != "": + return op{}, fmt.Errorf("%s: takes no size", name) + } + if args.Size != "" { + n, err := parseSize(args.Size) + if err != nil { + return op{}, fmt.Errorf("%s: size: %w", name, err) + } + o.bytes = n + } + if spec.millis { + if args.Millis <= 0 { + return op{}, fmt.Errorf("%s: millis must be positive", name) + } + if args.Parallel < 0 { + return op{}, fmt.Errorf("%s: parallel cannot be negative", name) + } + o.millis = args.Millis + o.parallel = max(args.Parallel, 1) + } else if args.Millis != 0 || args.Parallel != 0 { + return op{}, fmt.Errorf("%s: takes no millis or parallel", name) + } + return o, nil +} + +// parseSize reads a Kubernetes quantity as a positive byte count. +func parseSize(s string) (int64, error) { + q, err := resource.ParseQuantity(s) + if err != nil { + return 0, err + } + n, ok := q.AsInt64() + if !ok || n <= 0 { + return 0, fmt.Errorf("%q is not a positive whole byte count", s) + } + return n, nil +} + +func formatSize(n int64) string { + return resource.NewQuantity(n, resource.BinarySI).String() +} + +// Encode renders a Script as YAML in the form Decode reads, with one +// flow-style line per op so the file reads like the step table it is. +func Encode(s *Script) ([]byte, error) { + root := &yaml.Node{Kind: yaml.MappingNode} + addScalar(root, "name", s.Name) + addScalar(root, "min_actor_memory", formatSize(s.MinActorMemory)) + steps := &yaml.Node{Kind: yaml.SequenceNode} + for _, st := range s.Steps { + sn := &yaml.Node{Kind: yaml.MappingNode} + addScalar(sn, "name", st.Name) + addScalar(sn, "agent", st.Agent) + addScalar(sn, "think", st.Think.String()) + ops := &yaml.Node{Kind: yaml.SequenceNode} + for _, o := range st.Ops { + on, err := encodeOp(o) + if err != nil { + return nil, fmt.Errorf("step %q: %w", st.Name, err) + } + ops.Content = append(ops.Content, on) + } + sn.Content = append(sn.Content, scalar("ops"), ops) + steps.Content = append(steps.Content, sn) + } + root.Content = append(root.Content, scalar("steps"), steps) + + var buf bytes.Buffer + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err := enc.Encode(root); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func encodeOp(o op) (*yaml.Node, error) { + name, ok := opNames[o.kind] + if !ok { + return nil, fmt.Errorf("unknown op kind %d", o.kind) + } + spec := opSpecs[name] + args := &yaml.Node{Kind: yaml.MappingNode, Style: yaml.FlowStyle} + if spec.key { + addScalar(args, "key", o.key) + } + if spec.size { + addScalar(args, "size", formatSize(o.bytes)) + } + if spec.millis { + addScalar(args, "millis", strconv.FormatInt(o.millis, 10)) + addScalar(args, "parallel", strconv.Itoa(int(o.parallel))) + } + wrapper := &yaml.Node{Kind: yaml.MappingNode} + wrapper.Content = append(wrapper.Content, scalar(name), args) + return wrapper, nil +} + +func scalar(v string) *yaml.Node { + return &yaml.Node{Kind: yaml.ScalarNode, Value: v} +} + +func addScalar(m *yaml.Node, k, v string) { + m.Content = append(m.Content, scalar(k), scalar(v)) +} diff --git a/internal/benchmarking/boomer/agentsession/scripts.go b/internal/benchmarking/boomer/agentsession/scripts.go new file mode 100644 index 000000000..97f7d5b80 --- /dev/null +++ b/internal/benchmarking/boomer/agentsession/scripts.go @@ -0,0 +1,68 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agentsession + +import ( + "embed" + "fmt" + "io/fs" + "sort" + "strings" +) + +// DefaultScript is the variant a run uses when --agentsession-script is +// unset: the 20-step coding session. +const DefaultScript = "coding-session" + +// scriptFS holds the built-in script variants. A file scripts/.yaml is +// selectable as --agentsession-script=; TestEmbeddedScriptsAreValid +// keeps every one of them loadable. +// +//go:embed scripts/*.yaml +var scriptFS embed.FS + +// Names lists the built-in script variants. +func Names() []string { + entries, err := fs.ReadDir(scriptFS, "scripts") + if err != nil { + return nil + } + var names []string + for _, e := range entries { + if n, ok := strings.CutSuffix(e.Name(), ".yaml"); ok { + names = append(names, n) + } + } + sort.Strings(names) + return names +} + +// Load returns the built-in script variant called name. The file's name +// field must agree with the file name, so a knob value always matches what +// the stats rows and logs report. +func Load(name string) (*Script, error) { + data, err := scriptFS.ReadFile("scripts/" + name + ".yaml") + if err != nil { + return nil, fmt.Errorf("no built-in agent-session script %q (have %s)", name, strings.Join(Names(), ", ")) + } + s, err := Decode(data) + if err != nil { + return nil, fmt.Errorf("built-in script %q: %w", name, err) + } + if s.Name != name { + return nil, fmt.Errorf("built-in script file %q names itself %q", name, s.Name) + } + return s, nil +} diff --git a/internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml b/internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml new file mode 100644 index 000000000..b1f1fb564 --- /dev/null +++ b/internal/benchmarking/boomer/agentsession/scripts/coding-session.yaml @@ -0,0 +1,151 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The default agent-session script: a coding agent told to fetch a +# repository, get it building, fix a failing test, extend the test suite, +# refactor, and package the result, with an LLM round-trip (seconds of +# suspension) before every step. Each op is one glutton RPC; see +# scriptfile.go for the op kinds and their arguments. +# +# The script declares ~96Mi of resident RAM and ~110Mi of files (tmpfs, so +# also memory). The guest peak on top of that (kernel, kata-agent, Go +# allocator transients) has been observed around 320Mi, and 512Mi actors +# OOM under allocator churn, hence min_actor_memory: 1Gi. +name: coding-session +min_actor_memory: 1Gi +steps: + - name: 01_read_task + agent: Boots, reads the task prompt, loads its context window + think: 2s + ops: + - fill_ram: {key: agent_context, size: 32Mi} + - ping: {} + - name: 02_clone_repo + agent: git clone of the target repository + think: 3s + ops: + - ingest: {key: repo_tarball, size: 16Mi} + - burn_cpu: {millis: 500, parallel: 1} + - name: 03_explore_tree + agent: Lists directories, greps for entry points + think: 4s + ops: + - read_disk_digest: {key: repo_tarball} + - burn_cpu: {millis: 200, parallel: 1} + - name: 04_read_key_files + agent: Opens the files it plans to change; they enter the LLM context + think: 3s + ops: + - read_disk_data: {key: repo_tarball} + - churn_ram: {key: agent_context, size: 8Mi} + - name: 05_install_deps + agent: Installs dependencies (pip install / go mod download) + think: 2s + ops: + - ingest: {key: deps_cache, size: 32Mi} + - burn_cpu: {millis: 1500, parallel: 2} + - name: 06_first_build + agent: First full build of the project + think: 3s + ops: + - fill_ram: {key: compiler_ws, size: 64Mi} + - burn_cpu: {millis: 3000, parallel: 2} + - write_disk: {key: build_artifacts, size: 24Mi} + - name: 07_run_unit_tests + agent: Runs the existing unit tests; one fails + think: 2s + ops: + - read_disk_digest: {key: build_artifacts} + - burn_cpu: {millis: 2500, parallel: 2} + - name: 08_reason_about_failure + agent: Re-reads the failing test and traces the bug (mostly thinking) + think: 8s + ops: + - walk_ram: {key: agent_context} + - ping: {} + - name: 09_edit_source + agent: Applies the LLM's fix to the source files + think: 5s + ops: + - ingest: {key: patch, size: 64Ki} + - write_disk: {key: repo_tarball, size: 4Mi} + - name: 10_incremental_build + agent: Rebuilds just the changed packages + think: 2s + ops: + - burn_cpu: {millis: 1200, parallel: 2} + - write_disk: {key: build_artifacts, size: 8Mi} + - name: 11_rerun_failed_test + agent: Re-runs the previously failing test; it passes + think: 2s + ops: + - burn_cpu: {millis: 800, parallel: 1} + - name: 12_write_new_tests + agent: Writes regression tests for the fix + think: 6s + ops: + - ingest: {key: patch, size: 128Ki} + - write_disk: {key: test_fixtures, size: 2Mi} + - name: 13_run_new_tests + agent: Runs the new tests in isolation + think: 2s + ops: + - read_disk_digest: {key: test_fixtures} + - burn_cpu: {millis: 1000, parallel: 1} + - name: 14_full_test_suite + agent: Runs the entire test suite to check for regressions + think: 3s + ops: + - burn_cpu: {millis: 4000, parallel: 2} + - read_disk_digest: {key: build_artifacts} + - churn_ram: {key: agent_context, size: 8Mi} + - name: 15_lint_format + agent: Runs the linter and formatter over the tree + think: 2s + ops: + - read_disk_digest: {key: repo_tarball} + - burn_cpu: {millis: 900, parallel: 1} + - name: 16_refactor + agent: Applies the LLM's multi-file cleanup refactor + think: 8s + ops: + - ingest: {key: patch, size: 512Ki} + - write_disk: {key: repo_tarball, size: 6Mi} + - burn_cpu: {millis: 600, parallel: 1} + - name: 17_rebuild + agent: Full rebuild after the refactor + think: 2s + ops: + - churn_ram: {key: compiler_ws, size: 32Mi} + - burn_cpu: {millis: 2000, parallel: 2} + - write_disk: {key: build_artifacts, size: 16Mi} + - name: 18_final_test_suite + agent: Final full-suite run before shipping + think: 2s + ops: + - burn_cpu: {millis: 3500, parallel: 2} + - name: 19_package_artifact + agent: Builds the release package / container image + think: 3s + ops: + - read_disk_digest: {key: build_artifacts} + - burn_cpu: {millis: 1000, parallel: 1} + - write_disk: {key: release_package, size: 24Mi} + - name: 20_commit_and_summarize + agent: Commits, writes the summary, hands the session back + think: 5s + ops: + - write_disk: {key: repo_tarball, size: 256Ki} + - read_disk_data: {key: release_package} + - walk_ram: {key: agent_context} diff --git a/internal/benchmarking/boomer/agentsession/validate.go b/internal/benchmarking/boomer/agentsession/validate.go new file mode 100644 index 000000000..d33782ff6 --- /dev/null +++ b/internal/benchmarking/boomer/agentsession/validate.go @@ -0,0 +1,110 @@ +// Copyright 2026 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package agentsession + +import ( + "fmt" +) + +// Budget is what a script declares it will hold in the sandbox: resident +// RAM (the largest fill per array) and files in the data dir (the largest +// object per key). The data dir is tmpfs, so both come out of the actor's +// memory limit. It does not model the guest's real peak: kernel, kata-agent, +// and Go allocator transients sit on top. +type Budget struct { + RAM int64 + Disk int64 +} + +// Budgets sums the bytes a step sequence declares. +func Budgets(steps []Step) Budget { + ramMax := map[string]int64{} + diskMax := map[string]int64{} + for _, s := range steps { + for _, o := range s.Ops { + switch o.kind { + case opFillRAM: + ramMax[o.key] = max(ramMax[o.key], o.bytes) + case opIngest, opWriteDisk: + diskMax[o.key] = max(diskMax[o.key], o.bytes) + } + } + } + var b Budget + for _, v := range ramMax { + b.RAM += v + } + for _, v := range diskMax { + b.Disk += v + } + return b +} + +// Validate pins a script's invariants: a usable name, a memory floor that +// covers what the script declares, unique non-empty steps with positive +// think times, and no op that consumes a sandbox object before an earlier +// step created it. A broken ordering would fail at run time with NotFound +// from glutton; this catches it when the script is loaded. +func Validate(s *Script) error { + if !scriptNameRE.MatchString(s.Name) { + return fmt.Errorf("script name %q must match %s", s.Name, scriptNameRE) + } + if len(s.Steps) == 0 { + return fmt.Errorf("script %q has no steps", s.Name) + } + seen := map[string]bool{} + ramFilled := map[string]bool{} + diskWritten := map[string]bool{} + for i, st := range s.Steps { + where := fmt.Sprintf("step %d (%q)", i+1, st.Name) + if st.Name == "" || st.Agent == "" { + return fmt.Errorf("%s: name and agent must be set", where) + } + if seen[st.Name] { + return fmt.Errorf("%s: duplicate step name", where) + } + seen[st.Name] = true + if st.Think <= 0 { + return fmt.Errorf("%s: think time must be positive", where) + } + if len(st.Ops) == 0 { + return fmt.Errorf("%s: has no ops", where) + } + for j, o := range st.Ops { + switch o.kind { + case opFillRAM: + ramFilled[o.key] = true + case opChurnRAM, opWalkRAM: + if !ramFilled[o.key] { + return fmt.Errorf("%s op %d: %s of RAM array %q before any fill_ram", where, j+1, opNames[o.kind], o.key) + } + case opIngest, opWriteDisk: + diskWritten[o.key] = true + case opReadDiskDigest, opReadDiskData: + if !diskWritten[o.key] { + return fmt.Errorf("%s op %d: %s of %q before any ingest or write_disk", where, j+1, opNames[o.kind], o.key) + } + } + } + } + b := Budgets(s.Steps) + if s.MinActorMemory <= 0 { + return fmt.Errorf("script %q: min_actor_memory is required (declared RAM %s + disk %s)", s.Name, formatSize(b.RAM), formatSize(b.Disk)) + } + if declared := b.RAM + b.Disk; s.MinActorMemory < declared { + return fmt.Errorf("script %q: min_actor_memory %s is below the declared RAM %s + disk %s", s.Name, formatSize(s.MinActorMemory), formatSize(b.RAM), formatSize(b.Disk)) + } + return nil +} diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig.go b/internal/benchmarking/boomer/dynconfig/dynconfig.go index ad792ff6f..9b92e1ea9 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig.go @@ -69,6 +69,7 @@ type Config struct { SweperfNumCycles int // number of cycles to partition steps into; 0 falls back to default SweperfPollIntervalMs int // /status poll interval in ms; 0 falls back to default + AgentSessionScript string // built-in agent-session script variant; "" falls back to the default AgentSessionThinkScale float64 // multiplier on the script's per-step think times; 0 reads as 1.0 } @@ -119,6 +120,7 @@ type payload struct { SweperfNumCycles *float64 `json:"sweperf_num_cycles"` SweperfPollIntervalMs *float64 `json:"sweperf_poll_interval_ms"` + AgentSessionScript *string `json:"agentsession_script"` AgentSessionThinkScale *float64 `json:"agentsession_think_scale"` } @@ -285,6 +287,9 @@ func (p payload) merge(current Config) Config { if p.SweperfPollIntervalMs != nil { out.SweperfPollIntervalMs = int(*p.SweperfPollIntervalMs) } + if p.AgentSessionScript != nil { + out.AgentSessionScript = *p.AgentSessionScript + } if p.AgentSessionThinkScale != nil { out.AgentSessionThinkScale = *p.AgentSessionThinkScale } @@ -364,6 +369,7 @@ func StartPoll( slog.Int("sweperf_total_steps", next.SweperfTotalSteps), slog.Int("sweperf_num_cycles", next.SweperfNumCycles), slog.Int("sweperf_poll_interval_ms", next.SweperfPollIntervalMs), + slog.String("agentsession_script", next.AgentSessionScript), slog.Float64("agentsession_think_scale", next.AgentSessionThinkScale), ) } @@ -412,6 +418,7 @@ func SubscribeSpawn(url string, holder *Holder, sampler ProbabilityUpdater, fetc slog.Int("sweperf_total_steps", next.SweperfTotalSteps), slog.Int("sweperf_num_cycles", next.SweperfNumCycles), slog.Int("sweperf_poll_interval_ms", next.SweperfPollIntervalMs), + slog.String("agentsession_script", next.AgentSessionScript), slog.Float64("agentsession_think_scale", next.AgentSessionThinkScale), ) }) diff --git a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go index 99f1f3ba9..51a0d1cca 100644 --- a/internal/benchmarking/boomer/dynconfig/dynconfig_test.go +++ b/internal/benchmarking/boomer/dynconfig/dynconfig_test.go @@ -42,7 +42,8 @@ func TestParseValid(t *testing.T) { "sweperf_template": "swebench-astropy-7336", "sweperf_total_steps": 21, "sweperf_num_cycles": 4, - "sweperf_poll_interval_ms": 100 + "sweperf_poll_interval_ms": 100, + "agentsession_script": "coding-session" }`) cfg, err := Parse(jsonBlob, Config{}) @@ -92,6 +93,9 @@ func TestParseValid(t *testing.T) { if cfg.SweperfPollIntervalMs != 100 { t.Errorf("SweperfPollIntervalMs: got %d, want 100", cfg.SweperfPollIntervalMs) } + if cfg.AgentSessionScript != "coding-session" { + t.Errorf("AgentSessionScript: got %q, want coding-session", cfg.AgentSessionScript) + } } func TestParseInvalidValues(t *testing.T) {