From 445770e032daddd4886cb4b9343eb903d4e65312 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Tue, 29 Sep 2026 18:02:39 +0200 Subject: [PATCH] feat(durable): first coding-agent tool turn (Package 16) Generation offers active registered tools through positional system entries and hands a tool-calling answer to parallel or sequential pi.tool tasks and a pi.post-tools task that applies addTools/terminate and continues the run. Tool tasks validate, run beforeTool/afterTool, record intent with the replay policy, publish bounded, sanitized, adaptively throttled progress in pi.live.tools, and write results with a rendered diagnostics block. Hook dispatch with conversation and subtree scopes, prepareArguments, HarnessOptions.env, and read/bash/edit/write tools in pi-durable/tools. The env shell now streams raw output and spills past thresholds; its bounded capture and publisher are removed. --- package-lock.json | 4 +- packages/durable/CHANGELOG.md | 13 + packages/durable/README.md | 18 + packages/durable/docs/pico-v5-handoff.md | 18 +- packages/durable/docs/pico-v5.md | 38 +- packages/durable/package.json | 10 +- packages/durable/src/entries.ts | 8 +- packages/durable/src/env/index.ts | 46 +- packages/durable/src/env/node.ts | 100 +- .../src/env/utils/adaptive-publisher.ts | 87 -- .../durable/src/env/utils/output-capture.ts | 258 ----- packages/durable/src/harness/config.ts | 4 +- packages/durable/src/harness/generation.ts | 238 +++-- packages/durable/src/harness/harness.ts | 13 + packages/durable/src/harness/json.ts | 31 + packages/durable/src/harness/live.ts | 76 +- packages/durable/src/harness/output.ts | 288 ++++++ packages/durable/src/harness/post-tools.ts | 65 ++ packages/durable/src/harness/prompt.ts | 98 +- packages/durable/src/harness/registry.ts | 4 +- packages/durable/src/harness/scheduler.ts | 85 +- packages/durable/src/harness/tool.ts | 462 +++++++++ packages/durable/src/harness/types.ts | 88 +- packages/durable/src/index.ts | 17 +- packages/durable/src/tools/bash.ts | 89 ++ packages/durable/src/tools/edit-diff.ts | 500 +++++++++ packages/durable/src/tools/edit.ts | 144 +++ packages/durable/src/tools/env.ts | 8 + .../durable/src/tools/file-mutation-queue.ts | 61 ++ packages/durable/src/tools/image.ts | 89 ++ packages/durable/src/tools/index.ts | 10 + packages/durable/src/tools/path-utils.ts | 30 + packages/durable/src/tools/read.ts | 111 ++ packages/durable/src/tools/write.ts | 38 + .../durable/src/{env/utils => }/truncate.ts | 160 +-- packages/durable/src/types.ts | 33 +- packages/durable/test/chat-support.ts | 3 + .../test/env-adaptive-publisher.test.ts | 135 --- packages/durable/test/env-node-spill.test.ts | 12 +- packages/durable/test/env-node.test.ts | 115 +-- .../durable/test/env-output-capture.test.ts | 332 ------ packages/durable/test/env-truncate.test.ts | 163 +-- .../durable/test/examples/17-coding-tools.ts | 98 ++ .../utf8-byte-length-without-buffer.ts | 3 +- .../durable/test/harness-generation.test.ts | 11 - .../durable/test/harness-live-deltas.test.ts | 439 ++++++++ packages/durable/test/harness-output.test.ts | 272 +++++ packages/durable/test/harness-prompt.test.ts | 96 +- .../durable/test/harness-registry.test.ts | 13 +- .../test/harness-tools-recovery.test.ts | 350 +++++++ packages/durable/test/harness-tools.test.ts | 951 ++++++++++++++++++ .../test/storage-runtime-boundary.test.ts | 5 +- packages/durable/test/tool-output-bench.ts | 298 ++++++ packages/durable/test/tools.test.ts | 531 ++++++++++ scripts/check-entry-graphs.mjs | 5 +- 55 files changed, 5755 insertions(+), 1419 deletions(-) delete mode 100644 packages/durable/src/env/utils/adaptive-publisher.ts delete mode 100644 packages/durable/src/env/utils/output-capture.ts create mode 100644 packages/durable/src/harness/json.ts create mode 100644 packages/durable/src/harness/output.ts create mode 100644 packages/durable/src/harness/post-tools.ts create mode 100644 packages/durable/src/harness/tool.ts create mode 100644 packages/durable/src/tools/bash.ts create mode 100644 packages/durable/src/tools/edit-diff.ts create mode 100644 packages/durable/src/tools/edit.ts create mode 100644 packages/durable/src/tools/env.ts create mode 100644 packages/durable/src/tools/file-mutation-queue.ts create mode 100644 packages/durable/src/tools/image.ts create mode 100644 packages/durable/src/tools/index.ts create mode 100644 packages/durable/src/tools/path-utils.ts create mode 100644 packages/durable/src/tools/read.ts create mode 100644 packages/durable/src/tools/write.ts rename packages/durable/src/{env/utils => }/truncate.ts (51%) delete mode 100644 packages/durable/test/env-adaptive-publisher.test.ts delete mode 100644 packages/durable/test/env-output-capture.test.ts create mode 100644 packages/durable/test/examples/17-coding-tools.ts create mode 100644 packages/durable/test/harness-live-deltas.test.ts create mode 100644 packages/durable/test/harness-output.test.ts create mode 100644 packages/durable/test/harness-tools-recovery.test.ts create mode 100644 packages/durable/test/harness-tools.test.ts create mode 100644 packages/durable/test/tool-output-bench.ts create mode 100644 packages/durable/test/tools.test.ts diff --git a/package-lock.json b/package-lock.json index 40fad3ada..f8d726cf9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5812,7 +5812,9 @@ "license": "MIT", "dependencies": { "@earendil-works/chord": "^0.99.1", - "@earendil-works/pi-ai": "^0.99.1" + "@earendil-works/pi-ai": "^0.99.1", + "diff": "8.0.4", + "typebox": "1.3.27" }, "devDependencies": { "shx": "0.4.0", diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 8563502a4..67af05aae 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -24,6 +24,10 @@ - `createRegistry()` now pre-registers the built-in `pi.generation` task and `pi` conversation setup, which cannot be disposed or replaced, and `Harness.open()` rejects a registry whose snapshot lacks either. - `RegistrySnapshot` now requires `conversationSetups()`. - `Tx` now requires `settleSubmission()`. +- `TaskRuntime` now requires `env`, `hooks`, `getTask()`, `waitForTask()`, and `entry()`; `ToolExecutionApi` requires `env` and `diagnostic()` and no longer declares `conversation()` until owned-conversation handles land. +- `createRegistry()` also pre-registers the built-in `pi.tool` and `pi.post-tools` tasks. +- `ToolResultEntry` now carries `data: { diagnostics }`. +- The environment shell no longer keeps a bounded, throttled output view: `ShellExecOptions.capture` and `onUpdate` are replaced by raw `onOutput` chunks and `spill: { afterBytes, afterLines }`, `ShellExecResult` is `{ exitCode, spillPath? }`, and `ExecutionError.spillPath` reports the spill of a timed-out or aborted command. ### Added @@ -51,6 +55,15 @@ - Added `registry.conversations.setup()`: setups run in every Harness commit that creates or forks a conversation, including raw `Tx.createConversation()` and `Tx.forkConversation()`, before host `init`. The built-in `pi` setup runs first and stages the default configuration with every registered tool active (forks keep their copied configuration) and an empty `pi.live`. - `Conversation.submit()`, `Submission.wait()`, `waitForTask()`, and `waitForIdle()` now enable task scheduling, so they never wait on a Harness whose `resume()` was not called. - Scheduler-written `faulted` and `orphaned` outcomes of a run task now settle the run's input submissions `unanswered` and clear its run control in the same commit. +- Added the tool chain: generation offers active registered tools through positional system entries, answers calls to tools it did not offer with `tool_unavailable`, and hands the run to parallel or sequential `pi.tool` tasks and a `pi.post-tools` task that applies `addTools` and `terminate` and continues with the next generation. +- Added the `pi.tool` task: argument validation before and after `beforeTool`, durable intent with the replay policy, recovery that reruns only replay-safe tools, bounded `output()` and `details()` progress in `pi.live.tools` slots with adaptive throttling, output and details fallback, `afterTool`, and results with a rendered `` diagnostics block. +- Added tool diagnostics: `api.diagnostic()`, `ToolExecutionResult.diagnostics`, and Harness diagnostics for truncation and error results. +- Added hook dispatch with conversation and owned-subtree scopes: `beforeRequest`, `afterResponse`, and `onYield` continuations on generation, `beforeTool` and `afterTool` on tools, and `afterTools` on post-tools, with `HookApi` memos. +- Added `HarnessOptions.env`, `ToolRegistration.executionMode`, and the `toolExecution` configuration with `get/setToolExecution()`. +- Added `read`, `bash`, `edit`, and `write` tools in `@earendil-works/pi-durable/tools`; they use `api.env`. Bash streams raw output into `api.output()`, reports its spill file as a diagnostic, and throws on a nonzero exit or timeout. Reading images is not supported yet. +- Added `ToolRegistration.prepareArguments()` to repair malformed arguments before validation; the edit tool uses it for `edits` sent as a JSON string or a single object and for top-level `oldText`/`newText`. +- Tool output retained by `api.output()` is an exact slice of whole lines, sanitized of control characters. +- Added the `bench:tool-output` benchmark of tool output rates, retention, backends, replay, and 1 GiB throughput. ### Fixed diff --git a/packages/durable/README.md b/packages/durable/README.md index 878f4513d..78db0d989 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -31,6 +31,24 @@ The portable SQLite core, minimal database facade, and ordered schema migrations The Node adapter uses WAL mode with `synchronous = NORMAL` and checkpoints the WAL on close. Acknowledged commits survive process crashes, but the newest commits may be lost after a power or host failure. One `SqliteStorage` owner must serialize writes to a database file; cross-process ID allocation is not supported. +## Tools + +`@earendil-works/pi-durable/tools` provides `read`, `bash`, `edit`, and `write` tools. They reach files and processes only through the execution environment the Harness passes as `api.env` (`HarnessOptions.env`, or one a tool wrapper supplies). Reading images is not supported yet. + +```ts +import { NodeExecutionEnv } from "@earendil-works/pi-durable/env/node"; +import { createBashTool, createEditTool, createReadTool, createWriteTool } from "@earendil-works/pi-durable/tools"; + +registry.batch(() => { + for (const tool of [createReadTool(), createBashTool(), createEditTool(), createWriteTool()]) registry.tools.add(tool); +}); +const harness = await Harness.open(storage, { models, registry, env: new NodeExecutionEnv({ cwd }) }, context); +``` + +The package root loads TypeBox, because the built-in tool task validates tool arguments with pi-ai's `validateToolArguments()`. Unbundled, that costs about 23 MB of peak RSS; a tree-shaken bundle keeps about 4 MB. + +`npm run bench:tool-output` drives the tool task and its throttled `pi.live` progress commits at low, normal, and high output rates, with head and tail retention, one and four parallel tools, and all three backends. It reports commits, operation bytes, whole-window output writes, commit latency, stored size mid-round and after the round, replay time on reopen, peak RSS, and heap, plus 1 GiB throughput through `api.output()` and through the bash tool. + ## Storage conformance Storage adapters can register the runner-independent conformance cases through the testing entry. The convenience adapter accepts Vitest/Jest-compatible runner functions without importing either package: diff --git a/packages/durable/docs/pico-v5-handoff.md b/packages/durable/docs/pico-v5-handoff.md index c161df62b..03ade23cc 100644 --- a/packages/durable/docs/pico-v5-handoff.md +++ b/packages/durable/docs/pico-v5-handoff.md @@ -11,7 +11,7 @@ facades, membranes, document routing, view projection, events, or clone chains. - Obsolete `pico` and `pico4` prototypes were removed. - `pico3` remains. -- Packages 1–15 are implemented in `packages/durable`; Package 10 was already satisfied by Chord's canonical structural diff implementation. +- Packages 1–16 are implemented in `packages/durable`; Package 10 was already satisfied by Chord's canonical structural diff implementation. ## 1. Records, cursors, and memory tables @@ -475,7 +475,10 @@ Neither side uses a production fake successor. - `@earendil-works/pi-durable/tools`: copy `read`, `bash`, `edit`, and `write` with their helpers from `packages/agent/src/harness/tools` and adapt them to `ToolRegistration` (`api.env`, `api.output`, `api.details`). Image reading in - `read` is deferred; note it where the tool rejects or skips images. + `read` is deferred; note it where the tool rejects or skips images. The env + shell streams raw output (`onOutput`) and spills past thresholds; `output()` is + the only place output is bounded, sanitized, and throttled. `prepareArguments` + restores the edit tool's argument repair. Acceptance: input → model tool call → registered local read/bash/edit operation → tool result → model answer → durable submission settlement. Run that path once @@ -497,6 +500,10 @@ diagnostic ordering, the `` block, and entry `data`; and all positional tool-history cases. Also test the ported tools against `NodeExecutionEnv`, and a wrapper supplying a different `api.env`. +Document in the durable README that the package root loads TypeBox through the +tool task's argument validation (pi-ai `validateToolArguments()`): about 23 MB of +peak RSS unbundled, about 4 MB in a tree-shaken bundle. + Tool output benchmark (`test/*.bench.ts`, memory, SQLite, and JSONL): drive the real tool task, adaptive throttle, and `pi.live` commits with low (a line every few seconds), normal (a compiler or test run), and high (continuous `cat` of a @@ -526,6 +533,13 @@ self-head cuts, successor turns, queued reset/handoff, and every terminal cleanu Successful inputs still require an answer; writes settle on placement and never start generation. +Usage ledger (decided with the reviewers, needs Mario's approval before spec text): one conversation-scoped +`pi.usage` document (`latest`, `fork: "initial"`, base on every change, created lazily, mounted in the +view) holding pi-ai `Usage` totals per `provider/model` for entries appended in that conversation. The +totals update in the same commit as each assistant entry through generation's single `appendAssistant()` +helper (already in place). Session and ownership-subtree totals are summed on read, never stored. Entries +stay authoritative; the document is a derived index. + Define the headed reset/handoff entry once, used by `reset(handoff)` and by the post-tools `handoff` control deferred from Package 16, and implement that control. diff --git a/packages/durable/docs/pico-v5.md b/packages/durable/docs/pico-v5.md index ad3da8a32..7d32137de 100644 --- a/packages/durable/docs/pico-v5.md +++ b/packages/durable/docs/pico-v5.md @@ -1412,7 +1412,7 @@ type RunningTask = TaskRecord & { type NextTaskState = Extract, { status: "running" | "terminal" }>; interface HookRunner { - each(name: K, invoke: (handler: H[K]) => void | Promise): Promise; + each(name: K, invoke: (handler: NonNullable) => void | Promise): Promise; } type PhaseHandler = ( @@ -1429,6 +1429,8 @@ interface TaskRuntime extends DocumentObserver, Docum readonly registry: RegistrySnapshot; readonly models: Models; readonly hooks: HookRunner; + /** `HarnessOptions.env`; the tool task passes it to tools as `api.env`. */ + readonly env: ExecutionEnv | undefined; commit( change: ( @@ -1443,6 +1445,8 @@ interface TaskRuntime extends DocumentObserver, Docum conversation(id: ConversationId, context: Context): Promise; /** Committed task record. */ getTask(id: TaskId, context: Context): Promise | undefined>; + /** Terminal receipt; rejects when the invocation ends. */ + waitForTask(id: TaskId, context: Context): Promise>; /** Committed entry visible from the task's conversation. */ entry(id: EntryId, context: Context): Promise; entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; @@ -1534,7 +1538,8 @@ record. `sleep(until)` compares against the Harness `now` clock and rejects when the invocation is signalled or its context is cancelled. Watches acquired through the runtime stop when the invocation ends. `snapshot()`/`snapshotAsOf()` read committed documents, for example to supply `PromptInput.read` (section 7.4). -`getTask()` and `entry()` read committed records with one lookup each. +`getTask()` and `entry()` read committed records with one lookup each; `waitForTask()` waits for a terminal receipt, +for example a child task created by a tool. `context()` captures its bounds on the Session line and derives the view from immutable entries off the line, like `Conversation.context()`. Like every runtime operation, these reject after the invocation ends. @@ -1641,7 +1646,10 @@ abort handler commits terminal outcome A run invocation may not commit after its durable abort mark appears. Every runtime operation rejects after its owning invocation ends, even while the -Session remains open. Returning from one phase handler does not end an invocation +Session remains open, and the invocation's signal and handler context abort when +it ends for any reason. An invocation ends only after its last handler returned, +so this cancels only detached leftovers, such as an unawaited `waitForTask()` or +a fetch started with the handler's context; they could no longer write anything. Returning from one phase handler does not end an invocation that continues into another phase. Invocation mode is volatile and derived from the durable mark on reopen. Cancelling one caller's `Context` only cancels that call or wait; it does not durably abort shared work unless the invoked API @@ -2154,6 +2162,8 @@ interface ToolExecutionApi extends DocumentObserver, DocumentReader { type ToolRegistration = Tool & { readonly replay?: "safe" | "unsafe"; readonly executionMode?: ToolExecutionMode; + /** Pure repair of commonly malformed arguments; runs before validation and must not mutate `args`. */ + prepareArguments?(args: JsonValue): JsonValue; readonly outputLimits?: { readonly maxBytes?: number; readonly maxLines?: number; @@ -2198,7 +2208,11 @@ its final result, plus diagnostics (below): - `output(chunk)` appends running text output, like stdout. If `execute()` omits `content`, the final retained output becomes one text content item; no output - becomes an empty content list. + becomes an empty content list. Retained output is an exact slice of whole lines + of the stream (the first lines for `head`, the last for `tail`), trailing + newline included; a single line longer than `maxBytes` is cut at the byte limit + on a character boundary. Control characters other than tab and newline are + removed from the retained text; accepting a chunk does no per-chunk sanitizing. - `details(value)` replaces the running details with a complete JSON value; it does not merge keys. If `execute()` omits `details`, the last value becomes the final `details`, so a renderer handles one details shape from the first @@ -2245,8 +2259,13 @@ A `warn` diagnostic does not set `isError`. `output()` never spills complete output to a file because spilling requires a filesystem, which may be remote or unavailable. A tool that must preserve complete output spills through the `ExecutionEnv` or `FileSystem` it was given, -such as shell execution with spill capture, and reports the resulting path in a -diagnostic, and in its details when a renderer needs it. +and reports the resulting path in a diagnostic, and in its details when a +renderer needs it. The environment's shell streams raw output chunks and spills +the complete output to a file once it crosses byte or line thresholds; it keeps +no bounded view of its own, so `output()` is the one place output is bounded, +sanitized, and throttled. The `bash` tool pipes those chunks into `output()`, +reports the spill path as a diagnostic, and throws on a nonzero exit or timeout; +the error result still carries the retained output and diagnostics. The `details()` promise resolves after the corresponding or coalesced document commit. During normal settlement the tool task stops its throttle and awaits the @@ -2277,8 +2296,11 @@ once from its phase snapshot and uses that implementation until execution settles: one phase handler resolves, validates, runs `beforeTool`, records intent, executes, and commits the result, so no phase boundary separates resolution from settlement. An unregistered implementation produces `tool_unavailable`. -Arguments are validated against the resolved implementation's schema before and -after `beforeTool`; a failure produces an `invalid_arguments` error result. +The implementation's `prepareArguments`, if any, first repairs the call's +arguments, for example `edits` sent as a JSON string; the stored call keeps what +the model sent. Arguments are validated against the resolved implementation's +schema before and after `beforeTool`; a failure, or a throwing repair, produces an +`invalid_arguments` error result. After hooks and validation, the tool task durably records the final arguments and resolved replay policy before execution. Recovery does not rerun `beforeTool` diff --git a/packages/durable/package.json b/packages/durable/package.json index 0cc52f512..c1810e72f 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -21,6 +21,11 @@ "types": "./dist/env/node.d.ts", "import": "./dist/env/node.js" }, + "./tools": { + "source": "./src/tools/index.ts", + "types": "./dist/tools/index.d.ts", + "import": "./dist/tools/index.js" + }, "./storage/memory": { "source": "./src/storage/memory.ts", "types": "./dist/storage/memory.d.ts", @@ -65,6 +70,7 @@ "test": "vitest --run", "bench:storage": "vitest bench --config vitest.benchmark.config.ts", "bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts", + "bench:tool-output": "node --conditions=source --experimental-strip-types --expose-gc test/tool-output-bench.ts", "prepublishOnly": "npm run clean && npm run build" }, "keywords": [ @@ -85,7 +91,9 @@ }, "dependencies": { "@earendil-works/chord": "^0.99.1", - "@earendil-works/pi-ai": "^0.99.1" + "@earendil-works/pi-ai": "^0.99.1", + "diff": "8.0.4", + "typebox": "1.3.27" }, "devDependencies": { "shx": "0.4.0", diff --git a/packages/durable/src/entries.ts b/packages/durable/src/entries.ts index a7df70774..f17f3aa29 100644 --- a/packages/durable/src/entries.ts +++ b/packages/durable/src/entries.ts @@ -1,4 +1,5 @@ import type { JsonValue } from "@earendil-works/chord"; +import type { ToolDiagnostic } from "./harness/types.ts"; import type { Entry, EntryRecord, TypedEntry } from "./types.ts"; /** Define a typed entry kind whose `is()` guard narrows by `EntryRecord.kind`. */ @@ -16,5 +17,8 @@ export const UserEntry = defineEntry("pi.user"); export const AssistantEntry = defineEntry("pi.assistant"); /** Positional prompt and tool change: `model` is `[SystemMessage]` with empty `content`. */ export const SystemEntry = defineEntry("pi.system"); -/** Tool result: `model` is `[ToolResultMessage]`. Written by tool tasks. */ -export const ToolResultEntry = defineEntry("pi.tool-result"); +/** + * Tool result: `model` is `[ToolResultMessage]`, whose content ends with the rendered diagnostics block; `data` holds + * the structured diagnostics, possibly none. Written by tool tasks, and by generation for calls it did not offer. + */ +export const ToolResultEntry = defineEntry<{ diagnostics: ToolDiagnostic[] }>("pi.tool-result"); diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 6dc99608c..63a520f5d 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -1,5 +1,4 @@ import type { Context } from "@earendil-works/chord"; -import type { TruncationResult } from "./utils/truncate.ts"; /** Result of a fallible operation. Expected failures are returned instead of thrown. */ export type Result = { ok: true; value: TValue } | { ok: false; error: TError }; @@ -65,6 +64,8 @@ export type ExecutionErrorCode = export class ExecutionError extends Error { public code: ExecutionErrorCode; + /** Spill file of a command that timed out or was aborted after its output crossed the spill thresholds. */ + public spillPath?: string; constructor(code: ExecutionErrorCode, message: string, cause?: Error) { super(message, cause === undefined ? undefined : { cause }); @@ -133,39 +134,17 @@ export interface FileSystem { cleanup(context: Context): Promise; } -export type ShellOutputRetention = "head" | "tail"; - -export interface ShellOutputLimits { - maxBytes: number; - maxLines: number; - retain?: ShellOutputRetention; +/** Spill the complete output to a temporary file once it exceeds either threshold. */ +export interface ShellSpillOptions { + afterBytes: number; + /** Complete or partial lines. */ + afterLines: number; } -export interface ShellOutputCaptureOptions { - limits: ShellOutputLimits; - spill?: boolean; -} - -export type ShellOutputTruncation = Omit; - -export interface ShellOutputMetadata { - truncation: ShellOutputTruncation; - spillPath?: string; - lastLineBytes?: number; -} - -export interface ShellOutputView extends ShellOutputMetadata { - text: string; -} - -export type ShellOutputUpdate = - | { kind: "replace"; output: ShellOutputView } - | { kind: "append"; text: string; metadata: ShellOutputMetadata } - | { kind: "slide"; drop: number; text: string; metadata: ShellOutputMetadata } - | { kind: "metadata"; metadata: ShellOutputMetadata }; - -export interface ShellExecResult extends ShellOutputMetadata { +export interface ShellExecResult { exitCode: number; + /** Temporary file holding the complete raw output, when the spill thresholds were exceeded. */ + spillPath?: string; } export interface ShellExecOptions { @@ -173,8 +152,9 @@ export interface ShellExecOptions { env?: Record; inheritEnv?: boolean; timeout?: number; - capture?: ShellOutputCaptureOptions; - onUpdate?: (update: ShellOutputUpdate, context: Context) => void; + /** Every decoded chunk of combined stdout and stderr as it arrives: raw, unbounded, and unthrottled. */ + onOutput?: (text: string, context: Context) => void; + spill?: ShellSpillOptions; } export interface Shell { diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index abc2bea57..4d25aac24 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -34,12 +34,11 @@ import { type TextLineReader, toError, } from "./index.ts"; -import { OutputCapture } from "./utils/output-capture.ts"; const MAX_TIMEOUT_MS = 2_147_483_647; const MAX_TIMEOUT_SECONDS = MAX_TIMEOUT_MS / 1000; const EXIT_STDIO_GRACE_MS = 100; -const SPILL_HIGH_WATER_MARK = 8 * 1024 * 1024; +const SPILL_HIGH_WATER_MARK = 1024 * 1024; type SpillChunk = string | Uint8Array; @@ -505,17 +504,21 @@ export class NodeExecutionEnv implements ExecutionEnv { callbackError = new ExecutionError("callback_error", cause.message, cause); onAbort(); }; - let capture: OutputCapture; - try { - capture = new OutputCapture(options?.capture, context, { - onUpdate: options?.onUpdate, - onError: failCallback, - }); - } catch (error) { - const cause = toError(error); - resolvePromise(err(new ExecutionError("unknown", cause.message, cause))); - return; - } + // One decoder per stream, so a character split across chunks of one stream survives interleaving. + const stdoutDecoder = new TextDecoder(); + const stderrDecoder = new TextDecoder(); + // No output reaches the caller after exec() settled, for example from a descendant holding stdio open. + const emit = (text: string): void => { + if (settled || text === "" || options?.onOutput === undefined || callbackError !== undefined) return; + try { + options.onOutput(text, context); + } catch (error) { + failCallback(error); + } + }; + // Output seen before the spill starts: counted against the thresholds and kept for the spill's prefix. + let seenBytes = 0; + let seenNewlines = 0; const settle = (result: Result) => { if (settled) return; @@ -523,7 +526,6 @@ export class NodeExecutionEnv implements ExecutionEnv { if (timeoutId) clearTimeout(timeoutId); if (signal) signal.removeEventListener("abort", onAbort); if (child?.pid) this.activeChildPids.delete(child.pid); - capture.dispose(); resolvePromise(result); }; const pauseOutput = () => { @@ -568,7 +570,6 @@ export class NodeExecutionEnv implements ExecutionEnv { const created = await this.createTempFile({ prefix: "pi-output-", suffix: ".log" }, context); if (!created.ok) throw created.error; spillPath = created.value; - capture.setSpillPath(spillPath); spillStream = createWriteStream(spillPath, { flags: "a", highWaterMark: SPILL_HIGH_WATER_MARK }); spillStream.on("error", failSpill); for (const queued of spillQueue) writeSpill(queued); @@ -625,26 +626,27 @@ export class NodeExecutionEnv implements ExecutionEnv { else signal.addEventListener("abort", onAbort, { once: true }); } - const feed = (chunk: Uint8Array) => { - try { - const wasTruncated = capture.truncated; - capture.push(chunk); - if (!options?.capture?.spill || chunk.length === 0) return; - if (spillPath !== undefined || wasTruncated) { - startSpill(chunk); - } else if (capture.truncated) { - for (const prefix of spillPrefix) startSpill(prefix); - spillPrefix.length = 0; - startSpill(chunk); - } else { - spillPrefix.push(chunk); - } - } catch (error) { - failCallback(error); + const feed = (decoder: InstanceType) => (chunk: Uint8Array) => { + emit(decoder.decode(chunk, { stream: true })); + const spill = options?.spill; + if (spill === undefined || chunk.length === 0) return; + if (spillStart !== undefined) { + startSpill(chunk); + return; } + seenBytes += chunk.length; + for (let index = chunk.indexOf(0x0a); index !== -1; index = chunk.indexOf(0x0a, index + 1)) seenNewlines++; + const lines = seenNewlines + (chunk[chunk.length - 1] === 0x0a ? 0 : 1); + if (seenBytes <= spill.afterBytes && lines <= spill.afterLines) { + spillPrefix.push(chunk); + return; + } + for (const prefix of spillPrefix) startSpill(prefix); + spillPrefix.length = 0; + startSpill(chunk); }; - child.stdout?.on("data", feed); - child.stderr?.on("data", feed); + child.stdout?.on("data", feed(stdoutDecoder)); + child.stderr?.on("data", feed(stderrDecoder)); void waitForChildProcess( child, @@ -655,41 +657,31 @@ export class NodeExecutionEnv implements ExecutionEnv { ).then( async ({ code, signal: exitSignal }) => { await finishSpill(); - try { - capture.finish(); - capture.flush(); - } catch (error) { - failCallback(error); - } + emit(stdoutDecoder.decode()); + emit(stderrDecoder.decode()); if (callbackError) { settle(err(callbackError)); return; } - if (timedOut) { - settle(err(new ExecutionError("timeout", `timeout:${options?.timeout}`))); - return; - } - if (signal?.aborted) { - settle(err(new ExecutionError("aborted", "aborted"))); + const interrupted = timedOut + ? new ExecutionError("timeout", `timeout:${options?.timeout}`) + : signal?.aborted + ? new ExecutionError("aborted", "aborted") + : undefined; + if (interrupted !== undefined) { + if (spillPath !== undefined) interrupted.spillPath = spillPath; + settle(err(interrupted)); return; } if (spillError) { settle(err(spillError)); return; } - const output = capture.snapshot(); // A process killed by a signal (e.g. OOM killer) has no exit code; map it // to the conventional 128 + signal number so callers do not mistake it // for a successful exit. const exitCode = code ?? (exitSignal ? 128 + (osConstants.signals[exitSignal] ?? 0) : 1); - settle( - ok({ - exitCode, - truncation: output.truncation, - ...(output.spillPath === undefined ? {} : { spillPath: output.spillPath }), - ...(output.lastLineBytes === undefined ? {} : { lastLineBytes: output.lastLineBytes }), - }), - ); + settle(ok({ exitCode, ...(spillPath === undefined ? {} : { spillPath }) })); }, (error: Error) => settle(err(new ExecutionError("spawn_error", error.message, error))), ); diff --git a/packages/durable/src/env/utils/adaptive-publisher.ts b/packages/durable/src/env/utils/adaptive-publisher.ts deleted file mode 100644 index d971ee969..000000000 --- a/packages/durable/src/env/utils/adaptive-publisher.ts +++ /dev/null @@ -1,87 +0,0 @@ -export interface AdaptivePublisherOptions { - snapshot(): TValue; - update(previous: TValue | undefined, current: TValue): TUpdate | undefined; - measure(update: TUpdate): number; - publish(update: TUpdate): void; - onError(error: unknown): void; - minIntervalMs?: number; - targetBytesPerSecond?: number; -} - -/** - * Publishes the latest state without queuing intermediate mutations. - * - * The first dirty state after idle is immediate. Each publication then buys a - * delay proportional to its encoded size, with a minimum interval that also - * bounds event count. A single trailing timer guarantees eventual publication. - */ -export class AdaptivePublisher { - readonly #options: AdaptivePublisherOptions; - readonly #minIntervalMs: number; - readonly #targetBytesPerSecond: number; - #published: TValue | undefined; - #dirty = false; - #nextEmitAt = 0; - #timer: ReturnType | undefined; - #disposed = false; - - constructor(options: AdaptivePublisherOptions) { - this.#options = options; - this.#minIntervalMs = options.minIntervalMs ?? 100; - this.#targetBytesPerSecond = options.targetBytesPerSecond ?? 100 * 1024; - } - - markDirty(): void { - if (this.#disposed) return; - this.#dirty = true; - const wait = this.#nextEmitAt - Date.now(); - if (wait <= 0) { - this.flush(); - return; - } - this.#armTimer(wait); - } - - flush(force = false): void { - if (this.#disposed || !this.#dirty) return; - const now = Date.now(); - if (!force && now < this.#nextEmitAt) { - this.#armTimer(this.#nextEmitAt - now); - return; - } - if (this.#timer !== undefined) clearTimeout(this.#timer); - this.#timer = undefined; - const current = this.#options.snapshot(); - const update = this.#options.update(this.#published, current); - if (update === undefined) { - this.#published = current; - this.#dirty = false; - return; - } - const encodedBytes = this.#options.measure(update); - this.#published = current; - this.#dirty = false; - this.#nextEmitAt = now + Math.max(this.#minIntervalMs, (encodedBytes * 1000) / this.#targetBytesPerSecond); - // Commit before delivery. A consumer may apply the update and then throw or - // reenter the producer; retaining the old baseline would duplicate that delta. - this.#options.publish(update); - } - - dispose(): void { - if (this.#timer !== undefined) clearTimeout(this.#timer); - this.#timer = undefined; - this.#disposed = true; - } - - #armTimer(wait: number): void { - if (this.#timer !== undefined) return; - this.#timer = setTimeout(() => { - this.#timer = undefined; - try { - this.flush(); - } catch (error) { - this.#options.onError(error); - } - }, wait); - } -} diff --git a/packages/durable/src/env/utils/output-capture.ts b/packages/durable/src/env/utils/output-capture.ts deleted file mode 100644 index ded71a284..000000000 --- a/packages/durable/src/env/utils/output-capture.ts +++ /dev/null @@ -1,258 +0,0 @@ -import type { Context } from "@earendil-works/chord"; -import type { ShellOutputCaptureOptions, ShellOutputMetadata, ShellOutputUpdate, ShellOutputView } from "../index.ts"; -import { AdaptivePublisher } from "./adaptive-publisher.ts"; -import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, truncateHead, truncateTail, utf8ByteLength } from "./truncate.ts"; - -export const OUTPUT_MIN_EMIT_INTERVAL_MS = 100; -export const OUTPUT_TARGET_BYTES_PER_SECOND = 100 * 1024; - -const INVALID_SHELL_OUTPUT = /[\x00-\x08\x0b-\x1f\ufff9-\ufffb]/g; -const textEncoder = new TextEncoder(); -const textDecoder = new TextDecoder(); - -interface OutputCaptureHandlers { - onUpdate?: (update: ShellOutputUpdate, context: Context) => void; - onError(error: unknown): void; -} - -/** - * Maintains and publishes one bounded shell-output view. - * - * Writes received while publication is rate-limited collapse into the latest - * view. Small changes remain responsive; complete window turnovers purchase a - * proportionally longer delay. The first update after idle and an explicit - * final flush are immediate. - */ -export class OutputCapture { - readonly #maxBytes: number; - readonly #maxLines: number; - readonly #retain: "head" | "tail"; - readonly #context: Context; - readonly #onUpdate: OutputCaptureHandlers["onUpdate"]; - - readonly #decoder = new TextDecoder(); - #buffer = ""; - #bufferBytes = 0; - #totalBytes = 0; - #newlines = 0; - #endsWithNewline = true; - #currentLineBytes = 0; - #lastTerminatedLineBytes = 0; - #spillPath: string | undefined; - #disposed = false; - readonly #publisher: AdaptivePublisher; - - constructor(options: ShellOutputCaptureOptions | undefined, context: Context, handlers: OutputCaptureHandlers) { - this.#maxBytes = options?.limits.maxBytes ?? DEFAULT_MAX_BYTES; - this.#maxLines = options?.limits.maxLines ?? DEFAULT_MAX_LINES; - this.#retain = options?.limits.retain ?? "tail"; - this.#context = context; - this.#onUpdate = handlers.onUpdate; - if (!Number.isFinite(this.#maxBytes) || this.#maxBytes <= 0) { - throw new TypeError("Output maxBytes must be a positive finite number"); - } - if (!Number.isInteger(this.#maxLines) || this.#maxLines <= 0) { - throw new TypeError("Output maxLines must be a positive integer"); - } - this.#publisher = new AdaptivePublisher({ - snapshot: () => this.snapshot(), - update: updateFrom, - measure: (update) => utf8ByteLength(JSON.stringify(update)), - publish: (update) => this.#onUpdate?.(update, this.#context), - onError: handlers.onError, - minIntervalMs: OUTPUT_MIN_EMIT_INTERVAL_MS, - targetBytesPerSecond: OUTPUT_TARGET_BYTES_PER_SECOND, - }); - } - - get truncated(): boolean { - return this.#totalBytes > this.#maxBytes || this.#totalLines() > this.#maxLines; - } - - push(chunk: string | Uint8Array): void { - if (this.#disposed) return; - if (typeof chunk === "string") { - this.#appendText(this.#decoder.decode()); - this.#appendText(chunk); - return; - } - this.#appendText(this.#decoder.decode(chunk, { stream: true })); - } - - finish(): void { - if (this.#disposed) return; - this.#appendText(this.#decoder.decode()); - } - - setSpillPath(path: string): void { - if (this.#disposed || this.#spillPath === path) return; - this.#spillPath = path; - this.#publisher.markDirty(); - this.flush(); - } - - snapshot(): ShellOutputView { - const retained = - this.#retain === "head" - ? truncateHead(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines }) - : truncateTail(this.#buffer, { maxBytes: this.#maxBytes, maxLines: this.#maxLines }); - const totalLines = this.#totalLines(); - const truncated = this.truncated; - const { content, ...truncation } = retained; - // The working buffer preserves the retained edge, so its truncation decision reports the first limit reached. - const truncatedBy = !truncated - ? null - : retained.truncated - ? retained.truncatedBy - : totalLines > this.#maxLines - ? "lines" - : "bytes"; - const lastLineBytes = this.#endsWithNewline ? this.#lastTerminatedLineBytes : this.#currentLineBytes; - return { - text: sanitizeShellOutput(content), - truncation: { - ...truncation, - truncated, - truncatedBy, - totalBytes: this.#totalBytes, - totalLines, - }, - ...(this.#spillPath === undefined ? {} : { spillPath: this.#spillPath }), - ...(retained.lastLinePartial ? { lastLineBytes } : {}), - }; - } - - flush(): void { - if (this.#disposed) return; - this.#publisher.flush(true); - } - - dispose(): void { - this.#publisher.dispose(); - this.#disposed = true; - } - - #appendText(text: string): void { - if (text === "") return; - const textBytes = utf8ByteLength(text); - this.#totalBytes += textBytes; - this.#newlines += countNewlines(text); - this.#endsWithNewline = text.endsWith("\n"); - const lastNewline = text.lastIndexOf("\n"); - if (lastNewline === -1) { - this.#currentLineBytes += textBytes; - } else { - const previousNewline = lastNewline === 0 ? -1 : text.lastIndexOf("\n", lastNewline - 1); - this.#lastTerminatedLineBytes = - previousNewline === -1 - ? this.#currentLineBytes + utf8ByteLength(text.slice(0, lastNewline)) - : utf8ByteLength(text.slice(previousNewline + 1, lastNewline)); - this.#currentLineBytes = utf8ByteLength(text.slice(lastNewline + 1)); - } - this.#buffer += text; - this.#bufferBytes += textBytes; - - // Trimming stops at a UTF-8 character boundary. The four-byte allowance keeps at least twice the byte limit, - // preserving the evidence for limits and partial lines even when an edge character is multi-byte. - const guard = this.#maxBytes * 2 + 4; - if (this.#bufferBytes > guard * 2) { - this.#buffer = - this.#retain === "tail" - ? trimToLastUtf8Bytes(this.#buffer, guard) - : trimToFirstUtf8Bytes(this.#buffer, guard); - this.#bufferBytes = utf8ByteLength(this.#buffer); - } - this.#publisher.markDirty(); - } - - #totalLines(): number { - return this.#newlines + (this.#endsWithNewline || this.#totalBytes === 0 ? 0 : 1); - } -} - -export function applyShellOutputUpdate( - current: ShellOutputView | undefined, - update: ShellOutputUpdate, -): ShellOutputView { - switch (update.kind) { - case "replace": - return update.output; - case "append": - return { text: `${current?.text ?? ""}${update.text}`, ...update.metadata }; - case "slide": - return { text: `${current?.text.slice(update.drop) ?? ""}${update.text}`, ...update.metadata }; - case "metadata": - return { text: current?.text ?? "", ...update.metadata }; - } -} - -function updateFrom(previous: ShellOutputView | undefined, current: ShellOutputView): ShellOutputUpdate { - if (previous === undefined) return { kind: "replace", output: current }; - const metadata: ShellOutputMetadata = { - truncation: current.truncation, - ...(current.spillPath === undefined ? {} : { spillPath: current.spillPath }), - ...(current.lastLineBytes === undefined ? {} : { lastLineBytes: current.lastLineBytes }), - }; - if (current.text === previous.text) return { kind: "metadata", metadata }; - if (current.text.length > previous.text.length && current.text.slice(0, previous.text.length) === previous.text) { - return { kind: "append", text: current.text.slice(previous.text.length), metadata }; - } - const shared = suffixPrefixOverlap( - previous.text, - current.text, - Math.min(previous.text.length, current.text.length, current.truncation.maxBytes * 2), - ); - if (shared > 0) { - return { - kind: "slide", - drop: previous.text.length - shared, - text: current.text.slice(shared), - metadata, - }; - } - return { kind: "replace", output: current }; -} - -function suffixPrefixOverlap(before: string, after: string, scan: number): number { - if (before.length === 0 || after.length === 0 || scan === 0) return 0; - const tail = before.length > scan ? before.slice(before.length - scan) : before; - for (const probeLength of [Math.min(64, after.length), 1]) { - const probe = after.slice(0, probeLength); - let candidates = 0; - for (let index = tail.indexOf(probe); index !== -1; index = tail.indexOf(probe, index + 1)) { - if (++candidates > 8) break; - const overlapLength = tail.length - index; - if (overlapLength <= after.length && tail.slice(index) === after.slice(0, overlapLength)) { - return overlapLength; - } - } - if (probeLength === 1) break; - } - return 0; -} - -export function sanitizeShellOutput(text: string): string { - return text.replace(INVALID_SHELL_OUTPUT, ""); -} - -function countNewlines(text: string): number { - let count = 0; - for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++; - return count; -} - -function trimToLastUtf8Bytes(text: string, maxBytes: number): string { - const bytes = textEncoder.encode(text); - if (bytes.length <= maxBytes) return text; - let start = bytes.length - maxBytes; - while (start < bytes.length && ((bytes[start] ?? 0) & 0xc0) === 0x80) start++; - return textDecoder.decode(bytes.subarray(start)); -} - -function trimToFirstUtf8Bytes(text: string, maxBytes: number): string { - const bytes = textEncoder.encode(text); - if (bytes.length <= maxBytes) return text; - let end = maxBytes; - while (end > 0 && ((bytes[end] ?? 0) & 0xc0) === 0x80) end--; - return textDecoder.decode(bytes.subarray(0, end)); -} diff --git a/packages/durable/src/harness/config.ts b/packages/durable/src/harness/config.ts index 3d0b692eb..fb5faea54 100644 --- a/packages/durable/src/harness/config.ts +++ b/packages/durable/src/harness/config.ts @@ -1,6 +1,6 @@ import type { ModelThinkingLevel } from "@earendil-works/pi-ai"; import { defineDoc } from "../documents.ts"; -import type { ConversationRetryPolicy, ConversationStreamOptions } from "./types.ts"; +import type { ConversationRetryPolicy, ConversationStreamOptions, ToolExecutionMode } from "./types.ts"; /** Durable per-conversation model, thinking level, request options, and desired tool loadout. */ export type ConversationConfigState = { @@ -12,6 +12,8 @@ export type ConversationConfigState = { streamOptions?: ConversationStreamOptions; /** Durable generation attempt retries; absent uses `DEFAULT_RETRY_POLICY`. */ retry?: ConversationRetryPolicy; + /** Whether a round's tools run at once or in call order; absent means `parallel`. */ + toolExecution?: ToolExecutionMode; }; export const DEFAULT_RETRY_POLICY: ConversationRetryPolicy = { diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index 40bc2c0d0..d2715d2b9 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -7,15 +7,28 @@ import type { Model, ModelThinkingLevel, SimpleStreamOptions, + ToolCall, } from "@earendil-works/pi-ai"; import { isRetryableAssistantError, retryDelayMs } from "@earendil-works/pi-ai/utils/retry"; -import { AssistantEntry, SystemEntry } from "../entries.ts"; +import { getCurrentTools } from "@earendil-works/pi-ai/utils/transcript"; +import { AssistantEntry, SystemEntry, UserEntry } from "../entries.ts"; import { defineTask } from "../tasks.ts"; -import type { ConversationId, EntryId, NextTaskState, TaskRuntime, Tx } from "../types.ts"; +import type { ConversationId, EntryId, NextTaskState, TaskId, TaskRuntime, Tx, TypedEntry } from "../types.ts"; import { ConversationConfig, DEFAULT_RETRY_POLICY } from "./config.ts"; -import { endRun, LiveDoc, type LiveState } from "./live.ts"; -import { planSystemEntries, renderSections, replaySections } from "./prompt.ts"; -import type { ConversationStreamOptions, ModelRef } from "./types.ts"; +import { assignJson } from "./json.ts"; +import { endRun, LiveDoc, type LiveState, type ToolSlot } from "./live.ts"; +import { PostToolsTask } from "./post-tools.ts"; +import { desiredTools, planSystemEntries, renderSections, replaySections } from "./prompt.ts"; +import { appendToolResult, harnessError, ToolTask, type ToolTaskResult } from "./tool.ts"; +import type { + ConversationStreamOptions, + GenerationHooks, + ModelRef, + PromptInput, + ToolExecutionMode, + ToolRegistration, + UserInput, +} from "./types.ts"; export type GenerationInput = Record; @@ -28,47 +41,71 @@ export type GenerationCheckpoint = thinkingLevel: ModelThinkingLevel; /** Configured request options when preparation committed; a resend after recovery uses them unchanged. */ streamOptions: ConversationStreamOptions; + toolExecution: ToolExecutionMode; /** Newest entry included in the request. */ cutoff: EntryId; } | { phase: "retry"; attempt: number; until: number } - | { phase: "poll"; attempt: number; model: ModelRef; handle: DeferredHandle; pollAt: number }; + | { + phase: "poll"; + attempt: number; + model: ModelRef; + toolExecution: ToolExecutionMode; + cutoff: EntryId; + handle: DeferredHandle; + pollAt: number; + }; export type GenerationResult = { entryId: EntryId }; -type Runtime = TaskRuntime; +type Runtime = TaskRuntime; type Next = NextTaskState; +/** What classification needs from the request that produced a message. */ +type Request = { + readonly attempt: number; + readonly model: ModelRef; + readonly toolExecution: ToolExecutionMode; + readonly cutoff: EntryId; + /** Committed model context through `cutoff`, when the phase already derived it. */ + readonly messages?: readonly Message[]; + /** Set when the message came from polling, so a still deferred result polls strictly later. */ + readonly pollAt?: number; +}; + const PARTIAL_THROTTLE_MS = 100; const DEFAULT_POLL_AFTER_MS = 5000; /** - * Built-in generation task: prepares the positional system prompt, requests or polls the model, retries, and classifies - * the response. The run's inputs live in `pi.live.run`. + * Built-in generation task: prepares the positional system prompt and tool loadout, requests or polls the model, + * retries, and classifies the response. The run's inputs live in `pi.live.run`. */ -export const GenerationTask = defineTask({ +export const GenerationTask = defineTask({ name: "pi.generation", version: 1, initial: () => ({ phase: "prepare", attempt: 1 }), phases: { /** - * Render the system prompt and append the positional `pi.system` entries it needs, then move to `request`. Only - * the Harness writes to a busy conversation, so the transcript read here is still the tail at the commit. + * Render the system prompt and tool loadout and append the positional `pi.system` entries they need, then move to + * `request`. The configuration read here is fixed for this request. Only the Harness writes to a busy + * conversation, so the transcript read here is still the tail at the commit. */ prepare: async (task, runtime, context) => { const { conversationId, registry } = runtime; - for (const failure of registry.failures()) if (failure.kind === "section") runtime.report(failure.error); - const { model, thinkingLevel, streamOptions } = + for (const failure of registry.failures()) runtime.report(failure.error); + const config = (await runtime.snapshot(ConversationConfig, conversationId, context)) ?? ConversationConfig.definition.initial(); + const { model, thinkingLevel, streamOptions } = config; if (model === undefined || runtime.models.getModel(model.provider, model.modelId) === undefined) { return failNoModel(runtime, model, context); } const view = await runtime.context(conversationId, context); const shown = replaySections(view.messages); - const input = { + const tools = desiredTools(config.activeTools, (name) => registry.tool(name)); + const input: PromptInput = { conversationId, - tools: [], + tools, shown: Object.fromEntries(shown), model, thinkingLevel, @@ -76,18 +113,24 @@ export const GenerationTask = defineTask runtime.report(error); const desired = await renderSections(registry.sections(), input, shown, report, context); - const entries = planSystemEntries(view, desired, runtime.now()); + const entries = planSystemEntries(view, desired, tools, runtime.now()); await runtime.commit(async (tx) => { let cutoff = (await tx.scanEntries({ conversationId }, 1)).items[0]?.id; for (const entry of entries) cutoff = (await tx.appendEntry(SystemEntry, conversationId, entry)).id; if (cutoff === undefined) throw new Error(`Conversation ${conversationId} has no entries to send`); - const { attempt } = task.state.checkpoint; - const request = { attempt, model, thinkingLevel, streamOptions: streamOptions ?? {}, cutoff }; + const request = { + attempt: task.state.checkpoint.attempt, + model, + thinkingLevel, + streamOptions: streamOptions ?? {}, + toolExecution: config.toolExecution ?? "parallel", + cutoff, + }; return { status: "running", checkpoint: { phase: "request", ...request } }; }, context); }, request: async (task, runtime, context) => { - const { attempt, model: ref, thinkingLevel, streamOptions, cutoff } = task.state.checkpoint; + const { attempt, model: ref, thinkingLevel, streamOptions, toolExecution, cutoff } = task.state.checkpoint; const conversationId = runtime.conversationId; await runtime.commit(async (tx) => { const live = await tx.doc(LiveDoc, conversationId); @@ -98,13 +141,19 @@ export const GenerationTask = defineTask { + const replaced = await hook({ messages }, runtime, context); + if (replaced !== undefined) messages = replaced.messages; + }); const options: SimpleStreamOptions = { ...streamOptions, signal: runtime.signal, ...(thinkingLevel === "off" ? {} : { reasoning: thinkingLevel }), }; - const message = await streamResponse(runtime, model, view.messages, options, attempt, context); - await classify(runtime, attempt, ref, undefined, message, context); + const message = await streamResponse(runtime, model, messages, options, attempt, context); + const request = { attempt, model: ref, toolExecution, cutoff, messages: view.messages }; + await classify(runtime, request, message, context); }, retry: async (task, runtime, context) => { const { attempt, until } = task.state.checkpoint; @@ -115,12 +164,12 @@ export const GenerationTask = defineTask { - const { attempt, model: ref, handle, pollAt } = task.state.checkpoint; + const { attempt, model: ref, toolExecution, cutoff, handle, pollAt } = task.state.checkpoint; const model = runtime.models.getModel(ref.provider, ref.modelId); if (model === undefined) return failNoModel(runtime, ref, context); await runtime.sleep(pollAt, context); const message = await runtime.models.fetchDeferred(model, handle, { signal: runtime.signal }); - await classify(runtime, attempt, ref, pollAt, message, context); + await classify(runtime, { attempt, model: ref, toolExecution, cutoff, pollAt }, message, context); }, }, abort: async (task, runtime, context) => { @@ -161,7 +210,7 @@ async function convertPartial(tx: Tx, live: Draft, conversationId: Co const partial = live.generation?.message; if (partial === undefined) return; const message = copyJson(partial) as unknown as AssistantMessage; - await tx.appendEntry(AssistantEntry, conversationId, { model: [{ ...message, stopReason: "aborted" }] }); + await appendAssistant(tx, conversationId, { ...message, stopReason: "aborted" }); } /** @@ -219,33 +268,39 @@ async function streamResponse( } } -/** - * Classify a terminal provider message in one commit that also clears the partial. `previousPollAt` is set when the - * message came from polling, so a still deferred result polls strictly later. - */ +/** Classify a terminal provider message in one commit that also clears the partial. */ async function classify( runtime: Runtime, - attempt: number, - ref: ModelRef, - previousPollAt: number | undefined, + request: Request, message: AssistantMessage, context: Context, ): Promise { // An abort mark or close: the abort invocation or the reopened run handles the committed state. runtime.signal.throwIfAborted(); const conversationId = runtime.conversationId; + const { attempt, model: ref, toolExecution, cutoff } = request; if (message.stopReason === "deferred" && message.deferred !== undefined) { const handle = message.deferred; const pollAt = Math.max( runtime.now() + (handle.pollAfterMs ?? DEFAULT_POLL_AFTER_MS), - previousPollAt === undefined ? Number.NEGATIVE_INFINITY : previousPollAt + 1, + request.pollAt === undefined ? Number.NEGATIVE_INFINITY : request.pollAt + 1, ); await runtime.commit(async (tx) => { (await tx.doc(LiveDoc, conversationId)).generation = { attempt, deferred: { pollAt } }; - return { status: "running", checkpoint: { phase: "poll", attempt, model: ref, handle, pollAt } }; + const checkpoint = { phase: "poll", attempt, model: ref, toolExecution, cutoff, handle, pollAt } as const; + return { status: "running", checkpoint }; }, context); return; } + await runtime.hooks.each("afterResponse", (hook) => hook(message, runtime, context)); + const calls = message.content.filter((content): content is ToolCall => content.type === "toolCall"); + if (message.stopReason === "toolUse" && calls.length > 0) { + return startToolRound(runtime, request, message, calls, context); + } + if (message.stopReason === "stop" || message.stopReason === "length" || message.stopReason === "toolUse") { + return answer(runtime, message, context); + } + // The retry policy governs the next attempt, so it is read now rather than pinned at preparation. const policy = (await runtime.snapshot(ConversationConfig, conversationId, context))?.retry ?? DEFAULT_RETRY_POLICY; const retry = message.stopReason === "error" && @@ -255,15 +310,7 @@ async function classify( const until = retry ? runtime.now() + retryDelayMs(policy, attempt) : 0; await runtime.commit(async (tx): Promise => { const live = await tx.doc(LiveDoc, conversationId); - const entry = await tx.appendEntry(AssistantEntry, conversationId, { model: [message] }); - switch (message.stopReason) { - case "stop": - case "length": - // Until the tool chain exists, a tool call settles as the answer. - case "toolUse": - endRun(tx, live, runtime.taskId, { status: "done", answer: entry.id }); - return { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; - } + await appendAssistant(tx, conversationId, message); if (retry) { live.generation = { attempt, retry: { at: until, error: message.errorMessage ?? "" } }; return { status: "running", checkpoint: { phase: "retry", attempt, until } }; @@ -277,32 +324,93 @@ async function classify( }, context); } -type JsonContainer = Record | JsonValue[]; +/** + * A final answer. The first `onYield` continuation appends a user message and hands the run to a successor + * generation; otherwise the run's inputs settle `done` and the final boundary applies. + */ +async function answer(runtime: Runtime, message: AssistantMessage, context: Context): Promise { + let continuation: UserInput | undefined; + await runtime.hooks.each("onYield", async (hook) => { + if (continuation !== undefined) return; + continuation = (await hook(message, runtime, context))?.continue; + }); + const conversationId = runtime.conversationId; + await runtime.commit(async (tx): Promise => { + const live = await tx.doc(LiveDoc, conversationId); + const entry = await appendAssistant(tx, conversationId, message); + const result: Next = { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; + if (continuation === undefined) { + endRun(tx, live, runtime.taskId, { status: "done", answer: entry.id }); + return result; + } + const user = { role: "user", content: continuation, timestamp: runtime.now() } as const; + await tx.appendEntry(UserEntry, conversationId, { model: [user] }); + handOver(live, runtime.taskId, await tx.createTask(GenerationTask, {})); + delete live.generation; + return result; + }, context); +} /** - * Assign `value` at `target[key]` leaf by leaf. Chord records a container assignment as one full set and only emits an - * append when a string leaf is reassigned with a longer string, so writing the partial whole would store and publish the - * complete message on every flush. + * Append the tool-calling answer and start its tool round in one commit (spec §8.3). A call to a tool the request did + * not offer gets its `tool_unavailable` result here; every other call gets a tool task, chained in call order when the + * round is sequential. Post-tools waits for all of them and takes over the run. */ -function assignJson(target: JsonContainer, key: string | number, value: JsonValue): void { - const slots = target as Record; - const current = slots[key]; - if (isRecord(current) && isRecord(value)) { - for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name]; - for (const [name, child] of Object.entries(value)) assignJson(current, name, child); - return; - } - if (Array.isArray(current) && Array.isArray(value) && current.length <= value.length) { - const items = current as JsonValue[]; - for (let index = 0; index < value.length; index++) { - if (index < items.length) assignJson(items, index, value[index]!); - else items.push(value[index]!); +async function startToolRound( + runtime: Runtime, + request: Request, + message: AssistantMessage, + calls: readonly ToolCall[], + context: Context, +): Promise { + const conversationId = runtime.conversationId; + const messages = request.messages ?? (await runtime.context(conversationId, context, request.cutoff)).messages; + const offered = new Set(getCurrentTools(messages).map((tool) => tool.name)); + const sequential = + request.toolExecution === "sequential" || + calls.some((call) => offered.has(call.name) && runtime.registry.tool(call.name)?.executionMode === "sequential"); + await runtime.commit(async (tx): Promise => { + const live = await tx.doc(LiveDoc, conversationId); + const entry = await appendAssistant(tx, conversationId, message); + const slots: ToolSlot[] = []; + const tools: TaskId[] = []; + for (const call of calls) { + if (!offered.has(call.name)) { + const unavailable = harnessError("tool_unavailable", `Tool ${call.name} is not available`); + const result = await appendToolResult(tx, conversationId, call, unavailable, runtime.now()); + slots.push({ callId: call.id, name: call.name, status: "done", entry: result.id }); + continue; + } + const previous = tools.at(-1); + const after = sequential && previous !== undefined ? [previous] : []; + const taskId = await tx.createTask(ToolTask, { assistant: entry.id, callId: call.id }, { after }); + tools.push(taskId); + slots.push({ callId: call.id, name: call.name, taskId, status: "pending" }); } - return; - } - if (current !== value) slots[key] = value; + handOver( + live, + runtime.taskId, + await tx.createTask(PostToolsTask, { assistant: entry.id, tools }, { after: tools }), + ); + delete live.generation; + live.tools = slots; + return { status: "terminal", outcome: { status: "completed", result: { entryId: entry.id } } }; + }, context); } -function isRecord(value: JsonValue | undefined): value is Record { - return typeof value === "object" && value !== null && !Array.isArray(value); +/** + * Append a provider result. Every built-in writer of assistant entries goes through here. + * REMINDER: Package 17 adds the `pi.usage` totals update here, in the same commit as the entry. + */ +function appendAssistant( + tx: Tx, + conversationId: ConversationId, + message: AssistantMessage, +): Promise> { + return tx.appendEntry(AssistantEntry, conversationId, { model: [message] }); +} + +/** Hand run control from `from` to `to`; the run's inputs move with it. */ +export function handOver(live: Draft, from: TaskId, to: TaskId): void { + if (live.run?.taskId === from) live.run.taskId = to; } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 02d575b7a..7ba865ac3 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -41,6 +41,7 @@ import type { SettledTask, Submission, SubmissionDraft, + ToolExecutionMode, ToolRegistration, } from "./types.ts"; import { scanAll } from "./util.ts"; @@ -130,6 +131,17 @@ class ConversationImpl implements Conversation { }, context); } + async getToolExecution(context: Context): Promise { + return (await this.#config(context)).toolExecution ?? "parallel"; + } + + setToolExecution(mode: ToolExecutionMode | undefined, context: Context): Promise { + return this.#editConfig((config) => { + if (mode === undefined) delete config.toolExecution; + else config.toolExecution = mode; + }, context); + } + submit(submission: SubmissionDraft, context: Context): Promise { return this.#host.submissions.submit(this.id, submission, context); } @@ -203,6 +215,7 @@ class HarnessImpl extends SessionImpl implements storage, registry: options.registry, models: options.models, + env: options.env, now, report: options.onReport ?? (() => {}), settleOutcome: settleSchedulerOutcome, diff --git a/packages/durable/src/harness/json.ts b/packages/durable/src/harness/json.ts new file mode 100644 index 000000000..66cfcd891 --- /dev/null +++ b/packages/durable/src/harness/json.ts @@ -0,0 +1,31 @@ +import type { JsonValue } from "@earendil-works/chord"; + +type JsonContainer = Record | JsonValue[]; + +/** + * Assign `value` at `target[key]` leaf by leaf. Chord records a container assignment as one full set and only emits an + * append when a string leaf is reassigned with a longer string, so writing the partial whole would store and publish the + * complete message on every flush. + */ +export function assignJson(target: JsonContainer, key: string | number, value: JsonValue): void { + const slots = target as Record; + const current = slots[key]; + if (isRecord(current) && isRecord(value)) { + for (const name of Object.keys(current)) if (!Object.hasOwn(value, name)) delete current[name]; + for (const [name, child] of Object.entries(value)) assignJson(current, name, child); + return; + } + if (Array.isArray(current) && Array.isArray(value) && current.length <= value.length) { + const items = current as JsonValue[]; + for (let index = 0; index < value.length; index++) { + if (index < items.length) assignJson(items, index, value[index]!); + else items.push(value[index]!); + } + return; + } + if (current !== value) slots[key] = value; +} + +function isRecord(value: JsonValue | undefined): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 06d90cfbb..44fb832ec 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -2,12 +2,32 @@ import type { Draft, JsonRepresentation, JsonValue } from "@earendil-works/chord import type { AssistantMessage } from "@earendil-works/pi-ai"; import { defineDoc } from "../documents.ts"; import type { Transaction } from "../session/transaction.ts"; -import type { SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; +import type { EntryId, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; import type { SchedulerOutcome } from "./scheduler.ts"; +import type { ToolDiagnostic } from "./types.ts"; -/** Built-in live conversation state: run control and presentation of the current generation. */ +/** Presentation of one tool call of the current round. */ +export type ToolSlot = { + callId: string; + name: string; + /** Absent for a call its request did not offer; generation wrote its result. */ + taskId?: TaskId; + status: "pending" | "running" | "done"; + /** Retained running output and what the bounds dropped. */ + output?: string; + droppedBytes?: number; + droppedLines?: number; + /** Last `details()` value. */ + details?: JsonValue; + /** Diagnostics recorded through `api.diagnostic()`. */ + diagnostics?: ToolDiagnostic[]; + /** Result entry once done; absent when the tool task faulted or was orphaned. */ + entry?: EntryId; +}; + +/** Built-in live conversation state: run control and presentation of the current generation and tool round. */ export type LiveState = { - /** Run control: the task responsible for the run and its placed inputs; present exactly while busy. */ + /** Run control: the task that settles the run's inputs, and those inputs; present exactly while busy. */ run?: { taskId: TaskId; inputs: SubmissionId[] }; /** Presentation of the current generation attempt. */ generation?: { @@ -19,6 +39,8 @@ export type LiveState = { /** Provider-side deferred response being polled. */ deferred?: { pollAt: number }; }; + /** The current tool round in call order, from the tool-calling answer until post-tools. */ + tools?: ToolSlot[]; }; export const LiveDoc = defineDoc({ @@ -28,17 +50,21 @@ export const LiveDoc = defineDoc({ history: "latest", fork: "initial", initial: () => ({}), - // A complete base whenever nothing is in flight, so the delta chain spans at most one generation, including its - // retries and deferred polls, or one tool round. - checkpointWhen: (value) => value.generation === undefined, + // REMINDER: a complete base whenever nothing runs (spec §8.2): no generation and no running tool slot. That holds + // while idle, in the commit handing a generation over to its tool round, and between tools, so the delta chain + // spans at most one generation or the overlapping execution of one round's tools. A slot holds output only while + // running, so every base is small. Do not add a delta-count bound; the tool output benchmark checks this rule. + checkpointWhen: (value) => + value.generation === undefined && !(value.tools ?? []).some((slot) => slot.status === "running"), }); /** Built-in task kinds that can own `pi.live.run`. */ -const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation"]); +const RUN_TASK_KINDS: ReadonlySet = new Set(["pi.generation", "pi.post-tools"]); +const TOOL_TASK_KIND = "pi.tool"; /** - * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation`, whose - * presentation belongs to the ending task. + * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation` and `tools`, + * whose presentation belongs to the ending run. */ export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlement: SubmissionSettlement): void { if (live.run?.taskId === taskId) { @@ -46,17 +72,45 @@ export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlemen delete live.run; } delete live.generation; + delete live.tools; +} + +/** The slot of tool task `taskId` in the current round, if the round still lists it. */ +export function toolSlot(live: Draft, taskId: TaskId): Draft | undefined { + return live.tools?.find((slot) => slot.taskId === taskId); +} + +/** Mark a slot done: the result entry, if any, now carries its running output, details, and diagnostics. */ +export function finishSlot(slot: Draft, entry: EntryId | undefined): void { + slot.status = "done"; + if (entry !== undefined) slot.entry = entry; + clearProgress(slot); +} + +/** Remove what a tool published while running; its result entry or a rerun replaces it. */ +export function clearProgress(slot: Draft): void { + delete slot.output; + delete slot.droppedBytes; + delete slot.droppedLines; + delete slot.details; + delete slot.diagnostics; } /** - * Harness cleanup for a terminal outcome the scheduler writes itself (`faulted` or `orphaned`). Ignores non-run task - * kinds so it never creates `pi.live` elsewhere. Committed partials are discarded without a transcript entry. + * Harness cleanup for a terminal outcome the scheduler writes itself (`faulted` or `orphaned`). A run task ends its + * run; a tool task's slot is marked done without an entry, and context derivation synthesizes the missing result. + * Ignores other kinds so it never creates `pi.live` elsewhere. Committed partials are discarded without an entry. */ export async function settleSchedulerOutcome( tx: Transaction, record: TaskRecord, outcome: SchedulerOutcome, ): Promise { + if (record.kind === TOOL_TASK_KIND) { + const slot = toolSlot(await tx.doc(LiveDoc, record.conversationId), record.id); + if (slot !== undefined) finishSlot(slot, undefined); + return; + } if (!RUN_TASK_KINDS.has(record.kind)) return; const live = await tx.doc(LiveDoc, record.conversationId); if (live.run?.taskId !== record.id) return; diff --git a/packages/durable/src/harness/output.ts b/packages/durable/src/harness/output.ts new file mode 100644 index 000000000..c04475c8e --- /dev/null +++ b/packages/durable/src/harness/output.ts @@ -0,0 +1,288 @@ +import { utf8ByteLength } from "../truncate.ts"; + +/** Retention limits of one tool's output. */ +export type OutputLimits = { readonly maxBytes: number; readonly maxLines: number; readonly retain: "head" | "tail" }; + +/** Retained output and what the limits dropped. */ +export type BoundedOutput = { readonly text: string; readonly droppedBytes: number; readonly droppedLines: number }; + +/** An exact slice of the input within the limits, and what it left out. */ +export type OutputSlice = { + readonly text: string; + readonly bytes: number; + readonly droppedBytes: number; + readonly droppedLines: number; +}; + +const NEWLINE = 0x0a; +const INVALID_OUTPUT = /[\x00-\x08\x0b-\x1f\ufff9-\ufffb]/g; +const encoder = new TextEncoder(); +const decoder = new TextDecoder(); + +/** Remove control characters that break display and transcripts; tabs and newlines stay. */ +export function sanitizeOutput(text: string): string { + return text.replace(INVALID_OUTPUT, ""); +} + +/** + * Bound `text` to whole lines within the limits: the first lines for `head`, the last lines for `tail`. The result is an + * exact slice, trailing newline included. A single line longer than `maxBytes` is cut at the byte limit on a character + * boundary. + */ +export function boundOutput(text: string, limits: OutputLimits): OutputSlice { + const bytes = encoder.encode(text); + const [from, to] = limits.retain === "head" ? headRange(bytes, limits) : tailRange(bytes, limits); + const kept = bytes.subarray(from, to); + return { + text: kept.length === bytes.length ? text : decoder.decode(kept), + bytes: kept.length, + droppedBytes: bytes.length - kept.length, + droppedLines: lineCount(bytes) - lineCount(kept), + }; +} + +function headRange(bytes: Uint8Array, limits: OutputLimits): [number, number] { + if (limits.maxLines === 0 || limits.maxBytes === 0) return [0, 0]; + let end = bytes.length; + let lines = 0; + for (let index = bytes.indexOf(NEWLINE); index !== -1; index = bytes.indexOf(NEWLINE, index + 1)) { + if (++lines === limits.maxLines) { + end = index + 1; + break; + } + } + if (end > limits.maxBytes) { + const newline = bytes.lastIndexOf(NEWLINE, limits.maxBytes - 1); + end = newline === -1 ? characterEnd(bytes, limits.maxBytes) : newline + 1; + } + return [0, end]; +} + +function tailRange(bytes: Uint8Array, limits: OutputLimits): [number, number] { + if (limits.maxLines === 0 || limits.maxBytes === 0) return [bytes.length, bytes.length]; + // A trailing newline ends the last line rather than starting another. + const last = bytes[bytes.length - 1] === NEWLINE ? bytes.length - 2 : bytes.length - 1; + let start = 0; + let lines = 1; + for (let index = last < 0 ? -1 : bytes.lastIndexOf(NEWLINE, last); index !== -1; ) { + if (lines === limits.maxLines) { + start = index + 1; + break; + } + lines++; + index = index === 0 ? -1 : bytes.lastIndexOf(NEWLINE, index - 1); + } + if (bytes.length - start > limits.maxBytes) { + const from = bytes.length - limits.maxBytes; + const newline = bytes.indexOf(NEWLINE, from - 1); + // The first line starting inside the byte window, or a cut of the last line when it alone is too long. + start = newline !== -1 && newline + 1 < bytes.length ? newline + 1 : characterStart(bytes, from); + } + return [start, bytes.length]; +} + +/** The last character boundary at or before `index`. */ +export function characterEnd(bytes: Uint8Array, index: number): number { + let end = index; + while (end > 0 && ((bytes[end] ?? 0) & 0xc0) === 0x80) end--; + return end; +} + +/** The first character boundary at or after `index`. */ +function characterStart(bytes: Uint8Array, index: number): number { + let start = index; + while (start < bytes.length && ((bytes[start] ?? 0) & 0xc0) === 0x80) start++; + return start; +} + +function lineCount(bytes: Uint8Array): number { + if (bytes.length === 0) return 0; + let newlines = 0; + for (let index = bytes.indexOf(NEWLINE); index !== -1; index = bytes.indexOf(NEWLINE, index + 1)) newlines++; + return newlines + (bytes[bytes.length - 1] === NEWLINE ? 0 : 1); +} + +/** + * Bounded running output of one tool call. Accepting a chunk costs time proportional to the chunk: head retention stops + * storing once the window is full, and tail retention drops stored text the window no longer needs when it snapshots. + * Counts of the whole stream are kept so the dropped totals stay exact. + */ +export class OutputBuffer { + readonly #limits: OutputLimits; + readonly #decoder = new TextDecoder(); + /** Stored chunks: for head the start of the stream, for tail a suffix that still contains the next window. */ + #chunks: { readonly text: string; readonly bytes: number; readonly newlines: number }[] = []; + #storedBytes = 0; + #storedNewlines = 0; + #full = false; + #totalBytes = 0; + #totalNewlines = 0; + #endsWithNewline = true; + + constructor(limits: OutputLimits) { + this.#limits = limits; + } + + /** Bytes currently held; bounded by the limits plus one chunk. */ + get storedBytes(): number { + return this.#storedBytes; + } + + /** Accept a chunk; returns whether anything was accepted. */ + push(chunk: string | Uint8Array): boolean { + // Bytes of an incomplete character from an earlier byte chunk come first. + const text = + typeof chunk === "string" ? this.#decoder.decode() + chunk : this.#decoder.decode(chunk, { stream: true }); + return this.#accept(text); + } + + /** Flush an incomplete trailing character as a replacement character; call when the stream ends. */ + end(): void { + this.#accept(this.#decoder.decode()); + } + + #accept(text: string): boolean { + if (text.length === 0) return false; + const bytes = utf8ByteLength(text); + const newlines = countNewlines(text); + this.#totalBytes += bytes; + this.#totalNewlines += newlines; + this.#endsWithNewline = text.endsWith("\n"); + if (this.#full) return true; + this.#chunks.push({ text, bytes, newlines }); + this.#storedBytes += bytes; + this.#storedNewlines += newlines; + if (this.#limits.retain === "head") { + // Nothing past a full window is ever needed. + this.#full = this.#storedBytes > this.#limits.maxBytes || this.#storedNewlines >= this.#limits.maxLines; + return true; + } + // Drop leading chunks while the rest still holds more than a window: more than `maxBytes` bytes or `maxLines` + // newlines, plus one, so the window's line start can still be found. Each chunk is dropped once. + while (this.#chunks.length > 1) { + const first = this.#chunks[0]!; + const bytesAfter = this.#storedBytes - first.bytes; + const newlinesAfter = this.#storedNewlines - first.newlines; + if (bytesAfter <= this.#limits.maxBytes + 1 && newlinesAfter <= this.#limits.maxLines + 1) break; + this.#chunks.shift(); + this.#storedBytes = bytesAfter; + this.#storedNewlines = newlinesAfter; + } + return true; + } + + /** Retained, sanitized output and what the limits dropped from the whole stream. */ + snapshot(): BoundedOutput { + const stored = + this.#chunks.length === 1 ? this.#chunks[0]!.text : this.#chunks.map((chunk) => chunk.text).join(""); + const kept = boundOutput(stored, this.#limits); + const storedLines = lines(this.#storedNewlines, stored === "" || stored.endsWith("\n")); + const keptLines = storedLines - kept.droppedLines; + // Tail windows never reach back before this one, so only the kept slice needs storing. + if (this.#limits.retain === "tail" || this.#chunks.length > 1) { + const text = this.#limits.retain === "tail" ? kept.text : stored; + const bytes = this.#limits.retain === "tail" ? kept.bytes : this.#storedBytes; + this.#chunks = text === "" ? [] : [{ text, bytes, newlines: countNewlines(text) }]; + this.#storedBytes = bytes; + this.#storedNewlines = this.#chunks[0]?.newlines ?? 0; + } + return { + text: sanitizeOutput(kept.text), + droppedBytes: this.#totalBytes - kept.bytes, + droppedLines: lines(this.#totalNewlines, this.#endsWithNewline) - keptLines, + }; + } +} + +/** Lines of text with `newlines` newlines; a final unterminated line counts. */ +function lines(newlines: number, terminated: boolean): number { + return newlines + (terminated ? 0 : 1); +} + +function countNewlines(text: string): number { + let count = 0; + for (let index = text.indexOf("\n"); index !== -1; index = text.indexOf("\n", index + 1)) count++; + return count; +} + +/** Minimum pause between progress commits; each commit also buys a pause proportional to what it wrote. */ +const MIN_PROGRESS_INTERVAL_MS = 100; +const PROGRESS_BYTES_PER_SECOND = 100 * 1024; + +/** + * Adaptive progress commits, like the environment's shell output capture: the first change after an idle period + * commits at once; each commit then delays the next by at least 100 ms and by its written size at 100 KiB/s. At most + * one commit is in flight; changes made meanwhile coalesce into the next one. + */ +export class Progress { + readonly #write: () => Promise; + readonly #onError: (error: unknown) => void; + #waiters: PromiseWithResolvers[] = []; + #timer: ReturnType | undefined; + #inFlight: Promise | undefined; + #nextAt = 0; + #dirty = false; + #stopped = false; + + constructor(write: () => Promise, onError: (error: unknown) => void) { + this.#write = write; + this.#onError = onError; + } + + /** Schedule a commit. */ + mark(): void { + this.#dirty = true; + this.#schedule(); + } + + /** Schedule a commit; the promise settles with the commit that includes this change. */ + markAndWait(): Promise { + const waiter = Promise.withResolvers(); + this.#waiters.push(waiter); + this.mark(); + return waiter.promise; + } + + /** Stop committing and wait for the commit in flight; returns the waiters the final commit must settle. */ + async stop(): Promise[]> { + this.#stopped = true; + clearTimeout(this.#timer); + this.#timer = undefined; + await this.#inFlight; + return this.#waiters.splice(0); + } + + #schedule(): void { + if (this.#stopped || this.#timer !== undefined || this.#inFlight !== undefined) return; + const wait = this.#nextAt - Date.now(); + if (wait <= 0) this.#flush(); + else + this.#timer = setTimeout(() => { + this.#timer = undefined; + this.#flush(); + }, wait); + } + + #flush(): void { + if (this.#stopped || !this.#dirty) return; + this.#dirty = false; + const waiters = this.#waiters.splice(0); + const started = Date.now(); + this.#inFlight = this.#write() + .then( + (bytes) => { + this.#nextAt = started + Math.max(MIN_PROGRESS_INTERVAL_MS, (bytes * 1000) / PROGRESS_BYTES_PER_SECOND); + for (const waiter of waiters) waiter.resolve(); + }, + (error: unknown) => { + this.#nextAt = started + MIN_PROGRESS_INTERVAL_MS; + for (const waiter of waiters) waiter.reject(error); + this.#onError(error); + }, + ) + .finally(() => { + this.#inFlight = undefined; + if (this.#dirty) this.#schedule(); + }); + } +} diff --git a/packages/durable/src/harness/post-tools.ts b/packages/durable/src/harness/post-tools.ts new file mode 100644 index 000000000..1e2aedf7c --- /dev/null +++ b/packages/durable/src/harness/post-tools.ts @@ -0,0 +1,65 @@ +import { defineTask } from "../tasks.ts"; +import type { EntryId, TaskId } from "../types.ts"; +import { ConversationConfig } from "./config.ts"; +import { GenerationTask, handOver } from "./generation.ts"; +import { endRun, LiveDoc } from "./live.ts"; +import type { ToolTaskResult } from "./tool.ts"; +import type { PostToolsHooks, ToolControl } from "./types.ts"; + +export type PostToolsInput = { assistant: EntryId; tools: TaskId[] }; +export type PostToolsCheckpoint = { phase: "join" }; +export type PostToolsResult = Record; + +/** + * Built-in post-tools task: runs once every tool task of the round is terminal, applies the round's controls, and either + * ends the run at the final boundary or hands it to the next generation at the `postTools` boundary. + */ +export const PostToolsTask = defineTask({ + name: "pi.post-tools", + version: 1, + initial: () => ({ phase: "join" }), + phases: { + join: async (task, runtime, context) => { + const { assistant, tools } = task.input; + const conversationId = runtime.conversationId; + const controls = new Map(); + for (const id of tools) { + const state = (await runtime.getTask(id, context))?.state; + const outcome = state?.status === "terminal" ? state.outcome : undefined; + controls.set(id, outcome?.status === "completed" ? outcome.result.control : undefined); + } + const slots = (await runtime.snapshot(LiveDoc, conversationId, context))?.tools ?? []; + const results = slots.flatMap((slot) => (slot.entry === undefined ? [] : [slot.entry])); + await runtime.hooks.each("afterTools", (hook) => hook(assistant, results, runtime, context)); + // Every call of the round, including those answered without a task, must ask to terminate. + const terminate = + slots.length > 0 && + slots.every((slot) => slot.taskId !== undefined && controls.get(slot.taskId)?.terminate === true); + const added = [...controls.values()].flatMap((control) => control?.addTools ?? []); + await runtime.commit(async (tx) => { + if (added.length > 0) { + const config = await tx.doc(ConversationConfig, conversationId); + for (const name of added) if (!config.activeTools.includes(name)) config.activeTools.push(name); + } + const live = await tx.doc(LiveDoc, conversationId); + if (terminate) { + endRun(tx, live, runtime.taskId, { status: "done", answer: assistant }); + } else { + delete live.tools; + handOver(live, runtime.taskId, await tx.createTask(GenerationTask, {})); + } + return { status: "terminal", outcome: { status: "completed", result: {} } }; + }, context); + }, + }, + /** Ends the run; tool tasks still running are not stopped (spec §12). */ + abort: async (_task, runtime, context) => { + await runtime.commit(async (tx) => { + endRun(tx, await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId, { + status: "unanswered", + reason: "aborted", + }); + return { status: "terminal", outcome: { status: "aborted" } }; + }, context); + }, +}); diff --git a/packages/durable/src/harness/prompt.ts b/packages/durable/src/harness/prompt.ts index 1161785a5..8b7037cec 100644 --- a/packages/durable/src/harness/prompt.ts +++ b/packages/durable/src/harness/prompt.ts @@ -1,5 +1,6 @@ import type { Context } from "@earendil-works/chord"; -import type { Message, SystemMessage } from "@earendil-works/pi-ai"; +import type { Message, SystemMessage, Tool, ToolReference } from "@earendil-works/pi-ai"; +import { declarationsEqual, getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { SystemEntry } from "../entries.ts"; import type { ContextEdit, TypedEntryDraft } from "../types.ts"; import type { ContextView, PromptInput, PromptSection, ToolRegistration } from "./types.ts"; @@ -48,18 +49,37 @@ export async function renderSections( type SystemDraft = TypedEntryDraft; +/** Active names the snapshot resolves, first occurrence of each, in configured order, as composed by wrappers. */ +export function desiredTools( + activeTools: readonly string[], + resolve: (name: string) => T | undefined, +): T[] { + const tools = new Map(); + for (const name of activeTools) { + const tool = tools.has(name) ? undefined : resolve(name); + if (tool !== undefined) tools.set(name, tool); + } + return [...tools.values()]; +} + +type ToolChanges = { readonly toolsRemoved: ToolReference[]; readonly toolsAdded: Tool[] }; + /** - * Plan the `pi.system` entries that make the replayed sections of `view` equal `desired` in values and order. + * Plan the `pi.system` entries that make the replayed sections and tools of `view` equal `desired` and `tools` in values + * and order. * * - A head marker with no later `pi.system` entry in context: one complete baseline that omits every retained earlier * `pi.system` entry, written even when it restates the replayed values. - * - Otherwise, when a minimal patch would leave a different order: remove every shown section, then re-add every - * desired section in order. + * - Otherwise, when a minimal section patch would leave a different order: remove every shown section, then re-add + * every desired section in order. * - Otherwise the minimal patch of changed values and `null` removals, or nothing. + * + * Tool changes ride on the last planned entry, or on one entry of their own. */ export function planSystemEntries( view: ContextView, desired: ReadonlyMap, + tools: readonly Tool[], timestamp: number, ): SystemDraft[] { const head = view.head; @@ -67,21 +87,62 @@ export function planSystemEntries( const edits: ContextEdit[] = view.entries .filter((entry) => SystemEntry.is(entry)) .map((entry) => ({ target: entry.id, action: "omit" })); - const baseline = systemEntry(Object.fromEntries(desired), timestamp); + const baseline = systemEntry( + Object.fromEntries(desired), + { toolsRemoved: [], toolsAdded: tools.map(toToolDeclaration) }, + timestamp, + ); return [edits.length === 0 ? baseline : { ...baseline, edits }]; } + const sections = planSections(replaySections(view.messages), desired); + const changes = planTools(getCurrentTools(view.messages), tools); + if (changes.toolsRemoved.length === 0 && changes.toolsAdded.length === 0) { + return sections.map((patch) => systemEntry(patch, undefined, timestamp)); + } + if (sections.length === 0) return [systemEntry(undefined, changes, timestamp)]; + return sections.map((patch, index) => + systemEntry(patch, index === sections.length - 1 ? changes : undefined, timestamp), + ); +} - const shown = replaySections(view.messages); +/** + * Tool changes from `offered` to `desired`. A changed declaration is removed and re-added. Replay keeps retained tools + * in place and appends additions; when that would not yield the desired order, every offered tool is removed and + * every desired tool re-added in order. + */ +function planTools(offered: readonly Tool[], desired: readonly Tool[]): ToolChanges { + const wanted = new Map(desired.map((tool) => [tool.name, tool])); + const kept = offered.filter((tool) => { + const next = wanted.get(tool.name); + return next !== undefined && declarationsEqual(tool, next); + }); + const keptNames = new Set(kept.map((tool) => tool.name)); + const added = desired.filter((tool) => !keptNames.has(tool.name)); + const replayed = [...kept, ...added]; + if (replayed.some((tool, index) => tool.name !== desired[index]!.name)) { + return { + toolsRemoved: offered.map((tool) => ({ name: tool.name })), + toolsAdded: desired.map(toToolDeclaration), + }; + } + return { + toolsRemoved: offered.filter((tool) => !keptNames.has(tool.name)).map((tool) => ({ name: tool.name })), + toolsAdded: added.map(toToolDeclaration), + }; +} + +/** Section patches: none, the minimal patch, or a remove-all/re-add-all pair when the order would differ. */ +function planSections( + shown: ReadonlyMap, + desired: ReadonlyMap, +): Record[] { const patchedOrder = [ ...[...shown.keys()].filter((key) => desired.has(key)), ...[...desired.keys()].filter((key) => !shown.has(key)), ]; const desiredOrder = [...desired.keys()]; if (patchedOrder.some((key, index) => key !== desiredOrder[index])) { - return [ - systemEntry(Object.fromEntries([...shown.keys()].map((key) => [key, null])), timestamp), - systemEntry(Object.fromEntries(desired), timestamp), - ]; + return [Object.fromEntries([...shown.keys()].map((key) => [key, null])), Object.fromEntries(desired)]; } const patch: Record = {}; @@ -90,10 +151,21 @@ export function planSystemEntries( if (next !== value) patch[key] = next ?? null; } for (const [key, value] of desired) if (!shown.has(key)) patch[key] = value; - return Object.keys(patch).length === 0 ? [] : [systemEntry(patch, timestamp)]; + return Object.keys(patch).length === 0 ? [] : [patch]; } -function systemEntry(sections: Record, timestamp: number): SystemDraft { - const message: SystemMessage = { role: "system", content: "", sections, timestamp }; +function systemEntry( + sections: Record | undefined, + tools: ToolChanges | undefined, + timestamp: number, +): SystemDraft { + const message: SystemMessage = { + role: "system", + content: "", + ...(sections === undefined ? {} : { sections }), + ...(tools === undefined || tools.toolsRemoved.length === 0 ? {} : { toolsRemoved: tools.toolsRemoved }), + ...(tools === undefined || tools.toolsAdded.length === 0 ? {} : { toolsAdded: tools.toolsAdded }), + timestamp, + }; return { model: [message] }; } diff --git a/packages/durable/src/harness/registry.ts b/packages/durable/src/harness/registry.ts index b52cfa5e3..164cb9a03 100644 --- a/packages/durable/src/harness/registry.ts +++ b/packages/durable/src/harness/registry.ts @@ -2,6 +2,8 @@ import type { ConversationRecord, Tx } from "../types.ts"; import { ConversationConfig } from "./config.ts"; import { GenerationTask } from "./generation.ts"; import { LiveDoc } from "./live.ts"; +import { PostToolsTask } from "./post-tools.ts"; +import { ToolTask } from "./tool.ts"; import type { AnyTask, ConversationSetup, @@ -21,7 +23,7 @@ import type { const SECTION_KEY = /^[a-z][a-z0-9_-]*$/; /** Built-in task definitions every registry starts with; they cannot be disposed or replaced. */ -export const BUILTIN_TASKS: readonly AnyTask[] = [GenerationTask]; +export const BUILTIN_TASKS: readonly AnyTask[] = [GenerationTask, ToolTask, PostToolsTask]; export const BUILTIN_SETUP_KEY = "pi"; diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 1f69ea8cd..68b0f4794 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -1,12 +1,16 @@ import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context"; import type { Models } from "@earendil-works/pi-ai"; +import type { ExecutionEnv } from "../env/index.ts"; import type { SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { CommitPublication, ConversationId, DocumentWatch, + EntryId, + EntryRecord, + HookRunner, JsonObject, RunningTask, Storage, @@ -21,6 +25,7 @@ import { readContext } from "./context.ts"; import type { AnyTask, HarnessInspection, + HookScope, RegistryReader, RegistrySnapshot, SettledTask, @@ -89,6 +94,7 @@ export type TaskSchedulerOptions = { readonly storage: Storage; readonly registry: RegistryReader; readonly models: Models; + readonly env: ExecutionEnv | undefined; readonly now: () => number; readonly report: (error: unknown) => void; /** Harness cleanup staged in the same commit as every terminal outcome the scheduler writes itself. */ @@ -113,6 +119,7 @@ export class TaskScheduler { readonly #storage: Storage; readonly #registry: RegistryReader; readonly #models: Models; + readonly #env: ExecutionEnv | undefined; readonly #now: () => number; readonly #report: (error: unknown) => void; readonly #settleOutcome: TaskSchedulerOptions["settleOutcome"]; @@ -124,6 +131,8 @@ export class TaskScheduler { readonly #idleWaiters = new Waiters(); /** Definition whose migration failed per task; retried only once the registry resolves another definition. */ readonly #failedMigrations = new Map(); + /** Owner conversations of each conversation, nearest first; owner edges never change. */ + readonly #owners = new Map(); #unsubscribeRegistry: () => void = () => {}; #enabled = false; #closing = false; @@ -135,6 +144,7 @@ export class TaskScheduler { this.#storage = options.storage; this.#registry = options.registry; this.#models = options.models; + this.#env = options.env; this.#now = options.now; this.#report = options.report; this.#settleOutcome = options.settleOutcome; @@ -409,7 +419,11 @@ export class TaskScheduler { async #run(reservation: Reservation): Promise { const invocation = reservation.invocation; const state = { task: reservation.task, snapshot: reservation.snapshot, reported: undefined as ReportedTask }; - const runtime = this.#runtime(invocation, () => state.snapshot); + const runtime = this.#runtime( + invocation, + () => state.snapshot, + () => state.task, + ); let previous: PhaseResult | undefined; for (;;) { const current = await this.#step(invocation, (tx, current) => this.#decide(tx, current, previous, state)); @@ -471,7 +485,11 @@ export class TaskScheduler { if (current === undefined || this.#closing) return; let failure: { readonly error: unknown } | undefined; try { - const runtime = this.#runtime(invocation, () => reservation.snapshot); + const runtime = this.#runtime( + invocation, + () => reservation.snapshot, + () => reservation.task, + ); await erased(reservation.task).abort(current, runtime, invocation.context); } catch (error) { failure = { error }; @@ -515,12 +533,14 @@ export class TaskScheduler { return this.#settleOutcome(tx, record, outcome); } - /** End an invocation: its runtime operations reject from now on, its watches stop, and its task is free. */ + /** End an invocation: its runtime operations reject from now on, its signal aborts, its watches stop, and its task is free. */ #end(invocation: Invocation): void { if (invocation.ended) return; invocation.ended = true; if (this.#invocations.get(invocation.taskId) === invocation) this.#invocations.delete(invocation.taskId); for (const watch of invocation.watches) void watch.stop(); + // Pending waits bound to the invocation, such as a tool's waitForTask(), reject with it. + invocation.controller.abort(endedError(invocation)); } #idle(conversationId: ConversationId | undefined): boolean { @@ -533,12 +553,30 @@ export class TaskScheduler { // ─── Invocation runtime ────────────────────────────────────────────────── - #runtime(invocation: Invocation, snapshot: () => RegistrySnapshot): ErasedRuntime { + #runtime(invocation: Invocation, snapshot: () => RegistrySnapshot, task: () => AnyTask): ErasedRuntime { + const hooks: HookRunner> = { + each: async (name, invoke) => { + if (invocation.ended) throw endedError(invocation); + for (const { handlers, scope } of snapshot().hooks(task())) { + const handler = (handlers as Record)[name]; + if (typeof handler !== "function") continue; + if (scope !== undefined && !(await this.#inScope(invocation, scope))) continue; + try { + await invoke(handler.bind(handlers)); + } catch (error) { + if (invocation.controller.signal.aborted) throw error; + this.#report(error); + } + } + }, + }; return { taskId: invocation.taskId as TaskId, conversationId: invocation.conversationId, signal: invocation.controller.signal, models: this.#models, + env: this.#env, + hooks: hooks as ErasedRuntime["hooks"], get registry() { return snapshot(); }, @@ -577,6 +615,27 @@ export class TaskScheduler { this.#read(invocation, () => sessionMethod(this.#session, "snapshotAsOf")(...args), )) as ErasedRuntime["snapshotAsOf"], + getTask: ((id: TaskId, context: Context) => + this.#read(invocation, () => + this.#session.readOnLine(() => this.#storage.task(id, context)), + )) as ErasedRuntime["getTask"], + waitForTask: ((id: TaskId, context: Context) => + this.#read(invocation, () => + this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)), + )) as ErasedRuntime["waitForTask"], + entry: ((...args: readonly unknown[]) => { + const [token, id, context] = + args.length === 2 + ? [undefined, args[0] as EntryId, args[1] as Context] + : [args[0] as { readonly kind: string }, args[1] as EntryId, args[2] as Context]; + return this.#read(invocation, async () => { + const found = await this.#session.readOnLine(() => + this.#storage.entry(invocation.conversationId, id, context), + ); + const entry: EntryRecord | undefined = found?.entry; + return token === undefined || entry?.kind === token.kind ? entry : undefined; + }); + }) as ErasedRuntime["entry"], context: (conversationId, context, at) => this.#read(invocation, () => readContext(this.#session, this.#storage, conversationId, context, at)), now: () => this.#now(), @@ -584,6 +643,24 @@ export class TaskScheduler { }; } + /** Whether a scoped hook registration matches the invocation's conversation. */ + async #inScope(invocation: Invocation, scope: HookScope): Promise { + if (scope.conversationId === invocation.conversationId) return true; + if (scope.subtree !== true) return false; + return (await this.#ownersOf(invocation.conversationId)).includes(scope.conversationId); + } + + /** Conversations owning `conversationId` through tasks, transitively, nearest first. */ + async #ownersOf(conversationId: ConversationId): Promise { + const cached = this.#owners.get(conversationId); + if (cached !== undefined) return cached; + const record = await this.#session.readOnLine(() => this.#storage.conversation(conversationId, this.#context)); + const owner = record?.owner?.conversationId; + const owners = owner === undefined ? [] : [owner, ...(await this.#ownersOf(owner))]; + this.#owners.set(conversationId, owners); + return owners; + } + /** Run a committed-state read unless the invocation has ended. */ async #read(invocation: Invocation, read: () => Promise): Promise { if (invocation.ended) throw endedError(invocation); diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts new file mode 100644 index 000000000..1445be0a9 --- /dev/null +++ b/packages/durable/src/harness/tool.ts @@ -0,0 +1,462 @@ +import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; +import { awaitWithContext } from "@earendil-works/chord/context"; +import { overlap } from "@earendil-works/chord/delta"; +import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai"; +import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation"; +import { AssistantEntry, ToolResultEntry } from "../entries.ts"; +import { defineTask } from "../tasks.ts"; +import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, utf8ByteLength } from "../truncate.ts"; +import type { + ConversationId, + EntryId, + JsonObject, + Task, + TaskId, + TaskOptions, + TaskRuntime, + Tx, + TypedEntry, +} from "../types.ts"; +import { assignJson } from "./json.ts"; +import { clearProgress, finishSlot, LiveDoc, type ToolSlot, toolSlot } from "./live.ts"; +import { boundOutput, OutputBuffer, type OutputLimits, Progress } from "./output.ts"; +import type { + ToolControl, + ToolDiagnostic, + ToolExecutionApi, + ToolExecutionResult, + ToolHooks, + ToolRegistration, +} from "./types.ts"; + +export type ToolTaskInput = { assistant: EntryId; callId: string }; + +export type ToolTaskCheckpoint = + | { phase: "call" } + /** Durable intent: the final arguments and the replay policy recorded before execution. */ + | { phase: "execute"; arguments: JsonObject; replay: "safe" | "unsafe" }; + +export type ToolTaskResult = { entryId: EntryId; control?: ToolControl }; + +type Runtime = TaskRuntime; +type Content = (TextContent | ImageContent)[]; + +/** + * Built-in tool task: resolves the called tool from its phase snapshot, validates, runs `beforeTool`, records intent, + * executes, runs `afterTool`, and appends the result, all in one `call` handler so nothing separates resolution from + * settlement. `execute` is reached only by recovery and applies the replay rule. + */ +export const ToolTask = defineTask({ + name: "pi.tool", + version: 1, + initial: () => ({ phase: "call" }), + phases: { + call: async (task, runtime, context) => { + const call = await readCall(runtime, task.input, context); + const tool = runtime.registry.tool(call.name); + if (tool === undefined) { + const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`); + return settle(runtime, call, "completed", () => error, context); + } + const prepared = prepare(tool, call.arguments as JsonObject); + const checked = "error" in prepared ? prepared : validate(tool, call, prepared.args); + if ("error" in checked) return settle(runtime, call, "completed", () => invalid(checked.error), context); + let args = checked.args; + let block: string | undefined; + await runtime.hooks.each("beforeTool", async (hook) => { + if (block !== undefined) return; + try { + const decision = await hook({ ...call, arguments: args }, runtime, context); + if (decision?.block !== undefined) block = decision.block; + else if (decision?.arguments !== undefined) args = decision.arguments; + } catch (error) { + if (runtime.signal.aborted) throw error; + block = errorText(error); + } + }); + if (block !== undefined) { + const blocked = harnessError("blocked", `Tool call blocked: ${block}`); + return settle(runtime, call, "completed", () => blocked, context); + } + const validated = validate(tool, call, args); + if ("error" in validated) return settle(runtime, call, "completed", () => invalid(validated.error), context); + const final = validated.args; + await runtime.commit(async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot !== undefined) slot.status = "running"; + const intent = { phase: "execute", arguments: final, replay: tool.replay ?? "unsafe" } as const; + return { status: "running", checkpoint: intent }; + }, context); + await run(runtime, call, tool, final, context); + }, + /** Recovery after intent: rerun only when the stored and the current policy both say `safe`. */ + execute: async (task, runtime, context) => { + const { arguments: args, replay } = task.state.checkpoint; + const call = await readCall(runtime, task.input, context); + const tool = runtime.registry.tool(call.name); + if (replay === "safe" && tool?.replay === "safe") { + // The rerun reports from scratch; clear what the interrupted attempt published. + await runtime.commit(async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot !== undefined) clearProgress(slot); + return undefined; + }, context); + return run(runtime, call, tool, args, context); + } + const message = `Tool ${call.name} was interrupted and may have partially run`; + await settle(runtime, call, "completed", (slot) => fromSlot(slot, "interrupted", message), context); + }, + }, + abort: async (task, runtime, context) => { + const call = await readCall(runtime, task.input, context); + const message = `Tool ${call.name} was aborted`; + await settle(runtime, call, "aborted", (slot) => fromSlot(slot, "aborted", message), context); + }, +}); + +/** The tool call `callId` of the assistant entry. */ +async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise { + const entry = await runtime.entry(AssistantEntry, input.assistant, context); + const message = entry?.model?.[0]; + const call = + message?.role === "assistant" + ? message.content.find( + (content): content is ToolCall => content.type === "toolCall" && content.id === input.callId, + ) + : undefined; + if (call === undefined) throw new Error(`Entry ${input.assistant} has no tool call ${input.callId}`); + return call; +} + +/** Arguments, or why they are invalid. */ +type Checked = { readonly args: JsonObject } | { readonly error: string }; + +/** The call's arguments as repaired by the tool; a throwing repair makes them invalid. */ +function prepare(tool: ToolRegistration, args: JsonObject): Checked { + if (tool.prepareArguments === undefined) return { args }; + try { + return { args: tool.prepareArguments(args) as JsonObject }; + } catch (error) { + return { error: errorText(error) }; + } +} + +/** Arguments validated and coerced against the implementation's schema. */ +function validate(tool: ToolRegistration, call: ToolCall, args: JsonObject): Checked { + try { + return { args: validateToolArguments(tool, { ...call, arguments: args }) as JsonObject }; + } catch (error) { + return { error: errorText(error) }; + } +} + +function invalid(message: string): ToolExecutionResult { + return harnessError("invalid_arguments", message); +} + +/** What a running tool reported through its api: output, the last details, and diagnostics. */ +type Reported = { + readonly output: OutputBuffer; + readonly limits: OutputLimits; + readonly diagnostics: ToolDiagnostic[]; + details: JsonValue | undefined; +}; + +/** Execute with the resolved implementation, then settle its result. */ +async function run( + runtime: Runtime, + call: ToolCall, + tool: ToolRegistration, + args: JsonObject, + context: Context, +): Promise { + const limits: OutputLimits = { + maxBytes: tool.outputLimits?.maxBytes ?? DEFAULT_MAX_BYTES, + maxLines: tool.outputLimits?.maxLines ?? DEFAULT_MAX_LINES, + retain: tool.outputLimits?.retain ?? "head", + }; + const reported: Reported = { output: new OutputBuffer(limits), limits, diagnostics: [], details: undefined }; + const progress = publishProgress(runtime, reported, context); + let ended = false; + const assertLive = (): void => { + if (ended) throw new Error(`Tool call ${call.id} has settled`); + }; + const api: ToolExecutionApi = { + taskId: runtime.taskId, + conversationId: runtime.conversationId, + callId: call.id, + env: runtime.env, + output: (chunk) => { + assertLive(); + if (reported.output.push(chunk)) progress.mark(); + }, + diagnostic: (diagnostic) => { + assertLive(); + reported.diagnostics.push(copyJson(diagnostic, { omitUndefinedProperties: true }) as ToolDiagnostic); + progress.mark(); + }, + details: async (value, detailsContext) => { + assertLive(); + detailsContext.abortSignal?.throwIfAborted(); + reported.details = copyJson(value, { omitUndefinedProperties: true }); + const committed = progress.markAndWait(); + // Cancelling the wait leaves the update in place; the commit's own outcome stays observed. + committed.catch(() => {}); + return awaitWithContext(committed, detailsContext); + }, + commit: async (change, commitContext) => { + let result: Awaited> | undefined; + await runtime.commit(async (tx) => { + result = await change(tx); + return undefined; + }, commitContext); + return result as Awaited>; + }, + memo: runtime.memo, + createTask: async ( + task: Task, + input: I, + options: Omit, + taskContext: Context, + ): Promise> => { + let id: TaskId | undefined; + await runtime.commit(async (tx) => { + id = await tx.createTask(task, input, options); + return undefined; + }, taskContext); + return id!; + }, + getTask: runtime.getTask, + waitForTask: runtime.waitForTask, + snapshot: runtime.snapshot, + snapshotAsOf: runtime.snapshotAsOf, + watchDoc: runtime.watchDoc, + }; + + let result: ToolExecutionResult; + try { + result = await tool.execute(args, api, context); + } catch (error) { + if (runtime.signal.aborted) { + ended = true; + for (const waiter of await progress.stop()) waiter.reject(error); + throw error; + } + result = { isError: true, diagnostics: [toolDiagnostic("tool_error", errorText(error))] }; + } + ended = true; + reported.output.end(); + // Details still waiting for a progress commit settle with the terminal commit, the final flush. + const pending = await progress.stop(); + try { + const settled = await finalResult(runtime, call, result, reported, context); + await settle(runtime, call, "completed", () => settled, context); + } catch (error) { + for (const waiter of pending) waiter.reject(error); + throw error; + } + for (const waiter of pending) waiter.resolve(); +} + +/** + * Throttled commits of what the tool reported into its `pi.live.tools` slot, each writing only what changed since the + * last one. + */ +function publishProgress(runtime: Runtime, reported: Reported, context: Context): Progress { + let written = { text: "", details: undefined as JsonValue | undefined, diagnostics: 0 }; + return new Progress( + async () => { + // Capture everything synchronously: the tool keeps reporting while the commit is in flight. + const snapshot = reported.output.snapshot(); + const current = { text: snapshot.text, details: reported.details, diagnostics: reported.diagnostics.length }; + const added = reported.diagnostics.slice(written.diagnostics, current.diagnostics); + const detailsChanged = current.details !== written.details; + // What the commit writes, as Chord diffs the string: an append, a trim plus an append of what follows the shared + // part, or the whole window when its bounded overlap search finds nothing. + let bytes = 0; + if (snapshot.text !== written.text) { + const shared = snapshot.text.startsWith(written.text) + ? written.text.length + : overlap(written.text, snapshot.text, 65_536); + bytes += utf8ByteLength(snapshot.text.slice(shared)); + } + if (detailsChanged) bytes += utf8ByteLength(JSON.stringify(current.details ?? null)); + if (added.length > 0) bytes += utf8ByteLength(JSON.stringify(added)); + await runtime.commit(async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + if (slot === undefined) return undefined; + // REMINDER: assign `output` as one string field. Chord then diffs it into an append, or a trim plus an + // append for a sliding tail; replacing the slot object would record the whole window on every commit. + if ((slot.output ?? "") !== snapshot.text) slot.output = snapshot.text; + if (snapshot.droppedBytes > 0) slot.droppedBytes = snapshot.droppedBytes; + if (snapshot.droppedLines > 0) slot.droppedLines = snapshot.droppedLines; + // Diff details leaf by leaf and append new diagnostics, so each commit writes only what changed. + if (detailsChanged && current.details !== undefined) { + assignJson(slot as unknown as Record, "details", current.details); + } + if (added.length > 0) { + if (slot.diagnostics === undefined) slot.diagnostics = []; + for (const diagnostic of added) slot.diagnostics.push(diagnostic); + } + return undefined; + }, context); + written = current; + return bytes; + }, + (error) => { + // Rejections after an abort mark or close are expected; the committed state stays consistent. + if (!runtime.signal.aborted) runtime.report(error); + }, + ); +} + +/** + * The settled result: the tool's result with the retained output and last details as fallbacks, its diagnostics after + * those reported through the api, `afterTool` applied, and explicit text bounded, with the Harness's truncation + * diagnostic last. + */ +async function finalResult( + runtime: Runtime, + call: ToolCall, + result: ToolExecutionResult, + reported: Reported, + context: Context, +): Promise { + const harness: ToolDiagnostic[] = []; + const retained = result.content === undefined ? reported.output.snapshot() : undefined; + const content: Content = + retained === undefined ? result.content! : retained.text === "" ? [] : [{ type: "text", text: retained.text }]; + let final: ToolExecutionResult = { + ...result, + content, + details: result.details === undefined ? reported.details : result.details, + diagnostics: [...reported.diagnostics, ...(result.diagnostics ?? [])], + }; + await runtime.hooks.each("afterTool", async (hook) => { + final = (await hook(call, final, runtime, context)) ?? final; + }); + // The retained output's truncation applies only while afterTool kept that content. + if (retained !== undefined && retained.droppedBytes > 0 && final.content === content) { + harness.push(truncated(retained, reported.limits.retain)); + } + const bounded = boundContent(final.content ?? [], reported.limits); + if (bounded.droppedBytes > 0) harness.push(truncated(bounded, reported.limits.retain)); + return { ...final, content: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] }; +} + +/** + * Commit the tool's terminal state: append its result entry, mark its slot done, and complete or end aborted with the + * entry ID. `build` receives the slot so interruption and abort can report the durable partial output. + */ +async function settle( + runtime: Runtime, + call: ToolCall, + status: "completed" | "aborted", + build: (slot: Readonly | undefined) => ToolExecutionResult, + context: Context, +): Promise { + await runtime.commit(async (tx) => { + const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + const result = build(slot); + const entry = await appendToolResult(tx, runtime.conversationId, call, result, runtime.now()); + if (slot !== undefined) finishSlot(slot, entry.id); + const entryId = entry.id; + if (status === "aborted") return { status: "terminal", outcome: { status: "aborted", result: { entryId } } }; + // Tools build control objects freely; drop keys set to undefined so the task result is strict JSON. + const control = + result.control === undefined + ? {} + : { control: copyJson(result.control as JsonValue, { omitUndefinedProperties: true }) as ToolControl }; + return { status: "terminal", outcome: { status: "completed", result: { entryId, ...control } } }; + }, context); +} + +/** An error result from the slot's durable partial output, details, and diagnostics. */ +function fromSlot(slot: Readonly | undefined, code: string, message: string): ToolExecutionResult { + const diagnostics = [...(slot?.diagnostics ?? [])]; + const droppedBytes = slot?.droppedBytes ?? 0; + if (droppedBytes > 0) diagnostics.push(truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 })); + diagnostics.push(toolDiagnostic(code, message)); + return { + content: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }], + isError: true, + ...(slot?.details === undefined ? {} : { details: slot.details }), + diagnostics, + }; +} + +/** An error result the Harness writes itself: no content and one `error` diagnostic with `code`. */ +export function harnessError(code: string, message: string): ToolExecutionResult { + return { content: [], isError: true, diagnostics: [toolDiagnostic(code, message)] }; +} + +function toolDiagnostic(code: string, message: string): ToolDiagnostic { + return { severity: "error", code, message }; +} + +/** The Harness's truncation diagnostic; `retain` is unknown when rebuilt from a slot after recovery. */ +function truncated( + dropped: { readonly droppedLines: number; readonly droppedBytes: number }, + retain?: "head" | "tail", +): ToolDiagnostic { + const kept = retain === undefined ? "" : ` to its ${retain === "head" ? "beginning" : "end"}`; + return { + severity: "warn", + code: "truncated", + message: `Output truncated${kept}: ${dropped.droppedLines} lines, ${dropped.droppedBytes} bytes dropped`, + }; +} + +/** + * Append a `pi.tool-result` entry. The content ends with the rendered diagnostics, so the stored message is exactly + * what the model sees; `data` keeps the structured list. + */ +export async function appendToolResult( + tx: Tx, + conversationId: ConversationId, + call: ToolCall, + result: ToolExecutionResult, + timestamp: number, +): Promise> { + const diagnostics = [...(result.diagnostics ?? [])]; + const content: Content = [...(result.content ?? [])]; + if (diagnostics.length > 0) content.push({ type: "text", text: renderDiagnostics(diagnostics) }); + const message = { + role: "toolResult", + toolCallId: call.id, + toolName: call.name, + content, + ...(result.details === undefined ? {} : { details: result.details }), + isError: result.isError ?? false, + timestamp, + } as ToolResultMessage; + return tx.appendEntry(ToolResultEntry, conversationId, { model: [message], data: { diagnostics } }); +} + +function renderDiagnostics(diagnostics: readonly ToolDiagnostic[]): string { + return `\n${diagnostics.map((diagnostic) => `[${diagnostic.severity}] ${diagnostic.message}`).join("\n")}\n`; +} + +/** + * Bound the text of result content. When the joined text exceeds the limits, the text items are replaced by one bounded + * item at the position of the first (head) or last (tail) text item; other content is kept. + */ +function boundContent( + content: Content, + limits: OutputLimits, +): { content: Content; droppedBytes: number; droppedLines: number } { + const texts = content.filter((item): item is TextContent => item.type === "text"); + const bounded = boundOutput(texts.map((item) => item.text).join(""), limits); + if (bounded.droppedBytes === 0) return { content, droppedBytes: 0, droppedLines: 0 }; + const keep = limits.retain === "head" ? texts[0] : texts.at(-1); + const result: Content = []; + for (const item of content) { + if (item.type !== "text") result.push(item); + else if (item === keep) result.push({ ...item, text: bounded.text }); + } + return { content: result, droppedBytes: bounded.droppedBytes, droppedLines: bounded.droppedLines }; +} + +function errorText(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 915ba4b70..7a9a89285 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -1,14 +1,17 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import type { + AssistantMessage, CacheRetention, Message, Models, ModelThinkingLevel, Tool, + ToolCall, ToolResultMessage, Transport, UserMessage, } from "@earendil-works/pi-ai"; +import type { ExecutionEnv } from "../env/index.ts"; import type { ConversationId, ConversationOwnership, @@ -108,20 +111,41 @@ export type ToolControl = { readonly handoff?: string; }; +/** Remark about a call for the model and the UI, such as truncation or a spill path; never part of the tool's data. */ +export type ToolDiagnostic = { + readonly severity: "info" | "warn" | "error"; + readonly message: string; + readonly code?: string; +}; + export type ToolExecutionResult = { + /** Omitted: the retained `output()` text becomes the content. */ readonly content?: ToolResultMessage["content"]; readonly isError?: boolean; + /** Omitted: the last `details()` value becomes the details. */ readonly details?: JsonValue; + /** Added after those recorded through `api.diagnostic()`. */ + readonly diagnostics?: readonly ToolDiagnostic[]; readonly control?: ToolControl; }; -/** Operations available to one tool invocation. */ +/** Whether the tools of one round run at once or one after another in call order. */ +export type ToolExecutionMode = "parallel" | "sequential"; + +/** + * Operations available to one tool invocation. A plain object, so a wrapper can pass `{ ...api, env }` to the tool it + * wraps. Every operation rejects after the invocation ends. + */ export interface ToolExecutionApi extends DocumentObserver, DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; readonly callId: string; + /** `HarnessOptions.env` unless a wrapper supplies another environment. */ + readonly env: ExecutionEnv | undefined; /** Append running output; it becomes the result content when the result omits `content`. */ output(chunk: string | Uint8Array): void; + /** Record a model-visible remark about this call. */ + diagnostic(diagnostic: ToolDiagnostic): void; /** Replace running details; the last value becomes the result details when the result omits `details`. */ details(value: JsonValue, context: Context): Promise; commit(change: (tx: Tx) => T | Promise, context: Context): Promise; @@ -135,13 +159,20 @@ export interface ToolExecutionApi extends DocumentObserver, DocumentReader { ): Promise>; getTask(id: TaskId, context: Context): Promise | undefined>; waitForTask(id: TaskId, context: Context): Promise>; - conversation(id: ConversationId, context: Context): Promise; + // conversation() arrives with ConversationHandle in Package 18. } /** Executable tool registered in a registry. Only pi-ai `Tool` fields enter the transcript. */ export type ToolRegistration = Tool & { /** Whether an interrupted execution may rerun on recovery. Default `unsafe`. */ readonly replay?: "safe" | "unsafe"; + /** Default: the conversation's `toolExecution`. One sequential call makes its whole round sequential. */ + readonly executionMode?: ToolExecutionMode; + /** + * Repair arguments models commonly get wrong before validation, such as a JSON string where an array belongs. Must be + * pure and must not mutate `args`: it runs again when a call is retried before its intent is recorded. + */ + prepareArguments?(args: JsonValue): JsonValue; readonly outputLimits?: { readonly maxBytes?: number; readonly maxLines?: number; @@ -312,6 +343,8 @@ export type HarnessOptions = { /** pi-ai model access used by generation. */ readonly models: Models; readonly registry: RegistryReader; + /** Default execution environment offered to tools as `api.env`. */ + readonly env?: ExecutionEnv; readonly now?: () => number; /** Receives extension failures that do not fail the calling operation. Must not throw. */ readonly onReport?: (error: unknown) => void; @@ -371,6 +404,10 @@ export interface Conversation { getRetryPolicy(context: Context): Promise; /** `undefined` removes the configured policy. */ setRetryPolicy(policy: ConversationRetryPolicy | undefined, context: Context): Promise; + /** `parallel` when unset. */ + getToolExecution(context: Context): Promise; + /** `undefined` removes the configured mode. */ + setToolExecution(mode: ToolExecutionMode | undefined, context: Context): Promise; /** * Durably admit user input or a passive entry write. Until the inbox exists, a busy conversation rejects every @@ -430,3 +467,50 @@ export interface Harness extends Session { /** Resolve when no live non-background task exists. */ waitForIdle(context: Context): Promise; } + +/** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */ +export interface HookApi extends DocumentReader { + readonly taskId: TaskId; + readonly conversationId: ConversationId; + memo(name: string, context: Context): Promise; + memo(name: string, candidate: T, context: Context): Promise; +} + +export type HookResult = T | undefined | Promise; + +/** Hooks of the built-in generation task. */ +export interface GenerationHooks { + /** Before every request attempt, including recovery; the result is used for that request only. */ + beforeRequest( + request: { readonly messages: readonly Message[] }, + api: HookApi, + context: Context, + ): HookResult<{ readonly messages: readonly Message[] }>; + /** Every terminal provider message, before classification. */ + afterResponse(message: AssistantMessage, api: HookApi, context: Context): void | Promise; + /** A final answer; the first `continue` appends a user message and continues the run. */ + onYield(answer: AssistantMessage, api: HookApi, context: Context): HookResult<{ readonly continue: UserInput }>; +} + +/** Hooks of the built-in tool task. */ +export interface ToolHooks { + /** Before intent; the first `block` wins, otherwise `arguments` replace the call's arguments. A throw blocks. */ + beforeTool( + call: ToolCall, + api: HookApi, + context: Context, + ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; + /** After execution, before the result entry; replaces the result. */ + afterTool( + call: ToolCall, + result: ToolExecutionResult, + api: HookApi, + context: Context, + ): HookResult; +} + +/** Hooks of the built-in post-tools task. */ +export interface PostToolsHooks { + /** After every tool of the round is terminal; `results` are the round's result entries in call order. */ + afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; +} diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 84206fe78..8a0e19483 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -9,8 +9,15 @@ export { GenerationTask, } from "./harness/generation.ts"; export { Harness } from "./harness/harness.ts"; -export { LiveDoc, type LiveState } from "./harness/live.ts"; +export { LiveDoc, type LiveState, type ToolSlot } from "./harness/live.ts"; +export { + type PostToolsCheckpoint, + type PostToolsInput, + type PostToolsResult, + PostToolsTask, +} from "./harness/post-tools.ts"; export { createRegistry } from "./harness/registry.ts"; +export { ToolTask, type ToolTaskCheckpoint, type ToolTaskInput, type ToolTaskResult } from "./harness/tool.ts"; export type { AnyTask, ContextView, @@ -21,13 +28,17 @@ export type { ConversationRetryPolicy, ConversationSetup, ConversationStreamOptions, + GenerationHooks, HarnessInspection, HarnessOptions, + HookApi, HookRegistration, + HookResult, HookScope, HooksOf, InputSubmissionDraft, ModelRef, + PostToolsHooks, PromptInput, PromptSection, PromptSectionWrapper, @@ -42,8 +53,11 @@ export type { SubmissionDraft, TaskInspection, ToolControl, + ToolDiagnostic, ToolExecutionApi, + ToolExecutionMode, ToolExecutionResult, + ToolHooks, ToolRegistration, ToolWrapper, UserInput, @@ -87,6 +101,7 @@ export type { EntryId, EntryQuery, EntryRecord, + HookRunner, Id, JsonObject, LatestConversationSemantics, diff --git a/packages/durable/src/tools/bash.ts b/packages/durable/src/tools/bash.ts new file mode 100644 index 000000000..8ef2c9e87 --- /dev/null +++ b/packages/durable/src/tools/bash.ts @@ -0,0 +1,89 @@ +import type { Context } from "@earendil-works/chord"; +import { type Static, Type } from "typebox"; +import type { ToolExecutionApi, ToolRegistration } from "../harness/types.ts"; +import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts"; +import { requireEnv } from "./env.ts"; + +const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000; + +const bashSchema = Type.Object({ + command: Type.String({ description: "Bash command to execute" }), + timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), +}); + +export type BashToolInput = Static; + +export interface BashExecution { + command: string; + cwd: string; + env: Record; + inheritEnv: boolean; +} + +export type BashPrepare = (execution: BashExecution, api: ToolExecutionApi, context: Context) => void | Promise; + +export interface BashToolOptions { + commandPrefix?: string; + prepare?: BashPrepare; +} + +function validateTimeout(timeout: number | undefined): void { + if (timeout === undefined) return; + if (!Number.isFinite(timeout) || timeout <= 0) { + throw new Error("Invalid timeout: must be a finite number of seconds"); + } + if (timeout > MAX_TIMEOUT_SECONDS) { + throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`); + } +} + +/** + * Runs a command through the environment's shell. Its output streams to `api.output()`, where the Harness keeps the + * tail within the default limits; the result content is that retained output. Output beyond the limits is spilled to a + * file whose path is reported as a diagnostic. A nonzero exit or timeout throws, which makes an error result that still + * carries the output and diagnostics. + */ +export function createBashTool(options?: BashToolOptions): ToolRegistration { + return { + name: "bash", + description: `Execute a bash command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`, + parameters: bashSchema, + outputLimits: { retain: "tail" }, + async execute(args, api, context) { + const { command, timeout } = args as BashToolInput; + validateTimeout(timeout); + const env = requireEnv(api); + const execution: BashExecution = { + command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command, + cwd: env.cwd, + env: {}, + inheritEnv: true, + }; + await options?.prepare?.(execution, api, context); + const result = await env.exec( + execution.command, + { + cwd: execution.cwd, + env: execution.env, + inheritEnv: execution.inheritEnv, + ...(timeout === undefined ? {} : { timeout }), + onOutput: (text) => api.output(text), + spill: { afterBytes: DEFAULT_MAX_BYTES, afterLines: DEFAULT_MAX_LINES }, + }, + context, + ); + const spillPath = result.ok ? result.value.spillPath : result.error.spillPath; + if (spillPath !== undefined) { + api.diagnostic({ severity: "info", code: "full_output", message: `Full output: ${spillPath}` }); + } + if (!result.ok) { + if (result.error.code === "aborted" && context.abortSignal?.aborted) throw result.error; + if (result.error.code === "timeout") throw new Error(`Command timed out after ${timeout} seconds`); + if (result.error.code === "aborted") throw new Error("Command aborted"); + throw result.error; + } + if (result.value.exitCode !== 0) throw new Error(`Command exited with code ${result.value.exitCode}`); + return {}; + }, + }; +} diff --git a/packages/durable/src/tools/edit-diff.ts b/packages/durable/src/tools/edit-diff.ts new file mode 100644 index 000000000..c4534401c --- /dev/null +++ b/packages/durable/src/tools/edit-diff.ts @@ -0,0 +1,500 @@ +/** + * Shared diff computation utilities for the edit and similar tools. + */ + +import * as Diff from "diff"; + +export function detectLineEnding(content: string): "\r\n" | "\n" { + const crlfIdx = content.indexOf("\r\n"); + const lfIdx = content.indexOf("\n"); + if (lfIdx === -1) return "\n"; + if (crlfIdx === -1) return "\n"; + return crlfIdx < lfIdx ? "\r\n" : "\n"; +} + +export function normalizeToLF(text: string): string { + return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n"); +} + +export function restoreLineEndings(text: string, ending: "\r\n" | "\n"): string { + return ending === "\r\n" ? text.replace(/\n/g, "\r\n") : text; +} + +/** + * Normalize text for fuzzy matching. Applies progressive transformations: + * - Strip trailing whitespace from each line + * - Normalize smart quotes to ASCII equivalents + * - Normalize Unicode dashes/hyphens to ASCII hyphen + * - Normalize special Unicode spaces to regular space + */ +export function normalizeForFuzzyMatch(text: string): string { + return ( + text + .normalize("NFKC") + // Strip trailing whitespace per line + .split("\n") + .map((line) => line.trimEnd()) + .join("\n") + // Smart single quotes → ' + .replace(/[\u2018\u2019\u201A\u201B]/g, "'") + // Smart double quotes → " + .replace(/[\u201C\u201D\u201E\u201F]/g, '"') + // Various dashes/hyphens → - + // U+2010 hyphen, U+2011 non-breaking hyphen, U+2012 figure dash, + // U+2013 en-dash, U+2014 em-dash, U+2015 horizontal bar, U+2212 minus + .replace(/[\u2010\u2011\u2012\u2013\u2014\u2015\u2212]/g, "-") + // Special spaces → regular space + // U+00A0 NBSP, U+2002-U+200A various spaces, U+202F narrow NBSP, + // U+205F medium math space, U+3000 ideographic space + .replace(/[\u00A0\u2002-\u200A\u202F\u205F\u3000]/g, " ") + ); +} + +function splitLinesWithEndings(content: string): string[] { + return content.match(/[^\n]*\n|[^\n]+/g) ?? []; +} + +interface LineSpan { + start: number; + end: number; +} + +interface MatchedEdit { + editIndex: number; + matchIndex: number; + matchLength: number; + newText: string; +} + +type TextReplacement = Pick; + +function getLineSpans(content: string): LineSpan[] { + let offset = 0; + return splitLinesWithEndings(content).map((line) => { + const span = { start: offset, end: offset + line.length }; + offset = span.end; + return span; + }); +} + +function getReplacementLineRange(lines: LineSpan[], replacement: TextReplacement) { + const replacementStart = replacement.matchIndex; + const replacementEnd = replacement.matchIndex + replacement.matchLength; + + let startLine = -1; + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + if (replacementStart >= line.start && replacementStart < line.end) { + startLine = i; + break; + } + } + if (startLine === -1) { + throw new Error("Replacement range is outside the base content."); + } + + let endLine = startLine; + while (endLine < lines.length && lines[endLine].end < replacementEnd) { + endLine++; + } + if (endLine >= lines.length) { + throw new Error("Replacement range is outside the base content."); + } + + return { startLine, endLine: endLine + 1 }; +} + +function applyReplacements(content: string, replacements: TextReplacement[], offset = 0): string { + let result = content; + for (let i = replacements.length - 1; i >= 0; i--) { + const replacement = replacements[i]; + const matchIndex = replacement.matchIndex - offset; + result = + result.substring(0, matchIndex) + replacement.newText + result.substring(matchIndex + replacement.matchLength); + } + return result; +} + +/** + * Apply replacements matched against `baseContent` to `originalContent` while + * preserving unchanged line blocks from the original. + * + * This is useful when `baseContent` is a normalized view of the original. Each + * replacement is widened to the lines it actually touches, those touched lines + * are rewritten from the normalized base, and all other lines are copied back + * from `originalContent`. The actual replacement ranges drive preservation so + * duplicate normalized lines cannot be aligned to the wrong occurrence. + */ +export function applyReplacementsPreservingUnchangedLines( + originalContent: string, + baseContent: string, + replacements: TextReplacement[], +): string { + const originalLines = splitLinesWithEndings(originalContent); + const baseLines = getLineSpans(baseContent); + if (originalLines.length !== baseLines.length) { + throw new Error("Cannot preserve unchanged lines because the base content has a different line count."); + } + + const groups: Array<{ startLine: number; endLine: number; replacements: TextReplacement[] }> = []; + const sortedReplacements = [...replacements].sort((a, b) => a.matchIndex - b.matchIndex); + for (const replacement of sortedReplacements) { + const range = getReplacementLineRange(baseLines, replacement); + const current = groups[groups.length - 1]; + if (current && range.startLine < current.endLine) { + current.endLine = Math.max(current.endLine, range.endLine); + current.replacements.push(replacement); + continue; + } + groups.push({ ...range, replacements: [replacement] }); + } + + let originalLineIndex = 0; + let result = ""; + for (const group of groups) { + result += originalLines.slice(originalLineIndex, group.startLine).join(""); + + const groupStartOffset = baseLines[group.startLine].start; + const groupEndOffset = baseLines[group.endLine - 1].end; + result += applyReplacements( + baseContent.slice(groupStartOffset, groupEndOffset), + group.replacements, + groupStartOffset, + ); + originalLineIndex = group.endLine; + } + result += originalLines.slice(originalLineIndex).join(""); + + return result; +} + +export interface FuzzyMatchResult { + /** Whether a match was found */ + found: boolean; + /** The index where the match starts (in the content that should be used for replacement) */ + index: number; + /** Length of the matched text */ + matchLength: number; + /** Whether fuzzy matching was used (false = exact match) */ + usedFuzzyMatch: boolean; + /** + * The content to use for replacement operations. + * When exact match: original content. When fuzzy match: normalized content. + */ + contentForReplacement: string; +} + +export interface Edit { + oldText: string; + newText: string; +} + +export interface AppliedEditsResult { + baseContent: string; + newContent: string; +} + +/** + * Find oldText in content, trying exact match first, then fuzzy match. + * When fuzzy matching is used, the returned contentForReplacement is the + * fuzzy-normalized version of the content (trailing whitespace stripped, + * Unicode quotes/dashes normalized to ASCII). + */ +export function fuzzyFindText(content: string, oldText: string): FuzzyMatchResult { + // Try exact match first + const exactIndex = content.indexOf(oldText); + if (exactIndex !== -1) { + return { + found: true, + index: exactIndex, + matchLength: oldText.length, + usedFuzzyMatch: false, + contentForReplacement: content, + }; + } + + // Try fuzzy match - work entirely in normalized space + const fuzzyContent = normalizeForFuzzyMatch(content); + const fuzzyOldText = normalizeForFuzzyMatch(oldText); + const fuzzyIndex = fuzzyContent.indexOf(fuzzyOldText); + + if (fuzzyIndex === -1) { + return { + found: false, + index: -1, + matchLength: 0, + usedFuzzyMatch: false, + contentForReplacement: content, + }; + } + + // When fuzzy matching, return offsets in normalized space. Callers can use + // the normalized content to compute replacements, then decide how much of + // that normalized output should be written back. + return { + found: true, + index: fuzzyIndex, + matchLength: fuzzyOldText.length, + usedFuzzyMatch: true, + contentForReplacement: fuzzyContent, + }; +} + +/** Strip UTF-8 BOM if present, return both the BOM (if any) and the text without it */ +export function stripBom(content: string): { bom: string; text: string } { + return content.startsWith("\uFEFF") ? { bom: "\uFEFF", text: content.slice(1) } : { bom: "", text: content }; +} + +function countOccurrences(content: string, oldText: string): number { + const fuzzyContent = normalizeForFuzzyMatch(content); + const fuzzyOldText = normalizeForFuzzyMatch(oldText); + return fuzzyContent.split(fuzzyOldText).length - 1; +} + +function getNotFoundError(path: string, editIndex: number, totalEdits: number): Error { + if (totalEdits === 1) { + return new Error( + `Could not find the exact text in ${path}. The old text must match exactly including all whitespace and newlines.`, + ); + } + return new Error( + `Could not find edits[${editIndex}] in ${path}. The oldText must match exactly including all whitespace and newlines.`, + ); +} + +function getDuplicateError(path: string, editIndex: number, totalEdits: number, occurrences: number): Error { + if (totalEdits === 1) { + return new Error( + `Found ${occurrences} occurrences of the text in ${path}. The text must be unique. Please provide more context to make it unique.`, + ); + } + return new Error( + `Found ${occurrences} occurrences of edits[${editIndex}] in ${path}. Each oldText must be unique. Please provide more context to make it unique.`, + ); +} + +function getEmptyOldTextError(path: string, editIndex: number, totalEdits: number): Error { + if (totalEdits === 1) { + return new Error(`oldText must not be empty in ${path}.`); + } + return new Error(`edits[${editIndex}].oldText must not be empty in ${path}.`); +} + +function getNoChangeError(path: string, totalEdits: number): Error { + if (totalEdits === 1) { + return new Error( + `No changes made to ${path}. The replacement produced identical content. This might indicate an issue with special characters or the text not existing as expected.`, + ); + } + return new Error(`No changes made to ${path}. The replacements produced identical content.`); +} + +/** + * Apply one or more exact-text replacements to LF-normalized content. + * + * All edits are matched against the same original content. Replacements are + * then applied in reverse order so offsets remain stable. If any edit needs + * fuzzy matching, the operation runs in fuzzy-normalized content space and then + * overlays those line-level changes onto the original content so unchanged line + * blocks keep their original bytes. + */ +export function applyEditsToNormalizedContent( + normalizedContent: string, + edits: Edit[], + path: string, +): AppliedEditsResult { + const normalizedEdits = edits.map((edit) => ({ + oldText: normalizeToLF(edit.oldText), + newText: normalizeToLF(edit.newText), + })); + + for (let i = 0; i < normalizedEdits.length; i++) { + if (normalizedEdits[i].oldText.length === 0) { + throw getEmptyOldTextError(path, i, normalizedEdits.length); + } + } + + const initialMatches = normalizedEdits.map((edit) => fuzzyFindText(normalizedContent, edit.oldText)); + const usedFuzzyMatch = initialMatches.some((match) => match.usedFuzzyMatch); + const replacementBaseContent = usedFuzzyMatch ? normalizeForFuzzyMatch(normalizedContent) : normalizedContent; + + const matchedEdits: MatchedEdit[] = []; + for (let i = 0; i < normalizedEdits.length; i++) { + const edit = normalizedEdits[i]; + const matchResult = fuzzyFindText(replacementBaseContent, edit.oldText); + if (!matchResult.found) { + throw getNotFoundError(path, i, normalizedEdits.length); + } + + const occurrences = countOccurrences(replacementBaseContent, edit.oldText); + if (occurrences > 1) { + throw getDuplicateError(path, i, normalizedEdits.length, occurrences); + } + + matchedEdits.push({ + editIndex: i, + matchIndex: matchResult.index, + matchLength: matchResult.matchLength, + newText: edit.newText, + }); + } + + matchedEdits.sort((a, b) => a.matchIndex - b.matchIndex); + for (let i = 1; i < matchedEdits.length; i++) { + const previous = matchedEdits[i - 1]; + const current = matchedEdits[i]; + if (previous.matchIndex + previous.matchLength > current.matchIndex) { + throw new Error( + `edits[${previous.editIndex}] and edits[${current.editIndex}] overlap in ${path}. Merge them into one edit or target disjoint regions.`, + ); + } + } + + const baseContent = normalizedContent; + const newContent = usedFuzzyMatch + ? applyReplacementsPreservingUnchangedLines(normalizedContent, replacementBaseContent, matchedEdits) + : applyReplacements(replacementBaseContent, matchedEdits); + + if (baseContent === newContent) { + throw getNoChangeError(path, normalizedEdits.length); + } + + return { baseContent, newContent }; +} + +/** Generate a standard unified patch. */ +export function generateUnifiedPatch(path: string, oldContent: string, newContent: string, contextLines = 4): string { + return Diff.createTwoFilesPatch(path, path, oldContent, newContent, undefined, undefined, { + context: contextLines, + headerOptions: Diff.FILE_HEADERS_ONLY, + }); +} + +/** + * Generate a display-oriented diff string with line numbers and context. + * Returns both the diff string and the first changed line number (in the new file). + */ +export function generateDiffString( + oldContent: string, + newContent: string, + contextLines = 4, +): { diff: string; firstChangedLine: number | undefined } { + const parts = Diff.diffLines(oldContent, newContent); + const output: string[] = []; + + const oldLines = oldContent.split("\n"); + const newLines = newContent.split("\n"); + const maxLineNum = Math.max(oldLines.length, newLines.length); + const lineNumWidth = String(maxLineNum).length; + + let oldLineNum = 1; + let newLineNum = 1; + let lastWasChange = false; + let firstChangedLine: number | undefined; + + for (let i = 0; i < parts.length; i++) { + const part = parts[i]; + const raw = part.value.split("\n"); + if (raw[raw.length - 1] === "") { + raw.pop(); + } + + if (part.added || part.removed) { + // Capture the first changed line (in the new file) + if (firstChangedLine === undefined) { + firstChangedLine = newLineNum; + } + + // Show the change + for (const line of raw) { + if (part.added) { + const lineNum = String(newLineNum).padStart(lineNumWidth, " "); + output.push(`+${lineNum} ${line}`); + newLineNum++; + } else { + // removed + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(`-${lineNum} ${line}`); + oldLineNum++; + } + } + lastWasChange = true; + } else { + // Context lines - only show a few before/after changes + const nextPartIsChange = i < parts.length - 1 && (parts[i + 1].added || parts[i + 1].removed); + const hasLeadingChange = lastWasChange; + const hasTrailingChange = nextPartIsChange; + + if (hasLeadingChange && hasTrailingChange) { + if (raw.length <= contextLines * 2) { + for (const line of raw) { + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(` ${lineNum} ${line}`); + oldLineNum++; + newLineNum++; + } + } else { + const leadingLines = raw.slice(0, contextLines); + const trailingLines = raw.slice(raw.length - contextLines); + const skippedLines = raw.length - leadingLines.length - trailingLines.length; + + for (const line of leadingLines) { + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(` ${lineNum} ${line}`); + oldLineNum++; + newLineNum++; + } + + output.push(` ${"".padStart(lineNumWidth, " ")} ...`); + oldLineNum += skippedLines; + newLineNum += skippedLines; + + for (const line of trailingLines) { + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(` ${lineNum} ${line}`); + oldLineNum++; + newLineNum++; + } + } + } else if (hasLeadingChange) { + const shownLines = raw.slice(0, contextLines); + const skippedLines = raw.length - shownLines.length; + + for (const line of shownLines) { + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(` ${lineNum} ${line}`); + oldLineNum++; + newLineNum++; + } + + if (skippedLines > 0) { + output.push(` ${"".padStart(lineNumWidth, " ")} ...`); + oldLineNum += skippedLines; + newLineNum += skippedLines; + } + } else if (hasTrailingChange) { + const skippedLines = Math.max(0, raw.length - contextLines); + if (skippedLines > 0) { + output.push(` ${"".padStart(lineNumWidth, " ")} ...`); + oldLineNum += skippedLines; + newLineNum += skippedLines; + } + + for (const line of raw.slice(skippedLines)) { + const lineNum = String(oldLineNum).padStart(lineNumWidth, " "); + output.push(` ${lineNum} ${line}`); + oldLineNum++; + newLineNum++; + } + } else { + // Skip these context lines entirely + oldLineNum += raw.length; + newLineNum += raw.length; + } + + lastWasChange = false; + } + } + + return { diff: output.join("\n"), firstChangedLine }; +} diff --git a/packages/durable/src/tools/edit.ts b/packages/durable/src/tools/edit.ts new file mode 100644 index 000000000..b62040219 --- /dev/null +++ b/packages/durable/src/tools/edit.ts @@ -0,0 +1,144 @@ +import type { JsonValue } from "@earendil-works/chord"; +import { type Static, Type } from "typebox"; +import type { FileError } from "../env/index.ts"; +import type { ToolRegistration } from "../harness/types.ts"; +import { + applyEditsToNormalizedContent, + detectLineEnding, + type Edit, + generateDiffString, + generateUnifiedPatch, + normalizeToLF, + restoreLineEndings, + stripBom, +} from "./edit-diff.ts"; +import { requireEnv } from "./env.ts"; +import { withFileMutationQueue } from "./file-mutation-queue.ts"; +import { resolveToolPath } from "./path-utils.ts"; + +const replaceEditSchema = Type.Object({ + oldText: Type.String({ + description: + "Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.", + }), + newText: Type.String({ description: "Replacement text for this targeted edit." }), +}); + +const editSchema = Type.Object({ + path: Type.String({ description: "Path to the file to edit (relative or absolute)" }), + edits: Type.Array(replaceEditSchema, { + description: + "One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.", + }), +}); + +export type EditToolInput = Static; +type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown }; +type SingleEditInput = { oldText: string; newText: string }; + +function isSingleEditInput(value: unknown): value is SingleEditInput { + if (!value || typeof value !== "object" || Array.isArray(value)) return false; + const edit = value as Record; + return typeof edit.oldText === "string" && typeof edit.newText === "string"; +} + +/** + * Repair shapes models commonly send: `edits` as a JSON string or as a single edit object, and a top-level + * `oldText`/`newText` pair. Works on a copy; the call's arguments stay unchanged. + */ +function prepareEditArguments(input: JsonValue): JsonValue { + if (!input || typeof input !== "object" || Array.isArray(input)) return input; + const args: Record = { ...input }; + if (typeof args.edits === "string") { + try { + const parsed: unknown = JSON.parse(args.edits); + if (Array.isArray(parsed)) { + args.edits = parsed; + } else if (isSingleEditInput(parsed)) { + args.edits = [parsed]; + } + } catch {} + } else if (isSingleEditInput(args.edits)) { + args.edits = [args.edits]; + } + + const legacy = args as LegacyEditToolInput; + if (typeof legacy.oldText !== "string" || typeof legacy.newText !== "string") return args as JsonValue; + const edits = Array.isArray(legacy.edits) ? [...legacy.edits] : []; + edits.push({ oldText: legacy.oldText, newText: legacy.newText }); + const { oldText: _oldText, newText: _newText, ...rest } = legacy; + return { ...rest, edits } as JsonValue; +} + +export interface EditToolDetails { + diff: string; + patch: string; + firstChangedLine?: number; +} + +function validateEditInput(input: EditToolInput): { path: string; edits: Edit[] } { + if (!Array.isArray(input.edits) || input.edits.length === 0) { + throw new Error("Edit tool input is invalid. edits must contain at least one replacement."); + } + return { path: input.path, edits: input.edits }; +} + +function editAccessError(path: string, error: FileError): Error { + return new Error(`Could not edit file: ${path}. Error code: ${error.code}.`, { cause: error }); +} + +export function createEditTool(): ToolRegistration { + return { + name: "edit", + description: + "Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.", + parameters: editSchema, + prepareArguments: prepareEditArguments, + async execute(args, api, context) { + const { path, edits } = validateEditInput(args as EditToolInput); + const env = requireEnv(api); + const absolutePath = await resolveToolPath(env, path, context); + return withFileMutationQueue( + env, + absolutePath, + async () => { + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + const info = await env.fileInfo(absolutePath, context); + if (!info.ok) throw editAccessError(path, info.error); + if (info.value.kind !== "file" && info.value.kind !== "symlink") { + throw new Error(`Could not edit file: ${path}. Path is not a file.`); + } + + const readResult = await env.readTextFile(absolutePath, context); + if (!readResult.ok) throw editAccessError(path, readResult.error); + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + + const { bom, text: content } = stripBom(readResult.value); + const originalEnding = detectLineEnding(content); + const normalizedContent = normalizeToLF(content); + const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path); + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + + const finalContent = bom + restoreLineEndings(newContent, originalEnding); + const writeResult = await env.writeFile(absolutePath, finalContent, context); + if (!writeResult.ok) throw editAccessError(path, writeResult.error); + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + + const diffResult = generateDiffString(baseContent, newContent); + const details: EditToolDetails = { + diff: diffResult.diff, + patch: generateUnifiedPatch(path, baseContent, newContent), + ...(diffResult.firstChangedLine === undefined + ? {} + : { firstChangedLine: diffResult.firstChangedLine }), + }; + return { + content: [{ type: "text", text: `Successfully replaced ${edits.length} block(s) in ${path}.` }], + details: details as unknown as JsonValue, + }; + }, + context, + ); + }, + }; +} diff --git a/packages/durable/src/tools/env.ts b/packages/durable/src/tools/env.ts new file mode 100644 index 000000000..bfa40c3d9 --- /dev/null +++ b/packages/durable/src/tools/env.ts @@ -0,0 +1,8 @@ +import type { ExecutionEnv } from "../env/index.ts"; +import type { ToolExecutionApi } from "../harness/types.ts"; + +/** The call's execution environment; a tool without one fails with an ordinary error result. */ +export function requireEnv(api: ToolExecutionApi): ExecutionEnv { + if (api.env === undefined) throw new Error("No execution environment is configured"); + return api.env; +} diff --git a/packages/durable/src/tools/file-mutation-queue.ts b/packages/durable/src/tools/file-mutation-queue.ts new file mode 100644 index 000000000..0e4e6c43e --- /dev/null +++ b/packages/durable/src/tools/file-mutation-queue.ts @@ -0,0 +1,61 @@ +import type { Context } from "@earendil-works/chord"; +import { type ExecutionEnv, getOrThrow } from "../env/index.ts"; + +type MutationQueueState = { + queues: Map>; + registration: Promise; +}; + +const states = new WeakMap(); + +function getState(env: ExecutionEnv): MutationQueueState { + let state = states.get(env); + if (!state) { + state = { queues: new Map(), registration: Promise.resolve() }; + states.set(env, state); + } + return state; +} + +async function getMutationQueueKey(env: ExecutionEnv, path: string, context: Context): Promise { + const absolutePath = getOrThrow(await env.absolutePath(path, context)); + const canonicalPath = await env.canonicalPath(absolutePath, context); + if (canonicalPath.ok) return canonicalPath.value; + if (canonicalPath.error.code === "not_found" || canonicalPath.error.code === "not_supported") return absolutePath; + throw canonicalPath.error; +} + +/** Serialize file mutations targeting the same environment and canonical path. */ +export async function withFileMutationQueue( + env: ExecutionEnv, + path: string, + fn: () => Promise, + context: Context, +): Promise { + const state = getState(env); + const registration = state.registration.then(async () => { + const key = await getMutationQueueKey(env, path, context); + const currentQueue = state.queues.get(key) ?? Promise.resolve(); + + let releaseNext = () => {}; + const nextQueue = new Promise((resolve) => { + releaseNext = resolve; + }); + const chainedQueue = currentQueue.then(() => nextQueue); + state.queues.set(key, chainedQueue); + return { key, currentQueue, chainedQueue, releaseNext }; + }); + state.registration = registration.then( + () => undefined, + () => undefined, + ); + + const { key, currentQueue, chainedQueue, releaseNext } = await registration; + await currentQueue; + try { + return await fn(); + } finally { + releaseNext(); + if (state.queues.get(key) === chainedQueue) state.queues.delete(key); + } +} diff --git a/packages/durable/src/tools/image.ts b/packages/durable/src/tools/image.ts new file mode 100644 index 000000000..554192b5f --- /dev/null +++ b/packages/durable/src/tools/image.ts @@ -0,0 +1,89 @@ +const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]; + +export function detectSupportedImageMimeType(buffer: Uint8Array): string | undefined { + if (startsWith(buffer, [0xff, 0xd8, 0xff])) return buffer[3] === 0xf7 ? undefined : "image/jpeg"; + if (startsWith(buffer, PNG_SIGNATURE)) return isPng(buffer) && !isAnimatedPng(buffer) ? "image/png" : undefined; + if (startsWithAscii(buffer, 0, "GIF87a") || startsWithAscii(buffer, 0, "GIF89a")) return "image/gif"; + if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) return "image/webp"; + if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) return "image/bmp"; + return undefined; +} + +function isPng(buffer: Uint8Array): boolean { + return ( + buffer.length >= 16 && readUint32BE(buffer, PNG_SIGNATURE.length) === 13 && startsWithAscii(buffer, 12, "IHDR") + ); +} + +function isAnimatedPng(buffer: Uint8Array): boolean { + let offset = PNG_SIGNATURE.length; + while (offset + 8 <= buffer.length) { + const chunkLength = readUint32BE(buffer, offset); + const chunkTypeOffset = offset + 4; + if (startsWithAscii(buffer, chunkTypeOffset, "acTL")) return true; + if (startsWithAscii(buffer, chunkTypeOffset, "IDAT")) return false; + const nextOffset = offset + 8 + chunkLength + 4; + if (nextOffset <= offset || nextOffset > buffer.length) return false; + offset = nextOffset; + } + return false; +} + +function isBmp(buffer: Uint8Array): boolean { + if (buffer.length < 26) return false; + const declaredFileSize = readUint32LE(buffer, 2); + const pixelDataOffset = readUint32LE(buffer, 10); + const dibHeaderSize = readUint32LE(buffer, 14); + if (declaredFileSize !== 0 && declaredFileSize < 26) return false; + if (pixelDataOffset < 14 + dibHeaderSize) return false; + if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) return false; + + let colorPlanes: number; + let bitsPerPixel: number; + if (dibHeaderSize === 12) { + colorPlanes = readUint16LE(buffer, 22); + bitsPerPixel = readUint16LE(buffer, 24); + } else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) { + if (buffer.length < 30) return false; + colorPlanes = readUint16LE(buffer, 26); + bitsPerPixel = readUint16LE(buffer, 28); + } else { + return false; + } + return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel); +} + +function readUint16LE(buffer: Uint8Array, offset: number): number { + return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8); +} + +function readUint32BE(buffer: Uint8Array, offset: number): number { + return ( + (buffer[offset] ?? 0) * 0x1000000 + + ((buffer[offset + 1] ?? 0) << 16) + + ((buffer[offset + 2] ?? 0) << 8) + + (buffer[offset + 3] ?? 0) + ); +} + +function readUint32LE(buffer: Uint8Array, offset: number): number { + return ( + (buffer[offset] ?? 0) + + ((buffer[offset + 1] ?? 0) << 8) + + ((buffer[offset + 2] ?? 0) << 16) + + (buffer[offset + 3] ?? 0) * 0x1000000 + ); +} + +function startsWith(buffer: Uint8Array, bytes: number[]): boolean { + if (buffer.length < bytes.length) return false; + return bytes.every((byte, index) => buffer[index] === byte); +} + +function startsWithAscii(buffer: Uint8Array, offset: number, text: string): boolean { + if (buffer.length < offset + text.length) return false; + for (let index = 0; index < text.length; index++) { + if (buffer[offset + index] !== text.charCodeAt(index)) return false; + } + return true; +} diff --git a/packages/durable/src/tools/index.ts b/packages/durable/src/tools/index.ts new file mode 100644 index 000000000..7f5df8b4a --- /dev/null +++ b/packages/durable/src/tools/index.ts @@ -0,0 +1,10 @@ +export { + type BashExecution, + type BashPrepare, + type BashToolInput, + type BashToolOptions, + createBashTool, +} from "./bash.ts"; +export { createEditTool, type EditToolDetails, type EditToolInput } from "./edit.ts"; +export { createReadTool, type ReadToolDetails, type ReadToolInput } from "./read.ts"; +export { createWriteTool, type WriteToolInput } from "./write.ts"; diff --git a/packages/durable/src/tools/path-utils.ts b/packages/durable/src/tools/path-utils.ts new file mode 100644 index 000000000..42c7af507 --- /dev/null +++ b/packages/durable/src/tools/path-utils.ts @@ -0,0 +1,30 @@ +import type { Context } from "@earendil-works/chord"; +import { type ExecutionEnv, getOrThrow } from "../env/index.ts"; + +const UNICODE_SPACES = /[\u00A0\u2000-\u200A\u202F\u205F\u3000]/g; +const NARROW_NO_BREAK_SPACE = "\u202F"; + +function normalizeToolPath(path: string): string { + const normalized = path.replace(UNICODE_SPACES, " "); + return normalized.startsWith("@") ? normalized.slice(1) : normalized; +} + +export async function resolveToolPath(env: ExecutionEnv, path: string, context: Context): Promise { + return getOrThrow(await env.absolutePath(normalizeToolPath(path), context)); +} + +export async function resolveReadToolPath(env: ExecutionEnv, path: string, context: Context): Promise { + const resolved = await resolveToolPath(env, path, context); + const variants = [ + resolved, + resolved.replace(/ (AM|PM)\./gi, `${NARROW_NO_BREAK_SPACE}$1.`), + resolved.normalize("NFD"), + resolved.replace(/'/g, "\u2019"), + resolved.normalize("NFD").replace(/'/g, "\u2019"), + ]; + + for (const variant of new Set(variants)) { + if (getOrThrow(await env.exists(variant, context))) return variant; + } + return resolved; +} diff --git a/packages/durable/src/tools/read.ts b/packages/durable/src/tools/read.ts new file mode 100644 index 000000000..7253a6afa --- /dev/null +++ b/packages/durable/src/tools/read.ts @@ -0,0 +1,111 @@ +import type { JsonValue } from "@earendil-works/chord"; +import { type Static, Type } from "typebox"; +import { getOrThrow } from "../env/index.ts"; +import { characterEnd } from "../harness/output.ts"; +import type { ToolDiagnostic, ToolRegistration } from "../harness/types.ts"; +import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, formatSize, type TruncationResult, truncateHead } from "../truncate.ts"; +import { requireEnv } from "./env.ts"; +import { detectSupportedImageMimeType } from "./image.ts"; +import { resolveReadToolPath } from "./path-utils.ts"; + +const readSchema = Type.Object({ + path: Type.String({ description: "Path to the file to read (relative or absolute)" }), + offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })), + limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })), +}); + +export type ReadToolInput = Static; + +export interface ReadToolDetails { + /** How the shown text was cut; the text itself is the result content. */ + truncation?: Omit; +} + +/** Reads text files. Remarks about truncation and continuation are diagnostics; the content is only file text. */ +export function createReadTool(): ToolRegistration { + return { + name: "read", + description: `Read the contents of a text file. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`, + parameters: readSchema, + async execute(args, api, context) { + const { path, offset, limit } = args as ReadToolInput; + const env = requireEnv(api); + const absolutePath = await resolveReadToolPath(env, path, context); + const bytes = getOrThrow(await env.readBinaryFile(absolutePath, context)); + const mimeType = detectSupportedImageMimeType(bytes); + if (mimeType) { + // Image content is not supported yet. + return { + content: [], + isError: true, + diagnostics: [ + { + severity: "error", + code: "unsupported_image", + message: `${path} is an image (${mimeType}); reading images is not supported`, + }, + ], + }; + } + + const textContent = new TextDecoder().decode(bytes); + const allLines = textContent.split("\n"); + const totalFileLines = allLines.length; + const startLine = offset ? Math.max(0, offset - 1) : 0; + const startLineDisplay = startLine + 1; + if (startLine >= allLines.length) { + throw new Error(`Offset ${offset} is beyond end of file (${allLines.length} lines total)`); + } + + let selectedContent: string; + let userLimitedLines: number | undefined; + if (limit !== undefined) { + const endLine = Math.min(startLine + limit, allLines.length); + selectedContent = allLines.slice(startLine, endLine).join("\n"); + userLimitedLines = endLine - startLine; + } else { + selectedContent = allLines.slice(startLine).join("\n"); + } + + const { content: headText, ...truncation } = truncateHead(selectedContent); + const diagnostics: ToolDiagnostic[] = []; + let outputText = headText; + let details: ReadToolDetails | undefined; + if (truncation.firstLineExceedsLimit) { + // Show the start of the line, cut at the byte limit on a character boundary. + const lineBytes = new TextEncoder().encode(allLines[startLine]); + const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES); + outputText = new TextDecoder().decode(lineBytes.subarray(0, end)); + diagnostics.push({ + severity: "warn", + code: "truncated", + message: `Line ${startLineDisplay} is ${formatSize(lineBytes.byteLength)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`, + }); + details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } }; + } else if (truncation.truncated) { + const endLineDisplay = startLineDisplay + truncation.outputLines - 1; + const nextOffset = endLineDisplay + 1; + const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`; + diagnostics.push({ + severity: "info", + code: "truncated", + message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`, + }); + details = { truncation }; + } else if (userLimitedLines !== undefined && startLine + userLimitedLines < allLines.length) { + const remaining = allLines.length - (startLine + userLimitedLines); + const nextOffset = startLine + userLimitedLines + 1; + diagnostics.push({ + severity: "info", + message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`, + }); + } + + return { + content: outputText === "" ? [] : [{ type: "text", text: outputText }], + ...(details === undefined ? {} : { details: details as unknown as JsonValue }), + diagnostics, + }; + }, + }; +} diff --git a/packages/durable/src/tools/write.ts b/packages/durable/src/tools/write.ts new file mode 100644 index 000000000..0147d534d --- /dev/null +++ b/packages/durable/src/tools/write.ts @@ -0,0 +1,38 @@ +import { type Static, Type } from "typebox"; +import { getOrThrow } from "../env/index.ts"; +import type { ToolRegistration } from "../harness/types.ts"; +import { requireEnv } from "./env.ts"; +import { withFileMutationQueue } from "./file-mutation-queue.ts"; +import { resolveToolPath } from "./path-utils.ts"; + +const writeSchema = Type.Object({ + path: Type.String({ description: "Path to the file to write (relative or absolute)" }), + content: Type.String({ description: "Content to write to the file" }), +}); + +export type WriteToolInput = Static; + +export function createWriteTool(): ToolRegistration { + return { + name: "write", + description: + "Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.", + parameters: writeSchema, + async execute(args, api, context) { + const { path, content } = args as WriteToolInput; + const env = requireEnv(api); + const absolutePath = await resolveToolPath(env, path, context); + return withFileMutationQueue( + env, + absolutePath, + async () => { + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + getOrThrow(await env.writeFile(absolutePath, content, context)); + if (context.abortSignal?.aborted) throw new Error("Operation aborted"); + return { content: [{ type: "text", text: `Successfully wrote to ${path}` }] }; + }, + context, + ); + }, + }; +} diff --git a/packages/durable/src/env/utils/truncate.ts b/packages/durable/src/truncate.ts similarity index 51% rename from packages/durable/src/env/utils/truncate.ts rename to packages/durable/src/truncate.ts index 07bf7c4d7..43cdf412e 100644 --- a/packages/durable/src/env/utils/truncate.ts +++ b/packages/durable/src/truncate.ts @@ -5,12 +5,11 @@ * - Line limit (default: 2000 lines) * - Byte limit (default: 50KB) * - * Never returns partial lines (except bash tail truncation edge case). + * Never returns partial lines. Tool output streams are bounded by `src/harness/output.ts` instead. */ export const DEFAULT_MAX_LINES = 2000; export const DEFAULT_MAX_BYTES = 50 * 1024; // 50KB -export const GREP_MAX_LINE_LENGTH = 500; // Max chars per grep match line export interface TruncationResult { /** The truncated content */ @@ -86,29 +85,6 @@ function splitLinesForCounting(content: string): string[] { return lines; } -function replaceUnpairedSurrogates(content: string): string { - let output = ""; - for (let i = 0; i < content.length; i++) { - const code = content.charCodeAt(i); - if (code >= 0xd800 && code <= 0xdbff) { - if (i + 1 < content.length) { - const next = content.charCodeAt(i + 1); - if (next >= 0xdc00 && next <= 0xdfff) { - output += content[i] + content[i + 1]; - i++; - continue; - } - } - output += "�"; - } else if (code >= 0xdc00 && code <= 0xdfff) { - output += "�"; - } else { - output += content[i]; - } - } - return output; -} - /** * Format bytes as human-readable size. */ @@ -210,137 +186,3 @@ export function truncateHead(content: string, options: TruncationOptions = {}): maxBytes, }; } - -/** - * Truncate content from the tail (keep last N lines/bytes). - * Suitable for bash output where you want to see the end (errors, final results). - * - * May return partial first line if the last line of original content exceeds byte limit. - */ -export function truncateTail(content: string, options: TruncationOptions = {}): TruncationResult { - const maxLines = options.maxLines ?? DEFAULT_MAX_LINES; - const maxBytes = options.maxBytes ?? DEFAULT_MAX_BYTES; - - const totalBytes = utf8ByteLength(content); - const lines = splitLinesForCounting(content); - const totalLines = lines.length; - - // Check if no truncation needed - if (totalLines <= maxLines && totalBytes <= maxBytes) { - return { - content, - truncated: false, - truncatedBy: null, - totalLines, - totalBytes, - outputLines: totalLines, - outputBytes: totalBytes, - lastLinePartial: false, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; - } - - // Work backwards from the end - const outputLinesArr: string[] = []; - let outputBytesCount = 0; - let truncatedBy: "lines" | "bytes" = "lines"; - let lastLinePartial = false; - - for (let i = lines.length - 1; i >= 0 && outputLinesArr.length < maxLines; i--) { - const line = lines[i]; - const lineBytes = utf8ByteLength(line) + (outputLinesArr.length > 0 ? 1 : 0); // +1 for newline - - if (outputBytesCount + lineBytes > maxBytes) { - truncatedBy = "bytes"; - // Edge case: if we haven't added ANY lines yet and this line exceeds maxBytes, - // take the end of the line (partial) - if (outputLinesArr.length === 0) { - const truncatedLine = truncateStringToBytesFromEnd(line, maxBytes); - outputLinesArr.unshift(truncatedLine); - outputBytesCount = utf8ByteLength(truncatedLine); - lastLinePartial = true; - } - break; - } - - outputLinesArr.unshift(line); - outputBytesCount += lineBytes; - } - - // Without a byte break, only omitted lines prove the line limit was reached; otherwise a trailing newline exceeded bytes. - if (truncatedBy !== "bytes") truncatedBy = outputLinesArr.length < totalLines ? "lines" : "bytes"; - - const outputContent = outputLinesArr.join("\n"); - const finalOutputBytes = utf8ByteLength(outputContent); - - return { - content: outputContent, - truncated: true, - truncatedBy, - totalLines, - totalBytes, - outputLines: outputLinesArr.length, - outputBytes: finalOutputBytes, - lastLinePartial, - firstLineExceedsLimit: false, - maxLines, - maxBytes, - }; -} - -/** - * Truncate a string to fit within a byte limit (from the end). - * Handles multi-byte UTF-8 characters correctly. - */ -function truncateStringToBytesFromEnd(str: string, maxBytes: number): string { - if (maxBytes <= 0) return ""; - - let outputBytes = 0; - let start = str.length; - let needsReplacement = false; - for (let i = str.length; i > 0; ) { - let characterStart = i - 1; - const code = str.charCodeAt(characterStart); - let characterBytes: number; - let unpairedSurrogate = false; - if (code >= 0xdc00 && code <= 0xdfff && characterStart > 0) { - const previous = str.charCodeAt(characterStart - 1); - if (previous >= 0xd800 && previous <= 0xdbff) { - characterStart--; - characterBytes = 4; - } else { - characterBytes = 3; - unpairedSurrogate = true; - } - } else if (code >= 0xd800 && code <= 0xdfff) { - characterBytes = 3; - unpairedSurrogate = true; - } else { - characterBytes = code <= 0x7f ? 1 : code <= 0x7ff ? 2 : 3; - } - if (outputBytes + characterBytes > maxBytes) break; - outputBytes += characterBytes; - start = characterStart; - needsReplacement ||= unpairedSurrogate; - i = characterStart; - } - - const output = str.slice(start); - return needsReplacement ? replaceUnpairedSurrogates(output) : output; -} - -/** - * Truncate a single line to max characters, adding [truncated] suffix. - * Used for grep match lines. - */ -export function truncateLine( - line: string, - maxChars: number = GREP_MAX_LINE_LENGTH, -): { text: string; wasTruncated: boolean } { - if (line.length <= maxChars) { - return { text: line, wasTruncated: false }; - } - return { text: `${line.slice(0, maxChars)}... [truncated]`, wasTruncated: true }; -} diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index d20403afe..a02737ede 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1,7 +1,8 @@ import type { AttachedReplicatedState, Context, Draft, JsonValue } from "@earendil-works/chord"; import type { Op } from "@earendil-works/chord/delta"; import type { Message, Models } from "@earendil-works/pi-ai"; -import type { ContextView, RegistrySnapshot } from "./harness/types.ts"; +import type { ExecutionEnv } from "./env/index.ts"; +import type { ContextView, RegistrySnapshot, SettledTask } from "./harness/types.ts"; /** JSON object used as the root of every durable document. */ export type JsonObject = { [key: string]: JsonValue }; @@ -143,18 +144,34 @@ export type PhaseHandler = ( context: Context, ) => Promise; +/** Dispatches one hook of a task to every matching registered handler, in registry order of the phase snapshot. */ +export interface HookRunner { + /** + * Call `invoke` with each matching handler named `name`. An ordinary throw from `invoke` is reported and the next + * handler runs; once the invocation is signalled, the error propagates. Composition happens inside `invoke`. + */ + each(name: K, invoke: (handler: NonNullable) => void | Promise): Promise; +} + /** * Operations of one task invocation. Every operation rejects after the invocation ends; watches acquired through it - * stop at invocation end. `_H` is the task's hook map, consumed once the runtime gains its hook runner. + * stop at invocation end. */ -export interface TaskRuntime extends DocumentObserver, DocumentReader { +export interface TaskRuntime extends DocumentObserver, DocumentReader { readonly taskId: TaskId; readonly conversationId: ConversationId; - /** Aborted when the run is signalled by `abortTask()` or the Harness closes. */ + /** + * Aborted when the run is signalled by `abortTask()`, the Harness closes, or the invocation ends. Work still using it + * after the invocation ended, such as a detached wait, is cancelled; it could not write anything anyway. + */ readonly signal: AbortSignal; /** Registry snapshot of the current phase; refreshed at every phase boundary. */ readonly registry: RegistrySnapshot; readonly models: Models; + /** `HarnessOptions.env`; tools receive it as `api.env`. */ + readonly env: ExecutionEnv | undefined; + /** Handlers registered for this task's name whose scope matches its conversation. */ + readonly hooks: HookRunner; /** * Commit on the Session line after rereading the task. Rejects when the task is terminal, the invocation ended, the @@ -173,6 +190,14 @@ export interface TaskRuntime extends DocumentObserve memo(name: string, context: Context): Promise; /** Store `candidate` unless a memo already exists; return the durable winner. */ memo(name: string, candidate: T, context: Context): Promise; + /** Committed task record. */ + getTask(id: TaskId, context: Context): Promise | undefined>; + /** Resolve with the task's terminal receipt; rejects when the invocation ends. */ + waitForTask(id: TaskId, context: Context): Promise>; + /** Committed entry visible from the task's conversation. */ + entry(id: EntryId, context: Context): Promise; + /** Undefined when the entry is absent, not visible, or has another kind. */ + entry(token: Entry, id: EntryId, context: Context): Promise | undefined>; /** Committed raw active transcript and model context, optionally cut off at the visible entry `at`. */ context(conversationId: ConversationId, context: Context, at?: EntryId): Promise; /** The Harness clock. */ diff --git a/packages/durable/test/chat-support.ts b/packages/durable/test/chat-support.ts index 5175ab2c8..bba799511 100644 --- a/packages/durable/test/chat-support.ts +++ b/packages/durable/test/chat-support.ts @@ -17,6 +17,7 @@ import { type Registry, type Storage, } from "@earendil-works/pi-durable"; +import type { ExecutionEnv } from "../src/env/index.ts"; import { context } from "./session-support.ts"; /** Models and registry that survive a close/reopen, like a host process's own objects. */ @@ -39,12 +40,14 @@ export function chatSetup(options: RegisterFauxProviderOptions = {}): ChatSetup export async function openChat( storage: Storage, setup: ChatSetup, + options: { readonly env?: ExecutionEnv } = {}, ): Promise<{ readonly harness: Harness; readonly root: Conversation }> { const harness = await Harness.open( storage, { models: setup.models, registry: setup.registry, + ...(options.env === undefined ? {} : { env: options.env }), now: () => setup.now(), onReport: (error) => setup.reports.push(error), }, diff --git a/packages/durable/test/env-adaptive-publisher.test.ts b/packages/durable/test/env-adaptive-publisher.test.ts deleted file mode 100644 index 74f2ddc94..000000000 --- a/packages/durable/test/env-adaptive-publisher.test.ts +++ /dev/null @@ -1,135 +0,0 @@ -import { afterEach, describe, expect, it, vi } from "vitest"; -import { AdaptivePublisher } from "../src/env/utils/adaptive-publisher.ts"; - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("AdaptivePublisher", () => { - it("bounds event count and spaces large publications by encoded size", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - let value = "a"; - const updates: string[] = []; - const publisher = new AdaptivePublisher({ - snapshot: () => value, - update: (_previous, current) => current, - measure: (update) => update.length, - publish: (update) => updates.push(update), - onError: (error) => { - throw error; - }, - minIntervalMs: 100, - targetBytesPerSecond: 100, - }); - - publisher.markDirty(); - value = "x".repeat(100); - publisher.markDirty(); - vi.advanceTimersByTime(100); - expect(updates).toEqual(["a", "x".repeat(100)]); - - value = "held"; - publisher.markDirty(); - vi.advanceTimersByTime(999); - expect(updates).toHaveLength(2); - vi.advanceTimersByTime(1); - expect(updates).toEqual(["a", "x".repeat(100), "held"]); - }); - - it("commits its baseline before a consumer throws", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - let value = "a"; - const updates: Array<{ previous: string | undefined; current: string }> = []; - let throwAfterApply = false; - const publisher = new AdaptivePublisher({ - snapshot: () => value, - update: (previous, current) => ({ previous, current }), - measure: () => 1, - publish: (update) => { - updates.push(update); - if (throwAfterApply) throw new Error("consumer failed after apply"); - }, - onError: () => {}, - minIntervalMs: 100, - targetBytesPerSecond: 100, - }); - - publisher.markDirty(); - vi.advanceTimersByTime(100); - value = "ab"; - throwAfterApply = true; - expect(() => publisher.markDirty()).toThrow("consumer failed after apply"); - publisher.flush(true); - expect(updates).toEqual([ - { previous: undefined, current: "a" }, - { previous: "a", current: "ab" }, - ]); - - throwAfterApply = false; - vi.advanceTimersByTime(100); - value = "abc"; - publisher.markDirty(); - expect(updates.at(-1)).toEqual({ previous: "ab", current: "abc" }); - }); - - it("advances its baseline without publishing or delaying an undefined update", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - let value = "a"; - const previousValues: Array = []; - const published: string[] = []; - const publisher = new AdaptivePublisher({ - snapshot: () => value, - update: (previous, current) => { - previousValues.push(previous); - return current === "skip" ? undefined : current; - }, - measure: () => 1, - publish: (update) => published.push(update), - onError: () => {}, - }); - - value = "skip"; - publisher.markDirty(); - value = "b"; - publisher.markDirty(); - expect(published).toEqual(["b"]); - expect(previousValues).toEqual([undefined, "skip"]); - }); - - it("reports trailing publication failures and cancels pending work on dispose", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - let value = 0; - const errors: unknown[] = []; - const published: number[] = []; - const publisher = new AdaptivePublisher({ - snapshot: () => value, - update: (_previous, current) => current, - measure: () => 1, - publish: (update) => { - published.push(update); - if (update === 2) throw new Error("trailing failed"); - }, - onError: (error) => errors.push(error), - minIntervalMs: 100, - }); - - value = 1; - publisher.markDirty(); - value = 2; - publisher.markDirty(); - vi.advanceTimersByTime(100); - expect(errors).toEqual([new Error("trailing failed")]); - - value = 3; - publisher.markDirty(); - publisher.dispose(); - vi.advanceTimersByTime(1_000); - publisher.markDirty(); - publisher.flush(true); - expect(published).toEqual([1, 2]); - }); -}); diff --git a/packages/durable/test/env-node-spill.test.ts b/packages/durable/test/env-node-spill.test.ts index caa3c7f41..c451c11f8 100644 --- a/packages/durable/test/env-node-spill.test.ts +++ b/packages/durable/test/env-node-spill.test.ts @@ -57,21 +57,27 @@ describe("NodeExecutionEnv spill backpressure", () => { mkdirSync(root, { recursive: true }); tempDirs.push(root); const env = new NodeExecutionEnv({ cwd: root }); - // The shell exits after the first chunk crosses the capture limit and backpressures the spill. A background + // The shell exits after the first chunk crosses the spill threshold and backpressures the spill. A background // descendant retains stdout and writes after the post-exit grace period. Without the pending-spill guard, // settlement destroys stdout before that descendant output is read. const command = "printf '%020d' 0 | tr 0 a; (sleep 0.2; printf '%01000d' 0 | tr 0 b) &"; + let output = ""; const result = getOrThrow( await env.exec( command, - { capture: { limits: { maxBytes: 10, maxLines: 10, retain: "tail" }, spill: true }, onUpdate: () => {} }, + { + spill: { afterBytes: 10, afterLines: 10 }, + onOutput: (text) => { + output += text; + }, + }, BACKGROUND_CONTEXT, ), ); expect(spill.rejectedWrites).toBeGreaterThan(0); - expect(result.truncation.totalBytes).toBe(1020); + expect(output).toHaveLength(1020); expect(result.spillPath).toBeDefined(); tempDirs.push(join(result.spillPath!, "..")); expect(getOrThrow(await env.readTextFile(result.spillPath!, BACKGROUND_CONTEXT))).toBe( diff --git a/packages/durable/test/env-node.test.ts b/packages/durable/test/env-node.test.ts index c1c7843b7..a6689a247 100644 --- a/packages/durable/test/env-node.test.ts +++ b/packages/durable/test/env-node.test.ts @@ -6,9 +6,8 @@ import { delimiter, join } from "node:path"; import { pathToFileURL } from "node:url"; import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { FileError, getOrThrow, type ShellExecOptions, type ShellOutputView } from "../src/env/index.ts"; +import { FileError, getOrThrow, type ShellExecOptions } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; -import { applyShellOutputUpdate } from "../src/env/utils/output-capture.ts"; const tempDirs: string[] = []; const chmodRestorePaths: string[] = []; @@ -64,14 +63,14 @@ async function collectShellOutput( command: string, options: ShellExecOptions | undefined, context: Parameters[2], -): Promise<{ result: Awaited>; output: ShellOutputView | undefined }> { - let output: ShellOutputView | undefined; +): Promise<{ result: Awaited>; output: string }> { + let output = ""; const result = await env.exec( command, { ...options, - onUpdate: (update) => { - output = applyShellOutputUpdate(output, update); + onOutput: (text) => { + output += text; }, }, context, @@ -577,7 +576,7 @@ describe("NodeExecutionEnv shell", () => { BACKGROUND_CONTEXT, ); const result = getOrThrow(collected.result); - expect(collected.output?.text).toBe(`${await realpath(root)}:ok`); + expect(collected.output).toBe(`${await realpath(root)}:ok`); expect(result.exitCode).toBe(0); }); @@ -608,7 +607,7 @@ describe("NodeExecutionEnv shell", () => { BACKGROUND_CONTEXT, ); getOrThrow(collected.result); - expect(collected.output?.text).toBe(`${expectedSessionFile}|true|preserved`); + expect(collected.output).toBe(`${expectedSessionFile}|true|preserved`); }, ); @@ -628,7 +627,7 @@ describe("NodeExecutionEnv shell", () => { BACKGROUND_CONTEXT, ); getOrThrow(collected.result); - expect(collected.output?.text).toBe("::explicit"); + expect(collected.output).toBe("::explicit"); } finally { if (previousInherited === undefined) delete process.env[inheritedKey]; else process.env[inheritedKey] = previousInherited; @@ -669,8 +668,8 @@ describe("NodeExecutionEnv shell", () => { BACKGROUND_CONTEXT, ); const result = getOrThrow(collected.result); - expect(collected.output?.text).toContain("Hello, World!"); - expect(collected.output?.text).toContain("args:-s"); + expect(collected.output).toContain("Hello, World!"); + expect(collected.output).toContain("args:-s"); expect(result.exitCode).toBe(0); } finally { process.chdir(originalCwd); @@ -700,7 +699,7 @@ describe("NodeExecutionEnv shell", () => { () => controller.abort(), ); getOrThrow(collected.result); - expect(collected.output?.text).toContain("child-exiting"); + expect(collected.output).toContain("child-exiting"); } finally { controller.abort(); cleanupDetachedChild(pidFile); @@ -720,27 +719,27 @@ describe("NodeExecutionEnv shell", () => { await expect(withTimeout(execution, 3000)).resolves.toMatchObject({ ok: true }); }, 20_000); - it("combines stdout and stderr into one bounded view", async () => { + it("streams combined stdout and stderr", async () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); - const updates: string[] = []; - let output: ShellOutputView | undefined; - const result = getOrThrow( - await env.exec( - "printf out; printf err >&2", - { - onUpdate: (update) => { - updates.push(update.kind); - output = applyShellOutputUpdate(output, update); - }, - }, - BACKGROUND_CONTEXT, - ), + const collected = await collectShellOutput(env, "printf out; printf err >&2", undefined, BACKGROUND_CONTEXT); + expect(getOrThrow(collected.result)).toEqual({ exitCode: 0 }); + expect(collected.output).toContain("out"); + expect(collected.output).toContain("err"); + }); + + it("decodes UTF-8 split across raw process chunks", async () => { + const root = createTempDir(); + const env = new NodeExecutionEnv({ cwd: root }); + const script = + "const b=Buffer.from('😀');process.stdout.write(b.subarray(0,2));setTimeout(()=>process.stdout.write(b.subarray(2)),50)"; + const collected = await collectShellOutput( + env, + `${JSON.stringify(process.execPath)} -e ${JSON.stringify(script)}`, + undefined, + BACKGROUND_CONTEXT, ); - expect(result.exitCode).toBe(0); - expect(output?.text).toContain("out"); - expect(output?.text).toContain("err"); - expect(updates[0]).toBe("replace"); + expect(collected.output).toBe("😀"); }); it("reports a missing working directory before spawning", async () => { @@ -758,8 +757,7 @@ describe("NodeExecutionEnv shell", () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT)); - expect(result.exitCode).toBe(7); - expect(result.truncation.totalBytes).toBe(0); + expect(result).toEqual({ exitCode: 7 }); }); // Regression test for https://github.com/earendil-works/pi/issues/8992 @@ -778,7 +776,7 @@ describe("NodeExecutionEnv shell", () => { if (!result.ok) expect(result.error).toMatchObject({ code: "timeout" }); }); - it("rejects invalid timeouts and capture limits before spawning", async () => { + it("rejects invalid timeouts before spawning", async () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); for (const timeout of [0, -1, Number.NaN, Number.POSITIVE_INFINITY, 2_147_484]) { @@ -787,16 +785,6 @@ describe("NodeExecutionEnv shell", () => { error: { code: "timeout", message: expect.stringContaining("Invalid timeout") }, }); } - for (const limits of [ - { maxBytes: 0, maxLines: 1 }, - { maxBytes: 1, maxLines: 0 }, - { maxBytes: 1, maxLines: 1.5 }, - ]) { - expect(await env.exec("touch spawned", { capture: { limits } }, BACKGROUND_CONTEXT)).toMatchObject({ - ok: false, - error: { code: "unknown" }, - }); - } expect(getOrThrow(await env.exists("spawned", BACKGROUND_CONTEXT))).toBe(false); }); @@ -806,7 +794,7 @@ describe("NodeExecutionEnv shell", () => { const result = await env.exec( "printf out", { - onUpdate: () => { + onOutput: () => { throw new Error("callback failed"); }, }, @@ -893,15 +881,14 @@ describe("NodeExecutionEnv shell", () => { } }); - it("does not create a spill before bounded output crosses its limits", async () => { + it("does not create a spill before output crosses its thresholds", async () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); const result = getOrThrow( await env.exec( "printf short", { - capture: { limits: { maxBytes: 100, maxLines: 10, retain: "tail" }, spill: true }, - onUpdate: () => {}, + spill: { afterBytes: 100, afterLines: 10 }, }, BACKGROUND_CONTEXT, ), @@ -909,7 +896,7 @@ describe("NodeExecutionEnv shell", () => { expect(result.spillPath).toBeUndefined(); }); - it("preserves exact raw bytes in the spill while decoding a bounded text view", async () => { + it("preserves exact raw bytes in the spill while streaming decoded text", async () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); const expected = [0x66, 0x80, 0x00, 0x6f]; @@ -917,8 +904,7 @@ describe("NodeExecutionEnv shell", () => { await env.exec( `${JSON.stringify(process.execPath)} -e "process.stdout.write(Buffer.from([${expected.join(",")}]))"`, { - capture: { limits: { maxBytes: 1, maxLines: 10, retain: "tail" }, spill: true }, - onUpdate: () => {}, + spill: { afterBytes: 1, afterLines: 10 }, }, BACKGROUND_CONTEXT, ), @@ -928,14 +914,27 @@ describe("NodeExecutionEnv shell", () => { expect([...getOrThrow(await env.readBinaryFile(result.spillPath!, BACKGROUND_CONTEXT))]).toEqual(expected); }); + it("reports the spill of a command that times out", async () => { + const root = createTempDir(); + const env = new NodeExecutionEnv({ cwd: root }); + const result = await env.exec( + "printf 12345678901234567890; sleep 5", + { timeout: 0.3, spill: { afterBytes: 10, afterLines: 10 } }, + BACKGROUND_CONTEXT, + ); + expect(result).toMatchObject({ ok: false, error: { code: "timeout", spillPath: expect.any(String) } }); + const spillPath = (result as { error: { spillPath: string } }).error.spillPath; + tempDirs.push(join(spillPath, "..")); + expect(getOrThrow(await env.readTextFile(spillPath, BACKGROUND_CONTEXT))).toBe("12345678901234567890"); + }); + it("fails rather than silently losing a requested spill", async () => { const root = createTempDir(); const env = new FailingSpillExecutionEnv({ cwd: root }); const result = await env.exec( "printf 12345678901234567890", { - capture: { limits: { maxBytes: 10, maxLines: 10, retain: "tail" }, spill: true }, - onUpdate: () => {}, + spill: { afterBytes: 10, afterLines: 10 }, }, BACKGROUND_CONTEXT, ); @@ -953,8 +952,7 @@ describe("NodeExecutionEnv shell", () => { await env.exec( `${JSON.stringify(process.execPath)} -e "process.stdout.write('x'.repeat(${size}))"`, { - capture: { limits: { maxBytes: 10, maxLines: 10, retain: "tail" }, spill: true }, - onUpdate: () => {}, + spill: { afterBytes: 10, afterLines: 10 }, }, BACKGROUND_CONTEXT, ), @@ -964,20 +962,17 @@ describe("NodeExecutionEnv shell", () => { expect(getOrThrow(await env.readTextFile(result.spillPath!, BACKGROUND_CONTEXT))).toHaveLength(size); }); - it("bounds large head-retained output while spilling every line", async () => { + it("streams every line and spills them all once output crosses its line threshold", async () => { const root = createTempDir(); const env = new NodeExecutionEnv({ cwd: root }); const collected = await collectShellOutput( env, "i=1; while [ $i -le 15000 ]; do echo line-$i; i=$((i+1)); done", - { capture: { limits: { maxBytes: 1024 * 1024, maxLines: 100, retain: "head" }, spill: true } }, + { spill: { afterBytes: 1024 * 1024, afterLines: 100 } }, BACKGROUND_CONTEXT, ); const result = getOrThrow(collected.result); - expect(result.truncation).toMatchObject({ truncated: true, truncatedBy: "lines", totalLines: 15000 }); - expect(collected.output?.text.split("\n")).toEqual( - Array.from({ length: 100 }, (_, index) => `line-${index + 1}`), - ); + expect(collected.output).toBe(Array.from({ length: 15000 }, (_, index) => `line-${index + 1}\n`).join("")); expect(result.spillPath).toBeDefined(); tempDirs.push(join(result.spillPath!, "..")); const spilled = getOrThrow(await env.readTextLines(result.spillPath!, undefined, BACKGROUND_CONTEXT)); diff --git a/packages/durable/test/env-output-capture.test.ts b/packages/durable/test/env-output-capture.test.ts deleted file mode 100644 index 5ec895486..000000000 --- a/packages/durable/test/env-output-capture.test.ts +++ /dev/null @@ -1,332 +0,0 @@ -import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; -import { afterEach, describe, expect, it, vi } from "vitest"; -import type { ShellOutputUpdate, ShellOutputView } from "../src/env/index.ts"; -import { applyShellOutputUpdate, OutputCapture, sanitizeShellOutput } from "../src/env/utils/output-capture.ts"; - -function createCapture(options?: { maxBytes?: number; maxLines?: number; retain?: "head" | "tail" }) { - const updates: ShellOutputUpdate[] = []; - const errors: unknown[] = []; - const capture = new OutputCapture( - { - limits: { - maxBytes: options?.maxBytes ?? 50, - maxLines: options?.maxLines ?? 100, - retain: options?.retain ?? "tail", - }, - }, - BACKGROUND_CONTEXT, - { - onUpdate: (update) => updates.push(update), - onError: (error) => errors.push(error), - }, - ); - return { capture, updates, errors }; -} - -function fold(updates: ShellOutputUpdate[]): ShellOutputView | undefined { - let output: ShellOutputView | undefined; - for (const update of updates) output = applyShellOutputUpdate(output, update); - return output; -} - -afterEach(() => { - vi.useRealTimers(); -}); - -describe("OutputCapture", () => { - it("removes invalid control characters without changing text or line boundaries", () => { - const input = "a\0b\tc\nd\re\u0007f\ufff9g\ufffbh😀"; - expect(sanitizeShellOutput(input)).toBe("ab\tc\ndefgh😀"); - const { capture } = createCapture(); - capture.push(input); - expect(capture.snapshot().text).toBe("ab\tc\ndefgh😀"); - }); - - it("decodes UTF-8 split across raw process chunks", () => { - const { capture } = createCapture(); - const bytes = new TextEncoder().encode("😀"); - capture.push(bytes.subarray(0, 2)); - expect(capture.snapshot().text).toBe(""); - capture.push(bytes.subarray(2)); - capture.finish(); - expect(capture.snapshot().text).toBe("😀"); - }); - - it("flushes an incomplete byte sequence before a string chunk", () => { - const { capture } = createCapture(); - capture.push(new TextEncoder().encode("😀").subarray(0, 2)); - capture.push("x"); - expect(capture.snapshot()).toMatchObject({ text: "\ufffdx", truncation: { totalBytes: 4 } }); - }); - - it("publishes the first bounded view immediately and trickling appends responsively", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture(); - - capture.push("one"); - expect(updates).toHaveLength(1); - expect(updates[0]?.kind).toBe("replace"); - - vi.advanceTimersByTime(150); - capture.push(" two"); - expect(updates).toHaveLength(2); - expect(updates[1]).toMatchObject({ kind: "append", text: " two" }); - expect(fold(updates)?.text).toBe("one two"); - }); - - it("collapses a burst into one trailing update", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture(); - capture.push("a"); - capture.push("b"); - capture.push("c"); - expect(updates).toHaveLength(1); - - vi.advanceTimersByTime(100); - expect(updates).toHaveLength(2); - expect(updates[1]).toMatchObject({ kind: "append", text: "bc" }); - expect(fold(updates)?.text).toBe("abc"); - }); - - it("publishes a small slide for post-cap trickle", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture({ maxBytes: 10 }); - capture.push("abcdefghij"); - vi.advanceTimersByTime(150); - capture.push("k"); - - expect(updates[1]).toMatchObject({ kind: "slide", drop: 1, text: "k" }); - expect(fold(updates)?.text).toBe("bcdefghijk"); - expect(fold(updates)?.truncation.totalBytes).toBe(11); - }); - - it("keeps the exact byte count for a single line larger than its working buffer", () => { - vi.useFakeTimers(); - const { capture } = createCapture({ maxBytes: 10 }); - capture.push("x".repeat(100)); - expect(capture.snapshot()).toMatchObject({ - text: "x".repeat(10), - lastLineBytes: 100, - truncation: { lastLinePartial: true }, - }); - }); - - it("uses a cap-bounded replacement after complete turnover", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture({ maxBytes: 10 }); - capture.push("abcdefghij"); - capture.push("x".repeat(100)); - vi.advanceTimersByTime(100); - - expect(updates[1]?.kind).toBe("replace"); - expect(fold(updates)?.text).toHaveLength(10); - expect(fold(updates)?.truncation.totalBytes).toBe(110); - }); - - it("forces held state and cancels its trailing timer on dispose", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture(); - capture.push("a"); - capture.push("b"); - capture.flush(); - expect(fold(updates)?.text).toBe("ab"); - capture.dispose(); - vi.advanceTimersByTime(1_000); - expect(updates).toHaveLength(2); - }); - - it("ignores input and publication after dispose", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const { capture, updates } = createCapture(); - capture.push("kept"); - capture.dispose(); - capture.push(" dropped"); - capture.setSpillPath("/tmp/dropped.log"); - capture.flush(); - vi.advanceTimersByTime(1_000); - expect(updates).toHaveLength(1); - expect(capture.snapshot().text).toBe("kept"); - }); - - it("preserves the original head after its raw guard is crossed", () => { - vi.useFakeTimers(); - const { capture } = createCapture({ maxBytes: 100, maxLines: 2, retain: "head" }); - capture.push(`first\nsecond\n${"tail".repeat(100)}`); - expect(capture.snapshot().text).toBe("first\nsecond"); - }); - - it("reports exact totals and the first limit reached while retaining the head", () => { - vi.useFakeTimers(); - const byLines = createCapture({ maxBytes: 1_000, maxLines: 2, retain: "head" }).capture; - for (let index = 1; index <= 5; index++) byLines.push(`line-${index}\n`); - expect(byLines.snapshot()).toMatchObject({ - text: "line-1\nline-2", - truncation: { - truncated: true, - truncatedBy: "lines", - totalLines: 5, - totalBytes: 35, - outputLines: 2, - outputBytes: 13, - }, - }); - - const byBytes = createCapture({ maxBytes: 10, maxLines: 100, retain: "head" }).capture; - byBytes.push("abcd\nefgh\nijkl"); - expect(byBytes.snapshot()).toMatchObject({ - text: "abcd\nefgh", - truncation: { truncated: true, truncatedBy: "bytes", totalLines: 3, totalBytes: 14, outputBytes: 9 }, - }); - }); - - it("does not report truncation at exactly the configured limits", () => { - vi.useFakeTimers(); - const { capture } = createCapture({ maxBytes: 11, maxLines: 2, retain: "head" }); - capture.push("hello\nworld"); - expect(capture.truncated).toBe(false); - expect(capture.snapshot()).toMatchObject({ - text: "hello\nworld", - truncation: { truncated: false, truncatedBy: null, totalLines: 2, totalBytes: 11 }, - }); - capture.push("\n"); - expect(capture.truncated).toBe(true); - expect(capture.snapshot().truncation).toMatchObject({ truncatedBy: "bytes", totalLines: 2, totalBytes: 12 }); - }); - - it("reports the first limit reached when bytes and lines are both exceeded", () => { - vi.useFakeTimers(); - const head = createCapture({ maxBytes: 5, maxLines: 2, retain: "head" }).capture; - head.push("abcdef\nx\ny"); - expect(head.snapshot()).toMatchObject({ - text: "", - truncation: { - truncated: true, - truncatedBy: "bytes", - firstLineExceedsLimit: true, - totalLines: 3, - totalBytes: 10, - }, - }); - - const tail = createCapture({ maxBytes: 5, maxLines: 2, retain: "tail" }).capture; - tail.push("a\nb\nc\nlonglonglong"); - expect(tail.snapshot()).toMatchObject({ - text: "glong", - lastLineBytes: 12, - truncation: { truncated: true, truncatedBy: "bytes", lastLinePartial: true, totalLines: 4 }, - }); - }); - - it("keeps limit evidence when small byte limits trim multi-byte edge characters", () => { - vi.useFakeTimers(); - const head = createCapture({ maxBytes: 1, maxLines: 1, retain: "head" }).capture; - head.push(`😀\n${"x".repeat(100)}`); - expect(head.snapshot()).toMatchObject({ - text: "", - truncation: { truncatedBy: "bytes", firstLineExceedsLimit: true, totalLines: 2, totalBytes: 105 }, - }); - - const tail = createCapture({ maxBytes: 1, maxLines: 1, retain: "tail" }).capture; - tail.push(`${"x".repeat(100)}\n😀`); - expect(tail.snapshot()).toMatchObject({ - text: "", - lastLineBytes: 4, - truncation: { truncatedBy: "bytes", lastLinePartial: true, totalLines: 2, totalBytes: 105 }, - }); - }); - - it("reports the full byte length of an oversized newline-terminated retained tail line", () => { - vi.useFakeTimers(); - const single = createCapture({ maxBytes: 10 }).capture; - single.push(`${"x".repeat(100)}\n`); - expect(single.snapshot()).toMatchObject({ - text: "x".repeat(10), - lastLineBytes: 100, - truncation: { lastLinePartial: true, totalLines: 1, totalBytes: 101 }, - }); - - const chunked = createCapture({ maxBytes: 10 }).capture; - chunked.push("x".repeat(60)); - chunked.push(`${"x".repeat(40)}\n`); - expect(chunked.snapshot().lastLineBytes).toBe(100); - - const afterEarlierLine = createCapture({ maxBytes: 10 }).capture; - afterEarlierLine.push(`short\n${"é".repeat(50)}\n`); - expect(afterEarlierLine.snapshot().lastLineBytes).toBe(100); - - const leadingNewline = createCapture({ maxBytes: 10 }).capture; - leadingNewline.push("x".repeat(50)); - leadingNewline.push("\n"); - expect(leadingNewline.snapshot().lastLineBytes).toBe(50); - }); - - it("publishes spill metadata without resending text", () => { - vi.useFakeTimers(); - const { capture, updates, errors } = createCapture(); - capture.push("output"); - capture.setSpillPath("/tmp/output.log"); - expect(updates.at(-1)).toMatchObject({ kind: "metadata", metadata: { spillPath: "/tmp/output.log" } }); - expect(fold(updates)?.spillPath).toBe("/tmp/output.log"); - expect(errors).toEqual([]); - }); - - it("routes trailing publication failures to its error handler", () => { - vi.useFakeTimers(); - vi.setSystemTime(0); - const errors: unknown[] = []; - let fail = false; - const capture = new OutputCapture({ limits: { maxBytes: 50, maxLines: 10 } }, BACKGROUND_CONTEXT, { - onUpdate: () => { - if (fail) throw new Error("consumer failed"); - }, - onError: (error) => errors.push(error), - }); - capture.push("a"); - fail = true; - capture.push("b"); - vi.advanceTimersByTime(100); - expect(errors).toEqual([new Error("consumer failed")]); - capture.dispose(); - }); - - it.each([ - { maxBytes: 0, maxLines: 1 }, - { maxBytes: Number.POSITIVE_INFINITY, maxLines: 1 }, - { maxBytes: 1, maxLines: 0 }, - { maxBytes: 1, maxLines: 1.5 }, - ])("rejects invalid limits %o", (limits) => { - expect(() => new OutputCapture({ limits }, BACKGROUND_CONTEXT, { onError: () => {} })).toThrow(TypeError); - }); -}); - -describe("applyShellOutputUpdate", () => { - it("folds every update kind into the published view", () => { - const metadata = { - truncation: { - truncated: true, - truncatedBy: "bytes" as const, - totalLines: 1, - totalBytes: 9, - outputLines: 1, - outputBytes: 4, - lastLinePartial: true, - firstLineExceedsLimit: false, - maxLines: 10, - maxBytes: 4, - }, - }; - let view = applyShellOutputUpdate(undefined, { kind: "replace", output: { text: "ab", ...metadata } }); - view = applyShellOutputUpdate(view, { kind: "append", text: "cd", metadata }); - expect(view.text).toBe("abcd"); - view = applyShellOutputUpdate(view, { kind: "slide", drop: 2, text: "ef", metadata }); - expect(view.text).toBe("cdef"); - view = applyShellOutputUpdate(view, { kind: "metadata", metadata: { ...metadata, spillPath: "/tmp/full.log" } }); - expect(view).toMatchObject({ text: "cdef", spillPath: "/tmp/full.log" }); - }); -}); diff --git a/packages/durable/test/env-truncate.test.ts b/packages/durable/test/env-truncate.test.ts index 3c3f3fb55..7a2f3f5ef 100644 --- a/packages/durable/test/env-truncate.test.ts +++ b/packages/durable/test/env-truncate.test.ts @@ -1,7 +1,7 @@ import { execFileSync } from "node:child_process"; import { fileURLToPath } from "node:url"; import { describe, expect, it } from "vitest"; -import { formatSize, truncateHead, truncateLine, truncateTail } from "../src/env/utils/truncate.ts"; +import { formatSize, truncateHead } from "../src/truncate.ts"; const encoder = new TextEncoder(); @@ -9,60 +9,6 @@ function byteLength(content: string): number { return encoder.encode(content).length; } -function bufferTail(content: string, maxBytes: number): string { - const bytes = Buffer.from(content, "utf8"); - if (bytes.length <= maxBytes) return content; - let start = bytes.length - maxBytes; - while (start < bytes.length && (bytes[start] & 0xc0) === 0x80) start++; - return bytes.subarray(start).toString("utf8"); -} - -function assertMatchesBufferTail(input: string, maxByteValues?: readonly number[]): void { - const totalBytes = Buffer.byteLength(input, "utf8"); - const values = maxByteValues ?? Array.from({ length: totalBytes + 5 }, (_, maxBytes) => maxBytes); - for (const maxBytes of values) { - const result = truncateTail(input, { maxBytes, maxLines: 10 }); - const expected = bufferTail(input, maxBytes); - if (result.content !== expected) { - throw new Error( - `tail mismatch input=${JSON.stringify(input)} maxBytes=${maxBytes} expected=${JSON.stringify(expected)} actual=${JSON.stringify(result.content)}`, - ); - } - const outputBytes = Buffer.byteLength(result.content, "utf8"); - if (outputBytes > maxBytes) { - throw new Error( - `tail output exceeded byte limit input=${JSON.stringify(input)} maxBytes=${maxBytes} outputBytes=${outputBytes}`, - ); - } - } -} - -function sampledByteLimits(input: string): number[] { - const totalBytes = Buffer.byteLength(input, "utf8"); - const candidates = [ - 0, - 1, - 2, - 3, - 4, - 5, - 8, - Math.floor(totalBytes / 2) - 1, - Math.floor(totalBytes / 2), - Math.floor(totalBytes / 2) + 1, - totalBytes - 8, - totalBytes - 5, - totalBytes - 4, - totalBytes - 3, - totalBytes - 2, - totalBytes - 1, - totalBytes, - totalBytes + 1, - totalBytes + 4, - ]; - return [...new Set(candidates.filter((value) => value >= 0))].sort((a, b) => a - b); -} - describe("truncate utilities", () => { it("reports UTF-8 byte counts in truncation results", () => { const content = "aé🙂\nb"; @@ -90,24 +36,20 @@ describe("truncate utilities", () => { bufferAvailable: boolean; lengths: number[]; head: { content: string; outputBytes: number; truncatedBy: string | null }; - tail: { content: string; outputBytes: number; lastLinePartial: boolean }; }; expect(result.bufferAvailable).toBe(false); expect(result.lengths).toEqual(inputs.map(byteLength)); expect(result.head).toMatchObject({ content: "aé🙂", outputBytes: 7, truncatedBy: "bytes" }); - expect(result.tail).toMatchObject({ content: "🙂b", outputBytes: 5, lastLinePartial: true }); }); it("does not count a trailing newline as an extra line", () => { const content = `${Array.from({ length: 3 }, () => "line").join("\n")}\n`; const head = truncateHead(content, { maxBytes: 100, maxLines: 3 }); - const tail = truncateTail(content, { maxBytes: 100, maxLines: 3 }); expect(head).toMatchObject({ truncated: false, totalLines: 3, outputLines: 3 }); - expect(tail).toMatchObject({ truncated: false, totalLines: 3, outputLines: 3 }); }); - it("truncates head and tail by line limits", () => { + it("truncates head by line limits", () => { const content = "one\ntwo\nthree\nfour"; expect(truncateHead(content, { maxBytes: 100, maxLines: 2 })).toMatchObject({ content: "one\ntwo", @@ -116,16 +58,9 @@ describe("truncate utilities", () => { totalLines: 4, outputLines: 2, }); - expect(truncateTail(content, { maxBytes: 100, maxLines: 2 })).toMatchObject({ - content: "three\nfour", - truncated: true, - truncatedBy: "lines", - totalLines: 4, - outputLines: 2, - }); }); - it("reports bytes when only a trailing newline or oversized line exceeds limits at the line cap", () => { + it("reports bytes when only a trailing newline exceeds limits at the line cap", () => { expect(truncateHead("hello\nworld\n", { maxBytes: 11, maxLines: 2 })).toMatchObject({ content: "hello\nworld", truncated: true, @@ -133,19 +68,6 @@ describe("truncate utilities", () => { totalLines: 2, outputLines: 2, }); - expect(truncateTail("hello\nworld\n", { maxBytes: 11, maxLines: 2 })).toMatchObject({ - content: "hello\nworld", - truncated: true, - truncatedBy: "bytes", - totalLines: 2, - outputLines: 2, - }); - expect(truncateTail("x".repeat(100), { maxBytes: 10, maxLines: 1 })).toMatchObject({ - content: "x".repeat(10), - truncatedBy: "bytes", - lastLinePartial: true, - outputLines: 1, - }); }); it("truncates head on UTF-8 byte limits without partial lines", () => { @@ -168,86 +90,9 @@ describe("truncate utilities", () => { expect(result.firstLineExceedsLimit).toBe(true); }); - it("truncates tail on UTF-8 boundaries when only a partial last line fits", () => { - const result = truncateTail("aé🙂b", { maxBytes: 5, maxLines: 10 }); - - expect(result.content).toBe("🙂b"); - expect(result.truncated).toBe(true); - expect(result.truncatedBy).toBe("bytes"); - expect(result.lastLinePartial).toBe(true); - expect(result.outputBytes).toBe(5); - }); - - it("truncates an oversized single line with a trailing newline", () => { - const input = `${"X".repeat(300_000)}\n`; - const result = truncateTail(input, { maxBytes: 1024, maxLines: 100 }); - - expect(result.content).toBe("X".repeat(1024)); - expect(result.outputBytes).toBe(1024); - expect(result.outputLines).toBe(1); - expect(result.lastLinePartial).toBe(true); - expect(result.truncatedBy).toBe("bytes"); - }); - - it("drops an oversized trailing character when it cannot fit in tail byte limit", () => { - const result = truncateTail("abc🙂", { maxBytes: 3, maxLines: 10 }); - - expect(result.content).toBe(""); - expect(result.truncated).toBe(true); - expect(result.truncatedBy).toBe("bytes"); - expect(result.lastLinePartial).toBe(true); - expect(result.outputBytes).toBe(0); - }); - - it("matches Buffer tail truncation semantics for surrogate edge cases", () => { - const inputs = ["a\ud83d", "\ude42b", "a\ude42b", "\ud83d\ud83d\ude42", "\ud83d\ude42\ude42", "👩‍💻"]; - for (const input of inputs) assertMatchesBufferTail(input); - }); - - it("matches Buffer tail truncation semantics across deterministic fuzz cases", () => { - const alphabet = [ - "a", - "\u007f", - "\u0080", - "é", - "\u07ff", - "\u0800", - "中", - "\ud7ff", - "\ud800", - "\ud83d", - "\udc00", - "\ude42", - "🙂", - "\ue000", - "\uffff", - ]; - - function checkExhaustive(prefix: string, depth: number): void { - assertMatchesBufferTail(prefix, sampledByteLimits(prefix)); - if (depth === 0) return; - for (const character of alphabet) checkExhaustive(prefix + character, depth - 1); - } - checkExhaustive("", 3); - - let seed = 0x12345678; - function random(): number { - seed = (seed * 1664525 + 1013904223) >>> 0; - return seed / 0x100000000; - } - for (let i = 0; i < 1_000; i++) { - let input = ""; - const length = Math.floor(random() * 80); - for (let j = 0; j < length; j++) input += alphabet[Math.floor(random() * alphabet.length)]; - assertMatchesBufferTail(input, sampledByteLimits(input)); - } - }); - - it("formats sizes and truncates long single lines", () => { + it("formats sizes", () => { expect(formatSize(1023)).toBe("1023B"); expect(formatSize(1536)).toBe("1.5KB"); expect(formatSize(3 * 1024 * 1024)).toBe("3.0MB"); - expect(truncateLine("abc", 3)).toEqual({ text: "abc", wasTruncated: false }); - expect(truncateLine("abcdef", 3)).toEqual({ text: "abc... [truncated]", wasTruncated: true }); }); }); diff --git a/packages/durable/test/examples/17-coding-tools.ts b/packages/durable/test/examples/17-coding-tools.ts new file mode 100644 index 000000000..5863528ef --- /dev/null +++ b/packages/durable/test/examples/17-coding-tools.ts @@ -0,0 +1,98 @@ +// A coding-agent tool turn on JSONL storage: the model reads, edits, and runs commands, then answers. A hook times a +// `cat` of /tmp/1gb.txt (create it first to measure; without it that call ends in an error result). The storage +// directory is left behind for inspection. +// Run from packages/durable: +// node --conditions=source --experimental-strip-types test/examples/17-coding-tools.ts +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; +import type { ToolResultMessage } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { NodeExecutionEnv } from "../../src/env/node.ts"; +import { + AssistantEntry, + ConversationConfig, + createRegistry, + Harness, + ToolResultEntry, + ToolTask, +} from "../../src/index.ts"; +import { openNodeJsonlStorage } from "../../src/storage/jsonl/node.ts"; +import { type BashToolInput, createBashTool, createEditTool, createReadTool } from "../../src/tools/index.ts"; + +const context = BACKGROUND_CONTEXT; +const directory = await mkdtemp(join(tmpdir(), "pi-durable-example-")); +await writeFile(join(directory, "notes.txt"), "hello world\n"); + +// The faux provider plays the model: four tool-calling answers, then a final answer. +const faux = fauxProvider(); +const models = createModels(); +models.setProvider(faux.provider); +const toolCall = (name: string, args: Parameters[1], id: string) => + fauxAssistantMessage([fauxToolCall(name, args, { id })], { stopReason: "toolUse" }); +faux.setResponses([ + toolCall("read", { path: "notes.txt" }, "r"), + toolCall("edit", { path: "notes.txt", edits: [{ oldText: "world", newText: "durable" }] }, "e"), + toolCall("bash", { command: "cat notes.txt" }, "b"), + toolCall("bash", { command: "cat /tmp/1gb.txt" }, "c"), + fauxAssistantMessage([fauxText("The file now greets durable.")]), +]); + +// Tools come from the registry and reach files and processes only through the environment the Harness offers them. +const registry = createRegistry(); +registry.batch(() => { + registry.tools.add(createReadTool()); + registry.tools.add(createEditTool()); + registry.tools.add(createBashTool()); +}); + +// Hooks see every call before and after execution; these time the big `cat`. +const isBigCat = (call: { name: string; arguments: unknown }) => + call.name === "bash" && (call.arguments as BashToolInput).command.includes("1gb.txt"); +let start = 0; +registry.hooks.add(ToolTask, { + beforeTool: (call) => { + if (isBigCat(call)) start = Date.now(); + return undefined; + }, + afterTool: (call) => { + if (isBigCat(call)) console.log(`cat /tmp/1gb.txt took ${Date.now() - start} ms`); + return undefined; + }, +}); + +const env = new NodeExecutionEnv({ cwd: directory }); +const storage = await openNodeJsonlStorage(directory, context); +const harness = await Harness.open(storage, { models, registry, env }, context); +const root = await harness.root(context, { + init: async (tx, id) => { + (await tx.doc(ConversationConfig, id)).model = { provider: "faux", modelId: "faux-1" }; + }, +}); + +// Each tool call runs as a durable pi.tool task; post-tools continues the run with the next generation. +const settled = await (await root.submit({ type: "input", content: "Greet durable instead." }, context)).wait(context); +console.log("status:", settled.status); +const transcript = await root.entries({}, 20, undefined, context); +for (const entry of [...transcript.items].reverse()) { + if (!ToolResultEntry.is(entry)) { + console.log(entry.kind); + continue; + } + const result = entry.model![0] as ToolResultMessage; + const text = result.content.map((item) => (item.type === "text" ? item.text : "")).join(""); + console.log( + `${entry.kind} ${result.toolName}:`, + JSON.stringify(text.length > 200 ? `${text.slice(0, 200)}…` : text), + ); +} +if (settled.status === "done" && settled.type === "input") { + const answer = await root.commit((tx) => tx.entry(AssistantEntry, settled.answer), context); + const message = answer?.model?.[0]; + console.log("answer:", message?.role === "assistant" ? message.content : message); +} +console.log("file:", JSON.stringify(await readFile(join(directory, "notes.txt"), "utf8"))); +await harness.close(context); +console.log("storage:", directory); diff --git a/packages/durable/test/fixtures/utf8-byte-length-without-buffer.ts b/packages/durable/test/fixtures/utf8-byte-length-without-buffer.ts index 481112786..7136deda5 100644 --- a/packages/durable/test/fixtures/utf8-byte-length-without-buffer.ts +++ b/packages/durable/test/fixtures/utf8-byte-length-without-buffer.ts @@ -1,4 +1,4 @@ -import { truncateHead, truncateTail, utf8ByteLength } from "../../src/env/utils/truncate.ts"; +import { truncateHead, utf8ByteLength } from "../../src/truncate.ts"; const inputs: string[] = JSON.parse(process.argv[2] ?? "[]"); @@ -7,6 +7,5 @@ process.stdout.write( bufferAvailable: "Buffer" in globalThis, lengths: inputs.map((input) => utf8ByteLength(input)), head: truncateHead("aé🙂\nb", { maxBytes: 7, maxLines: 10 }), - tail: truncateTail("aé🙂b", { maxBytes: 5, maxLines: 10 }), }), ); diff --git a/packages/durable/test/harness-generation.test.ts b/packages/durable/test/harness-generation.test.ts index 78f8b04e0..9b05948e0 100644 --- a/packages/durable/test/harness-generation.test.ts +++ b/packages/durable/test/harness-generation.test.ts @@ -1,7 +1,6 @@ import { type AssistantMessage, fauxAssistantMessage, - fauxToolCall, type Message, type Models, type SimpleStreamOptions, @@ -155,16 +154,6 @@ describe("generation", () => { await harness.close(context); }); - it("settles a tool call as the answer until the tool chain exists", async () => { - const setup = chatSetup(); - setup.faux.setResponses([fauxAssistantMessage([fauxToolCall("read", { path: "a" })], { stopReason: "toolUse" })]); - const { harness, root } = await openChat(new MemoryStorage(), setup); - harness.resume(); - const settled = await (await root.submit({ type: "input", content: "read a" }, context)).wait(context); - expect(settled.status).toBe("done"); - await harness.close(context); - }); - it("fails with no_model when no model is configured or the model is unknown", async () => { const setup = chatSetup(); const { harness, root } = await openChat(new MemoryStorage(), setup); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts new file mode 100644 index 000000000..01e7b7613 --- /dev/null +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -0,0 +1,439 @@ +import type { Op } from "@earendil-works/chord/delta"; +import { fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + type CommitPublication, + type Harness, + LiveDoc, + type LiveState, + MemoryStorage, + type ToolExecutionApi, + type ToolRegistration, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { chatSetup, openChat, waitFor } from "./chat-support.ts"; +import { context, documentChanges } from "./session-support.ts"; + +type Action = (api: ToolExecutionApi) => void | Promise; + +/** + * Drive one tool call step by step and capture the exact Chord operations of every `pi.live` commit. Each `step()` + * runs one action inside the tool and returns the operations of the commit it caused. + */ +async function drive(outputLimits: ToolRegistration["outputLimits"] = {}) { + const setup = chatSetup(); + const actions: ((api: ToolExecutionApi) => Promise)[] = []; + let wake: (() => void) | undefined; + setup.registry.tools.add({ + name: "drive", + description: "Driven by the test", + parameters: Type.Object({}), + outputLimits, + execute: async (_args, api, callContext) => { + for (;;) { + while (actions.length === 0) { + await new Promise((resolve, reject) => { + wake = resolve; + const signal = callContext.abortSignal!; + signal.addEventListener("abort", () => reject(signal.reason), { once: true }); + }); + } + if (!(await actions.shift()!(api))) return {}; + } + }, + }); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("drive", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const commits: Op[][] = []; + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }); + const submission = await root.submit({ type: "input", content: "go" }, context); + await waitFor(async () => (await live(harness))?.tools?.[0]?.status === "running"); + const push = (action: (api: ToolExecutionApi) => Promise): void => { + actions.push(action); + wake?.(); + }; + return { + harness, + commits, + /** Operations of every `pi.live` commit so far. */ + async step(action: Action): Promise { + const before = commits.length; + push(async (api) => { + await action(api); + return true; + }); + await waitFor(() => commits.length > before); + return commits.at(-1)!; + }, + /** Let the tool return and the run finish; returns the commits made meanwhile. */ + async finish(): Promise { + const before = commits.length; + push(async () => false); + await submission.wait(context); + return commits.slice(before); + }, + }; +} + +function live(harness: Harness): Promise | undefined> { + return harness.snapshot(LiveDoc, 1 as never, context); +} + +const OUTPUT = ["tools", 0, "output"]; + +function isOutput(op: Op): boolean { + return JSON.stringify(op[1]) === JSON.stringify(OUTPUT); +} + +describe("pi.live deltas", () => { + it("hands a generation over to its tool round and starts a tool with one field write each", async () => { + const setup = chatSetup(); + setup.registry.tools.add({ + name: "noop", + description: "noop", + parameters: Type.Object({}), + execute: async () => ({ content: [] }), + }); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("noop", {}, { id: "c1" })], { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const commits: Op[][] = []; + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); + const id = (kind: string) => tasks.items.filter((task) => task.kind === kind).map((task) => task.id); + const [firstGeneration, secondGeneration] = id("pi.generation").sort((a, b) => a - b); + const [tool] = id("pi.tool"); + const [postTools] = id("pi.post-tools"); + const entries = await root.entries({}, 10, undefined, context); + const result = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; + expect(commits).toEqual([ + // submission + [["s", ["run"], { taskId: firstGeneration, inputs: [expect.any(Number)] }]], + // request + [["s", ["generation"], { attempt: 1 }]], + // handover to the tool round + expect.arrayContaining([ + ["s", ["run", "taskId"], postTools], + ["d", ["generation"]], + ["s", ["tools"], [{ callId: "c1", name: "noop", taskId: tool, status: "pending" }]], + ]), + // intent + [["s", ["tools", 0, "status"], "running"]], + // result + expect.arrayContaining([ + ["s", ["tools", 0, "status"], "done"], + ["s", ["tools", 0, "entry"], result], + ]), + // post-tools hands over to the next generation + expect.arrayContaining([ + ["d", ["tools"]], + ["s", ["run", "taskId"], secondGeneration], + ]), + [["s", ["generation"], { attempt: 1 }]], + // the answer ends the run + expect.arrayContaining([ + ["d", ["run"]], + ["d", ["generation"]], + ]), + ]); + expect(commits[2]).toHaveLength(3); + expect(commits[4]).toHaveLength(2); + expect(commits[5]).toHaveLength(2); + await harness.close(context); + }); + + it("appends head output and then only updates the dropped counts once the window is full", async () => { + const run = await drive({ maxLines: 2 }); + expect(await run.step((api) => api.output("one\n"))).toEqual([["s", OUTPUT, "one\n"]]); + expect(await run.step((api) => api.output("two\n"))).toEqual([["a", OUTPUT, "two\n"]]); + // The window is full: the retained text stays; only the counts change. + expect(await run.step((api) => api.output("three\n"))).toEqual([ + ["s", ["tools", 0, "droppedBytes"], 6], + ["s", ["tools", 0, "droppedLines"], 1], + ]); + expect(await run.step((api) => api.output("four\n"))).toEqual([ + ["s", ["tools", 0, "droppedBytes"], 11], + ["s", ["tools", 0, "droppedLines"], 2], + ]); + await run.finish(); + await run.harness.close(context); + }); + + it("slides a tail window as a front trim plus an append", async () => { + const run = await drive({ maxLines: 3, retain: "tail" }); + expect(await run.step((api) => api.output("line 1\nline 2\nline 3\n"))).toEqual([ + ["s", OUTPUT, "line 1\nline 2\nline 3\n"], + ]); + expect(await run.step((api) => api.output("line 4\n"))).toEqual([ + ["t", OUTPUT, 7], + ["a", OUTPUT, "line 4\n"], + ["s", ["tools", 0, "droppedBytes"], 7], + ["s", ["tools", 0, "droppedLines"], 1], + ]); + // The buffer keeps only the window, and later slides stay minimal and exact. + expect(await run.step((api) => api.output("line 5\nline 6\n"))).toEqual([ + ["t", OUTPUT, 14], + ["a", OUTPUT, "line 5\nline 6\n"], + ["s", ["tools", 0, "droppedBytes"], 21], + ["s", ["tools", 0, "droppedLines"], 3], + ]); + expect((await live(run.harness))?.tools?.[0]?.output).toBe("line 4\nline 5\nline 6\n"); + await run.finish(); + await run.harness.close(context); + }); + + it("writes the whole window when Chord's overlap search cannot find the shared part", async () => { + // A retained window beyond the 64 KiB overlap scan. + const wide = await drive({ maxBytes: 100 * 1024, maxLines: 1_000_000, retain: "tail" }); + const line = (index: number) => `${String(index).padStart(10, "0")} ${"x".repeat(989)}\n`; + let text = ""; + for (let index = 0; index < 100; index++) text += line(index); + await wide.step((api) => api.output(text)); + const slid = await wide.step((api) => api.output(line(100) + line(101) + line(102) + line(103))); + expect(slid.filter(isOutput).map((op) => op[0])).toEqual(["s"]); + await wide.finish(); + await wide.harness.close(context); + + // Repetitive output still finds an overlap here; Chord's bounded candidate search can give up on other inputs + // and then writes one window. + const repetitive = await drive({ maxLines: 50, retain: "tail" }); + await repetitive.step((api) => api.output("y\n".repeat(50))); + const repeated = await repetitive.step((api) => api.output("z\n")); + expect(repeated.filter(isOutput)).toEqual([ + ["t", OUTPUT, 2], + ["a", OUTPUT, "z\n"], + ]); + await repetitive.finish(); + await repetitive.harness.close(context); + }); + + it("diffs details leaf by leaf and appends diagnostics", async () => { + const run = await drive(); + const details = ["tools", 0, "details"]; + expect(await run.step((api) => api.details({ step: 1, log: "a" }, context))).toEqual([ + ["s", details, { step: 1, log: "a" }], + ]); + expect(await run.step((api) => api.details({ step: 2, log: "ab" }, context))).toEqual( + expect.arrayContaining([ + ["s", [...details, "step"], 2], + ["a", [...details, "log"], "b"], + ]), + ); + expect(await run.step((api) => api.details({ step: 2 }, context))).toEqual([["d", [...details, "log"]]]); + const diagnostics = ["tools", 0, "diagnostics"]; + const first = { severity: "info", message: "first" } as const; + const second = { severity: "warn", message: "second" } as const; + expect(await run.step((api) => api.diagnostic(first))).toEqual([["s", diagnostics, [first]]]); + expect(await run.step((api) => api.diagnostic(second))).toEqual([["p", diagnostics, 1, 0, [second]]]); + // Settlement moves everything into the result entry and keeps the slot small. + const [settled] = await run.finish(); + expect(settled).toEqual( + expect.arrayContaining([ + ["s", ["tools", 0, "status"], "done"], + ["d", details], + ["d", diagnostics], + ]), + ); + await run.harness.close(context); + }); + + it("streams partial text as appends", async () => { + const setup = chatSetup({ tokensPerSecond: 400, tokenSize: { min: 4, max: 4 } }); + setup.faux.setResponses([fauxAssistantMessage([fauxText("word ".repeat(250))])]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const commits: Op[][] = []; + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const partials = commits.slice(2, -1); + expect(partials.length).toBeGreaterThan(2); + expect(partials[0]).toEqual([["s", ["generation", "message"], expect.any(Object)]]); + for (const ops of partials.slice(1)) { + expect(ops).toEqual([["a", ["generation", "message", "content", 0, "text"], expect.any(String)]]); + } + await harness.close(context); + }); + + it("stores a complete base exactly in the commits where nothing runs", async () => { + // Storage that remembers whether each commit wrote pi.live as a base or a delta. + const written = new Map(); + let liveId: number | undefined; + class RecordingStorage extends MemoryStorage { + override async commit( + writes: Parameters[0], + context: Parameters[1], + ) { + let kind: "base" | "delta" | undefined; + for (const write of writes) { + if (write.type === "document.change" && write.id === liveId) kind = write.content.kind; + } + const seq = await super.commit(writes, context); + if (kind !== undefined) written.set(seq, kind); + return seq; + } + } + const setup = chatSetup(); + for (const name of ["first", "second"]) { + setup.registry.tools.add({ + name, + description: name, + parameters: Type.Object({}), + execute: async (_args, api) => { + api.output(`${name} output\n`); + await new Promise((resolve) => setTimeout(resolve, 150)); + api.output(`${name} more\n`); + return {}; + }, + }); + } + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("first", {}, { id: "a" }), fauxToolCall("second", {}, { id: "b" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new RecordingStorage(), setup); + await root.setToolExecution("sequential", context); + const values = new Map(); + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind !== "pi.live") continue; + liveId = change.record.id; + if (change.value !== null) values.set(publication.seq, change.value as LiveState); + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const nothingRuns = (value: LiveState) => + value.generation === undefined && !(value.tools ?? []).some((slot) => slot.status === "running"); + const kinds = [...written].map(([seq, kind]) => { + expect(kind === "base").toBe(nothingRuns(values.get(seq)!)); + return kind; + }); + // Bases at the handover, after each sequential tool, after the round, and when the run ends. + expect(kinds.filter((kind) => kind === "base").length).toBeGreaterThanOrEqual(5); + expect(kinds).toContain("delta"); + await harness.close(context); + }); + + it("starts calls the request did not offer as done and marks a faulted tool's slot done without an entry", async () => { + const setup = chatSetup(); + setup.registry.tools.add({ + name: "bad", + description: "bad", + parameters: Type.Object({}), + // Not strict JSON: the result commit throws and the scheduler faults the task. + execute: async () => ({ content: [], details: { fn: (() => 1) as never } }), + }); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("ghost", {}, { id: "g" }), fauxToolCall("bad", {}, { id: "b" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const commits: Op[][] = []; + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind === "pi.live" && change.ops.length > 0) commits.push([...change.ops]); + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const entries = await root.entries({}, 10, undefined, context); + const ghostResult = entries.items.find((entry) => entry.kind === "pi.tool-result")!.id; + const handover = commits.find((ops) => ops.some((op) => op[0] === "s" && op[1][0] === "tools"))!; + expect(handover).toContainEqual([ + "s", + ["tools"], + [ + { callId: "g", name: "ghost", status: "done", entry: ghostResult }, + { callId: "b", name: "bad", taskId: expect.any(Number), status: "pending" }, + ], + ]); + // The fault cleanup writes only the status. + expect(commits).toContainEqual([["s", ["tools", 1, "status"], "done"]]); + await harness.close(context); + }); + + it("commits the tool-calling answer, its tool tasks, post-tools, and the tool round in one commit", async () => { + const setup = chatSetup(); + setup.registry.tools.add({ + name: "noop", + description: "noop", + parameters: Type.Object({}), + execute: async () => ({ content: [] }), + }); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("noop", {}, { id: "a" }), fauxToolCall("noop", {}, { id: "b" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + let handover: CommitPublication | undefined; + harness.subscribeCommits((publication) => { + for (const change of documentChanges(publication)) { + if (change.record.kind === "pi.live" && (change.value as LiveState | null)?.tools?.length === 2) { + handover ??= publication; + } + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const changes = handover!.changes; + const kinds = changes.flatMap((change) => + change.type === "entry" ? [change.value.kind] : change.type === "task" ? [change.value.kind] : [], + ); + expect(kinds.sort()).toEqual(["pi.assistant", "pi.generation", "pi.post-tools", "pi.tool", "pi.tool"]); + await harness.close(context); + }); + + it("writes an aborted tool's slot with field-level ops", async () => { + const run = await drive(); + await run.step((api) => api.output("partial\n")); + await run.step((api) => api.details({ n: 1 }, context)); + const taskId = (await live(run.harness))!.tools![0]!.taskId!; + const before = run.commits.length; + await run.harness.abortTask(taskId, context); + const isAbortCommit = (ops: readonly Op[]) => + ops.some((op) => op[0] === "s" && JSON.stringify(op[1]) === JSON.stringify(["tools", 0, "status"])); + await waitFor(() => run.commits.slice(before).some(isAbortCommit)); + const abortCommit = run.commits.slice(before).find(isAbortCommit)!; + expect(abortCommit).toEqual( + expect.arrayContaining([ + ["s", ["tools", 0, "status"], "done"], + ["s", ["tools", 0, "entry"], expect.any(Number)], + ["d", OUTPUT], + ["d", ["tools", 0, "details"]], + ]), + ); + expect(abortCommit).toHaveLength(4); + await run.harness.close(context); + }); + + it("keeps a complete base exactly while nothing runs", () => { + const base = (value: LiveState) => + LiveDoc.definition.checkpointWhen!(value, [], { deltasSinceBase: 1000 } as never); + const slot = (status: "pending" | "running" | "done") => ({ callId: "c", name: "n", status }); + const run = { taskId: 1 as never, inputs: [] }; + expect(base({})).toBe(true); + expect(base({ run, generation: { attempt: 1 } })).toBe(false); + expect(base({ run, tools: [slot("pending"), slot("done")] })).toBe(true); + expect(base({ run, tools: [slot("done"), slot("running")] })).toBe(false); + expect(base({ run })).toBe(true); + }); +}); diff --git a/packages/durable/test/harness-output.test.ts b/packages/durable/test/harness-output.test.ts new file mode 100644 index 000000000..2af4ae0f2 --- /dev/null +++ b/packages/durable/test/harness-output.test.ts @@ -0,0 +1,272 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { boundOutput, OutputBuffer, type OutputLimits, Progress, sanitizeOutput } from "../src/harness/output.ts"; + +const head = (maxLines: number, maxBytes = 1000): OutputLimits => ({ maxBytes, maxLines, retain: "head" }); +const tail = (maxLines: number, maxBytes = 1000): OutputLimits => ({ maxBytes, maxLines, retain: "tail" }); + +function bufferTail(content: string, maxBytes: number): string { + const bytes = Buffer.from(content, "utf8"); + if (bytes.length <= maxBytes) return content; + let start = bytes.length - maxBytes; + while (start < bytes.length && (bytes[start]! & 0xc0) === 0x80) start++; + return bytes.subarray(start).toString("utf8"); +} + +/** A single line longer than the byte limit is cut like Buffer's tail slice on a character boundary. */ +function assertMatchesBufferTail(input: string, maxByteValues?: readonly number[]): void { + const totalBytes = Buffer.byteLength(input, "utf8"); + const values = maxByteValues ?? Array.from({ length: totalBytes + 5 }, (_, maxBytes) => maxBytes); + for (const maxBytes of values) { + const kept = boundOutput(input, { maxBytes, maxLines: 10, retain: "tail" }).text; + const expected = bufferTail(input, maxBytes); + if (kept !== expected) { + throw new Error( + `tail mismatch input=${JSON.stringify(input)} maxBytes=${maxBytes} expected=${JSON.stringify(expected)} actual=${JSON.stringify(kept)}`, + ); + } + if (Buffer.byteLength(kept, "utf8") > maxBytes) throw new Error(`tail output exceeded ${maxBytes} bytes`); + } +} + +function sampledByteLimits(input: string): number[] { + const totalBytes = Buffer.byteLength(input, "utf8"); + const candidates = [ + 0, + 1, + 2, + 3, + 4, + 5, + 8, + Math.floor(totalBytes / 2), + totalBytes - 4, + totalBytes - 1, + totalBytes, + totalBytes + 1, + ]; + return [...new Set(candidates.filter((value) => value >= 0))].sort((a, b) => a - b); +} + +function bound(text: string, limits: OutputLimits) { + const { text: kept, droppedBytes, droppedLines } = boundOutput(text, limits); + return { kept, droppedBytes, droppedLines }; +} + +describe("tool output bounds", () => { + it("removes control characters but keeps tabs, newlines, and other text", () => { + expect(sanitizeOutput("a\0b\tc\nd\re\u0007f\ufff9g\ufffbh😀")).toBe("ab\tc\ndefgh😀"); + }); + + it("keeps output within the limits unchanged", () => { + expect(bound("a\nb\n", head(2))).toEqual({ kept: "a\nb\n", droppedBytes: 0, droppedLines: 0 }); + expect(bound("a\nb", tail(2))).toEqual({ kept: "a\nb", droppedBytes: 0, droppedLines: 0 }); + expect(bound("", tail(2))).toEqual({ kept: "", droppedBytes: 0, droppedLines: 0 }); + }); + + it("keeps nothing with a zero limit", () => { + expect(bound("ab\ncd\n", head(10, 0))).toEqual({ kept: "", droppedBytes: 6, droppedLines: 2 }); + expect(bound("ab\ncd\n", tail(0))).toEqual({ kept: "", droppedBytes: 6, droppedLines: 2 }); + }); + + it("keeps exact slices of whole lines, trailing newline included", () => { + expect(bound("a\nb\nc\n", head(2))).toEqual({ kept: "a\nb\n", droppedBytes: 2, droppedLines: 1 }); + expect(bound("a\nb\nc\n", tail(2))).toEqual({ kept: "b\nc\n", droppedBytes: 2, droppedLines: 1 }); + expect(bound("a\nb\nc", tail(2))).toEqual({ kept: "b\nc", droppedBytes: 2, droppedLines: 1 }); + // Blank lines are lines. + expect(bound("a\nb\nc\n\n", tail(3))).toEqual({ kept: "b\nc\n\n", droppedBytes: 2, droppedLines: 1 }); + }); + + it("cuts at the byte limit on whole lines when possible", () => { + expect(bound("aa\nbb\ncc\n", head(10, 7))).toEqual({ kept: "aa\nbb\n", droppedBytes: 3, droppedLines: 1 }); + expect(bound("aa\nbb\ncc\n", tail(10, 7))).toEqual({ kept: "bb\ncc\n", droppedBytes: 3, droppedLines: 1 }); + }); + + it("cuts a single line longer than the byte limit on a character boundary", () => { + // "é" is two bytes; five bytes hold two whole characters. + expect(bound("ééé\n", head(10, 5))).toEqual({ kept: "éé", droppedBytes: 3, droppedLines: 0 }); + expect(bound("x\néééé", tail(10, 5))).toEqual({ kept: "éé", droppedBytes: 6, droppedLines: 1 }); + }); + + it("cuts tails of surrogate edge cases exactly like Buffer", () => { + const inputs = ["a\ud83d", "\ude42b", "a\ude42b", "\ud83d\ud83d\ude42", "\ud83d\ude42\ude42", "👩‍💻"]; + for (const input of inputs) assertMatchesBufferTail(input); + }); + + it("cuts tails exactly like Buffer across deterministic fuzz cases", () => { + const alphabet = [ + "a", + "\u007f", + "\u0080", + "é", + "\u07ff", + "\u0800", + "中", + "\ud7ff", + "\ud800", + "\ud83d", + "\udc00", + "\ude42", + "🙂", + "\ue000", + "\uffff", + ]; + function checkExhaustive(prefix: string, depth: number): void { + assertMatchesBufferTail(prefix, sampledByteLimits(prefix)); + if (depth === 0) return; + for (const character of alphabet) checkExhaustive(prefix + character, depth - 1); + } + checkExhaustive("", 3); + let seed = 0x12345678; + const random = (): number => { + seed = (seed * 1664525 + 1013904223) >>> 0; + return seed / 0x100000000; + }; + for (let i = 0; i < 1_000; i++) { + let input = ""; + const length = Math.floor(random() * 80); + for (let j = 0; j < length; j++) input += alphabet[Math.floor(random() * alphabet.length)]; + assertMatchesBufferTail(input, sampledByteLimits(input)); + } + }); +}); + +describe("OutputBuffer", () => { + it("keeps exact totals across chunks and decodes UTF-8 split across byte chunks", () => { + const buffer = new OutputBuffer(tail(2)); + const bytes = new TextEncoder().encode("😀\n"); + buffer.push("a\nb\n"); + buffer.push(bytes.subarray(0, 2)); + buffer.push(bytes.subarray(2)); + expect(buffer.snapshot()).toEqual({ text: "b\n😀\n", droppedBytes: 2, droppedLines: 1 }); + }); + + it("sanitizes the retained text but counts the raw stream", () => { + const buffer = new OutputBuffer(tail(1)); + buffer.push("a\u0007\n"); + expect(buffer.snapshot()).toEqual({ text: "a\n", droppedBytes: 0, droppedLines: 0 }); + buffer.push("b\u001b\n"); + expect(buffer.snapshot()).toEqual({ text: "b\n", droppedBytes: 3, droppedLines: 1 }); + }); + + it("flushes an incomplete character before a string chunk and at the end", () => { + const buffer = new OutputBuffer(tail(10)); + const euro = new TextEncoder().encode("€"); + buffer.push(euro.subarray(0, 1)); + buffer.push("x"); + buffer.push(euro.subarray(0, 2)); + buffer.end(); + expect(buffer.snapshot().text).toBe("\ufffdx\ufffd"); + }); + + it("matches bounding the whole stream when several chunks arrive between snapshots", () => { + for (const limits of [head(3, 40), tail(3, 40), head(50, 25), tail(50, 25)]) { + const buffer = new OutputBuffer(limits); + let stream = ""; + for (let index = 0; index < 300; index++) { + const chunk = index % 7 === 0 ? `${"é".repeat(index % 30)}\n` : `line ${index}\n`; + stream += chunk; + buffer.push(chunk); + if (index % 5 !== 4) continue; + const expected = boundOutput(stream, limits); + expect(buffer.snapshot()).toEqual({ + text: expected.text, + droppedBytes: expected.droppedBytes, + droppedLines: expected.droppedLines, + }); + } + } + }); + + it("stops storing head output once the window is full", () => { + const buffer = new OutputBuffer(head(2)); + for (let index = 0; index < 1000; index++) buffer.push(`line ${index}\n`); + expect(buffer.storedBytes).toBeLessThan(20); + expect(buffer.snapshot()).toEqual({ text: "line 0\nline 1\n", droppedBytes: 8876, droppedLines: 998 }); + }); + + it("stores only the tail window after each snapshot", () => { + const buffer = new OutputBuffer(tail(3, 100)); + let stream = ""; + for (let index = 0; index < 2000; index++) { + const chunk = `line ${index}\n\n`; + stream += chunk; + buffer.push(chunk); + const snapshot = buffer.snapshot(); + expect(buffer.storedBytes).toBeLessThanOrEqual(100); + expect(snapshot.text).toBe(boundOutput(stream, tail(3, 100)).text); + } + }); +}); + +describe("Progress", () => { + afterEach(() => { + vi.useRealTimers(); + }); + + it("commits the first change at once, then waits at least 100 ms and the written size at 100 KiB/s", async () => { + vi.useFakeTimers({ now: 0 }); + const commits: number[] = []; + let size = 50 * 1024; + const progress = new Progress( + async () => { + commits.push(Date.now()); + return size; + }, + () => {}, + ); + progress.mark(); + await vi.advanceTimersByTimeAsync(0); + expect(commits).toEqual([0]); + // 50 KiB buys 500 ms; changes meanwhile coalesce into one commit. + size = 10; + progress.mark(); + progress.mark(); + await vi.advanceTimersByTimeAsync(499); + expect(commits).toEqual([0]); + await vi.advanceTimersByTimeAsync(1); + expect(commits).toEqual([0, 500]); + // A small commit still waits the minimum 100 ms. + progress.mark(); + await vi.advanceTimersByTimeAsync(99); + expect(commits).toEqual([0, 500]); + await vi.advanceTimersByTimeAsync(1); + expect(commits).toEqual([0, 500, 600]); + }); + + it("rejects the waiters of a failed commit and reports its error", async () => { + const errors: unknown[] = []; + const failure = new Error("commit failed"); + const progress = new Progress( + async () => { + throw failure; + }, + (error) => errors.push(error), + ); + await expect(progress.markAndWait()).rejects.toBe(failure); + expect(errors).toEqual([failure]); + }); + + it("stops: waits for the commit in flight and hands back waiters no commit covered yet", async () => { + vi.useFakeTimers({ now: 0 }); + let release!: () => void; + const inFlight = new Promise((resolve) => { + release = resolve; + }); + const progress = new Progress( + async () => { + await inFlight; + return 0; + }, + () => {}, + ); + const first = progress.markAndWait(); + const second = progress.markAndWait(); + const stopped = progress.stop(); + release(); + const pending = await stopped; + await first; + expect(pending).toHaveLength(1); + pending[0]!.resolve(); + await second; + }); +}); diff --git a/packages/durable/test/harness-prompt.test.ts b/packages/durable/test/harness-prompt.test.ts index 9870ad616..bf14f128b 100644 --- a/packages/durable/test/harness-prompt.test.ts +++ b/packages/durable/test/harness-prompt.test.ts @@ -1,4 +1,5 @@ -import type { SystemMessage } from "@earendil-works/pi-ai"; +import { type SystemMessage, type Tool, Type } from "@earendil-works/pi-ai"; +import { getCurrentTools, toToolDeclaration } from "@earendil-works/pi-ai/utils/transcript"; import { type Conversation, createRegistry, @@ -10,7 +11,7 @@ import { type ToolRegistration, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; -import { planSystemEntries, renderSections, replaySections } from "../src/harness/prompt.ts"; +import { desiredTools, planSystemEntries, renderSections, replaySections } from "../src/harness/prompt.ts"; import { openHarness, user } from "./harness-support.ts"; import { context } from "./session-support.ts"; @@ -18,7 +19,7 @@ type Planned = { readonly sections: Record; readonly omit /** Plan against the current context, append the plan, and check that replay then yields `desired` in order. */ async function apply(conversation: Conversation, desired: Record): Promise { - const drafts = planSystemEntries(await conversation.context(context), new Map(Object.entries(desired)), 7); + const drafts = planSystemEntries(await conversation.context(context), new Map(Object.entries(desired)), [], 7); await conversation.commit(async (tx) => { for (const draft of drafts) await tx.appendEntry(SystemEntry, conversation.id, draft); }, context); @@ -194,3 +195,92 @@ describe("system prompt preparation", () => { expect(await apply(conversation, {})).toEqual([]); }); }); + +describe("tool loadout preparation", () => { + const declaration = (name: string, description = name): Tool => ({ + name, + description, + parameters: Type.Object({}), + }); + + /** Plan tools only, append the plan, check that replay offers `tools` in order, and return each message's changes. */ + async function applyTools( + conversation: Conversation, + tools: readonly Tool[], + sections: Record = {}, + ): Promise<{ removed?: string[]; added?: string[]; sections?: Record }[]> { + const drafts = planSystemEntries( + await conversation.context(context), + new Map(Object.entries(sections)), + tools, + 7, + ); + await conversation.commit(async (tx) => { + for (const draft of drafts) await tx.appendEntry(SystemEntry, conversation.id, draft); + }, context); + const offered = getCurrentTools((await conversation.context(context)).messages); + expect(offered).toEqual(tools.map(toToolDeclaration)); + return drafts.map((draft) => { + const message = draft.model![0] as SystemMessage; + return { + ...(message.toolsRemoved === undefined ? {} : { removed: message.toolsRemoved.map((tool) => tool.name) }), + ...(message.toolsAdded === undefined ? {} : { added: message.toolsAdded.map((tool) => tool.name) }), + ...(message.sections === undefined ? {} : { sections: message.sections }), + }; + }); + } + + it("adds, removes, replaces changed declarations, and rewrites the order when needed", async () => { + const conversation = await root(); + const [a, b, c] = [declaration("a"), declaration("b"), declaration("c")]; + expect(await applyTools(conversation, [a, b])).toEqual([{ added: ["a", "b"] }]); + expect(await applyTools(conversation, [a, b])).toEqual([]); + expect(await applyTools(conversation, [a, b, c])).toEqual([{ added: ["c"] }]); + expect(await applyTools(conversation, [a, c])).toEqual([{ removed: ["b"] }]); + // A changed declaration at the end is removed and re-added in place. + const c2 = declaration("c", "changed"); + expect(await applyTools(conversation, [a, c2])).toEqual([{ removed: ["c"], added: ["c"] }]); + // A changed declaration in the middle would move to the end, so the whole order is rewritten. + const a2 = declaration("a", "changed"); + expect(await applyTools(conversation, [a2, c2])).toEqual([{ removed: ["a", "c"], added: ["a", "c"] }]); + // Order-only change. + expect(await applyTools(conversation, [c2, a2])).toEqual([{ removed: ["a", "c"], added: ["c", "a"] }]); + expect(await applyTools(conversation, [])).toEqual([{ removed: ["c", "a"] }]); + }); + + it("puts tool changes on the last section entry and re-declares every tool after a head cut", async () => { + const conversation = await root(); + const [a, b] = [declaration("a"), declaration("b")]; + expect(await applyTools(conversation, [a], { x: "1", y: "2" })).toEqual([ + { added: ["a"], sections: { x: "1", y: "2" } }, + ]); + // Section order changes need two entries; the tool change rides on the second. + expect(await applyTools(conversation, [a, b], { y: "2", x: "1" })).toEqual([ + { sections: { x: null, y: null } }, + { added: ["b"], sections: { y: "2", x: "1" } }, + ]); + await marker(conversation, "self"); + expect(await applyTools(conversation, [a, b], { y: "2", x: "1" })).toEqual([ + { added: ["a", "b"], sections: { y: "2", x: "1" } }, + ]); + }); + + it("offers each active name once, only when registered, as composed by wrappers", () => { + const registry = createRegistry(); + const base = { ...declaration("a"), execute: async () => ({}) }; + registry.tools.add(base); + registry.tools.add({ ...declaration("b"), execute: async () => ({}) }); + registry.tools.add({ ...declaration("broken"), execute: async () => ({}) }); + registry.tools.wrap("a", "describe", (tool) => ({ ...tool, description: "wrapped" })); + registry.tools.wrap("broken", "fail", () => { + throw new Error("wrapper failed"); + }); + const snapshot = registry.snapshot(); + const tools = desiredTools(["b", "missing", "a", "b", "broken"], (name) => snapshot.tool(name)); + expect(tools.map((tool) => [tool.name, tool.description])).toEqual([ + ["b", "b"], + ["a", "wrapped"], + ]); + expect(snapshot.failures().map((failure) => failure.name)).toEqual(["broken"]); + }); +}); diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index 18988661c..71f386ccd 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -1,6 +1,13 @@ import type { JsonValue } from "@earendil-works/chord"; import { Type } from "@earendil-works/pi-ai"; -import { createRegistry, defineTask, GenerationTask, type ToolRegistration } from "@earendil-works/pi-durable"; +import { + createRegistry, + defineTask, + GenerationTask, + PostToolsTask, + type ToolRegistration, + ToolTask, +} from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; type AppTool = ToolRegistration & { readonly snippet?: string }; @@ -202,7 +209,7 @@ describe("registry", () => { const worker = task("worker"); registry.tasks.add(worker); expect(() => registry.tasks.add(task("worker"))).toThrow("Task worker is already registered"); - expect(registry.tasks.list()).toEqual([GenerationTask, worker]); + expect(registry.tasks.list()).toEqual([GenerationTask, ToolTask, PostToolsTask, worker]); const first = { beforeRun: () => {} }; const second = { beforeRun: () => {} }; @@ -224,7 +231,7 @@ describe("registry", () => { it("starts with undisposable, non-overridable built-in tasks", () => { const registry = createRegistry(); - expect(registry.tasks.list()).toEqual([GenerationTask]); + expect(registry.tasks.list()).toEqual([GenerationTask, ToolTask, PostToolsTask]); expect(registry.snapshot().task("pi.generation")).toBe(GenerationTask); expect(() => registry.tasks.add({ definition: { ...GenerationTask.definition } })).toThrow( "Task pi.generation is already registered", diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts new file mode 100644 index 000000000..184e1c55d --- /dev/null +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -0,0 +1,350 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + type EntryRecord, + type Harness, + LiveDoc, + PostToolsTask, + type Registration, + type TaskId, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import type { ExecutionEnv } from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { createBashTool } from "../src/tools/index.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, waitFor } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +const directories = new Set(); + +afterEach(async () => { + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-tools-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +async function open(path: string, setup: ChatSetup, env?: ExecutionEnv) { + const opened = await openChat(await openNodeSqliteStorage(path), setup, env === undefined ? {} : { env }); + opened.harness.resume(); + return opened; +} + +function tool( + name: string, + execute: ToolRegistration["execute"], + extra: Partial = {}, +): ToolRegistration { + return { name, description: name, parameters: Type.Object({}), execute, ...extra }; +} + +function call(name: string, id = "c1"): AssistantMessage { + return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +function results(entries: readonly EntryRecord[]): ToolResultMessage[] { + return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); +} + +function text(message: ToolResultMessage | undefined): string { + return (message?.content ?? []).map((item) => (item.type === "text" ? item.text : "")).join("|"); +} + +async function toolTaskId(harness: Harness): Promise { + let id: TaskId | undefined; + await waitFor(async () => { + id = (await harness.snapshot(LiveDoc, 1 as never, context))?.tools?.[0]?.taskId; + return id !== undefined; + }); + return id!; +} + +/** + * A tool that writes output, then blocks until its invocation is cancelled the first `blocking` times it runs. + * `started` resolves once the output is durable. + */ +function blockingTool(name: string, extra: Partial = {}) { + const started = deferred(); + const state = { runs: 0, blocking: 1 }; + const registration = tool( + name, + async (_args, api, callContext) => { + state.runs++; + api.output(`run ${state.runs}\n`); + await api.details({ run: state.runs }, callContext); + if (state.runs <= state.blocking) { + started.resolve(); + await aborted(callContext.abortSignal!); + } + return {}; + }, + extra, + ); + return { registration, started, state }; +} + +describe("tool recovery", () => { + it("answers an unsafe tool interrupted after intent with its durable partial output", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const { registration, started, state } = blockingTool("work"); + setup.registry.tools.add(registration); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + const taskId = await toolTaskId(opened.harness); + await opened.harness.close(context); + + opened = await open(path, setup); + expect((await opened.harness.getTask(taskId, context))?.state).toMatchObject({ + checkpoint: { phase: "execute", arguments: {}, replay: "unsafe" }, + }); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(state.runs).toBe(1); + const [result] = results(await allEntries(opened.root)); + expect(result).toMatchObject({ isError: true, details: { run: 1 } }); + expect(text(result)).toBe( + "run 1\n|\n[error] Tool work was interrupted and may have partially run\n", + ); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + it("reruns a tool only when both the stored and the current replay policy are safe", async () => { + const cases: [stored: "safe" | "unsafe", current: "safe" | "unsafe", reruns: boolean][] = [ + ["safe", "safe", true], + ["safe", "unsafe", false], + ["unsafe", "safe", false], + ]; + for (const [stored, current, reruns] of cases) { + const path = await sqlitePath(); + const setup = chatSetup(); + const blocking = blockingTool("work", { replay: stored }); + const registration: Registration = setup.registry.tools.add(blocking.registration); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await blocking.started.promise; + await opened.harness.close(context); + + registration.dispose(); + setup.registry.tools.add({ ...blocking.registration, replay: current }); + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + const [result] = results(await allEntries(opened.root)); + expect(blocking.state.runs).toBe(reruns ? 2 : 1); + expect(result!.isError).toBe(!reruns); + if (reruns) expect(text(result)).toBe("run 2\n"); + await opened.harness.close(context); + } + }); + + it("reruns beforeTool when interrupted before intent, and executes once", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + let runs = 0; + setup.registry.tools.add( + tool("work", async () => { + runs++; + return { content: [] }; + }), + ); + const reached = deferred(); + let asked = 0; + const decisions: string[] = []; + setup.registry.hooks.add(ToolTask, { + beforeTool: async (_call, api, callContext) => { + asked++; + // A durable first-writer-wins decision survives the rerun. + decisions.push(await api.memo("test:decision", `attempt ${asked}`, callContext)); + if (asked === 1) { + reached.resolve(); + await aborted(callContext.abortSignal!); + } + return undefined; + }, + }); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await reached.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect([asked, runs]).toEqual([2, 1]); + expect(decisions).toEqual(["attempt 1", "attempt 1"]); + await opened.harness.close(context); + }); + + it("reruns post-tools interrupted before its commit", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + setup.registry.tools.add(tool("work", async () => ({ content: [] }))); + const reached = deferred(); + let observed = 0; + setup.registry.hooks.add(PostToolsTask, { + afterTools: async (_assistant, _results, _api, callContext) => { + observed++; + if (observed === 1) { + reached.resolve(); + await aborted(callContext.abortSignal!); + } + }, + }); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await reached.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(observed).toBe(2); + expect((await allEntries(opened.root)).map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.system", + "pi.assistant", + "pi.tool-result", + "pi.assistant", + ]); + await opened.harness.close(context); + }); + + it("answers an aborted tool with its partial output and continues the run", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const { registration, started } = blockingTool("work"); + setup.registry.tools.add(registration); + setup.faux.setResponses([call("work"), DONE]); + const opened = await open(path, setup); + const submission = await opened.root.submit({ type: "input", content: "go" }, context); + await started.promise; + const taskId = await toolTaskId(opened.harness); + expect(await opened.harness.abortTask(taskId, context)).toBe("marked"); + expect((await opened.harness.waitForTask(taskId, context)).state.outcome).toMatchObject({ + status: "aborted", + result: { entryId: expect.any(Number) }, + }); + expect((await submission.wait(context)).status).toBe("done"); + const [result] = results(await allEntries(opened.root)); + expect(text(result)).toBe("run 1\n|\n[error] Tool work was aborted\n"); + await opened.harness.close(context); + }); + + it("lets context derivation answer a faulted tool and continues the run", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + // A result that is not strict JSON makes the result commit throw, so the scheduler faults the task. + setup.registry.tools.add( + tool("bad", async () => ({ content: [], details: { fn: (() => 1) as unknown as JsonValue } })), + ); + const requests: string[] = []; + setup.faux.setResponses([ + call("bad"), + (request) => { + const result = request.messages.find((message) => message.role === "toolResult"); + requests.push(result?.role === "toolResult" ? text(result) : "none"); + return DONE; + }, + ]); + const opened = await open(path, setup); + const settled = await (await opened.root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + expect(results(await allEntries(opened.root))).toEqual([]); + expect(requests).toEqual(["Tool result unavailable: history ends before this call completed."]); + const tasks = await opened.harness.commit((tx) => tx.scanTasks({ conversationId: opened.root.id }, 20), context); + const faulted = tasks.items.find((task) => task.kind === "pi.tool")!; + expect(faulted.state).toMatchObject({ status: "terminal", outcome: { status: "faulted" } }); + await opened.harness.close(context); + }); + + it("answers a real bash command interrupted by close and reopen, then finishes the run", async () => { + const path = await sqlitePath(); + const env = new NodeExecutionEnv({ cwd: dirname(path) }); + const setup = chatSetup(); + setup.registry.tools.add(createBashTool()); + setup.faux.setResponses([ + fauxAssistantMessage([fauxToolCall("bash", { command: "echo started; sleep 30" }, { id: "b" })], { + stopReason: "toolUse", + }), + DONE, + ]); + let opened = await open(path, setup, env); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await waitFor(async () => { + const slot = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.tools?.[0]; + return slot?.output === "started\n"; + }); + await opened.harness.close(context); + + opened = await open(path, setup, env); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + const [result] = results(await allEntries(opened.root)); + expect(text(result)).toBe( + "started\n|\n[error] Tool bash was interrupted and may have partially run\n", + ); + await opened.harness.close(context); + }); + + it("clears the interrupted attempt's progress before a safe rerun", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const started = [deferred(), deferred()]; + let runs = 0; + setup.registry.tools.add( + tool( + "work", + async (_args, api, callContext) => { + const run = runs++; + if (run === 0) { + api.diagnostic({ severity: "info", message: "first a" }); + api.diagnostic({ severity: "info", message: "first b" }); + await api.details({ run: 1, extra: true }, callContext); + } else { + api.diagnostic({ severity: "info", message: "second" }); + await api.details({ run: 2 }, callContext); + } + started[run]!.resolve(); + await aborted(callContext.abortSignal!); + return {}; + }, + { replay: "safe" }, + ), + ); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + await opened.root.submit({ type: "input", content: "go" }, context); + await started[0]!.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + await started[1]!.promise; + const slot = (await opened.harness.snapshot(LiveDoc, opened.root.id, context))?.tools?.[0]; + expect(slot?.diagnostics).toEqual([{ severity: "info", message: "second" }]); + expect(slot?.details).toEqual({ run: 2 }); + await opened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts new file mode 100644 index 000000000..da558be3a --- /dev/null +++ b/packages/durable/test/harness-tools.test.ts @@ -0,0 +1,951 @@ +import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { + type AssistantMessage, + type FauxResponseStep, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type SystemMessage, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + type Conversation, + defineTask, + type EntryRecord, + GenerationTask, + type Harness, + LiveDoc, + MemoryStorage, + PostToolsTask, + type Registration, + type SubmissionId, + type ToolRegistration, + ToolResultEntry, + ToolTask, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { createBashTool, createEditTool, createReadTool } from "../src/tools/index.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat, textOf } from "./chat-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred } from "./task-support.ts"; + +type Execute = ToolRegistration["execute"]; + +function tool(name: string, execute: Execute, extra: Partial = {}): ToolRegistration { + return { + name, + description: `The ${name} tool`, + parameters: Type.Object({ text: Type.Optional(Type.String()) }), + execute, + ...extra, + }; +} + +/** A tool-calling answer with one call per `[name, args, id]`. */ +function calls(...list: readonly (readonly [string, Record, string])[]): AssistantMessage { + return fauxAssistantMessage( + list.map(([name, args, id]) => fauxToolCall(name, args, { id })), + { stopReason: "toolUse" }, + ); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +async function run( + setup: ChatSetup, + responses: FauxResponseStep[], + prepare?: (harness: Harness, root: Conversation) => Promise, +): Promise<{ harness: Harness; root: Conversation; entries: EntryRecord[]; status: string }> { + setup.faux.setResponses(responses); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await prepare?.(harness, root); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + return { harness, root, entries: await allEntries(root), status: settled.status }; +} + +function results(entries: readonly EntryRecord[]): ToolResultMessage[] { + return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); +} + +function resultText(message: ToolResultMessage): string { + return message.content.map((item) => (item.type === "text" ? item.text : `[${item.type}]`)).join("|"); +} + +describe("tool round", () => { + it("runs input, tool call, tool result, and answer, and settles the input", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool("echo", async (args) => ({ + content: [{ type: "text", text: `echo ${(args as { text: string }).text}` }], + })), + ); + const { harness, root, entries, status } = await run(setup, [calls(["echo", { text: "hi" }, "c1"]), DONE]); + expect(status).toBe("done"); + expect(entries.map((entry) => entry.kind)).toEqual([ + "pi.user", + "pi.system", + "pi.assistant", + "pi.tool-result", + "pi.assistant", + ]); + const system = entries[1]!.model![0] as SystemMessage; + expect(system.toolsAdded).toEqual([ + { name: "echo", description: "The echo tool", parameters: expect.any(Object) }, + ]); + const [result] = results(entries); + expect(result).toMatchObject({ toolCallId: "c1", toolName: "echo", isError: false }); + expect(resultText(result!)).toBe("echo hi"); + expect(ToolResultEntry.is(entries[3]) && entries[3].data).toEqual({ diagnostics: [] }); + expect(entries[3]!.byTaskId).toBeDefined(); + // The second request sees the tool result right after its call. + expect(setup.faux.state.callCount).toBe(2); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("answers calls to tools the request did not offer without a task", async () => { + const setup = chatSetup(); + setup.registry.tools.add(tool("echo", async () => ({ content: [] }))); + const { harness, root, entries, status } = await run(setup, [ + calls(["ghost", {}, "c1"], ["echo", {}, "c2"]), + DONE, + ]); + expect(status).toBe("done"); + const [ghost, echo] = results(entries); + expect(ghost).toMatchObject({ toolCallId: "c1", isError: true }); + expect(resultText(ghost!)).toBe("\n[error] Tool ghost is not available\n"); + expect(echo).toMatchObject({ toolCallId: "c2", isError: false }); + const ghostEntry = entries.find((entry) => ToolResultEntry.is(entry) && entry.model![0]!.role === "toolResult"); + expect(ghostEntry?.data).toEqual({ + diagnostics: [{ severity: "error", code: "tool_unavailable", message: "Tool ghost is not available" }], + }); + const tasks = await harness.commit((tx) => tx.scanTasks({ conversationId: root.id }, 20), context); + expect(tasks.items.filter((task) => task.kind === "pi.tool")).toHaveLength(1); + await harness.close(context); + }); + + it("still executes a tool that was deactivated after preparation", async () => { + const setup = chatSetup(); + const seen: string[] = []; + setup.registry.tools.add( + tool("echo", async () => { + seen.push("ran"); + return { content: [] }; + }), + ); + let root!: Conversation; + const deactivate: FauxResponseStep = async () => { + await root.setActiveTools([], context); + return calls(["echo", {}, "c1"]); + }; + const result = await run(setup, [deactivate, DONE], async (_harness, conversation) => { + root = conversation; + }); + expect(seen).toEqual(["ran"]); + expect(results(result.entries)[0]!.isError).toBe(false); + // The next preparation removes it. + const systems = result.entries.filter((entry) => entry.kind === "pi.system"); + expect((systems.at(-1)!.model![0] as SystemMessage).toolsRemoved).toEqual([{ name: "echo" }]); + await result.harness.close(context); + }); + + it("removes unregistered active tools from the offer and adds them back after re-registration", async () => { + const setup = chatSetup(); + const echo = tool("echo", async () => ({ content: [] })); + const registration = setup.registry.tools.add(echo); + const first = await run(setup, [DONE]); + registration.dispose(); + setup.faux.setResponses([calls(["echo", {}, "c1"]), DONE]); + const second = await (await first.root.submit({ type: "input", content: "again" }, context)).wait(context); + expect(second.status).toBe("done"); + let entries = await allEntries(first.root); + const removal = entries.filter((entry) => entry.kind === "pi.system").at(-1)!.model![0] as SystemMessage; + expect(removal.toolsRemoved).toEqual([{ name: "echo" }]); + expect(results(entries)[0]).toMatchObject({ isError: true }); + expect(await first.root.getActiveTools(context)).toEqual(["echo"]); + + setup.registry.tools.add(echo); + setup.faux.setResponses([DONE]); + await (await first.root.submit({ type: "input", content: "back" }, context)).wait(context); + entries = await allEntries(first.root); + const addition = entries.filter((entry) => entry.kind === "pi.system").at(-1)!.model![0] as SystemMessage; + expect(addition.toolsAdded?.map((declared) => declared.name)).toEqual(["echo"]); + await first.harness.close(context); + }); + + it("produces tool_unavailable when the implementation is unregistered before its task runs", async () => { + const setup = chatSetup(); + let second!: Registration; + setup.registry.tools.add( + tool( + "first", + async () => { + second.dispose(); + return { content: [] }; + }, + { executionMode: "sequential" }, + ), + ); + second = setup.registry.tools.add(tool("second", async () => ({ content: [{ type: "text", text: "ran" }] }))); + const { harness, entries, status } = await run(setup, [calls(["first", {}, "c1"], ["second", {}, "c2"]), DONE]); + expect(status).toBe("done"); + const [, late] = results(entries); + expect(resultText(late!)).toBe("\n[error] Tool second is not available\n"); + await harness.close(context); + }); + + it("runs a round in parallel by default and sequentially when configured or required by a tool", async () => { + const trace = async (setup: ChatSetup, prepare?: (root: Conversation) => Promise): Promise => { + const events: string[] = []; + const slow = + (name: string): Execute => + async () => { + events.push(`start ${name}`); + await new Promise((resolve) => setTimeout(resolve, 20)); + events.push(`end ${name}`); + return { content: [] }; + }; + if (setup.registry.snapshot().tool("a") === undefined) setup.registry.tools.add(tool("a", slow("a"))); + if (setup.registry.snapshot().tool("b") === undefined) setup.registry.tools.add(tool("b", slow("b"))); + const result = await run(setup, [calls(["a", {}, "c1"], ["b", {}, "c2"]), DONE], (_harness, root) => + prepare === undefined ? Promise.resolve() : prepare(root), + ); + await result.harness.close(context); + return events; + }; + expect((await trace(chatSetup())).slice(0, 2)).toEqual(["start a", "start b"]); + const sequential = await trace(chatSetup(), (root) => root.setToolExecution("sequential", context)); + expect(sequential).toEqual(["start a", "end a", "start b", "end b"]); + const perTool = chatSetup(); + perTool.registry.tools.add( + tool( + "a", + async () => { + await new Promise((resolve) => setTimeout(resolve, 20)); + return { content: [] }; + }, + { executionMode: "sequential" }, + ), + ); + const events: string[] = []; + perTool.registry.tools.add( + tool("b", async () => { + events.push("b"); + return { content: [] }; + }), + ); + const result = await run(perTool, [calls(["a", {}, "c1"], ["b", {}, "c2"]), DONE]); + const tasks = await result.harness.commit((tx) => tx.scanTasks({ conversationId: result.root.id }, 20), context); + const tools = tasks.items.filter((task) => task.kind === "pi.tool").sort((a, b) => a.id - b.id); + expect(tools[1]!.after).toEqual([tools[0]!.id]); + await result.harness.close(context); + }); +}); + +describe("tool results", () => { + it("uses retained output and the last details when the result omits them, with diagnostics in order", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool( + "log", + async (_args, api) => { + api.output("line 1\n"); + api.output(new TextEncoder().encode("line 2\nline 3\n")); + api.diagnostic({ severity: "info", message: "from api" }); + await api.details({ step: 1 }, context); + await api.details({ step: 2 }, context); + return { diagnostics: [{ severity: "warn", message: "from result" }] }; + }, + { outputLimits: { maxLines: 2 } }, + ), + ); + const { harness, entries } = await run(setup, [calls(["log", {}, "c1"]), DONE]); + const [result] = results(entries); + expect(result!.details).toEqual({ step: 2 }); + expect(resultText(result!)).toBe( + "line 1\nline 2\n|\n[info] from api\n[warn] from result\n[warn] Output truncated to its beginning: 1 lines, 7 bytes dropped\n", + ); + const entry = entries.find((candidate) => ToolResultEntry.is(candidate)); + expect(ToolResultEntry.is(entry) && entry.data.diagnostics.map((diagnostic) => diagnostic.code)).toEqual([ + undefined, + undefined, + "truncated", + ]); + await harness.close(context); + }); + + it("bounds explicit text content and keeps other content", async () => { + const setup = chatSetup(); + const image = { type: "image", data: "AAAA", mimeType: "image/png" } as const; + setup.registry.tools.add( + tool( + "big", + async () => ({ + content: [{ type: "text", text: "a\nb\n" }, image, { type: "text", text: "c\nd\n" }], + }), + { outputLimits: { maxLines: 2, retain: "tail" } }, + ), + ); + const { harness, entries } = await run(setup, [calls(["big", {}, "c1"]), DONE]); + expect(resultText(results(entries)[0]!)).toBe( + "[image]|c\nd\n|\n[warn] Output truncated to its end: 2 lines, 4 bytes dropped\n", + ); + await harness.close(context); + }); + + it("turns a throw into a tool_error result with the partial output", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool("fail", async (_args, api) => { + api.output("partial\n"); + throw new Error("boom"); + }), + ); + const { harness, entries, status } = await run(setup, [calls(["fail", {}, "c1"]), DONE]); + expect(status).toBe("done"); + const [result] = results(entries); + expect(result!.isError).toBe(true); + expect(resultText(result!)).toBe("partial\n|\n[error] boom\n"); + await harness.close(context); + }); + + it("validates arguments before and after beforeTool and applies blocks and replacements", async () => { + const setup = chatSetup(); + const seen: JsonValue[] = []; + setup.registry.tools.add( + tool("echo", async (args) => { + seen.push(args); + return { content: [] }; + }), + ); + setup.registry.hooks.add(ToolTask, { + beforeTool: (call) => { + if (call.id === "block") return { block: "not today" }; + if (call.id === "throw") throw new Error("hook failed"); + if (call.id === "bad") return { arguments: { text: { not: "a string" } } }; + return { arguments: { text: `${call.arguments.text}!` } }; + }, + }); + const { harness, entries } = await run(setup, [ + calls( + ["echo", { text: 1 as unknown as string }, "coerced"], + ["echo", { text: { not: "a string" } }, "invalid"], + ["echo", {}, "block"], + ["echo", {}, "throw"], + ["echo", { text: "x" }, "bad"], + ["echo", { text: "x" }, "ok"], + ), + DONE, + ]); + // Parallel tools append their results in completion order. + const byId = new Map(results(entries).map((result) => [result.toolCallId, [result.isError, resultText(result)]])); + expect(byId.get("block")).toEqual([true, "\n[error] Tool call blocked: not today\n"]); + expect(byId.get("throw")).toEqual([true, "\n[error] Tool call blocked: hook failed\n"]); + expect(byId.get("bad")![0]).toBe(true); + expect(byId.get("bad")![1]).toContain("Validation failed"); + expect(byId.get("ok")).toEqual([false, ""]); + expect(byId.get("invalid")![0]).toBe(true); + expect(byId.get("invalid")![1]).toContain("Validation failed"); + // pi-ai coerces a number to a string before the first validation. + expect(byId.get("coerced")).toEqual([false, ""]); + expect(seen).toEqual(expect.arrayContaining([{ text: "1!" }, { text: "x!" }])); + expect(seen).toHaveLength(2); + await harness.close(context); + }); + + it("repairs arguments with prepareArguments before validation, and a throwing repair is invalid", async () => { + const setup = chatSetup(); + const seen: JsonValue[] = []; + setup.registry.tools.add( + tool( + "echo", + async (args) => { + seen.push(args); + return { content: [] }; + }, + { + prepareArguments: (args) => { + const text = (args as { text?: unknown }).text; + if (text === "throw") throw new Error("cannot repair"); + return typeof text === "number" ? { text: `#${text}` } : args; + }, + }, + ), + ); + const { harness, entries } = await run(setup, [ + calls(["echo", { text: 7 as unknown as string }, "fixed"], ["echo", { text: "throw" }, "broken"]), + DONE, + ]); + const byId = new Map(results(entries).map((result) => [result.toolCallId, resultText(result)])); + expect(byId.get("fixed")).toBe(""); + expect(byId.get("broken")).toBe("\n[error] cannot repair\n"); + expect(seen).toEqual([{ text: "#7" }]); + // The stored call keeps what the model sent. + const call = (entries[2]!.model![0] as AssistantMessage).content.find((item) => item.type === "toolCall"); + expect(call?.type === "toolCall" && call.arguments).toEqual({ text: 7 }); + await harness.close(context); + }); + + it("lets the first beforeTool block win and skips later handlers", async () => { + const setup = chatSetup(); + setup.registry.tools.add(tool("echo", async () => ({ content: [] }))); + const asked: string[] = []; + setup.registry.hooks.add(ToolTask, { + beforeTool: () => { + asked.push("first"); + return { block: "first says no" }; + }, + }); + setup.registry.hooks.add(ToolTask, { + beforeTool: () => { + asked.push("second"); + return { block: "second says no" }; + }, + }); + const { harness, entries } = await run(setup, [calls(["echo", {}, "c1"]), DONE]); + expect(asked).toEqual(["first"]); + expect(resultText(results(entries)[0]!)).toBe("\n[error] Tool call blocked: first says no\n"); + await harness.close(context); + }); + + it("chains afterTool replacements and observes the round with afterTools", async () => { + const setup = chatSetup(); + setup.registry.tools.add(tool("echo", async () => ({ content: [{ type: "text", text: "raw" }] }))); + setup.registry.hooks.add(ToolTask, { + afterTool: (_call, result) => ({ ...result, content: [{ type: "text", text: "first" }] }), + }); + setup.registry.hooks.add(ToolTask, { + afterTool: (_call, result) => ({ ...result, details: { replaced: resultText(result as ToolResultMessage) } }), + }); + const observed: unknown[] = []; + setup.registry.hooks.add(PostToolsTask, { + afterTools: (assistant, entries) => void observed.push(assistant, entries), + }); + const { harness, entries } = await run(setup, [calls(["echo", {}, "c1"]), DONE]); + const [result] = results(entries); + expect(resultText(result!)).toBe("first"); + expect(result!.details).toEqual({ replaced: "first" }); + const resultEntry = entries.find((entry) => ToolResultEntry.is(entry))!; + expect(observed).toEqual([entries[2]!.id, [resultEntry.id]]); + await harness.close(context); + }); + + it("matches hook scopes to the conversation and its owned subtree", async () => { + const setup = chatSetup(); + setup.registry.tools.add(tool("echo", async () => ({ content: [] }))); + const calledIn: number[] = []; + const first = await run(setup, [DONE]); + // An owner task no registered definition takes stays live and pending. + const owner = defineTask, { phase: "never" }, null>({ + name: "test.owner", + version: 1, + initial: () => ({ phase: "never" }), + phases: { never: async () => {} }, + abort: async () => {}, + }); + const childId = await first.harness.commit(async (tx) => { + const taskId = await tx.createTask(owner, {}, { conversationId: first.root.id }); + return (await tx.createConversation({ ownership: { kind: "task", taskId } })).id; + }, context); + const child = (await first.harness.conversation(childId, context))!; + const other = await first.harness.createConversation({ ownership: { kind: "ownerless" } }, context); + for (const conversation of [child, other]) { + await conversation.setModel({ provider: "faux", modelId: "faux-1" }, context); + } + setup.registry.hooks.add( + ToolTask, + { beforeTool: (_call, api) => void calledIn.push(api.conversationId) }, + { scope: { conversationId: first.root.id, subtree: true } }, + ); + setup.registry.hooks.add( + ToolTask, + { beforeTool: (_call, api) => void calledIn.push(-api.conversationId) }, + { scope: { conversationId: first.root.id } }, + ); + for (const conversation of [first.root, child, other]) { + setup.faux.setResponses([calls(["echo", {}, "c1"]), DONE]); + await (await conversation.submit({ type: "input", content: "go" }, context)).wait(context); + } + expect(calledIn).toEqual([first.root.id, -first.root.id, child.id]); + await first.harness.close(context); + }); + + it("applies addTools and terminates only when every result of the round asks to", async () => { + const setup = chatSetup(); + setup.registry.tools.add(tool("stop", async () => ({ content: [], control: { terminate: true } }))); + setup.registry.tools.add(tool("grow", async () => ({ content: [], control: { addTools: ["extra", "stop"] } }))); + setup.registry.tools.add(tool("extra", async () => ({ content: [] }))); + const first = await run(setup, [calls(["stop", {}, "c1"])], async (_harness, root) => { + await root.setActiveTools(["stop", "grow"], context); + }); + expect(first.status).toBe("done"); + expect(first.entries.at(-1)!.kind).toBe("pi.tool-result"); + const settled = await first.harness.commit((tx) => tx.scanTasks({ conversationId: first.root.id }, 20), context); + expect(settled.items.every((task) => task.state.status === "terminal")).toBe(true); + + setup.faux.setResponses([calls(["stop", {}, "c1"], ["grow", {}, "c2"]), DONE]); + const second = await (await first.root.submit({ type: "input", content: "again" }, context)).wait(context); + expect(second.status).toBe("done"); + expect((await allEntries(first.root)).at(-1)!.kind).toBe("pi.assistant"); + expect(await first.root.getActiveTools(context)).toEqual(["stop", "grow", "extra"]); + await first.harness.close(context); + }); +}); + +describe("generation hooks", () => { + it("replaces request messages, observes responses, and continues on yield", async () => { + const setup = chatSetup(); + const requests: string[][] = []; + const record: FauxResponseStep = (request) => { + requests.push(request.messages.map((message) => `${message.role}:${textOf(message as never) ?? ""}`)); + return fauxAssistantMessage([fauxText(`answer ${requests.length}`)]); + }; + setup.registry.hooks.add(GenerationTask, { + beforeRequest: ({ messages }) => ({ + messages: [...messages, { role: "user", content: "injected", timestamp: 0 }], + }), + }); + const responses: string[] = []; + setup.registry.hooks.add(GenerationTask, { + afterResponse: (message) => void responses.push(textOf(message) ?? ""), + }); + let yields = 0; + setup.registry.hooks.add(GenerationTask, { + onYield: () => (yields++ === 0 ? { continue: [{ type: "text", text: "keep going" }] } : undefined), + }); + const { harness, entries, status } = await run(setup, [record, record]); + expect(status).toBe("done"); + expect(requests[0]!.slice(-2)).toEqual(["user:go", "user:injected"]); + expect(responses).toEqual(["answer 1", "answer 2"]); + expect(entries.map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant", "pi.user", "pi.assistant"]); + // The injected message was used for the request only. + expect(entries.some((entry) => textOf(entry.model?.[0]) === "injected")).toBe(false); + await harness.close(context); + }); + + it("keeps the run's input open across an onYield continuation and answers it with the final answer", async () => { + const setup = chatSetup(); + let yields = 0; + let harness!: Harness; + let input: SubmissionId | undefined; + let statusAtSecondRequest: string | undefined; + setup.registry.hooks.add(GenerationTask, { + onYield: () => (yields++ === 0 ? { continue: "again" } : undefined), + }); + const second: FauxResponseStep = async () => { + input = (await harness.snapshot(LiveDoc, 1 as never, context))!.run!.inputs[0]; + statusAtSecondRequest = (await (await harness.submission(input!, context))!.status(context)).status; + return fauxAssistantMessage([fauxText("second")]); + }; + const result = await run(setup, [fauxAssistantMessage([fauxText("first")]), second], async (opened) => { + harness = opened; + }); + expect(statusAtSecondRequest).toBe("placed"); + const answers = result.entries.filter((entry) => entry.kind === "pi.assistant"); + expect(await (await harness.submission(input!, context))!.status(context)).toMatchObject({ + status: "done", + answer: answers[1]!.id, + }); + await harness.close(context); + }); + + it("observes responses that arrive by polling a deferred request", async () => { + const setup = chatSetup({ deferred: { pendingFetches: 1, pollAfterMs: 1 } }); + const observed: string[] = []; + setup.registry.hooks.add(GenerationTask, { + afterResponse: (message) => void observed.push(`${message.stopReason}:${textOf(message) ?? ""}`), + }); + const result = await run(setup, [fauxAssistantMessage([fauxText("late")])], (_harness, conversation) => + conversation.setStreamOptions({ deferred: true }, context), + ); + expect(result.status).toBe("done"); + // The still-deferred results are not terminal. + expect(observed).toEqual(["stop:late"]); + await result.harness.close(context); + }); + + it("lets the first onYield continuation win and reports throws without stopping later handlers", async () => { + const setup = chatSetup(); + const called: string[] = []; + setup.registry.hooks.add(GenerationTask, { + afterResponse: () => { + called.push("throwing observer"); + throw new Error("observer failed"); + }, + }); + setup.registry.hooks.add(GenerationTask, { afterResponse: () => void called.push("next observer") }); + let yields = 0; + setup.registry.hooks.add(GenerationTask, { + onYield: () => (yields++ === 0 ? { continue: "first" } : undefined), + }); + setup.registry.hooks.add(GenerationTask, { + onYield: () => { + called.push("second onYield"); + return called.filter((name) => name === "second onYield").length === 1 ? { continue: "second" } : undefined; + }, + }); + const { harness, entries } = await run(setup, [ + fauxAssistantMessage([fauxText("a")]), + fauxAssistantMessage([fauxText("b")]), + fauxAssistantMessage([fauxText("c")]), + ]); + // The first continuation skips the second handler; on the next answer the second handler's continuation wins. + const users = entries.filter((entry) => entry.kind === "pi.user").map((entry) => textOf(entry.model![0])); + expect(users).toEqual(["go", "first", "second"]); + expect(called.filter((name) => name === "second onYield")).toHaveLength(2); + expect(called.filter((name) => name === "next observer")).toHaveLength(3); + expect(setup.reports.map((error) => (error as Error).message)).toContain("observer failed"); + await harness.close(context); + }); + + it("keeps durable hook decisions in task memos", async () => { + const setup = chatSetup(); + let asked = 0; + setup.registry.tools.add(tool("echo", async () => ({ content: [] }))); + setup.registry.hooks.add(ToolTask, { + beforeTool: async (_call, api) => { + asked++; + const decision = await api.memo("approval:decision", "approved", context); + expect(await api.memo("approval:decision", "denied", context)).toBe(decision); + return undefined; + }, + }); + const { harness } = await run(setup, [calls(["echo", {}, "c1"]), DONE]); + expect(asked).toBe(1); + await harness.close(context); + }); +}); + +describe("tool execution api", () => { + it("provides the Harness environment, lets wrappers replace it, and runs commits, memos, and child tasks", async () => { + const setup = chatSetup(); + const env = new NodeExecutionEnv({ cwd: "/tmp" }); + const other = new NodeExecutionEnv({ cwd: "/" }); + const child = defineTask<{ n: number }, { phase: "run" }, number>({ + name: "test.child", + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, callContext) => { + await runtime.commit( + () => ({ status: "terminal", outcome: { status: "completed", result: task.input.n * 2 } }), + callContext, + ); + }, + }, + abort: async () => {}, + }); + setup.registry.tasks.add(child); + const seen: unknown[] = []; + setup.registry.tools.add( + tool("probe", async (_args, api) => { + seen.push(api.env?.cwd); + const entry = await api.commit( + (tx) => tx.appendEntry(api.conversationId, { kind: "test.note", data: api.callId }), + context, + ); + seen.push(entry.byTaskId === api.taskId); + seen.push(await api.memo("m", 1, context), await api.memo("m", 2, context)); + const id = await api.createTask(child, { n: 21 }, {}, context); + const done = await api.waitForTask(id, context); + seen.push(done.state.outcome); + return { content: [] }; + }), + ); + setup.registry.tools.wrap("probe", "cwd", (base) => ({ + ...base, + execute: (args, api, callContext) => + api.callId === "c2" + ? base.execute(args, { ...api, env: other }, callContext) + : base.execute(args, api, callContext), + })); + setup.faux.setResponses([calls(["probe", {}, "c1"], ["probe", {}, "c2"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup, { env }); + await root.setToolExecution("sequential", context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const outcome = { status: "completed", result: 42 }; + expect(seen).toEqual(["/tmp", true, 1, 1, outcome, "/", true, 1, 1, outcome]); + await harness.close(context); + }); +}); + +describe("coding tools", () => { + it("answers a failing command with its retained tail and diagnostics in order", async () => { + const directory = mkdtempSync(join(tmpdir(), "pi-durable-coding-")); + try { + const setup = chatSetup(); + setup.registry.tools.add(createBashTool()); + const command = "i=1; while [ $i -le 3000 ]; do echo line-$i; i=$((i + 1)); done; exit 7"; + setup.faux.setResponses([calls(["bash", { command }, "b"]), DONE]); + const env = new NodeExecutionEnv({ cwd: directory }); + const { harness, root } = await openChat(new MemoryStorage(), setup, { env }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const entry = (await allEntries(root)).find((candidate) => ToolResultEntry.is(candidate))!; + const [result] = results([entry]); + expect(result!.isError).toBe(true); + const text = resultText(result!); + expect(text.startsWith("line-1001\n")).toBe(true); + expect(ToolResultEntry.is(entry) && entry.data.diagnostics.map((diagnostic) => diagnostic.code)).toEqual([ + "full_output", + "tool_error", + "truncated", + ]); + expect(text).toContain("line-3000\n|\n[info] Full output: "); + expect(text).toContain( + "\n[error] Command exited with code 7\n[warn] Output truncated to its end: 1000 lines, ", + ); + await harness.close(context); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); + + it("reads, edits, and runs a command in one run, then answers", async () => { + const directory = mkdtempSync(join(tmpdir(), "pi-durable-coding-")); + try { + writeFileSync(join(directory, "notes.txt"), "hello world\n"); + const setup = chatSetup(); + setup.registry.batch(() => { + setup.registry.tools.add(createReadTool()); + setup.registry.tools.add(createEditTool()); + setup.registry.tools.add(createBashTool()); + }); + setup.faux.setResponses([ + calls(["read", { path: "notes.txt" }, "r"]), + calls(["edit", { path: "notes.txt", edits: [{ oldText: "world", newText: "durable" }] }, "e"]), + calls(["bash", { command: "cat notes.txt" }, "b"]), + DONE, + ]); + const env = new NodeExecutionEnv({ cwd: directory }); + const { harness, root } = await openChat(new MemoryStorage(), setup, { env }); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + const entries = await allEntries(root); + expect(results(entries).map((result) => [result.toolName, result.isError, resultText(result)])).toEqual([ + ["read", false, "hello world\n"], + ["edit", false, "Successfully replaced 1 block(s) in notes.txt."], + ["bash", false, "hello durable\n"], + ]); + expect(entries.at(-1)!.kind).toBe("pi.assistant"); + expect(readFileSync(join(directory, "notes.txt"), "utf8")).toBe("hello durable\n"); + await harness.close(context); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); + +describe("tool progress and lifetime", () => { + it("applies the default output limits", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool("lines", async (_args, api) => { + for (let index = 1; index <= 2500; index++) api.output(`${index}\n`); + return {}; + }), + ); + const { harness, entries } = await run(setup, [calls(["lines", {}, "c1"]), DONE]); + const text = resultText(results(entries)[0]!); + expect(text.startsWith("1\n2\n")).toBe(true); + expect( + text.endsWith( + "\n2000\n|\n[warn] Output truncated to its beginning: 500 lines, 2500 bytes dropped\n", + ), + ).toBe(true); + await harness.close(context); + }); + + it("sanitizes running output but keeps explicit result content as the tool returned it", async () => { + const setup = chatSetup(); + let slotOutput: string | undefined; + setup.registry.tools.add( + tool("noisy", async (_args, api) => { + api.output("a\u0007b\r\n"); + await api.details({ ready: true }, context); + slotOutput = (await api.snapshot(LiveDoc, api.conversationId, context))?.tools?.[0]?.output; + return {}; + }), + ); + setup.registry.tools.add(tool("explicit", async () => ({ content: [{ type: "text", text: "c\u001bd" }] }))); + const { harness, entries } = await run(setup, [calls(["noisy", {}, "c1"], ["explicit", {}, "c2"]), DONE]); + expect(slotOutput).toBe("ab\n"); + const byId = new Map(results(entries).map((result) => [result.toolCallId, resultText(result)])); + expect(byId.get("c1")).toBe("ab\n"); + expect(byId.get("c2")).toBe("c\u001bd"); + await harness.close(context); + }); + + it("drops control keys set to undefined instead of faulting", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool("grow", async () => ({ content: [], control: { addTools: ["extra"], terminate: undefined } })), + ); + setup.registry.tools.add(tool("extra", async () => ({ content: [] }))); + const { harness, root, status } = await run(setup, [calls(["grow", {}, "c1"]), DONE], (_harness, conversation) => + conversation.setActiveTools(["grow"], context), + ); + expect(status).toBe("done"); + expect(await root.getActiveTools(context)).toEqual(["grow", "extra"]); + await harness.close(context); + }); + + it("uses explicit null details instead of the last reported value", async () => { + const setup = chatSetup(); + setup.registry.tools.add( + tool("null", async (_args, api) => { + await api.details({ old: 1 }, context); + return { content: [], details: null }; + }), + ); + const { harness, entries } = await run(setup, [calls(["null", {}, "c1"]), DONE]); + expect(results(entries)[0]!.details).toBeNull(); + await harness.close(context); + }); + + it("settles details() promises with coalesced progress commits and the terminal commit", async () => { + const setup = chatSetup(); + const settled: string[] = []; + setup.registry.tools.add( + tool("details", async (_args, api) => { + // Three updates in one throttle window coalesce; the last is still pending when execute() returns. + const first = api.details({ n: 1 }, context).then(() => settled.push("first")); + const second = api.details({ n: 2 }, context).then(() => settled.push("second")); + await first; + void second; + void api.details({ n: 3 }, context).then(() => settled.push("third")); + return { content: [] }; + }), + ); + const { harness, entries } = await run(setup, [calls(["details", {}, "c1"]), DONE]); + expect(settled).toEqual(["first", "second", "third"]); + expect(results(entries)[0]!.details).toEqual({ n: 3 }); + await harness.close(context); + }); + + it("finishes a call under the implementation it resolved when the tool is replaced mid-call", async () => { + const setup = chatSetup(); + let release!: () => void; + const started = new Promise((resolve) => { + release = resolve; + }); + let finish!: () => void; + const finished = new Promise((resolve) => { + finish = resolve; + }); + const v1 = tool("work", async () => { + release(); + await finished; + return { content: [{ type: "text", text: "v1" }] }; + }); + let registration = setup.registry.tools.add(v1); + setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await started; + const previous = registration; + registration = setup.registry.batch(() => { + previous.dispose(); + setup.registry.tools.add(tool("work", async () => ({ content: [{ type: "text", text: "v2" }] }))); + }); + finish(); + await submission.wait(context); + expect(resultText(results(await allEntries(root))[0]!)).toBe("v1"); + await harness.close(context); + }); + + it("rejects invocation-bound waits and stops watches when the tool's invocation ends", async () => { + const setup = chatSetup(); + const never = defineTask, { phase: "never" }, null>({ + name: "test.never", + version: 1, + initial: () => ({ phase: "never" }), + phases: { never: async () => {} }, + abort: async () => {}, + }); + let wait!: Promise; + let watchClosed!: Promise; + setup.registry.tools.add( + tool("detach", async (_args, api) => { + // The child's definition is not registered, so it stays pending. + const child = await api.createTask(never, {}, {}, context); + wait = api.waitForTask(child, context); + wait.catch(() => {}); + const watch = await api.watchDoc(LiveDoc, api.conversationId, context); + watchClosed = watch!.closed; + return { content: [] }; + }), + ); + const { harness } = await run(setup, [calls(["detach", {}, "c1"]), DONE]); + await expect(wait).rejects.toThrow("invocation has ended"); + await watchClosed; + await harness.close(context); + }); + + it("rejects details() still waiting when the call is aborted during afterTool", async () => { + const setup = chatSetup(); + let pendingDetails!: Promise; + const inAfterTool = deferred(); + setup.registry.tools.add( + tool("slow", async (_args, api) => { + api.output("first\n"); + // The output commit is in flight, so these details wait for the next throttle window. + pendingDetails = api.details({ step: 1 }, context); + pendingDetails.catch(() => {}); + return {}; + }), + ); + setup.registry.hooks.add(ToolTask, { + afterTool: async (_call, _result, _api, callContext) => { + inAfterTool.resolve(); + await aborted(callContext.abortSignal!); + return undefined; + }, + }); + setup.faux.setResponses([calls(["slow", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await inAfterTool.promise; + const taskId = (await harness.snapshot(LiveDoc, root.id, context))!.tools![0]!.taskId!; + await harness.abortTask(taskId, context); + await expect(pendingDetails).rejects.toBeDefined(); + await submission.wait(context); + await harness.close(context); + }); + + it("answers an aborted tool with only its durable output, discarding buffered output", async () => { + const setup = chatSetup(); + let reached!: () => void; + const buffered = new Promise((resolve) => { + reached = resolve; + }); + setup.registry.tools.add( + tool("slow", async (_args, api, callContext) => { + api.output("durable\n"); + // The first output commits at once; this one waits for the next throttle window. + await new Promise((resolve) => setTimeout(resolve, 20)); + api.output("buffered\n"); + reached(); + await new Promise((_, reject) => + callContext.abortSignal!.addEventListener("abort", () => reject(callContext.abortSignal!.reason)), + ); + return {}; + }), + ); + setup.faux.setResponses([calls(["slow", {}, "c1"]), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await buffered; + const taskId = (await harness.snapshot(LiveDoc, root.id, context))!.tools![0]!.taskId!; + await harness.abortTask(taskId, context); + await submission.wait(context); + expect(resultText(results(await allEntries(root))[0]!)).toBe( + "durable\n|\n[error] Tool slow was aborted\n", + ); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/storage-runtime-boundary.test.ts b/packages/durable/test/storage-runtime-boundary.test.ts index e36a50465..84ef2d512 100644 --- a/packages/durable/test/storage-runtime-boundary.test.ts +++ b/packages/durable/test/storage-runtime-boundary.test.ts @@ -14,8 +14,9 @@ async function sourceGraph(entry: string): Promise> { visited.add(path); const source = await readFile(path, "utf8"); expect(source, `${path} imports a Node built-in`).not.toMatch(/(?:from\s+|import\s*)["']node:/); - for (const match of source.matchAll(/(?:from\s+|import\s*)["'](\.[^"']+)["']/g)) { - pending.push(resolve(dirname(path), match[1])); + // Type-only imports are erased and load nothing. + for (const match of source.matchAll(/\b(import|export)(\s+type\b)?[^;"']*?(?:from\s*)?["'](\.[^"']+)["']/g)) { + if (match[2] === undefined) pending.push(resolve(dirname(path), match[3])); } } return visited; diff --git a/packages/durable/test/tool-output-bench.ts b/packages/durable/test/tool-output-bench.ts new file mode 100644 index 000000000..2cee4ff8b --- /dev/null +++ b/packages/durable/test/tool-output-bench.ts @@ -0,0 +1,298 @@ +/** + * Tool output benchmark: drives the real tool task, adaptive progress throttle, and `pi.live` commits. + * + * node --conditions=source --experimental-strip-types --expose-gc test/tool-output-bench.ts + * + * Each scenario runs in its own child process so peak RSS is its own. Rate scenarios write output for a fixed time, + * then close the Harness mid-round (persistent backends) to measure replay of the round's delta chain on reopen, and + * finish the run to measure the stored size after the round's base and reclamation. Throughput scenarios push 1 GiB. + */ +import { execFile } from "node:child_process"; +import { mkdtemp, open, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { Type } from "typebox"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { ConversationConfig, createRegistry, Harness, LiveDoc, MemoryStorage, type Storage } from "../src/index.ts"; +import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { createBashTool } from "../src/tools/index.ts"; + +type Backend = "memory" | "sqlite" | "jsonl"; +type Rate = "low" | "normal" | "high"; +type Scenario = + | { kind: "rate"; backend: Backend; rate: Rate; retain: "head" | "tail"; tools: number; seconds?: number } + | { kind: "firehose"; backend: Backend; retain: "head" | "tail"; repetitive?: boolean } + | { kind: "cat"; backend: Backend }; + +const RATE_SECONDS = 3; +const GIB = 1 << 30; +const LINE = "The quick brown fox jumps over the lazy dog. Unique identifier"; +const BIG_FILE = join(tmpdir(), "pi-durable-bench-1gib.txt"); + +/** Output one tool writes per tick, and the pause between ticks. */ +const RATES: Record = { + // A line every 200 ms, like a slow build step. + low: { lines: 1, pauseMs: 200 }, + // Bursts of 25 lines every 50 ms (~40 KB/s), like a compiler or test run. + normal: { lines: 25, pauseMs: 50 }, + // Continuous 64 KiB chunks, yielding to the event loop between them. + high: { lines: 800, pauseMs: 0 }, +}; + +async function main(): Promise { + const [, , encoded] = process.argv; + if (encoded !== undefined) { + const result = await runScenario(JSON.parse(encoded) as Scenario); + process.stdout.write(`${JSON.stringify(result)}\n`); + return; + } + const scenarios: Scenario[] = []; + for (const backend of ["memory", "sqlite", "jsonl"] as const) { + for (const rate of ["low", "normal", "high"] as const) { + for (const retain of ["head", "tail"] as const) { + for (const tools of [1, 4]) scenarios.push({ kind: "rate", backend, rate, retain, tools }); + } + } + for (const retain of ["head", "tail"] as const) scenarios.push({ kind: "firehose", backend, retain }); + // Identical lines: how often Chord's bounded overlap search falls back to writing the whole tail window. + scenarios.push({ kind: "firehose", backend, retain: "tail", repetitive: true }); + // One long round: the delta chain a chatty tail tool builds before the round's base. + if (backend !== "memory") + scenarios.push({ kind: "rate", backend, rate: "high", retain: "tail", tools: 1, seconds: 30 }); + scenarios.push({ kind: "cat", backend }); + } + await ensureBigFile(); + const rows: Record[] = []; + for (const scenario of scenarios) { + const { stdout } = await promisify(execFile)( + process.execPath, + [...process.execArgv, fileURLToPath(import.meta.url), JSON.stringify(scenario)], + { maxBuffer: 1 << 20 }, + ); + rows.push({ ...label(scenario), ...JSON.parse(stdout.trim().split("\n").at(-1)!) }); + } + console.table(rows); +} + +function label(scenario: Scenario): Record { + if (scenario.kind === "rate") { + const { backend, rate, retain, tools, seconds } = scenario; + return { scenario: `${rate} ${retain} x${tools}${seconds === undefined ? "" : ` ${seconds}s`}`, backend }; + } + if (scenario.kind === "cat") return { scenario: "bash cat 1 GiB", backend: scenario.backend }; + const repetitive = scenario.repetitive === true ? " repetitive" : ""; + return { scenario: `api.output 1 GiB ${scenario.retain}${repetitive}`, backend: scenario.backend }; +} + +async function ensureBigFile(): Promise { + if ((await stat(BIG_FILE).catch(() => undefined))?.size === GIB) return; + // Unique lines, written in 64 MiB blocks and cut at exactly 1 GiB. + const file = await open(BIG_FILE, "w"); + try { + let index = 0; + for (let written = 0; written < GIB; ) { + let block = ""; + while (block.length < 64 << 20) block += `Line ${index++}: ${LINE}: ${(index * 7919).toString(16)}\n`; + const bytes = Buffer.from(block).subarray(0, GIB - written); + await file.write(bytes); + written += bytes.length; + } + } finally { + await file.close(); + } +} + +type Metrics = { + ms: number; + liveCommits: number; + liveOpKiB: number; + /** Output writes recorded as a whole-string set, including each slot's first write; the rest are appends/trims. */ + outputSets: number; + commitP50Ms: number; + commitP99Ms: number; + midRoundKiB?: number; + replayMs?: number; + finalKiB?: number; + peakRssMiB: number; + heapAfterGcMiB?: number; +}; + +async function runScenario(scenario: Scenario): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-bench-")); + try { + const path = join(directory, scenario.backend === "sqlite" ? "session.sqlite" : "session"); + const latencies: number[] = []; + const openTimed = async (): Promise => timed(await openStorage(scenario.backend, path), latencies); + let stop = false; + const faux = fauxProvider(); + const models = createModels(); + models.setProvider(faux.provider); + const registry = createRegistry(); + const toolCount = scenario.kind === "rate" ? scenario.tools : 1; + if (scenario.kind === "cat") { + registry.tools.add(createBashTool()); + } else { + const rate = scenario.kind === "rate" ? RATES[scenario.rate] : RATES.high; + registry.tools.add({ + name: "emit", + description: "", + parameters: Type.Object({}), + outputLimits: { retain: scenario.retain }, + execute: async (_args, api, callContext) => { + let counter = 0; + let written = 0; + // The firehose reuses one prebuilt chunk so it measures the Harness, not string building. + const prebuilt = scenario.kind === "firehose" ? lines(api.callId, 0, rate.lines) : undefined; + // Repetitive: identical lines with one numbered line per chunk, so the tail window keeps sliding. + const repeated = "y\n".repeat(32 * 1024 - 8); + while (!stop && (scenario.kind === "rate" || written < GIB)) { + const chunk = + scenario.kind === "firehose" && scenario.repetitive === true + ? `${repeated}#${counter}\n` + : (prebuilt ?? lines(api.callId, counter, rate.lines)); + counter += rate.lines; + api.output(chunk); + written += chunk.length; + await new Promise((resolve) => + rate.pauseMs === 0 ? setImmediate(resolve) : setTimeout(resolve, rate.pauseMs), + ); + } + // A persistent rate scenario stays in flight until close, so reopen replays the round. + if (scenario.kind === "rate" && scenario.backend !== "memory") { + await new Promise((_, reject) => + callContext.abortSignal!.addEventListener("abort", () => reject(callContext.abortSignal!.reason)), + ); + } + return {}; + }, + }); + } + const call = + scenario.kind === "cat" + ? [fauxToolCall("bash", { command: `cat ${BIG_FILE}` }, { id: "c0" })] + : Array.from({ length: toolCount }, (_, index) => fauxToolCall("emit", {}, { id: `c${index}` })); + faux.setResponses([ + fauxAssistantMessage(call, { stopReason: "toolUse" }), + fauxAssistantMessage([fauxText("done")]), + ]); + const env = new NodeExecutionEnv({ cwd: directory }); + let harness = await Harness.open(await openTimed(), { models, registry, env }, context); + const root = await harness.root(context, { + init: async (tx, id) => { + (await tx.doc(ConversationConfig, id)).model = { provider: "faux", modelId: "faux-1" }; + }, + }); + let liveCommits = 0; + let liveOpBytes = 0; + let outputSets = 0; + const observe = (target: Harness): void => { + target.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type !== "document" || change.record.kind !== "pi.live") continue; + liveCommits++; + liveOpBytes += JSON.stringify(change.ops).length; + for (const op of change.ops) if (op[0] === "s" && op[1].at(-1) === "output") outputSets++; + } + }); + }; + observe(harness); + const started = performance.now(); + const submission = await root.submit({ type: "input", content: "go" }, context); + const metrics: Partial = {}; + if (scenario.kind === "rate") { + await new Promise((resolve) => setTimeout(resolve, (scenario.seconds ?? RATE_SECONDS) * 1000)); + stop = true; + if (scenario.backend !== "memory") { + // Let the last progress commit land, then close mid-round and time replay on reopen. + await new Promise((resolve) => setTimeout(resolve, 600)); + metrics.midRoundKiB = await footprint(scenario.backend, path); + await harness.close(context); + const reopened = performance.now(); + harness = await Harness.open(await openTimed(), { models, registry, env }, context); + await harness.snapshot(LiveDoc, root.id, context); + observe(harness); + metrics.replayMs = Math.round(performance.now() - reopened); + harness.resume(); + await (await harness.submission(submission.id, context))!.wait(context); + } else { + await submission.wait(context); + } + } else { + await submission.wait(context); + } + const ms = Math.round(performance.now() - started); + if (scenario.backend !== "memory") metrics.finalKiB = await footprint(scenario.backend, path); + await harness.close(context); + latencies.sort((a, b) => a - b); + const gc = (globalThis as { gc?: () => void }).gc; + gc?.(); + return { + ms, + liveCommits, + liveOpKiB: Math.round(liveOpBytes / 1024), + outputSets, + commitP50Ms: round(latencies[Math.floor(latencies.length * 0.5)] ?? 0), + commitP99Ms: round(latencies[Math.floor(latencies.length * 0.99)] ?? 0), + ...metrics, + peakRssMiB: Math.round(process.resourceUsage().maxRSS / 1024), + heapAfterGcMiB: gc === undefined ? undefined : Math.round(process.memoryUsage().heapUsed / 2 ** 20), + }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +async function openStorage(backend: Backend, path: string): Promise { + if (backend === "memory") return new MemoryStorage(); + if (backend === "sqlite") return openNodeSqliteStorage(path); + return openNodeJsonlStorage(path, context); +} + +/** Measure every Storage commit. */ +function timed(storage: Storage, latencies: number[]): Storage { + return new Proxy(storage, { + get(target, property) { + const value = Reflect.get(target, property, target); + if (property !== "commit" || typeof value !== "function") { + return typeof value === "function" ? value.bind(target) : value; + } + return async (...args: unknown[]) => { + const start = performance.now(); + try { + return await value.apply(target, args); + } finally { + latencies.push(performance.now() - start); + } + }; + }, + }); +} + +async function footprint(backend: Backend, path: string): Promise { + if (backend === "sqlite") { + let bytes = 0; + for (const suffix of ["", "-wal", "-shm"]) bytes += (await stat(path + suffix).catch(() => undefined))?.size ?? 0; + return Math.round(bytes / 1024); + } + let bytes = 0; + for (const file of await readdir(path)) bytes += (await stat(join(path, file))).size; + return Math.round(bytes / 1024); +} + +function lines(prefix: string, first: number, count: number): string { + let text = ""; + for (let index = first; index < first + count; index++) text += `${prefix} ${index}: ${LINE}\n`; + return text; +} + +function round(value: number): number { + return Math.round(value * 100) / 100; +} + +await main(); diff --git a/packages/durable/test/tools.test.ts b/packages/durable/test/tools.test.ts new file mode 100644 index 000000000..f67ae2c3c --- /dev/null +++ b/packages/durable/test/tools.test.ts @@ -0,0 +1,531 @@ +// Ported from packages/agent/test/harness/tools.test.ts and adapted to ToolRegistration: tools take the environment +// from `api.env`, stream through `api.output()`, and report notices as diagnostics instead of content text. +import { mkdirSync, rmSync } from "node:fs"; +import { symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Context, JsonValue } from "@earendil-works/chord"; +import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context"; +import type { + ToolDiagnostic, + ToolExecutionApi, + ToolExecutionResult, + ToolRegistration, +} from "@earendil-works/pi-durable"; +import { applyPatch } from "diff"; +import { afterAll, describe, expect, it } from "vitest"; +import { + type ExecutionEnv, + ExecutionError, + err, + type FileError, + getOrThrow, + type Result, + type ShellExecOptions, + type ShellExecResult, +} from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { detectSupportedImageMimeType } from "../src/tools/image.ts"; +import { createBashTool, createEditTool, createReadTool, createWriteTool } from "../src/tools/index.ts"; +import { DEFAULT_MAX_LINES } from "../src/truncate.ts"; + +const tempDirs: string[] = []; + +afterAll(() => { + for (const dir of tempDirs) rmSync(dir, { recursive: true, force: true }); +}); + +function createTempDir(): string { + const dir = join(tmpdir(), `pi-durable-tools-${Date.now()}-${Math.random().toString(36).slice(2)}`); + mkdirSync(dir, { recursive: true }); + tempDirs.push(dir); + return dir; +} + +function createEnv(): NodeExecutionEnv { + return new NodeExecutionEnv({ cwd: createTempDir() }); +} + +/** A minimal execution API: the environment, collected output and diagnostics, and nothing durable. */ +function fakeApi(env: ExecutionEnv | undefined): { + api: ToolExecutionApi; + output: string[]; + diagnostics: ToolDiagnostic[]; +} { + const output: string[] = []; + const diagnostics: ToolDiagnostic[] = []; + const api = { + taskId: 1, + conversationId: 1, + callId: "call", + env, + output: (chunk: string | Uint8Array) => + output.push(typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)), + diagnostic: (diagnostic: ToolDiagnostic) => diagnostics.push(diagnostic), + details: async () => {}, + } as unknown as ToolExecutionApi; + return { api, output, diagnostics }; +} + +async function run( + tool: ToolRegistration, + args: JsonValue, + env: ExecutionEnv | undefined, + context: Context = BACKGROUND_CONTEXT, +): Promise { + const { api, output, diagnostics } = fakeApi(env); + return { ...(await tool.execute(args, api, context)), output, reported: diagnostics }; +} + +/** Run a tool expected to throw; returns the error with what it streamed and reported first. */ +async function runFailing( + tool: ToolRegistration, + args: JsonValue, + env: ExecutionEnv, +): Promise<{ error: Error; output: string[]; reported: ToolDiagnostic[] }> { + const { api, output, diagnostics } = fakeApi(env); + try { + await tool.execute(args, api, BACKGROUND_CONTEXT); + } catch (error) { + return { error: error as Error, output, reported: diagnostics }; + } + throw new Error("Expected the tool to throw"); +} + +function textOutput(result: ToolExecutionResult): string { + return (result.content ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n"); +} + +function diagnosticText(result: ToolExecutionResult): string { + return (result.diagnostics ?? []).map((diagnostic) => diagnostic.message).join("\n"); +} + +function deferred(): { promise: Promise; resolve: () => void } { + let resolve = () => {}; + const promise = new Promise((resolvePromise) => { + resolve = resolvePromise; + }); + return { promise, resolve }; +} + +function delay(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +class SlowReadExecutionEnv extends NodeExecutionEnv { + override async readTextFile(path: string, context: Context): Promise> { + await delay(20); + return super.readTextFile(path, context); + } +} + +class BlockingWriteExecutionEnv extends NodeExecutionEnv { + readonly firstWriteStarted = deferred(); + readonly finishFirstWrite = deferred(); + secondWriteStarted = false; + + override async writeFile( + path: string, + content: string | Uint8Array, + context: Context, + ): Promise> { + if (content === "first\n") { + this.firstWriteStarted.resolve(); + await this.finishFirstWrite.promise; + } else if (content === "second\n") { + this.secondWriteStarted = true; + } + return super.writeFile(path, content, context); + } +} + +class BlockingEditExecutionEnv extends NodeExecutionEnv { + readonly firstEditWriteStarted = deferred(); + readonly finishFirstEditWrite = deferred(); + firstEditWriteSettled = false; + secondEditWriteStarted = false; + + override async writeFile( + path: string, + content: string | Uint8Array, + context: Context, + ): Promise> { + if (content === "ALPHA\nbeta\n") { + this.firstEditWriteStarted.resolve(); + await this.finishFirstEditWrite.promise; + const result = await super.writeFile(path, content, BACKGROUND_CONTEXT); + this.firstEditWriteSettled = true; + return result; + } + if (content === "ALPHA\nBETA\n" || content === "alpha\nBETA\n") { + this.secondEditWriteStarted = true; + } + return super.writeFile(path, content, context); + } +} + +const TRUNCATED_OUTPUT_LINES = DEFAULT_MAX_LINES + 1; + +class TimeoutOutputExecutionEnv extends NodeExecutionEnv { + override async exec( + _command: string, + options: ShellExecOptions | undefined, + context: Context, + ): Promise> { + const output = `${Array.from({ length: TRUNCATED_OUTPUT_LINES }, (_, index) => `line-${index + 1}`).join("\n")}\n`; + const spillPath = getOrThrow(await this.createTempFile({ prefix: "timeout-", suffix: ".log" }, context)); + getOrThrow(await this.writeFile(spillPath, output, context)); + options?.onOutput?.(output, context); + const error = new ExecutionError("timeout", `timeout:${options?.timeout}`); + error.spillPath = spillPath; + return err(error); + } +} + +describe("durable tools", () => { + it("fail with an ordinary error when no environment is configured", async () => { + await expect(run(createReadTool(), { path: "x" }, undefined)).rejects.toThrow("No execution environment"); + }); + + describe("read", () => { + it.each(["GIF87a", "GIF89a"])("detects the complete %s signature", (signature) => { + expect(detectSupportedImageMimeType(Buffer.from(signature, "ascii"))).toBe("image/gif"); + }); + + it("reads text with offsets and limits and reports continuation as a diagnostic", async () => { + const env = createEnv(); + getOrThrow( + await env.writeFile( + "test.txt", + Array.from({ length: 100 }, (_, index) => `Line ${index + 1}`).join("\n"), + BACKGROUND_CONTEXT, + ), + ); + const result = await run(createReadTool(), { path: "test.txt", offset: 41, limit: 20 }, env); + const output = textOutput(result); + expect(output).not.toContain("Line 40"); + expect(output).toContain("Line 41"); + expect(output).toContain("Line 60"); + expect(output).not.toContain("Line 61"); + expect(output).not.toContain("more lines"); + expect(diagnosticText(result)).toBe("40 more lines in file. Use offset=61 to continue."); + }); + + it("truncates large text by line count", async () => { + const env = createEnv(); + getOrThrow( + await env.writeFile( + "large.txt", + Array.from({ length: 2500 }, (_, index) => `Line ${index + 1}`).join("\n"), + BACKGROUND_CONTEXT, + ), + ); + const result = await run(createReadTool(), { path: "large.txt" }, env); + expect(diagnosticText(result)).toBe("Showing lines 1-2000 of 2500. Use offset=2001 to continue."); + expect(result.diagnostics?.[0]?.code).toBe("truncated"); + expect((result.details as { truncation?: unknown } | undefined)?.truncation).toMatchObject({ + truncated: true, + truncatedBy: "lines", + totalLines: 2500, + outputLines: 2000, + }); + }); + + it("does not count a trailing newline as an extra line at the truncation limit", async () => { + const env = createEnv(); + getOrThrow( + await env.writeFile( + "exact.txt", + `${Array.from({ length: 2000 }, () => "x").join("\n")}\n`, + BACKGROUND_CONTEXT, + ), + ); + const result = await run(createReadTool(), { path: "exact.txt" }, env); + expect(result.details).toBeUndefined(); + expect(result.diagnostics).toEqual([]); + }); + + it("shows the start of a line longer than the byte limit", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("long.txt", `${"é".repeat(40_000)}\nnext\n`, BACKGROUND_CONTEXT)); + const result = await run(createReadTool(), { path: "long.txt" }, env); + const text = textOutput(result); + // Two-byte characters: the cut lands on a character boundary at or below the limit. + expect(text).toBe("é".repeat(25_600)); + expect(diagnosticText(result)).toBe( + "Line 1 is 78.1KB, exceeds the 50.0KB limit; showing its first 50.0KB. Use bash: sed -n '1p' long.txt | tail -c +51201", + ); + expect((result.details as { truncation: object }).truncation).toMatchObject({ + truncated: true, + firstLineExceedsLimit: true, + outputBytes: 51_200, + outputLines: 1, + }); + expect(result.details).not.toHaveProperty("truncation.content"); + }); + + it("rejects offsets beyond the file", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("short.txt", "one\ntwo\nthree", BACKGROUND_CONTEXT)); + await expect(run(createReadTool(), { path: "short.txt", offset: 100 }, env)).rejects.toThrow( + "Offset 100 is beyond end of file (3 lines total)", + ); + }); + + it("reports images by content as unsupported", async () => { + const env = createEnv(); + const png = Uint8Array.from( + Buffer.from( + "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGD4DwABBAEAX+XDSwAAAABJRU5ErkJggg==", + "base64", + ), + ); + getOrThrow(await env.writeFile("image.txt", png, BACKGROUND_CONTEXT)); + const result = await run(createReadTool(), { path: "image.txt" }, env); + expect(result).toMatchObject({ content: [], isError: true }); + expect(diagnosticText(result)).toBe("image.txt is an image (image/png); reading images is not supported"); + }); + }); + + describe("write", () => { + it("writes files and creates parent directories", async () => { + const env = createEnv(); + const result = await run(createWriteTool(), { path: "nested/dir/file.txt", content: "hello" }, env); + expect(textOutput(result)).toBe("Successfully wrote to nested/dir/file.txt"); + expect(getOrThrow(await env.readTextFile("nested/dir/file.txt", BACKGROUND_CONTEXT))).toBe("hello"); + }); + + it("keeps the mutation queue locked until an aborted write settles", async () => { + const env = new BlockingWriteExecutionEnv({ cwd: createTempDir() }); + const tool = createWriteTool(); + const controller = new AbortController(); + const firstWrite = run( + tool, + { path: "file.txt", content: "first\n" }, + env, + withAbortSignal(controller.signal, BACKGROUND_CONTEXT), + ); + await env.firstWriteStarted.promise; + controller.abort(); + const secondWrite = run(tool, { path: "file.txt", content: "second\n" }, env); + await delay(20); + expect(env.secondWriteStarted).toBe(false); + env.finishFirstWrite.resolve(); + await expect(firstWrite).rejects.toThrow(); + await secondWrite; + expect(getOrThrow(await env.readTextFile("file.txt", BACKGROUND_CONTEXT))).toBe("second\n"); + }); + }); + + describe("edit", () => { + it("applies disjoint edits and returns both diff formats", async () => { + const env = createEnv(); + const original = "alpha\nbeta\ngamma\ndelta\n"; + getOrThrow(await env.writeFile("edit.txt", original, BACKGROUND_CONTEXT)); + const result = await run( + createEditTool(), + { + path: "edit.txt", + edits: [ + { oldText: "alpha\n", newText: "ALPHA\n" }, + { oldText: "gamma\n", newText: "GAMMA\n" }, + ], + }, + env, + ); + const details = result.details as { diff: string; patch: string }; + expect(textOutput(result)).toBe("Successfully replaced 2 block(s) in edit.txt."); + expect(details.diff).toContain("ALPHA"); + expect(details.diff).toContain("GAMMA"); + expect(applyPatch(original, details.patch)).toBe("ALPHA\nbeta\nGAMMA\ndelta\n"); + expect(getOrThrow(await env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe("ALPHA\nbeta\nGAMMA\ndelta\n"); + }); + + it("repairs edits sent as a JSON string, a single object, or top-level oldText/newText without mutating them", () => { + const prepare = createEditTool().prepareArguments!; + const edit = { oldText: "a", newText: "b" }; + const asString = { path: "f", edits: JSON.stringify([edit]) }; + expect(prepare(asString)).toEqual({ path: "f", edits: [edit] }); + expect(asString.edits).toBe(JSON.stringify([edit])); + expect(prepare({ path: "f", edits: JSON.stringify(edit) })).toEqual({ path: "f", edits: [edit] }); + expect(prepare({ path: "f", edits: edit })).toEqual({ path: "f", edits: [edit] }); + expect(prepare({ path: "f", edits: [edit], oldText: "c", newText: "d" })).toEqual({ + path: "f", + edits: [edit, { oldText: "c", newText: "d" }], + }); + expect(prepare({ path: "f", edits: "not json" })).toEqual({ path: "f", edits: "not json" }); + }); + + it("matches all edits against the original and rejects overlaps", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("edit.txt", "one\ntwo\nthree\n", BACKGROUND_CONTEXT)); + await expect( + run( + createEditTool(), + { + path: "edit.txt", + edits: [ + { oldText: "one\ntwo\n", newText: "ONE\nTWO\n" }, + { oldText: "two\nthree\n", newText: "TWO\nTHREE\n" }, + ], + }, + env, + ), + ).rejects.toThrow(/overlap/); + expect(getOrThrow(await env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe("one\ntwo\nthree\n"); + }); + + it("rejects missing and duplicate target text", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("edit.txt", "foo foo foo", BACKGROUND_CONTEXT)); + const tool = createEditTool(); + await expect( + run(tool, { path: "edit.txt", edits: [{ oldText: "bar", newText: "baz" }] }, env), + ).rejects.toThrow(/Could not find the exact text/); + await expect( + run(tool, { path: "edit.txt", edits: [{ oldText: "foo", newText: "bar" }] }, env), + ).rejects.toThrow(/Found 3 occurrences/); + }); + + it("keeps the mutation queue locked until an aborted edit write settles", async () => { + const env = new BlockingEditExecutionEnv({ cwd: createTempDir() }); + getOrThrow(await env.writeFile("file.txt", "alpha\nbeta\n", BACKGROUND_CONTEXT)); + const tool = createEditTool(); + const controller = new AbortController(); + const firstEdit = run( + tool, + { path: "file.txt", edits: [{ oldText: "alpha", newText: "ALPHA" }] }, + env, + withAbortSignal(controller.signal, BACKGROUND_CONTEXT), + ); + await env.firstEditWriteStarted.promise; + controller.abort(); + const secondEdit = run(tool, { path: "file.txt", edits: [{ oldText: "beta", newText: "BETA" }] }, env); + await delay(20); + expect(env.secondEditWriteStarted).toBe(false); + env.finishFirstEditWrite.resolve(); + await expect(firstEdit).rejects.toThrow("Operation aborted"); + await secondEdit; + expect(env.firstEditWriteSettled).toBe(true); + expect(getOrThrow(await env.readTextFile("file.txt", BACKGROUND_CONTEXT))).toBe("ALPHA\nBETA\n"); + }); + + it("serializes concurrent edits through canonical and symlink paths", async () => { + const env = new SlowReadExecutionEnv({ cwd: createTempDir() }); + getOrThrow(await env.writeFile("target.txt", "alpha\nbeta\ngamma\n", BACKGROUND_CONTEXT)); + await symlink("target.txt", `${env.cwd}/link.txt`); + const tool = createEditTool(); + await Promise.all([ + run(tool, { path: "target.txt", edits: [{ oldText: "alpha", newText: "ALPHA" }] }, env), + run(tool, { path: "link.txt", edits: [{ oldText: "beta", newText: "BETA" }] }, env), + ]); + expect(getOrThrow(await env.readTextFile("target.txt", BACKGROUND_CONTEXT))).toBe("ALPHA\nBETA\ngamma\n"); + }); + + it("edits regular files through symlinks", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("target.txt", "before\n", BACKGROUND_CONTEXT)); + await symlink("target.txt", `${env.cwd}/link.txt`); + await run(createEditTool(), { path: "link.txt", edits: [{ oldText: "before", newText: "after" }] }, env); + expect(getOrThrow(await env.readTextFile("target.txt", BACKGROUND_CONTEXT))).toBe("after\n"); + }); + + it("preserves BOM and CRLF line endings", async () => { + const env = createEnv(); + getOrThrow(await env.writeFile("edit.txt", "\uFEFFone\r\ntwo\r\n", BACKGROUND_CONTEXT)); + await run(createEditTool(), { path: "edit.txt", edits: [{ oldText: "two", newText: "TWO" }] }, env); + expect(getOrThrow(await env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe("\uFEFFone\r\nTWO\r\n"); + }); + }); + + describe("bash", () => { + it("streams combined stdout and stderr and returns no content of its own", async () => { + const result = await run(createBashTool(), { command: "printf out; printf err >&2" }, createEnv()); + expect(result.output.join("")).toContain("out"); + expect(result.output.join("")).toContain("err"); + expect(result.content).toBeUndefined(); + }); + + it("throws on nonzero exits and timeouts after streaming the output", async () => { + const env = createEnv(); + const tool = createBashTool(); + const failed = await runFailing(tool, { command: "printf failed; exit 7" }, env); + expect(failed.error.message).toBe("Command exited with code 7"); + expect(failed.output.join("")).toBe("failed"); + const slow = await runFailing(tool, { command: "sleep 2", timeout: 0.01 }, env); + expect(slow.error.message).toBe("Command timed out after 0.01 seconds"); + }); + + it("reports the spill of a command that times out", async () => { + const env = new TimeoutOutputExecutionEnv({ cwd: createTempDir() }); + const failed = await runFailing( + createBashTool(), + { command: "emit-output-then-time-out", timeout: 0.05 }, + env, + ); + expect(failed.error.message).toBe("Command timed out after 0.05 seconds"); + const fullOutputPath = failed.reported[0]?.message.match(/^Full output: (.+)$/)?.[1]; + expect(fullOutputPath).toBeDefined(); + const fullOutput = getOrThrow(await env.readTextFile(fullOutputPath!, BACKGROUND_CONTEXT)); + expect(fullOutput).toContain("line-1\nline-2"); + expect(fullOutput).toContain(`line-${DEFAULT_MAX_LINES}\nline-${TRUNCATED_OUTPUT_LINES}`); + }); + + it("prepares command, cwd, and an explicit environment with the call's api", async () => { + const env = new NodeExecutionEnv({ + cwd: createTempDir(), + shellEnv: { PI_BASH_PREPARE_INHERITED: "inherited" }, + }); + getOrThrow(await env.createDir("workspace", undefined, BACKGROUND_CONTEXT)); + const workspace = `${env.cwd}/workspace`; + const controller = new AbortController(); + let receivedEnv: ExecutionEnv | undefined; + let receivedSignal: AbortSignal | undefined; + const tool = createBashTool({ + commandPrefix: "prefix=ready", + prepare: async (execution, api, callContext) => { + receivedEnv = api.env; + receivedSignal = callContext.abortSignal; + execution.cwd = workspace; + execution.env = { PI_BASH_PREPARE_EXPLICIT: "explicit" }; + execution.inheritEnv = false; + execution.command += `\nprintf '%s:%s:%s:%s' "$prefix" "\${PI_BASH_PREPARE_INHERITED-}" "$PI_BASH_PREPARE_EXPLICIT" "$PWD"`; + }, + }); + const result = await run(tool, { command: ":" }, env, withAbortSignal(controller.signal, BACKGROUND_CONTEXT)); + expect(receivedEnv).toBe(env); + expect(receivedSignal).toBe(controller.signal); + expect(result.output.join("")).toBe( + `ready::explicit:${getOrThrow(await env.canonicalPath(workspace, BACKGROUND_CONTEXT))}`, + ); + }); + + it("supports command prefixes", async () => { + const result = await run( + createBashTool({ commandPrefix: "value=hello" }), + { command: "printf $value" }, + createEnv(), + ); + expect(result.output.join("")).toBe("hello"); + }); + + it("streams every byte and spills complete output beyond the default limits", async () => { + const env = createEnv(); + const result = await run( + createBashTool(), + { command: "i=1; while [ $i -le 3000 ]; do echo line-$i; i=$((i + 1)); done" }, + env, + ); + const expected = Array.from({ length: 3000 }, (_, index) => `line-${index + 1}\n`).join(""); + expect(result.output.join("")).toBe(expected); + const fullOutputPath = result.reported[0]?.message.match(/^Full output: (.+)$/)?.[1]; + expect(fullOutputPath).toBeDefined(); + expect(getOrThrow(await env.readTextFile(fullOutputPath!, BACKGROUND_CONTEXT))).toBe(expected); + }); + + it("does not spill output within the limits", async () => { + const result = await run(createBashTool(), { command: "printf small" }, createEnv()); + expect(result.reported).toEqual([]); + }); + }); +}); diff --git a/scripts/check-entry-graphs.mjs b/scripts/check-entry-graphs.mjs index c101a2805..dbde063c4 100644 --- a/scripts/check-entry-graphs.mjs +++ b/scripts/check-entry-graphs.mjs @@ -43,8 +43,9 @@ const BUDGETS = { }, "packages/durable": { ".": { - maxFiles: 50, - forbid: ["packages/ai/src/index.ts", "packages/ai/src/utils/validation.ts", "packages/ai/src/utils/typebox-helpers.ts"], + // The built-in tool task validates arguments with pi-ai's TypeBox-based validation, so TypeBox is allowed. + maxFiles: 52, + forbid: ["packages/ai/src/index.ts", "packages/ai/src/utils/typebox-helpers.ts"], }, }, "packages/agent": {