Files
openbot/scripts/opencode-token-stream-e2e.ts
T
Aleksander DygońandClaude Opus 5.5 168cb1198d Keep an OpenCode answer whole when reasoning streams between its tokens (#3361)
* Keep an ACP answer whole when reasoning streams between its tokens (#3351)

A thought chunk closed the open answer as commentary. A model that streams
its reasoning between the tokens of one answer (seen on a custom OpenCode
endpoint) had its answer cut mid-word into commentary items, so words showed
with a space inside them, such as "B aca". A thought no longer closes the
answer. A tool call or the end of the turn still sets its phase.

Add scripts/opencode-token-stream-e2e.ts: it runs the real OpenCode CLI and
driver against a fake OpenAI-compatible endpoint that streams sub-word
tokens, with and without reasoning between them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Limit the whole-answer rule to OpenCode and keep the harness retry path

Review findings on #3361:
- The harness error report (#1471) needed #appendThought to end the partial
  answer. That branch now ends it itself, so the retried answer replaces it.
- A thought keeps the answer open only for the OpenCode flavor
  (thoughtsInsideAnswer). Other ACP agents keep the old step boundary, so the
  Antigravity failure check on the answer text is not changed.
- Add a fake OpenCode agent and opencode-thought-interleave-e2e to the
  provider fakes CI job. It covers interleaved thoughts and the harness retry.
- The token stream control case now checks the answer text.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-27 09:48:48 +01:00

186 lines
7.6 KiB
TypeScript

// Checks that an answer streamed in sub-word tokens through a custom OpenCode endpoint keeps its words
// whole (#3351). Runs the real OpenCode driver and CLI against a fake OpenAI-compatible server on
// loopback. Cases: plain tokens, tokens with an empty `reasoning_content` beside each one, tokens with
// reasoning between them, and a control where the endpoint itself sends the spaces.
// `bun scripts/opencode-token-stream-e2e.ts [<opencode binary>]`. Without an argument it uses
// `OPENBOT_OPENCODE_PATH`. Writes .openbot-build/opencode-token-stream-e2e/report.json.
import assert from "node:assert/strict";
import { mkdirSync, writeFileSync } from "node:fs";
import { mkdtemp, rm } from "node:fs/promises";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-token-stream-e2e");
const binary = process.argv[2] ?? process.env.OPENBOT_OPENCODE_PATH;
if (!binary) throw new Error("Give the OpenCode binary as the first argument or in OPENBOT_OPENCODE_PATH.");
const root = await mkdtemp(join(tmpdir(), "openbot-opencode-token-stream-"));
const home = join(root, "home");
mkdirSync(home, { recursive: true });
// Before the driver loads: OpenCode keeps its state under the home folder.
process.env.HOME = home;
process.env.XDG_CONFIG_HOME = join(home, ".config");
process.env.XDG_DATA_HOME = join(home, ".local/share");
process.env.XDG_CACHE_HOME = join(home, ".cache");
process.env.XDG_STATE_HOME = join(home, ".local/state");
const { requireProviderDriver } = await import("../src/backend/provider-drivers");
const { runTurn, startThread, stopClients } = await import("./provider-fakes/host");
const { getString, isRecord } = await import("../src/backend/protocol");
type AgentClient = import("../src/backend/agent-client").AgentClient;
const MODEL = "token-model";
const PROVIDER = "token-endpoint";
/** The start of the reported message, cut as a sub-word tokenizer cuts it. */
const TOKENS = [
"B",
"aca",
" gambar",
".",
" G",
"ambar",
":",
" tabel",
" R",
"enc",
"ana",
" (",
"prot",
"otype",
").",
];
const ANSWER = TOKENS.join("");
type Delta = Record<string, string>;
const CASES: Record<string, Delta[]> = {
plain: TOKENS.map((content) => ({ content })),
"empty-reasoning": TOKENS.map((content) => ({ content, reasoning_content: "" })),
"reasoning-between": TOKENS.flatMap((content, index): Delta[] =>
index % 4 === 1 ? [{ reasoning_content: "hmm" }, { content }] : [{ content }],
),
"endpoint-spaces": TOKENS.map((content, index) => ({ content: index === 0 ? content : ` ${content.trimStart()}` })),
};
/** What the control case sends: the space in "B aca" comes from the endpoint, and stays. */
const SPACED = (CASES["endpoint-spaces"] ?? []).map((delta) => delta.content).join("");
function stream(response: ServerResponse, deltas: readonly Delta[]): void {
response.writeHead(200, { "content-type": "text/event-stream" });
const chunk = (delta: unknown, finish: string | null) =>
`data: ${JSON.stringify({ id: "c1", object: "chat.completion.chunk", created: 0, model: MODEL, choices: [{ index: 0, delta, finish_reason: finish }] })}\n\n`;
response.write(chunk({ role: "assistant", content: "" }, null));
for (const delta of deltas) response.write(chunk(delta, null));
response.write(chunk({}, "stop"));
response.write("data: [DONE]\n\n");
response.end();
}
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
let body = "";
request.on("data", (part) => {
body += part;
});
request.on("end", () => {
if (!request.url?.endsWith("/chat/completions")) return void response.writeHead(404).end();
const name = Object.keys(CASES).find((key) => body.includes(`case:${key}`));
// OpenCode's title request also holds the prompt, so it gets the stream of the same case.
stream(response, name ? (CASES[name] ?? []) : [{ content: "Title" }]);
});
});
await new Promise<void>((done) => server.listen(0, "127.0.0.1", done));
const address = server.address();
assert.ok(address && typeof address === "object", "the fake server has no port");
const context = {
apiKey: () => null,
customProviders: () => [
{
id: PROVIDER,
name: "Token endpoint",
baseUrl: `http://127.0.0.1:${address.port}/v1`,
apiKey: "test-key",
models: [{ id: MODEL, name: "Token model" }],
headers: [],
},
],
mcpServers: () => [],
};
interface ItemEvent {
method: string;
id: string | null;
phase: string | null;
text: string | null;
}
interface CaseReport {
name: string;
events: ItemEvent[];
answers: string[];
answer: string;
error: string | null;
}
const report: { binary: string; expected: string; cases: CaseReport[]; result?: "pass" | "fail"; error?: string } = {
binary,
expected: ANSWER,
cases: [],
};
const clients: AgentClient[] = [];
try {
const workspace = join(root, "workspace");
mkdirSync(workspace);
const driver = requireProviderDriver("opencode");
const cli = { executable: binary, version: "test", source: "system" as const };
const client = driver.createClient(cli, 60_000, context, undefined);
clients.push(client);
client.start();
for (const name of Object.keys(CASES)) {
const thread = await startThread(client, { cwd: workspace, model: `${PROVIDER}/${MODEL}` });
const events: ItemEvent[] = [];
const listener = (notification: { method: string; params?: unknown }) => {
if (!notification.method.startsWith("item/") || !isRecord(notification.params)) return;
if (getString(notification.params, "threadId") !== thread) return;
const item = isRecord(notification.params.item) ? notification.params.item : null;
events.push({
method: notification.method,
id: item ? getString(item, "id") : getString(notification.params, "itemId"),
phase: item ? getString(item, "phase") : null,
text: item ? getString(item, "text") : getString(notification.params, "delta"),
});
};
client.on("notification", listener);
const turn = await runTurn(client, thread, { text: `case:${name}`, timeoutMs: 60_000 });
client.off("notification", listener);
const answers = events
.filter((event) => event.method === "item/completed" && event.phase !== "commentary" && event.text)
.map((event) => event.text ?? "");
report.cases.push({ name, events, answers, answer: answers.join(""), error: turn.error });
assert.equal(turn.error, null, `${name}: the turn failed`);
if (name === "endpoint-spaces") {
assert.deepEqual(answers, [SPACED], `${name}: the answer keeps the endpoint's own spaces`);
continue;
}
assert.deepEqual(answers, [ANSWER], `${name}: the answer is one item with whole words`);
if (name === "reasoning-between") {
const thoughts = events.filter((event) => event.method === "item/completed" && event.phase === "commentary");
assert.ok(thoughts.length > 0, `${name}: the reasoning stays a thought`);
assert.ok(
thoughts.every((event) => event.id?.includes(":thought")),
`${name}: no answer text becomes commentary`,
);
}
}
report.result = "pass";
} catch (error) {
report.result = "fail";
report.error = error instanceof Error ? error.message : String(error);
throw error;
} finally {
await stopClients(clients);
server.close();
await rm(root, { recursive: true, force: true });
mkdirSync(OUT, { recursive: true });
writeFileSync(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
}
process.stdout.write(`PASS opencode-token-stream-e2e: ${join(OUT, "report.json")}\n`);