Files
openbot/scripts/opencode-custom-model-e2e.ts
T
Aleksander DygońandClaude Opus 5.5 0ced348d93 Let each custom provider model set its reasoning efforts (#5138)
* Let each custom provider model set its reasoning efforts

A custom model on OpenCode always showed Low, Medium and High, and a custom
model on Pi could not reason, because Pi got `reasoning: false`. The form had
no field for the efforts that an endpoint accepts.

- Add an optional `reasoningEfforts` list to each custom model (none, minimal,
  low, medium, high, xhigh, max). It stays on this computer, like the context
  window: the Team API save and list keep their released model fields.
- OpenCode gets the list as `variants`. OpenCode 2.2.5 drops a `reasoning`
  key from this config shape, and listed variants replace its own set.
- Pi gets `reasoning: true` and a `thinkingLevelMap` that sets each level that
  the list does not have to null, so Pi offers only the listed levels.
- The form shows the efforts under each model row. With none selected, the
  harness keeps its own behavior.
- Instance transfer keeps the list.
- The OpenCode and Pi custom endpoint E2E scripts check the listed efforts and
  the `reasoning_effort` that a turn sends.

Input kinds (images, PDF) from the same issue are not in this change.

Refs #4154

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* Fix review findings on custom model efforts

- Send the invalid efforts error with `sourceText`, in all 14 languages.
- Keep the efforts of a found model when the user clears and checks it again.
- Make the preview list the saved efforts, as OpenCode and Pi do.
- Change the efforts list in place, and give each row its index as an
  accessor, so that a removed row does not rebuild the rows after it.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-05 12:35:25 +01:00

327 lines
15 KiB
TypeScript

// Runs the real `opencode` provider driver and the real OpenCode binary against a fake
// OpenAI-compatible server with two models, as a llama-swap user has them: the user's own OpenCode
// config names one as `model` and `small_model`, and the agent runs on the other. A local server like llama-swap loads
// the model that each request names, so every request must name the agent's model.
// A second endpoint needs an API key: a turn must send the saved key, and a refused or missing key
// must name the endpoint, not an OpenCode sign-in.
// Three more models carry #4234's shapes: a saved output cap must reach the request verbatim, a
// blank cap on a 128K window (the corrected repro) must get the 16,384 guess rather than OpenCode's
// 32,000, and a small window must get half the window.
// One more model saves its efforts (#4154): OpenCode must list exactly those efforts, and a turn at
// `max` must send `reasoning_effort: "max"`. A model with no saved efforts keeps OpenCode's own set.
// `OPENCODE_CLI=<opencode> bun run test:opencode-custom-model`.
// Writes .openbot-build/opencode-custom-model/report.json.
import assert from "node:assert/strict";
import { execFile } from "node:child_process";
import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises";
import { createServer, type IncomingMessage, type ServerResponse } from "node:http";
import { createServer as createTcpServer } from "node:net";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { promisify } from "node:util";
import { sourceText } from "@openbot/i18n/source";
import type { AgentClient } from "../src/backend/agent-client";
import { resolveOpencodeCli } from "../src/backend/cli";
import { runCauseEffect } from "../src/backend/effect-boundary";
import type { CustomProviderConfig } from "../src/backend/opencode-config";
import { getArray, getString, isRecord } from "../src/backend/protocol";
import { requireProviderDriver } from "../src/backend/provider-drivers";
import { runTurn, startThread, stopClients } from "./provider-fakes/host";
const OUT = resolve(import.meta.dirname, "../.openbot-build/opencode-custom-model");
const ENDPOINT = "spark";
const DEFAULT_MODEL = "default-model";
const AGENT_MODEL = "agent-model";
const KEYED_ENDPOINT = "vault";
const KEYED_PATH = "/keyed/v1";
const KEY = "e2e-endpoint-key";
/** #4234's corrected repro with the cap saved: a 128K window whose real output cap is 16K. */
const CAPPED_MODEL = "capped-model";
/** The same 128K window with the cap left blank: the guess must not exceed 16,384. */
const WIDE_MODEL = "wide-model";
/** #4234 as first filed: a 16K window, where half the window is the only budget that fits. */
const GUESSED_MODEL = "guessed-model";
/** #4154: an endpoint that accepts Low, High and Max for this model, and no other effort. */
const EFFORT_MODEL = "effort-model";
const SAVED_EFFORTS = ["low", "high", "max"] as const;
interface Request {
path: string;
model: string | null;
stream: boolean;
/** For the keyed endpoint: whether the request carried its key. */
auth?: "key" | "other" | "none";
/** The output budget OpenCode asked for; absent when the endpoint protocol has no such field. */
maxCompletionTokens: number | null;
/** The `reasoning_effort` that the request sent, or null. */
reasoningEffort: string | null;
}
interface Report {
passed: boolean;
openCodeVersion?: string;
models?: string[];
/** The efforts that OpenBot listed for each model. */
efforts?: Record<string, string[]>;
requests: Request[];
turns: { reply: string; error: string | null; status: string | null }[];
error?: string;
}
const report: Report = { passed: false, requests: [], turns: [] };
const root = await mkdtemp(join(tmpdir(), "openbot-opencode-model-e2e-"));
const clients: AgentClient[] = [];
const run = promisify(execFile);
/** The model and the stream flag of one chat request. */
async function readRequest(request: IncomingMessage): Promise<Omit<Request, "path" | "auth">> {
const chunks: Buffer[] = [];
for await (const part of request) chunks.push(Buffer.from(part));
const text = Buffer.concat(chunks).toString("utf8");
const value = text ? JSON.parse(text) : null;
const budget = isRecord(value) ? value.max_completion_tokens : null;
return {
model: getString(value, "model"),
stream: isRecord(value) && value.stream === true,
maxCompletionTokens: typeof budget === "number" ? budget : null,
reasoningEffort: getString(value, "reasoning_effort"),
};
}
function chunk(model: string, content: string | null): string {
const choice = content
? { index: 0, delta: { role: "assistant", content }, finish_reason: null }
: { index: 0, delta: {}, finish_reason: "stop" };
const value = { id: "chatcmpl-e2e", object: "chat.completion.chunk", created: 0, model, choices: [choice] };
return `data: ${JSON.stringify(value)}\n\n`;
}
const server = createServer((request: IncomingMessage, response: ServerResponse) => {
void (async () => {
const read = await readRequest(request);
const { model, stream } = read;
const path = request.url ?? "";
if (!path.startsWith(KEYED_PATH)) report.requests.push({ path, ...read });
else {
const header = request.headers.authorization;
const auth = header === `Bearer ${KEY}` ? "key" : header ? "other" : "none";
report.requests.push({ path, ...read, auth });
if (auth !== "key") {
// TokenMe's answer to a missing or refused key, which OpenCode 2.2.5 reports as `-32000`.
response.writeHead(401, { "content-type": "application/json" });
response.end(JSON.stringify({ error: { message: "The API key is invalid", type: "authentication_error" } }));
return;
}
}
const reply = `reply from ${model}`;
if (!stream) {
response.writeHead(200, { "content-type": "application/json" });
response.end(
JSON.stringify({
id: "chatcmpl-e2e",
object: "chat.completion",
created: 0,
model,
choices: [{ index: 0, message: { role: "assistant", content: reply }, finish_reason: "stop" }],
}),
);
return;
}
response.writeHead(200, { "content-type": "text/event-stream", "cache-control": "no-cache" });
response.write(chunk(model ?? "", reply));
response.write(chunk(model ?? "", null));
response.end("data: [DONE]\n\n");
})().catch((error: unknown) => {
response.writeHead(500);
response.end(String(error));
});
});
/** A free TCP port for OpenCode's managed service, so the test does not meet a running OpenBot's. */
async function freePort(): Promise<number> {
const probe = createTcpServer();
await new Promise<void>((listening) => probe.listen(0, "127.0.0.1", listening));
const address = probe.address();
await new Promise<void>((closed) => probe.close(() => closed()));
if (!address || typeof address === "string") throw new Error("The port probe has no port.");
return address.port;
}
let cliPath: string | null = null;
try {
await new Promise<void>((listening) => server.listen(0, "127.0.0.1", listening));
const address = server.address();
if (!address || typeof address === "string") throw new Error("The fake endpoint has no port.");
const origin = `http://127.0.0.1:${address.port}`;
const baseUrl = `${origin}/v1`;
// OpenCode must not read the developer's own config, keys or sessions.
for (const [name, folder] of [
["HOME", "home"],
["XDG_CONFIG_HOME", "config"],
["XDG_DATA_HOME", "data"],
["XDG_STATE_HOME", "state"],
["XDG_CACHE_HOME", "cache"],
] as const) {
process.env[name] = join(root, folder);
await mkdir(join(root, folder), { recursive: true });
}
// The user's own OpenCode config names a default and a small model of the same server, as a
// llama-swap user's can. OpenCode asks the small model for a title unless OpenBot turns titles off.
await mkdir(join(root, "config", "opencode"), { recursive: true });
await writeFile(
join(root, "config", "opencode", "opencode.json"),
`${JSON.stringify(
{
$schema: "https://opencode.ai/config.json",
model: `${ENDPOINT}/${DEFAULT_MODEL}`,
small_model: `${ENDPOINT}/${DEFAULT_MODEL}`,
},
null,
2,
)}\n`,
);
const workspace = join(root, "workspace");
await mkdir(workspace, { recursive: true });
const endpoint: CustomProviderConfig = {
id: ENDPOINT,
name: "Spark",
baseUrl,
apiKey: null,
headers: [],
models: [
{ id: DEFAULT_MODEL, name: "Default model" },
{ id: AGENT_MODEL, name: "Agent model" },
{ id: CAPPED_MODEL, name: "Capped model", contextWindow: 131_072, maxOutput: 16_384 },
{ id: WIDE_MODEL, name: "Wide model", contextWindow: 131_072 },
{ id: GUESSED_MODEL, name: "Guessed model", contextWindow: 16_384 },
{ id: EFFORT_MODEL, name: "Effort model", reasoningEfforts: [...SAVED_EFFORTS] },
],
};
const cli = await runCauseEffect(resolveOpencodeCli({ bundledExecutable: process.env.OPENCODE_CLI }));
report.openCodeVersion = cli.version;
cliPath = cli.executable;
await run(cli.executable, ["service", "set", "port", String(await freePort())], { env: process.env });
const client = requireProviderDriver("opencode").createClient(cli, 60_000, {
apiKey: () => null,
customProviders: () => [endpoint],
mcpServers: () => [],
providerStateDirectory: join(root, "provider-state"),
});
clients.push(client);
client.start();
const listed = await runCauseEffect(client.request("model/list", {}, (value) => value));
report.models = getArray(listed, "data").flatMap((model) => getString(model, "model") ?? []);
assert.ok(report.models.includes(`${ENDPOINT}/${AGENT_MODEL}`), "OpenCode must list the agent's model.");
report.efforts = Object.fromEntries(
getArray(listed, "data").map((model) => [
getString(model, "model") ?? "",
getArray(model, "supportedReasoningEfforts").flatMap((effort) => getString(effort, "reasoningEffort") ?? []),
]),
);
assert.deepEqual(
report.efforts[`${ENDPOINT}/${EFFORT_MODEL}`],
[...SAVED_EFFORTS],
"A model with saved efforts must list exactly those efforts.",
);
assert.deepEqual(
report.efforts[`${ENDPOINT}/${AGENT_MODEL}`],
["low", "medium", "high"],
"A model with no saved efforts must keep OpenCode's own efforts.",
);
const thread = await startThread(client, { cwd: workspace, model: `${ENDPOINT}/${AGENT_MODEL}` });
for (const text of ["Say one word.", "Say another word."]) {
const turn = await runTurn(client, thread, { text, timeoutMs: 120_000 });
report.turns.push(turn);
assert.equal(turn.error, null, `The turn failed: ${turn.error}`);
}
// #4234's shapes: the saved cap verbatim, and the half-window guess for a small window. An
// explicit limit is served as written on both the pinned 2.1.1 runtime and 2.2.5; only an
// unset one falls back to OpenCode's own pick (#4234's `max_completion_tokens: 32000`).
const budgets: ReadonlyArray<readonly [string, number]> = [
[CAPPED_MODEL, 16_384],
[WIDE_MODEL, 16_384],
[GUESSED_MODEL, 8_192],
];
for (const [model] of budgets) {
const modelThread = await startThread(client, { cwd: workspace, model: `${ENDPOINT}/${model}` });
const turn = await runTurn(client, modelThread, { text: "Say one word.", timeoutMs: 120_000 });
report.turns.push(turn);
assert.equal(turn.error, null, `The turn on ${model} failed: ${turn.error}`);
}
const effortThread = await startThread(client, { cwd: workspace, model: `${ENDPOINT}/${EFFORT_MODEL}` });
const effortTurn = await runTurn(client, effortThread, { effort: "max", text: "Say one word.", timeoutMs: 120_000 });
report.turns.push(effortTurn);
assert.equal(effortTurn.error, null, `The turn on ${EFFORT_MODEL} failed: ${effortTurn.error}`);
const effortRequests = report.requests.filter((request) => request.model === EFFORT_MODEL);
assert.ok(effortRequests.length > 0, `The endpoint got no request for ${EFFORT_MODEL}.`);
for (const request of effortRequests) {
assert.equal(request.reasoningEffort, "max", `${EFFORT_MODEL} must send the effort of the turn.`);
}
assert.ok(report.requests.length > 0, "The endpoint got no request.");
const known = new Set([AGENT_MODEL, EFFORT_MODEL, ...budgets.map(([model]) => model)]);
const others = report.requests.filter(
(request) => request.auth === undefined && request.model !== null && !known.has(request.model),
);
assert.deepEqual(others, [], "Each request must name a model OpenBot configured.");
for (const [model, budget] of budgets) {
const seen = report.requests.filter((request) => request.model === model);
assert.ok(seen.length > 0, `The endpoint got no request for ${model}.`);
for (const request of seen) {
assert.equal(
request.maxCompletionTokens,
budget,
`${model} must ask for the ${budget}-token budget OpenBot wrote, not ${request.maxCompletionTokens}.`,
);
}
}
const keyedModel = `${KEYED_ENDPOINT}/${AGENT_MODEL}`;
const keyedTurn = async (apiKey: string | null) => {
const keyed = requireProviderDriver("opencode").createClient(cli, 60_000, {
apiKey: () => null,
customProviders: () => [
{ ...endpoint, id: KEYED_ENDPOINT, name: "Vault", baseUrl: `${origin}${KEYED_PATH}`, apiKey },
],
mcpServers: () => [],
providerStateDirectory: join(root, "provider-state"),
});
clients.push(keyed);
keyed.start();
const keyedThread = await startThread(keyed, { cwd: workspace, model: keyedModel });
const turn = await runTurn(keyed, keyedThread, { text: "Say one word.", timeoutMs: 120_000 });
report.turns.push(turn);
return turn;
};
const sent = await keyedTurn(KEY);
assert.equal(sent.error, null, `The turn with the saved key failed: ${sent.error}`);
assert.ok(
report.requests.some((request) => request.auth === "key"),
"The endpoint must get the saved key.",
);
const refused = await keyedTurn("e2e-refused-key");
assert.equal(refused.error, sourceText("error.provider.customEndpointKeyRefused", { provider: "Vault" }));
const missing = await keyedTurn(null);
assert.equal(missing.error, sourceText("error.provider.customEndpointKeyMissing", { provider: "Vault" }));
report.passed = true;
} catch (error) {
report.error = error instanceof Error ? (error.stack ?? error.message) : String(error);
process.exitCode = 1;
} finally {
await stopClients(clients);
if (cliPath) await run(cliPath, ["service", "stop"], { env: process.env }).catch(() => undefined);
server.close();
await rm(root, { recursive: true, force: true });
await mkdir(OUT, { recursive: true });
await writeFile(join(OUT, "report.json"), `${JSON.stringify(report, null, 2)}\n`);
process.stdout.write(`${report.passed ? "passed" : "failed"}: ${join(OUT, "report.json")}\n`);
}