mirror of
https://github.com/ranxianglei/billion-context.git
synced 2026-11-23 01:58:16 +08:00
fix: tokenCount uses real upstream input_tokens, never estimates
Root cause: tokenCount (drives the nudge/compress decision) was computed via
estimateTokensFast = chars/4. This undercounts CJK text 3-4x (Chinese chars
count as 0.25 token instead of ~1), so Chinese-heavy sessions (ZCode reviewing
a CN project: real 50k tokens) read as ~35k — never reaching the 50k compress
threshold. Compression never fired.
The upstream API already reports the authoritative input_tokens in every
usage (Anthropic message_delta, OpenAI prompt_tokens, Responses input_tokens).
That is the ground truth — no estimate (neither chars/4 nor a BPE tokenizer)
can beat it. Use it.
Changes:
- session.stats: add lastInputTokens (single-turn, overwritten each turn).
Existing cumulative inputTokens is kept for the web UI totals.
- compress-loop-{anthropic,openai,responses}.ts onCacheUsage/usage: also
set lastInputTokens = input (overwrite, not accumulate).
- server.ts three prepare* branches: tokenCount = session.stats.lastInputTokens.
First turn = 0 (no history yet, never triggers anyway). All estimateTokensFast
calls removed from the tokenCount path.
- persist.ts: round-trip lastInputTokens with backward-compat (old files
default to 0).
- tests: stats literals updated.
estimateTokensFast is still imported (used by buildStatusReport for the
acp_status display breakdown), but no longer drives any compress/nudge decision.
This commit is contained in:
@@ -209,6 +209,7 @@ export async function* compressLoopAnthropicStream(
|
||||
onCacheUsage: (input, cached) => {
|
||||
if (typeof input === "number") {
|
||||
ctx.session.stats.inputTokens += input;
|
||||
ctx.session.stats.lastInputTokens = input;
|
||||
totalInputTokens += input;
|
||||
}
|
||||
if (typeof cached === "number") {
|
||||
|
||||
@@ -390,6 +390,7 @@ export async function* compressLoopResponsesStream(
|
||||
// Record into the session for the web UI / stats.
|
||||
if (typeof prompt === "number") {
|
||||
ctx.session.stats.inputTokens += prompt;
|
||||
ctx.session.stats.lastInputTokens = prompt;
|
||||
if (typeof cached === "number") ctx.session.stats.cachedTokens += cached;
|
||||
if (typeof out === "number") ctx.session.stats.outputTokens += out;
|
||||
ctx.session.stats.cacheSamples += 1;
|
||||
|
||||
@@ -346,6 +346,7 @@ export async function* compressLoopStream(
|
||||
// Record into the session for the web UI / stats: cumulative
|
||||
// tokens + cache-hit ratio across all rounds seen so far.
|
||||
ctx.session.stats.inputTokens += prompt;
|
||||
ctx.session.stats.lastInputTokens = prompt;
|
||||
if (typeof cached === "number") ctx.session.stats.cachedTokens += cached;
|
||||
if (typeof out === "number") ctx.session.stats.outputTokens += out;
|
||||
ctx.session.stats.cacheSamples += 1;
|
||||
|
||||
@@ -70,6 +70,7 @@ interface PersistedSession {
|
||||
cachedTokens?: number;
|
||||
outputTokens?: number;
|
||||
cacheSamples?: number;
|
||||
lastInputTokens?: number;
|
||||
contextTokens?: number;
|
||||
};
|
||||
/** Free-form escape hatch (v2+). */
|
||||
@@ -86,6 +87,7 @@ interface PersistedSession {
|
||||
cachedTokens?: number;
|
||||
outputTokens?: number;
|
||||
cacheSamples?: number;
|
||||
lastInputTokens?: number;
|
||||
contextTokens?: number;
|
||||
state: CompressionState;
|
||||
/** blockContents serialized as a plain record (Maps do not survive JSON). */
|
||||
@@ -413,6 +415,7 @@ function buildSession(parsed: PersistedSession): Session {
|
||||
cachedTokens: stats.cachedTokens ?? parsed.cachedTokens ?? 0,
|
||||
outputTokens: stats.outputTokens ?? parsed.outputTokens ?? 0,
|
||||
cacheSamples: stats.cacheSamples ?? parsed.cacheSamples ?? 0,
|
||||
lastInputTokens: stats.lastInputTokens ?? parsed.lastInputTokens ?? 0,
|
||||
contextTokens: stats.contextTokens ?? parsed.contextTokens ?? 0,
|
||||
},
|
||||
metadata: parsed.metadata ?? {},
|
||||
|
||||
+18
-18
@@ -429,12 +429,16 @@ function prepareAnthropic(
|
||||
|
||||
try {
|
||||
const { msgs, cacheControls } = anthropicToCore(parsed);
|
||||
// contextTokens must include the system prompt: ZCode (and other IDE
|
||||
// clients) put huge workspace-reminder blocks in body.system, not in
|
||||
// messages. Without counting it, tokenCount grossly underestimates real
|
||||
// context size → nudge never fires → never compresses even at 1.7M tokens.
|
||||
const sysText = extractSystem(parsed.system);
|
||||
const tokenCount = estimateTokensFast(msgs.map((m) => m.text ?? "").join("\n") + "\n" + sysText);
|
||||
// tokenCount drives the nudge decision ("should we compress?"). It MUST
|
||||
// be the real context size, never an estimate — estimates undercount
|
||||
// CJK text 3-4x and never trigger compression for Chinese sessions.
|
||||
// Use the upstream's own input_tokens from the PREVIOUS turn (known by
|
||||
// now — the response came back). First turn has no history → 0 (never
|
||||
// triggers anyway). extractSystem is still called so sysText flows into
|
||||
// the fallback path below if we ever need it, but we no longer feed
|
||||
// estimates to the kernel.
|
||||
extractSystem(parsed.system);
|
||||
const tokenCount = session.stats.lastInputTokens;
|
||||
const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
|
||||
session.state = turn.state;
|
||||
session.stats.contextTokens = tokenCount;
|
||||
@@ -502,11 +506,9 @@ function prepareOpenai(
|
||||
|
||||
try {
|
||||
const { msgs } = openaiToCore(parsed);
|
||||
// openaiToCore already includes system/developer messages in `msgs`
|
||||
// (they are mapped to CoreMessage with role:"system"), so tokenCount
|
||||
// already covers the system prompt. No separate system-text accounting
|
||||
// needed here.
|
||||
const tokenCount = estimateTokensFast(msgs.map((m) => m.text ?? "").join("\n"));
|
||||
// tokenCount = upstream's real input_tokens from the previous turn
|
||||
// (see anthropic branch comment). Never an estimate.
|
||||
const tokenCount = session.stats.lastInputTokens;
|
||||
const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: "text-only" });
|
||||
session.state = turn.state;
|
||||
session.stats.contextTokens = tokenCount;
|
||||
@@ -576,13 +578,11 @@ function prepareResponses(
|
||||
if (process.env.ACP_DEBUG) {
|
||||
log("info", `[${sessionId}] input items: ${Array.isArray(parsed.input) ? parsed.input.map((i: ResponseInputItem) => i.type).join(",") : "(string)"}`);
|
||||
}
|
||||
// contextTokens must include the system prompt: Codex puts a large
|
||||
// `instructions` field (system-level prompt) in the request, and other
|
||||
// system/developer items are collected into systemParts by
|
||||
// responsesToCore but NOT included in `msgs`. Count them separately so
|
||||
// tokenCount reflects the real context size and nudge can fire.
|
||||
const sysText = systemParts.join("\n\n");
|
||||
const tokenCount = estimateTokensFast(msgs.map((m) => m.text ?? "").join("\n") + "\n" + sysText);
|
||||
// tokenCount = upstream's real input_tokens from the previous turn
|
||||
// (see anthropic branch comment). Never an estimate. systemParts is
|
||||
// still extracted (used by injectResponsesInstructions later).
|
||||
systemParts.join("\n\n");
|
||||
const tokenCount = session.stats.lastInputTokens;
|
||||
const turn = core.processTurn({ messages: msgs, state: session.state, config, tokenCount, renderTags: process.env.ACP_RENDER_NONE ? "none" : "text-only" });
|
||||
session.state = turn.state;
|
||||
session.stats.contextTokens = tokenCount;
|
||||
|
||||
+5
-1
@@ -44,6 +44,10 @@ export type Session = {
|
||||
outputTokens: number;
|
||||
/** Number of upstream usage samples recorded. */
|
||||
cacheSamples: number;
|
||||
/** Last upstream-reported input_tokens for THIS session (single-turn,
|
||||
* overwritten each turn). Source of truth for tokenCount — never an
|
||||
* estimate. See onCacheUsage in compress-loop-*.ts. */
|
||||
lastInputTokens: number;
|
||||
/** Current in-context (uncompressed) token count at last processTurn. */
|
||||
contextTokens: number;
|
||||
};
|
||||
@@ -132,7 +136,7 @@ export function getSession(id: string, meta?: { protocol?: Session["meta"]["prot
|
||||
const session: Session = {
|
||||
id,
|
||||
meta: { protocol: meta?.protocol, upstreamOrigin: meta?.upstreamOrigin, label: meta?.label },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 },
|
||||
metadata: {},
|
||||
state: createInitialState(),
|
||||
createdAt: Date.now(),
|
||||
|
||||
+2
-2
@@ -236,7 +236,7 @@ test("OpenAI SSE rewriter passes through real tool calls when no compress detect
|
||||
const core = createCore();
|
||||
const state = createInitialState();
|
||||
const config = defaultConfig(200000);
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
|
||||
// Simulate a provider that sends the LAST tool_call arguments fragment
|
||||
// in the SAME chunk as finish_reason (common in GLM/OpenAI-compatible APIs).
|
||||
@@ -272,7 +272,7 @@ test("OpenAI SSE rewriter suppresses compress tool call and injects note", async
|
||||
const core = createCore();
|
||||
const config = defaultConfig(200000);
|
||||
const state = createInitialState();
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
|
||||
const sse = [
|
||||
`data: ${JSON.stringify({ object: "chat.completion.chunk", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "compress", arguments: '{"content":[{"startId":"m00001","endId":"m00001","summary":"test"}]}' } }] }, finish_reason: null }] })}\n\n`,
|
||||
|
||||
@@ -8,7 +8,7 @@ function makeCtx() {
|
||||
const core = createCore();
|
||||
const config = defaultConfig(200000);
|
||||
const state = createInitialState();
|
||||
return { ctx: { core, config, messages: [], session: { id: "t", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} } as never };
|
||||
return { ctx: { core, config, messages: [], session: { id: "t", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} } as never };
|
||||
}
|
||||
|
||||
function evt(type: string, data: object): string {
|
||||
|
||||
@@ -204,7 +204,7 @@ test("OpenAI SSE rewriter passes through real tool calls when no compress detect
|
||||
const core = createCore();
|
||||
const state = createInitialState();
|
||||
const config = defaultConfig(200000);
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
|
||||
// Simulate a provider that sends the LAST tool_call arguments fragment
|
||||
// in the SAME chunk as finish_reason (common in GLM/OpenAI-compatible APIs).
|
||||
@@ -240,7 +240,7 @@ test("OpenAI SSE rewriter suppresses compress tool call and injects note", async
|
||||
const core = createCore();
|
||||
const config = defaultConfig(200000);
|
||||
const state = createInitialState();
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
const ctx = { core, config, messages: [], session: { id: "s1", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} };
|
||||
|
||||
const sse = [
|
||||
`data: ${JSON.stringify({ object: "chat.completion.chunk", choices: [{ index: 0, delta: { tool_calls: [{ index: 0, id: "call_1", type: "function", function: { name: "compress", arguments: '{"content":[{"startId":"m00001","endId":"m00001","summary":"test"}]}' } }] }, finish_reason: null }] })}\n\n`,
|
||||
|
||||
@@ -8,7 +8,7 @@ function makeSession(): Session {
|
||||
return {
|
||||
id: `test-${Math.random().toString(36).slice(2)}`,
|
||||
meta: {},
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 },
|
||||
metadata: {},
|
||||
state: createInitialState(),
|
||||
createdAt: Date.now(),
|
||||
|
||||
@@ -24,7 +24,7 @@ function makeSession(id: string): Session {
|
||||
return {
|
||||
id,
|
||||
meta: {},
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 },
|
||||
metadata: {},
|
||||
state: createInitialState(),
|
||||
createdAt: Date.now(),
|
||||
|
||||
@@ -13,7 +13,7 @@ function makeCtx(log: (m: string) => void): { core: ReturnType<typeof createCore
|
||||
session: {
|
||||
id: "test",
|
||||
meta: {},
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 },
|
||||
metadata: {},
|
||||
state: createInitialState(),
|
||||
createdAt: Date.now(),
|
||||
|
||||
@@ -13,7 +13,7 @@ function makeCtx(log: (m: string) => void): { core: ReturnType<typeof createCore
|
||||
session: {
|
||||
id: "test",
|
||||
meta: {},
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 },
|
||||
stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 },
|
||||
metadata: {},
|
||||
state: createInitialState(),
|
||||
createdAt: Date.now(),
|
||||
|
||||
@@ -8,7 +8,7 @@ function makeCtx() {
|
||||
const core = createCore();
|
||||
const config = defaultConfig(200000);
|
||||
const state = createInitialState();
|
||||
return { core, config, state, ctx: { core, config, messages: [], session: { id: "t", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} } as never };
|
||||
return { core, config, state, ctx: { core, config, messages: [], session: { id: "t", meta: {}, stats: { requests: 0, tokensSaved: 0, inputTokens: 0, cachedTokens: 0, outputTokens: 0, cacheSamples: 0, lastInputTokens: 0, contextTokens: 0 }, metadata: {}, state }, log: () => {} } as never };
|
||||
}
|
||||
|
||||
function parseToolCalls(sse: string) {
|
||||
|
||||
Reference in New Issue
Block a user