mirror of
https://github.com/ranxianglei/billion-context.git
synced 2026-11-23 01:58:16 +08:00
This commit is contained in:
@@ -10,6 +10,7 @@ Versions follow the merge of a `*_release-v*` branch; CI publishes to npm on tag
|
||||
- **系统架构文档体系落地(owner 指令 2026-11-08):`docs/architecture/` 成为实时维护的架构枢纽**:新增 `docs/architecture.md` / `docs/architecture.zh-CN.md` 总枢纽(整体架构图中英双版转正 + 8 模块总表),首个样板模块页 `docs/architecture/storage.{md,zh-CN.md}`(统一存储:职责/原理/工作机制/硬不变量,配模块图中英双版)。文件级地图 `reference/architecture.md` 补齐 125 个缺失条目(新存储层 / marker / fastpath / lanes 等全部新模块)并升级为 CI 强制全量覆盖。新增 CI 车道 `architecture-docs`(`tools/check-architecture-docs.mjs`):地图覆盖全 `src/` 文件、总表↔页面成对、图 svg 优先 + png 备选齐备、SVG 基础良构(未转义 `&` 曾让英文图线上破图)。AGENTS.md「Documentation Layout」新增实时维护规则:改结构 ⇒ 同 PR 更新页 + 图 + 总表。其余 7 个模块页(pipeline / lanes / kernel / identity / fastpaths / webui / update)同一 PR 全部补齐——总表 8 行全 ✅(每页 = 中英双版文档 + 中英双版图)。
|
||||
|
||||
### Fixed
|
||||
- **超窗请求不再裸发上游:三道转发闸按「真实出站体」计量(#5000 #4543 #5885),未压缩大请求纳入 32MiB 成本带准入(#5794)**:旧判据只量「消息核心文本的字节」,看不见 system/tools/tool_calls 参数——#5000 的 55.2MiB 体(84 个工具)与 #4543 的 4.1MiB CJK 体(1,121,674 tok vs 1M 窗口)都能让全部闸门沉默,然后「转发一次取证」把必然 400 的体真发出去烧钱。现在三个「转发仲裁点」(压缩后再检、#1839 仲裁臂、出站熔断)改按出站体的**逐字符 token 计量**(system+tools+arguments+messages,CJK 感知,#554 侧车道同源尺子)对照**真实模型窗口**本地 413,不再裸发;取证探针闸保持 #2490 字节契约不动(那里的一次 400 是设计内的证据回路,见 #2313/#1001)。另:成本准入旧阈值 = 5GiB 收件上限,2 槽位预算形同虚设,#5794 的 51.2MiB/轮未压缩请求(每轮独占事件循环 5.5–7.2s、占本机 CPU 54.6% → 747s FATAL 自重启)从此被拦;未压缩体现在按 32MiB 成本带准入(运维显式调低收件上限仍逐字节生效)。tests:issue4079/issue4185/request-body-budget 新增实测档 e2e 与 32MiB 顺序用例;既有 89+67 用例全绿。#5885 的 modelContextLimit 被忽略属独立缺陷,另行跟进。
|
||||
- **dsh 闸门:必然被上游拒绝的超窗请求不再裸发直连(#5613 安全网 ①②③)**:`none`/`threw` 归属车道(如经网关的 ds-harness-remote)上的超窗历史请求,旧行为二选一——要么按乐观的 bytes/4 估算判「不超窗」而直接裸发上游(实测计费偏差≈×1.14,估算系统性偏低;真实现场把 max_tokens 挤到 1、乃至上游 400 后整会话报废),要么即便超窗也没有任何持续留痕(只记第一行之后全静默)。现在双档计价:原有 bytes/4 下限继续标记「必然超窗」,另加 ×1.14 校准估算档(会话 ledger 实测偏差,常数 `CALIBRATED_TEXT_TOKEN_RATIO`,src/util.ts),过线即本地 422 拒绝而不是骑 #1117 去赌上游——小体量的合法直通语义(sidecar 短调用等)逐字节保留,未知尺寸流式体保持 fail-open。每次裸发 DIRECT 现在都带体积(`this body N B`)或「尺寸未知、超窗熔断未跑」明示,按 首次/状态变化/每第10次 落 durable 日志(bili.log,[dsh-client] 通道)——连续泄漏环路从此必有痕。tests/dsh-native.test.ts:#2924/#4068 双臂升级为双档断言+校准边界钉(3,508,768 B 直通 / 3,508,772 B 拒)、新增留痕节拍回归、#4111/#4912 谓词改名对齐。
|
||||
- **全量压缩与 persona 文本漂移同轮发生时,主会话不再分叉到 `|sub:` 副车道(#5719)**:dsh 全量压缩 checkpoint(#3733 authored checkpoint,把整段历史头替换成一条 user 消息)与系统提示漂移(如 dsh 重启后 GUI 端口重分配写进 system prompt)落在同一轮时,既有的全部位置型 re-anchor 逃生结构性不可达——被记录的尾部按定义已被抹掉、incoming 比记录更短、压缩后首个模型步零 assistant——于是活着的会话确定性地分叉到 `<id>|sub:<fp16(新tag)>`:父车道记录冻结、副车道从零起步(resume-inheritance miss)、事故当天又触发 3 次再压缩 + 1 次 REFUSED turn + 2 次 max-tokens 截断。根因=判定器要求一种全量压缩按定义抹掉的字节级续接证据,而唯一能存活压缩的自足证明——bili 自己铸造的 checkpoint 内嵌的 `conv="<rawId>"` 自声明——从未被读取。现在 tagChanged 分支最前新增 E5 级:**第一条** user 消息携带我方 `\x3cbili-snapshot v1 …\x3e` 身份头且其 conv 与当前 id 逐字相等 → 直接 re-anchor 回主车道(record() 同步更新 tagHash,后续请求即普通 continue)。外部 persona 引用 checkpoint 的情形照旧分叉:要么根本到不了判定层(sideRequestLike 门,#4438),要么标记 conv 不符 / 不在第一条 user 消息。**旧行为→新行为**:该形状从「必然分叉」变为「带自声明时 re-anchor 回主车道」;无标记的形状逐字节不变(回归测试钉住每一既有 rung 的 reason 文案)。tests/issue5719-self-checkpoint-reanchor.test.ts(验收标准 1/2/3 + 四种 wire 形状 + foreign-quote 边界)。
|
||||
|
||||
|
||||
@@ -5,6 +5,19 @@ export const MAX_DECODED_REQUEST_BYTES = 2 * MAX_REQUEST_BYTES;
|
||||
const MAX_LARGE_DECODED_REQUESTS = 2;
|
||||
let largeDecodedRequests = 0;
|
||||
|
||||
// #5794: the slot budget above was keyed on limitsReceiveBytes() — whose #3128
|
||||
// default is 5 GiB — so it could only count a body OVER 5 GiB and the 2-slot
|
||||
// cap was effectively dead: #5794's 51.2 MiB / 6s-ingest request (29,619
|
||||
// messages, 54.6% of all local CPU, 747 s event-loop FATAL → #4762 self-restart)
|
||||
// never consumed a slot. Key the budget on a cost-meaningful threshold
|
||||
// instead: a decoded body this size costs seconds of single-threaded
|
||||
// ingest+parse per request, so at most TWO such bodies may be admitted at once
|
||||
// and the third gets 503 retry-after. The Math.min with limitsReceiveBytes()
|
||||
// keeps any operator-set SMALLER receive cap binding (strictly stricter, never
|
||||
// looser than before).
|
||||
|
||||
export const LARGE_DECODED_ADMISSION_BYTES = 32 * 1024 * 1024;
|
||||
|
||||
export class DecodedRequestBusyError extends Error {
|
||||
constructor() {
|
||||
super("large decoded request budget is busy; retry after an active request finishes");
|
||||
@@ -12,11 +25,13 @@ export class DecodedRequestBusyError extends Error {
|
||||
}
|
||||
}
|
||||
|
||||
/** 只限制超过配置的接收预算(limitsReceiveBytes)的解压请求;普通请求不占超大正文名额。 */
|
||||
/** #5794:超过 min(limitsReceiveBytes(), 32MiB) 的解码请求占用超大正文名额;
|
||||
* 普通请求不占。阈值取 min 保持单调:运维把接收预算调小时仍按更小者生效,
|
||||
* 同时补上 32MiB 成本带(51.2MiB 的请求体一次摄入口约 6s)——旧阈值默认 5GiB,名额实际从未生效。 */
|
||||
export class DecodedRequestAdmission {
|
||||
private admitted = false;
|
||||
observe(bytes: number): void {
|
||||
if (bytes <= limitsReceiveBytes() || this.admitted) return;
|
||||
if (bytes <= Math.min(limitsReceiveBytes(), LARGE_DECODED_ADMISSION_BYTES) || this.admitted) return;
|
||||
if (largeDecodedRequests >= MAX_LARGE_DECODED_REQUESTS) throw new DecodedRequestBusyError();
|
||||
largeDecodedRequests++;
|
||||
this.admitted = true;
|
||||
|
||||
+91
-9
@@ -3733,6 +3733,32 @@ export function buildWindowShrinkNote(args: {
|
||||
return head + `it was aligned down to codex's own window perception; set compress.modelContextLimit explicitly if your upstream serves the larger window.`;
|
||||
}
|
||||
|
||||
// #5000/#4543: the core-text byte floor (estimateCoreMessagesUpperBytes) prices
|
||||
// only m.text — tools[], tool_calls[].function.arguments, the system prompt and
|
||||
// the JSON envelope are invisible to it, so a body can ride multi-MB over the
|
||||
// window on the wire while every projection-based doom gate stays silent
|
||||
// (incident #5000: 55.2 MiB wire body whose core text cleared the floor;
|
||||
// #4543: a 4.1 MiB CJK body the 7.5 B/tok floor can only see past ~7.5 MB on a
|
||||
// 1M window). Price the ACTUAL outbound body instead: the same CJK-aware leaf
|
||||
// walk the #554 side-request guard uses (estimateRawBodyTokens) counts every
|
||||
// string field of the serialized request — the conservative overcount (ids,
|
||||
// roles, structural strings) is the right bias for a fail-closed gate. Image
|
||||
// tokens are charged separately by the caller (they sit in binary-carrying
|
||||
// keys the walk excludes by design). Unparseable bodies return undefined —
|
||||
// callers keep the byte-floor fallback for them.
|
||||
export function measuredOutboundTokens(prepared: { body: string | Buffer; parsedBody?: unknown }): number | undefined {
|
||||
let parsed: unknown = prepared.parsedBody;
|
||||
if (parsed === undefined) {
|
||||
const raw = typeof prepared.body === "string" ? prepared.body : prepared.body.toString("utf8");
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
return estimateRawBodyTokens(parsed);
|
||||
}
|
||||
|
||||
export async function preflightCompressIfNeeded(
|
||||
prepared: Prepared,
|
||||
runPrepare: () => Promise<Prepared>,
|
||||
@@ -3986,6 +4012,17 @@ export async function preflightCompressIfNeeded(
|
||||
// the operator can verify it by hand. Bytes, not chars: CJK monsters
|
||||
// are char-cheap (1 char ≈ 1 token already) but byte-doomed; every legit
|
||||
// fitting English payload sits ~4× under the ratio bar.
|
||||
// #5000/#4543: this probe gate keeps the #2490 byte-floor contract
|
||||
// (7.5 B/tok = the densest REAL billing observed, the most conservative
|
||||
// density; the 2MB absolute keeps experiment-sized payloads probing —
|
||||
// the #1001 shape, where real tokenizers compress past any byte-ratio
|
||||
// bar, must stay able to bet one request and settle a usage baseline).
|
||||
// A measured-token tier lives at the gates where a forward is
|
||||
// arbitration of a body preflight already tried to shrink: the #1839
|
||||
// forward-once arm below, the #3146/#4185 post-fold block further down,
|
||||
// and the egress fuse in relay.ts — not here, where the 400 of a probe
|
||||
// is the designed #2313 evidence (refusal record, model/origin-scoped,
|
||||
// TTL) rather than a guaranteed waste.
|
||||
const coreTextBytes = estimateCoreMessagesUpperBytes(prepared.processedMessages);
|
||||
if (coreTextBytes > Math.max(DOOMED_PROBE_ABSOLUTE_BYTES, limit * DOOMED_PROBE_BYTES_PER_TOKEN)) {
|
||||
const byteFloorTokens = Math.ceil(coreTextBytes / DOOMED_PROBE_BYTES_PER_TOKEN);
|
||||
@@ -4347,6 +4384,25 @@ export async function preflightCompressIfNeeded(
|
||||
}
|
||||
return outbound;
|
||||
}
|
||||
// #5000/#4543/#5885: parse the rebuilt outbound ONCE, up front, for the
|
||||
// measured-body doom tiers below (the #1839 arm and the #3146/#4185 gate)
|
||||
// AND for the output-budget read nativeFitAfterClamp needs. The meter is
|
||||
// the #554 side-request guard's own: it walks every string field of the
|
||||
// serialized body (system + tools + arguments + messages, CJK-aware), so
|
||||
// unlike the m.text byte floor it sees the mass that actually rides the
|
||||
// wire. Parse failure leaves both readings undefined — the byte-floor
|
||||
// fallbacks stay binding for those bodies.
|
||||
let outboundBudgetTokens: number | undefined;
|
||||
let outboundRawTokens: number | undefined;
|
||||
try {
|
||||
const outBody: unknown = JSON.parse(typeof outbound.body === "string" ? outbound.body : outbound.body.toString("utf8"));
|
||||
outboundRawTokens = estimateRawBodyTokens(outBody);
|
||||
const outField = outputBudgetField(outBody);
|
||||
const rawBudget = outField !== null ? readOutputBudget(outBody as Record<string, unknown>, outField) : undefined;
|
||||
if (typeof rawBudget === "number" && rawBudget > 0) outboundBudgetTokens = rawBudget;
|
||||
} catch {
|
||||
outboundBudgetTokens = undefined;
|
||||
}
|
||||
// #1839: the two measurements disagree — preflight's own final view
|
||||
// (post-fold content + images + wire overhead) fits, but the fresh
|
||||
// normal-config rebuild measures over. That divergence produced the
|
||||
@@ -4357,6 +4413,25 @@ export async function preflightCompressIfNeeded(
|
||||
// payload is no longer refused on a stale measurement. One forward
|
||||
// only — the #330 relaxed-zone caveat still bounds the risk.
|
||||
if (result.compressedRanges > 0 && !unknownBaseline && result.payloadEstimate < limit && !localRefusalHere) {
|
||||
// #5000/#4543/#5885: the disagreement above is between two ESTIMATES — a
|
||||
// MEASURED body is authoritative. If the rebuilt body itself exceeds the
|
||||
// window on the #554 meter, "forwarding once for arbitration" is a
|
||||
// guaranteed 400 (#4543: a 4.1 MiB CJK body measuring 1,121,674 tokens
|
||||
// against a 1M window) and the log's "NOT forwarded" claim would be a
|
||||
// lie (#5885 defect 3) — answer it 413 instead of burning the send.
|
||||
// Bar: the NATIVE window when bili knows it (a body over the upstream's
|
||||
// real window can never fit, on any tokenizer), the headroom limit as
|
||||
// fallback — a meter over-count against the POLICY headroom must not
|
||||
// swallow the #1839/#4185 "fits the native window" forwards.
|
||||
const armMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit;
|
||||
if (outboundRawTokens !== undefined && outboundRawTokens + imageTokens > armMeasuredBar) {
|
||||
const outMsgs = outbound.processedMessages.length > 0 ? outbound.processedMessages : outbound.originalMessages;
|
||||
log("error", `[${session.id}] preflight 413 measured-body profile (post-fold): ${describeCoreTextDuplication(outMsgs)}`);
|
||||
const measuredDoomMessage = `rebuilt outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} tokens (system + tools + arguments + messages; CJK-aware count, the #554 guard meter; ~${imageTokens.toLocaleString("en-US")} of it image) vs the model window ${armMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded for upstream arbitration. Remove or trim the oversized content (e.g. a giant paste or tool result) and retry.`;
|
||||
log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`);
|
||||
noteLocalRefusal(session, currentOrigin, model, limit);
|
||||
return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded };
|
||||
}
|
||||
log("warn", `[${session.id}] preflight view fits (~${result.payloadEstimate}/${limit}) but the rebuilt payload measures over — forwarding once for upstream arbitration (#1839)`);
|
||||
return outbound;
|
||||
}
|
||||
@@ -4433,20 +4508,27 @@ export async function preflightCompressIfNeeded(
|
||||
// native-fit arm closed; doom bytes (#2490) and #3846 guard every arm.
|
||||
const deterministicExhaustion = f?.kind === "exhausted" && f.deterministic;
|
||||
const noUpstreamEvidence = session.stats.lastInputTokens <= 0 && statedWall <= 0 && deterministicExhaustion;
|
||||
let outboundBudgetTokens: number | undefined;
|
||||
try {
|
||||
const outBody: unknown = JSON.parse(typeof outbound.body === "string" ? outbound.body : outbound.body.toString("utf8"));
|
||||
const outField = outputBudgetField(outBody);
|
||||
const rawBudget = outField !== null ? readOutputBudget(outBody as Record<string, unknown>, outField) : undefined;
|
||||
if (typeof rawBudget === "number" && rawBudget > 0) outboundBudgetTokens = rawBudget;
|
||||
} catch {
|
||||
outboundBudgetTokens = undefined;
|
||||
}
|
||||
const outboundCalibrated = foldedCalibratedSize ?? calibratedPayload;
|
||||
const nativeFitAfterClamp = statedWall <= 0 && outboundBudgetTokens !== undefined && typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0
|
||||
&& outboundCalibrated + outboundBudgetTokens <= resolvedNativeWindow;
|
||||
if ((unknownBaseline || noUpstreamEvidence || nativeFitAfterClamp) && !growthArmed) {
|
||||
const outMsgs = outbound.processedMessages.length > 0 ? outbound.processedMessages : outbound.originalMessages;
|
||||
// #5000/#4543: the measured body is the stronger reading — if it exceeds
|
||||
// the window no upstream can accept it, whatever the byte floor says
|
||||
// (the floor only sees m.text; #5000's 55.2 MiB wire body and #4543's
|
||||
// 4.1 MiB CJK body both cleared it). Same bar as the #1839 arm above:
|
||||
// the native window when known (a #4185 headroom over-count must not
|
||||
// false-refuse a natively-fitting body), the headroom limit as
|
||||
// fallback. Measured tier first, byte floor below stays as the
|
||||
// fallback for unparseable bodies.
|
||||
const gateMeasuredBar = typeof resolvedNativeWindow === "number" && resolvedNativeWindow > 0 ? resolvedNativeWindow : limit;
|
||||
if (outboundRawTokens !== undefined && outboundRawTokens + imageTokens > gateMeasuredBar) {
|
||||
log("error", `[${session.id}] preflight 413 measured-body profile: ${describeCoreTextDuplication(outMsgs)}`);
|
||||
const measuredDoomMessage = `outbound body measures ~${outboundRawTokens.toLocaleString("en-US")} tokens even after preflight compression (system + tools + arguments + messages; CJK-aware count, the #554 guard meter; ~${imageTokens.toLocaleString("en-US")} of it image) vs the model window ${gateMeasuredBar.toLocaleString("en-US")} (model=${model}); the body ITSELF exceeds the window, so it was NOT forwarded. Remove or trim the oversized content (e.g. a giant paste or tool result) and retry.`;
|
||||
log("error", `[${session.id}] preflight fail-fast 413 (retryable=false): ${measuredDoomMessage}`);
|
||||
noteLocalRefusal(session, currentOrigin, model, limit);
|
||||
return { failFast: true, status: 413, message: measuredDoomMessage, retryable: false, respond: !res.writableEnded };
|
||||
}
|
||||
const doomBytes = estimateCoreMessagesUpperBytes(outMsgs);
|
||||
if (doomBytes > Math.max(DOOMED_PROBE_ABSOLUTE_BYTES, limit * DOOMED_PROBE_BYTES_PER_TOKEN)) {
|
||||
const byteFloorTokens = Math.ceil(doomBytes / DOOMED_PROBE_BYTES_PER_TOKEN);
|
||||
|
||||
@@ -429,6 +429,13 @@ async function handleRequest(
|
||||
if (!passthroughMark && protocol !== null && bodyBuffer.length > 0) {
|
||||
try {
|
||||
const decoded = await decodeRequestBody(headerValue(req, "content-encoding"), bodyBuffer, knobLimitsDecodedBytes(), { signal: decodingSignal, onBytes: (bytes) => admission.observe(bytes) });
|
||||
// #5794: identity (uncompressed) bodies never fire the onBytes
|
||||
// hook above — it only runs inside the decompression path — so
|
||||
// they were invisible to the large-body slot budget (a 51.2 MiB
|
||||
// plain-JSON body consumed no slot at all). Observe the decoded
|
||||
// size here; idempotent — a compressed body already observed
|
||||
// via onBytes is admitted and this no-ops.
|
||||
admission.observe(decoded.body.byteLength);
|
||||
bodyBuffer = decoded.body;
|
||||
if (decoded.decoded) delete req.headers["content-encoding"];
|
||||
} catch (decErr) {
|
||||
|
||||
+28
-1
@@ -69,7 +69,7 @@ import { buildForwardHeaders, connectionNamedHeaders, RESPONSE_ONLY_STRIP_HEADER
|
||||
import { bodyDumpEnabled, logDumpFailure } from "./observability.js";
|
||||
import { effectiveAbsorbBlock } from "./prepare-responses.js";
|
||||
import { awaitDrain, bufferToStream, dumpStreamToFile, pipeThrough, readStreamToBuffer } from "./stream-io.js";
|
||||
import { buildForwardTarget, makeFoldPointReasoningDroppable, outboundPayloadBreakdown, registerRequestAbort, beginStreamKeepalive, effectiveTokenCount, googlePathKind, imageBillingFor, imageTokenCapFor, inboundResignCredential, outboundContextEstimates, repairResponsesAssistantOrdering, resolveUpstream, stripKernelSummaries, usageGradeInputBaseline, DOOMED_PROBE_ABSOLUTE_BYTES, DOOMED_PROBE_BYTES_PER_TOKEN, type Prepared } from "../server.js";
|
||||
import { buildForwardTarget, makeFoldPointReasoningDroppable, measuredOutboundTokens, outboundPayloadBreakdown, registerRequestAbort, beginStreamKeepalive, effectiveTokenCount, googlePathKind, imageBillingFor, imageTokenCapFor, inboundResignCredential, outboundContextEstimates, repairResponsesAssistantOrdering, resolveUpstream, stripKernelSummaries, usageGradeInputBaseline, DOOMED_PROBE_ABSOLUTE_BYTES, DOOMED_PROBE_BYTES_PER_TOKEN, type Prepared } from "../server.js";
|
||||
import { isStrictReasoningEcho, modelIdOf } from "../strict-echo.js";
|
||||
|
||||
/** Infer the wire protocol from the request path for compat-role rewrites on
|
||||
@@ -725,6 +725,33 @@ async function forwardCaptured(
|
||||
const fuseWindow = Math.max(declaredWindow, MIN_ASSUMED_WINDOW_TOKENS);
|
||||
if (declaredWindow > 0) {
|
||||
const fuseMsgs = prepared.processedMessages.length > 0 ? prepared.processedMessages : prepared.originalMessages;
|
||||
// #5000/#4543: the byte floor below prices only m.text — a body whose
|
||||
// mass sits in tools[] / tool arguments / the system prompt clears it
|
||||
// (incident #5000: a 55.2 MiB wire body whose core text cleared the
|
||||
// floor; #4543: a 4.1 MiB CJK body measuring ~1.12M tokens that the
|
||||
// 7.5 B/tok floor cannot see below ~8.4 MB). Price the ACTUAL
|
||||
// outbound body first with the #554 side-request meter (every string
|
||||
// field, CJK-aware) and judge it against the SAME floored bar the
|
||||
// byte floor uses (fuseWindow): the fuse's #4079 contract is
|
||||
// "never block what a largest-common-window upstream might accept",
|
||||
// and that contract binds a measurement just as much as a density
|
||||
// guess — a wrong-small declaration must not false-block a body a
|
||||
// 1M window would take. #4543 still lands: its declared 1,048,580
|
||||
// window sits ABOVE the 1M floor, so floored or not the 1.12M-token
|
||||
// body exceeds it. The byte floor stays as the fallback for
|
||||
// unparseable bodies. No refusal record either way (#3974).
|
||||
const fuseRawTokens = measuredOutboundTokens(prepared);
|
||||
if (fuseRawTokens !== undefined && fuseRawTokens > fuseWindow) {
|
||||
log("warn", `413: [${prepared.session.id}] egress window fuse measured-body profile: ${describeCoreTextDuplication(fuseMsgs)}`);
|
||||
const fuseMessage = `outbound body measures ~${fuseRawTokens.toLocaleString("en-US")} tokens (system + tools + arguments + messages; CJK-aware count, the #554 guard meter) vs the effective model window ${declaredWindow.toLocaleString("en-US")} (bars never assume below ${MIN_ASSUMED_WINDOW_TOKENS} tokens, #4079) (model=${prepared.model ?? "unknown"}); the body ITSELF exceeds the window, so the send was stopped at the egress fuse instead of being forwarded (#5000/#4543/#4068). Remove or trim the oversized content (e.g. a giant paste or tool result) and retry.`;
|
||||
log("warn", `413: [${prepared.session.id}] egress window fuse: ${fuseMessage}`);
|
||||
if (res.headersSent) emitStreamError(res, prepared.protocol, fuseMessage, undefined, opts.streamErrorShape);
|
||||
else {
|
||||
res.writeHead(413, { "content-type": "application/json" });
|
||||
res.end(JSON.stringify({ error: { type: "request_too_large", code: "egress_window_fuse", stage: "forward", message: fuseMessage, retryable: false } }));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const fuseBytes = estimateCoreMessagesUpperBytes(fuseMsgs);
|
||||
if (fuseBytes > Math.max(DOOMED_PROBE_ABSOLUTE_BYTES, fuseWindow * DOOMED_PROBE_BYTES_PER_TOKEN)) {
|
||||
const fuseFloorTokens = Math.ceil(fuseBytes / DOOMED_PROBE_BYTES_PER_TOKEN);
|
||||
|
||||
@@ -148,6 +148,11 @@ test("e2e #4079: a body that provably cannot fit any window is stopped at the eg
|
||||
try {
|
||||
// No `model` in the body → preflight's `!model` early return skips its
|
||||
// doom math and folding entirely; forward() is the last line (#4079).
|
||||
// The measured tier (the #554 meter on the ACTUAL body, #5000/#4543)
|
||||
// fires first for this parseable body: ~2.17M measured tokens > the
|
||||
// floored 1M bar. The byte floor below it stays as the fallback for
|
||||
// unparseable bodies and is still covered by the both-ways test's
|
||||
// 8.68MB-class arithmetic.
|
||||
// 2000 msgs × 4.4K chars ≈ 8.68MB core text — past the floored ratio bar
|
||||
// (max(20K declared, 1M assumed) × 7.5 B/token = 7.5MB; the 2MB absolute
|
||||
// bar is dominated).
|
||||
@@ -159,7 +164,7 @@ test("e2e #4079: a body that provably cannot fit any window is stopped at the eg
|
||||
assert.equal(parsed.error?.type, "request_too_large", "same structured too-large family as the byte budget");
|
||||
assert.equal(parsed.error?.code, "egress_window_fuse", "the fuse names itself for log greppability");
|
||||
assert.equal(parsed.error?.retryable, false, "4xx non-retryable class — retrying re-sends the same oversized history (#2847 semantics)");
|
||||
assert.match(parsed.error?.message ?? "", /outbound core text alone is ~\d{7} bytes/, "the byte math is in the operator-facing message");
|
||||
assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ tokens/, "the measured token math is in the operator-facing message");
|
||||
assert.match(parsed.error?.message ?? "", /bars never assume below 1000000 tokens/, "the assumed-window floor is stated for the operator (#4079)");
|
||||
assert.match(parsed.error?.message ?? "", /stopped at the egress fuse instead of being forwarded/, "the remedy wording is present");
|
||||
// THE behavior under test: the doomed payload NEVER reaches the upstream.
|
||||
@@ -170,6 +175,75 @@ test("e2e #4079: a body that provably cannot fit any window is stopped at the eg
|
||||
}
|
||||
});
|
||||
|
||||
test("e2e #5000: a body whose mass rides tools[] — core text clearing the byte floor — is still stopped by the measured tier, not forwarded", async () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "bili-4079t-"));
|
||||
const { ctx, calls, close } = await setup();
|
||||
try {
|
||||
// Incident #5000's shape: the wire body's mass sits in tool definitions,
|
||||
// which the core-text byte floor never sees (it prices m.text only).
|
||||
// 120 tools × 40KB descriptions ≈ 4.8MB of tool text ≈ ~1.2M measured
|
||||
// tokens — over the floored 1M bar — while the core text is ~1.5KB,
|
||||
// ~5000× under the 7.5MB byte floor. With `model` present the preflight
|
||||
// measured gate fires before the relay fuse; same verdict either way.
|
||||
const tools = Array.from({ length: 120 }, (_, i) => ({
|
||||
type: "function",
|
||||
function: { name: `tool_${i}`, description: "d".repeat(40_000) },
|
||||
}));
|
||||
const messages = bigMessages(3, 500);
|
||||
const r = await fetch(ctx.url, {
|
||||
method: "POST",
|
||||
headers: { ...ctx.headers },
|
||||
body: JSON.stringify({ model: "gpt-test", stream: true, tools, messages }),
|
||||
});
|
||||
const body = await r.text();
|
||||
const parsed = JSON.parse(body) as { error?: { type?: string; code?: string; retryable?: boolean; message?: string } };
|
||||
|
||||
assert.equal(r.status, 413, `tools-mass body must fail fast (got ${r.status}): ${body.slice(0, 300)}`);
|
||||
assert.equal(parsed.error?.code, "preflight_compress_failed", "the preflight fail-fast house shape (the measured gate answers before the relay)");
|
||||
assert.equal(parsed.error?.retryable, false, "4xx non-retryable class — retrying re-sends the same oversized history (#2847 semantics)");
|
||||
assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ tokens/, "the measured tier names itself");
|
||||
assert.match(parsed.error?.message ?? "", /NOT forwarded/, "the remedy wording is present");
|
||||
// THE behavior under test: the doomed payload NEVER reaches the upstream
|
||||
// (before the measured tier it cleared the byte floor and rode forward
|
||||
// to a guaranteed 400 — incident #5000's 55.2 MiB send).
|
||||
assert.equal(calls.length, 0, `upstream must see zero calls (got ${JSON.stringify(calls)})`);
|
||||
} finally {
|
||||
await close();
|
||||
rmrf(root);
|
||||
}
|
||||
});
|
||||
|
||||
test("e2e #4543: a CJK body whose CORE TEXT alone measures over the floored fuse bar — while its BYTES clear the byte floor — is stopped at the fuse (the incident's 4.1 MiB sailed to a guaranteed 400)", async () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "bili-4079c-"));
|
||||
const { ctx, calls, close } = await setup();
|
||||
try {
|
||||
// Incident #4543's shape: dense CJK history. 1.12M CJK chars ≈ 3.36MB —
|
||||
// ~55% under the 7.5MB floored byte floor (which the meter-based floor
|
||||
// at ~7.5 B/token cannot see at CJK's 3.4 B/tok) — but CJK bills ~1
|
||||
// token per character, so the #554 meter prices ~1.12M tokens: over the
|
||||
// 1M floored bar. No `model` in the body → preflight skipped → the
|
||||
// relay fuse is the last line. On master this body forwarded to a
|
||||
// guaranteed upstream 400 (incident logs: 4.1 MiB, 1,121,674 tokens,
|
||||
// grep "egress window fuse" = 0).
|
||||
const cjk = "数".repeat(1_120_000);
|
||||
const messages = [{ role: "user", content: cjk + "\n" + "x".repeat(3_072) }] as Array<{ role: string; content: string }>;
|
||||
const r = await post(ctx, messages, false);
|
||||
const body = await r.text();
|
||||
const parsed = JSON.parse(body) as { error?: { type?: string; code?: string; retryable?: boolean; message?: string } };
|
||||
|
||||
assert.equal(r.status, 413, `the CJK-measured body must stop at the fuse (got ${r.status}): ${body.slice(0, 300)}`);
|
||||
assert.equal(parsed.error?.code, "egress_window_fuse", "the fuse names itself");
|
||||
assert.equal(parsed.error?.retryable, false, "non-retryable — retrying re-sends the same oversized history (#2847 semantics)");
|
||||
assert.match(parsed.error?.message ?? "", /outbound body measures ~[\d,]+ tokens/, "the measured token math is in the operator-facing message");
|
||||
assert.match(parsed.error?.message ?? "", /stopped at the egress fuse instead of being forwarded/, "the remedy wording is present");
|
||||
// THE behavior under test: the doomed payload NEVER reaches the upstream.
|
||||
assert.equal(calls.length, 0, `upstream must see zero calls (got ${JSON.stringify(calls)})`);
|
||||
} finally {
|
||||
await close();
|
||||
rmrf(root);
|
||||
}
|
||||
});
|
||||
|
||||
test("e2e #4079 both-ways: bodies under the doom bars still forward — including a multi-MB one an estimator would call over-window", async () => {
|
||||
const root = mkdtempSync(path.join(tmpdir(), "bili-4079b-"));
|
||||
const { ctx, calls, close } = await setup();
|
||||
|
||||
@@ -209,12 +209,26 @@ test("usage-backed session: over the headroom limit, zero compressible → forwa
|
||||
});
|
||||
});
|
||||
|
||||
test("identified fresh session whose MEASURED body exceeds the native window but clears the 2MB byte floor → still fail-fast 413, nothing forwarded (#5000/#4185 measured tier)", async () => {
|
||||
await withFixture(async (proxyPort, upstreamPort, calls) => {
|
||||
// 300K chars of tool-result text ≈ ~75K measured tokens: over the 40K
|
||||
// native window, but only ~300KB — ~7× under the 2MB core-text byte
|
||||
// floor, which the old path alone let ride to a guaranteed 400.
|
||||
const r = await post(proxyPort, upstreamPort, "s-4185-measured", incidentConversation(300_000));
|
||||
const body = await r.text();
|
||||
assert.equal(r.status, 413, `measured-over body must fail fast: ${body.slice(0, 300)}`);
|
||||
assert.match(body, /outbound body measures ~[\d,]+ tokens even after preflight compression/, `measured diagnostic present: ${body.slice(0, 300)}`);
|
||||
assert.ok(body.includes("the body ITSELF exceeds the window"), `doom wording present: ${body.slice(0, 300)}`);
|
||||
assert.equal(calls.filter((c) => c.body.includes("TOOLMARKER_4185_")).length, 0, "the giant payload was never forwarded");
|
||||
});
|
||||
});
|
||||
|
||||
test("identified fresh session with mathematically doomed bytes (>2MB core) → still fail-fast 413, nothing forwarded (#4185 negative)", async () => {
|
||||
await withFixture(async (proxyPort, upstreamPort, calls) => {
|
||||
const r = await post(proxyPort, upstreamPort, "s-4185-doomed", incidentConversation(2_100_000));
|
||||
const body = await r.text();
|
||||
assert.equal(r.status, 413, "doomed bytes must still fail fast");
|
||||
assert.ok(body.includes("no upstream can accept it"), `doom diagnostic present: ${body.slice(0, 300)}`);
|
||||
assert.ok(body.includes("the body ITSELF exceeds the window"), `doom diagnostic present: ${body.slice(0, 300)}`);
|
||||
assert.equal(calls.length, 0, "nothing forwarded and no summarization spent");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2,7 +2,7 @@ import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import { gzipSync, deflateRawSync, brotliCompressSync } from "node:zlib";
|
||||
import { decodeRequestBody, DecompressedTooLargeError } from "../src/content-encoding.ts";
|
||||
import { checkOutboundBody, DecodedRequestAdmission, DecodedRequestBusyError, MAX_REQUEST_BYTES, MAX_DECODED_REQUEST_BYTES, OutboundBodyTooLargeError } from "../src/request-body-budget.ts";
|
||||
import { checkOutboundBody, DecodedRequestAdmission, DecodedRequestBusyError, LARGE_DECODED_ADMISSION_BYTES, MAX_REQUEST_BYTES, MAX_DECODED_REQUEST_BYTES, OutboundBodyTooLargeError } from "../src/request-body-budget.ts";
|
||||
import { fetchWithTimeout, _liveUpstreamTimersForTest } from "../src/fetch-util.ts";
|
||||
|
||||
// #3128: assert the LEGACY byte-budget behavior at the old 100/200/100 MiB caps. Pin the
|
||||
@@ -13,18 +13,50 @@ process.env.BILI_RECEIVE_BODY_BYTES = "104857600";
|
||||
process.env.BILI_DECODED_BODY_BYTES = "209715200";
|
||||
process.env.BILI_OUTBOUND_BODY_BYTES = "104857600";
|
||||
|
||||
test("large decoded admission keeps normal requests free, limits two large requests, and releases idempotently", () => {
|
||||
test("large decoded admission: normal requests free, two large slots, third busy — at both the receive cap and the #5794 32MiB cost band", () => {
|
||||
// The slot counter is module-global, so the whole budget's behavior is
|
||||
// pinned in ONE sequential test (top-level tests in a file run
|
||||
// concurrently; two tests each grabbing two slots would starve each
|
||||
// other regardless of release hygiene).
|
||||
//
|
||||
// Phase 1 — legacy receive-cap shape (pinned 100MiB above). Under the
|
||||
// #5794 cost band the 100MiB-class body is itself LARGE (the band is
|
||||
// below it), so "normal requests stay free" is pinned at a genuinely
|
||||
// normal size and the two-slot/behind-it semantics keep their shape.
|
||||
const a = new DecodedRequestAdmission(), b = new DecodedRequestAdmission(), c = new DecodedRequestAdmission();
|
||||
try {
|
||||
c.observe(MAX_REQUEST_BYTES);
|
||||
// Normal-sized requests stay free regardless of the large budget.
|
||||
c.observe(1_000_000);
|
||||
// Two large slots, third busy.
|
||||
a.observe(MAX_REQUEST_BYTES + 1);
|
||||
a.observe(MAX_DECODED_REQUEST_BYTES);
|
||||
b.observe(MAX_REQUEST_BYTES + 1);
|
||||
b.observe(MAX_DECODED_REQUEST_BYTES);
|
||||
assert.throws(() => c.observe(MAX_REQUEST_BYTES + 1), DecodedRequestBusyError);
|
||||
a.release(); a.release();
|
||||
c.observe(MAX_REQUEST_BYTES + 1);
|
||||
assert.throws(() => a.observe(MAX_REQUEST_BYTES + 1), DecodedRequestBusyError);
|
||||
} finally { a.release(); b.release(); c.release(); }
|
||||
|
||||
// Phase 2 — #5794 cost band: the 51.2 MiB incident body (29,619 messages,
|
||||
// ~6s of event loop per round, 747s FATAL → self-restart → every session's
|
||||
// cache cold) sits BELOW the 100MiB receive cap pinned above — the old
|
||||
// threshold let it through the slot budget entirely. The cost band closes
|
||||
// that gap without touching the operator's receive limit:
|
||||
// min(receive cap, 32MiB) is monotonic, so a smaller operator cap still
|
||||
// binds (the same expression the legacy phase already exercises at
|
||||
// 100MiB > 32MiB — here the band itself is the binding side).
|
||||
const band = LARGE_DECODED_ADMISSION_BYTES;
|
||||
assert.ok(band < MAX_REQUEST_BYTES, "the band sits below the pinned 100MiB receive cap");
|
||||
const f = new DecodedRequestAdmission(), g = new DecodedRequestAdmission(), h = new DecodedRequestAdmission();
|
||||
try {
|
||||
// Exactly at the band: free (same >-not-≥ convention as the receive cap).
|
||||
f.observe(band);
|
||||
// Just over: two slots, third busy.
|
||||
f.observe(band + 1); // f.admitted is still false (band was free) → takes slot 1
|
||||
g.observe(band + 1); // slot 2
|
||||
assert.throws(() => h.observe(band + 1), DecodedRequestBusyError);
|
||||
f.release();
|
||||
f.observe(band + 1); // re-admit after release (idempotent within a request)
|
||||
} finally { f.release(); g.release(); h.release(); }
|
||||
});
|
||||
|
||||
test("every stacked decode retains its hard cap and resource failures do not fall back to raw deflate", async () => {
|
||||
|
||||
Reference in New Issue
Block a user