Merge remote-tracking branch 'origin/main' into fix/kompress-rust-shared-cache

# Conflicts:
#	crates/headroom-core/src/relevance/embedding.rs
This commit is contained in:
Kenneth Wong
2026-11-01 23:31:32 +08:00
192 changed files with 13236 additions and 2581 deletions
+13 -84
View File
@@ -1,96 +1,25 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 240" width="880" height="240" role="img"
aria-label="Headroom Enterprise for teams: 3 months free with a 12-month plan. Adds tool search, model routing, attack removal and org-wide savings. Claim the perk.">
<title>Headroom Enterprise — 3 months free with a 12-month plan</title>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 200" width="880" height="200" role="img"
aria-label="Headroom for teams and enterprises. Want more savings? Headroom OSS shows what is possible; Enterprise saves more. More cost savings, tokens secured, on premise, deep attribution and many more features. Talk to us: hello@headroomlabs.ai.">
<title>Headroom for teams and enterprises — talk to us</title>
<defs>
<!-- Brand palette as in hero.svg: Mint #00F0B5, Obsidian #070A0F,
Midnight #0C1118, Cyan #65D8FF, White, Slate #98A4B3.
Dark in both GitHub themes, matching the hero. -->
<pattern id="pgrid" width="44" height="44" patternUnits="userSpaceOnUse">
<path d="M44 0H0V44" fill="none" stroke="#00F0B5" stroke-opacity="0.05" stroke-width="1"/>
</pattern>
<radialGradient id="pglow" cx="12%" cy="0%" r="80%">
<stop offset="0%" stop-color="#00F0B5" stop-opacity="0.16"/>
<stop offset="100%" stop-color="#00F0B5" stop-opacity="0"/>
</radialGradient>
<linearGradient id="pedge" x1="0" y1="0" x2="1" y2="0">
<stop offset="0%" stop-color="#00F0B5" stop-opacity="0.65"/>
<stop offset="55%" stop-color="#65D8FF" stop-opacity="0.35"/>
<stop offset="100%" stop-color="#65D8FF" stop-opacity="0.10"/>
</linearGradient>
<linearGradient id="pbtn" x1="0" y1="0" x2="1" y2="1">
<stop offset="0%" stop-color="#00F0B5"/>
<stop offset="100%" stop-color="#0FD9AE"/>
</linearGradient>
<linearGradient id="pshine" gradientUnits="userSpaceOnUse" x1="0" y1="0" x2="70" y2="0">
<stop offset="0%" stop-color="#FFFFFF" stop-opacity="0"/>
<stop offset="50%" stop-color="#FFFFFF" stop-opacity="0.45"/>
<stop offset="100%" stop-color="#FFFFFF" stop-opacity="0"/>
</linearGradient>
<clipPath id="pbtnclip"><rect x="646" y="94" width="208" height="58" rx="29"/></clipPath>
<!-- Brand palette and type scale as in hero.svg: Mint #00F0B5, Obsidian #070A0F,
Slate #98A4B3, White; title 42/700, lead 14.5, eyebrow 10.5 tracked. -->
<style>
.s { font-family: "Segoe UI", Inter, -apple-system, BlinkMacSystemFont, Helvetica, Arial, sans-serif; }
/* One light pass over the button per cycle draws the eye to the action. */
.shine { animation: shine 5s ease-in-out infinite; }
@keyframes shine { 0%, 55% { transform: translateX(-80px) } 85%, 100% { transform: translateX(240px) } }
.dot { animation: beat 2.4s ease-in-out infinite; }
@keyframes beat { 0%, 100% { opacity: 1 } 50% { opacity: .3 } }
@media (prefers-reduced-motion: reduce) {
.shine, .dot { animation: none; }
.shine { opacity: 0; }
}
</style>
</defs>
<!-- Card. One font family, three sizes: 52 headline, 20 lead and button, 16 everything else. -->
<rect x="1" y="1" width="878" height="238" rx="18" fill="#070A0F"/>
<rect x="1" y="1" width="878" height="238" rx="18" fill="url(#pgrid)"/>
<rect x="1" y="1" width="878" height="238" rx="18" fill="url(#pglow)"/>
<rect x="1" y="1" width="878" height="238" rx="18" fill="none" stroke="url(#pedge)" stroke-width="1.5"/>
<rect x="1" y="1" width="878" height="198" rx="18" fill="#070A0F" stroke="#00F0B5" stroke-opacity="0.35" stroke-width="1.5"/>
<g class="s">
<!-- Eyebrow -->
<circle class="dot" cx="46" cy="47" r="5" fill="#00F0B5"/>
<text x="60" y="53" font-size="16" font-weight="600" letter-spacing="1.5" fill="#00F0B5">HEADROOM ENTERPRISE · FOR TEAMS</text>
<text x="40" y="48" font-size="10.5" letter-spacing="1.6" fill="#00F0B5">HEADROOM FOR TEAMS &amp; ENTERPRISES</text>
<text x="40" y="96" font-size="42" font-weight="700" letter-spacing="-0.8" fill="#FFFFFF">Want more savings?</text>
<text x="40" y="128" font-size="14.5" fill="#98A4B3">Headroom OSS shows what is possible. Enterprise saves more, and adds:</text>
<text x="40" y="160" font-size="14.5" fill="#FFFFFF">More cost savings · Tokens secured · On premise · Deep attribution · and more</text>
<!-- Headline -->
<text x="40" y="116" font-size="52" font-weight="700" fill="#FFFFFF">3 months free</text>
<text x="40" y="152" font-size="20" fill="#98A4B3">with a 12-month plan · one perk per company</text>
<!-- What Enterprise adds -->
<g font-size="16" fill="#E6EDF3">
<rect x="40" y="178" width="118" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
<text x="99" y="202" text-anchor="middle">Tool search</text>
<rect x="168" y="178" width="134" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
<text x="235" y="202" text-anchor="middle">Model routing</text>
<rect x="312" y="178" width="140" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
<text x="382" y="202" text-anchor="middle">Attack removal</text>
<rect x="462" y="178" width="156" height="36" rx="18" fill="#0C1118" stroke="#65D8FF" stroke-opacity="0.30"/>
<text x="540" y="202" text-anchor="middle">Org-wide savings</text>
</g>
<!-- Divider -->
<line x1="634" y1="36" x2="634" y2="204" stroke="#98A4B3" stroke-opacity="0.18"/>
<!-- Call to action -->
<text x="750" y="72" font-size="16" text-anchor="middle" fill="#98A4B3">SSO · VPC or managed</text>
<g clip-path="url(#pbtnclip)">
<rect x="646" y="94" width="208" height="58" rx="29" fill="url(#pbtn)"/>
<rect class="shine" x="646" y="94" width="70" height="58" fill="url(#pshine)"/>
</g>
<text x="750" y="130" font-size="20" font-weight="700" text-anchor="middle" fill="#070A0F">Claim the perk →</text>
<text x="750" y="190" font-size="16" text-anchor="middle" fill="#98A4B3">headroom-perks.vercel.app</text>
<rect x="676" y="72" width="164" height="44" rx="22" fill="#00F0B5"/>
<text x="758" y="99" font-size="14.5" font-weight="700" text-anchor="middle" fill="#070A0F">Talk to us →</text>
<text x="758" y="140" font-size="13" text-anchor="middle" fill="#98A4B3">hello@headroomlabs.ai</text>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 4.8 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

+2 -2
View File
@@ -268,11 +268,11 @@ jobs:
run: |
python -m pip install --upgrade pip maturin
maturin build --profile ci --out dist --interpreter python3.14
- name: Install headroom-ai[sandbox] from prebuilt wheels only
- name: Install headroom-ai[lean] from prebuilt wheels only
run: |
python -m pip install --upgrade pip
WHEEL="$(ls dist/*.whl)"
python -m pip install --only-binary=:all: "${WHEEL}[sandbox]"
python -m pip install --only-binary=:all: "${WHEEL}[lean]"
- name: Proxy dependencies and LiteLLM pricing are available
working-directory: ${{ runner.temp }}
run: |
+20 -11
View File
@@ -813,6 +813,11 @@ jobs:
node-version: ">=24.16.0 <25"
registry-url: ${{ env.NPM_REGISTRY_URL }}
- name: Verify npm publishing authentication
env:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: npm whoami --registry "${NPM_REGISTRY_URL}"
# No artifact download required: the publish steps below pack and
# publish directly from the checked-out source tree (sdk/typescript
# and plugins/openclaw) via `npm pack` + `npm publish`. The
@@ -828,12 +833,15 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
version="${{ needs.detect-version.outputs.npm_version }}"
if npm view "${NPM_SDK_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
echo "${NPM_SDK_PACKAGE}@${version} is already published; continuing."
exit 0
fi
cd sdk/typescript
npm ci
npm run build
npm version "$version" --no-git-tag-version --allow-same-version
npm publish --access public
continue-on-error: true
- name: Publish ${{ env.NPM_OPENCLAW_PACKAGE }} to npmjs.org
id: npm-openclaw-publish
@@ -841,6 +849,10 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
version="${{ needs.detect-version.outputs.npm_version }}"
if npm view "${NPM_OPENCLAW_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
echo "${NPM_OPENCLAW_PACKAGE}@${version} is already published; continuing."
exit 0
fi
cd plugins/openclaw
npm ci
npm run build
@@ -854,7 +866,6 @@ jobs:
EOF
node prepare-dist.mjs
npm publish --access public
continue-on-error: true
- name: Publish ${{ env.NPM_OPENCODE_PACKAGE }} to npmjs.org
id: npm-opencode-publish
@@ -862,6 +873,10 @@ jobs:
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
run: |
version="${{ needs.detect-version.outputs.npm_version }}"
if npm view "${NPM_OPENCODE_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
echo "${NPM_OPENCODE_PACKAGE}@${version} is already published; continuing."
exit 0
fi
cd plugins/opencode
npm ci
npm run build
@@ -874,12 +889,6 @@ jobs:
fs.writeFileSync("package.json", `${JSON.stringify(pkg, null, 2)}\n`);
EOF
npm publish --access public
continue-on-error: true
- name: npm publish notice
if: steps.npm-sdk-publish.outcome == 'failure' || steps.npm-openclaw-publish.outcome == 'failure' || steps.npm-opencode-publish.outcome == 'failure'
run: |
echo "::notice::One or more npm publishes failed. Set NPM_SKIP=true in repo Variables to skip npm publishes if tokens are not configured."
publish-github-packages:
needs: [detect-version, build]
@@ -1067,7 +1076,7 @@ jobs:
# Appends once: a re-run finds this step's own marker and leaves the notes
# alone. Keyed on the marker, not the link, so a changelog entry that
# happens to mention the perks page does not suppress the footer.
# happens to mention the contact link does not suppress the footer.
# Release notes stay the changelog; this is one line under it.
- name: Add Headroom for Teams footer to release notes
env:
@@ -1079,8 +1088,8 @@ jobs:
if ! grep -qF "$MARKER" release-notes.md; then
{
printf '\n%s\n\n---\n\n' "$MARKER"
printf '**Headroom for Teams:** 3 months of Headroom Enterprise free with a 12-month plan. '
printf '[Claim the perk →](https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=release-notes&utm_content=%s)\n' "$TAG"
printf '**Want more savings?** Headroom Enterprise saves more for teams and enterprises. Talk to us at '
printf '[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) or [headroomlabs.ai](https://headroomlabs.ai).\n'
} >> release-notes.md
gh release edit "$TAG" --notes-file release-notes.md
fi
+1 -1
View File
@@ -122,7 +122,7 @@ Enable or disable automatic Copilot review in **Settings → Rules → Rulesets
- [Ruff](https://github.com/astral-sh/ruff) for lint + format, line length 100, PEP 8.
- Type hints on public functions; Google-style docstrings.
- Cover new behavior + edge cases; aim >80% coverage on new code.
- Python 3.10+. Optional features go behind extras.
- Python 3.11+. Optional features go behind extras.
- **Headroom writes LF and normalizes on read, on every platform.** Pass
`newline="\n"` to every `write_text`/`open` that writes a context, memory,
or state file, and normalize `\r\n`/`\r` when you read one back. Without
+4 -3
View File
@@ -50,11 +50,12 @@ COPY headroom/ headroom/
# The standalone Dockerfile must support every backend advertised by
# `headroom proxy --backend`, including Bedrock temporary/SSO credentials.
# Those credentials require botocore (GH #1551), supplied by [bedrock].
ARG HEADROOM_EXTRAS=proxy,code,bedrock
# [redis] backs the replica-shared stores: without it HEADROOM_COMPRESS_SESSION_STORE
# stops startup and HEADROOM_CCR_BACKEND=redis falls back to a per-process store.
ARG HEADROOM_EXTRAS=proxy,code,bedrock,redis
# `uv pip install` ignores [tool.uv] override-dependencies (uv.lock only), so
# pass them explicitly: the image gets the security overrides the lock has
# (pillow 13 over fastembed's <13 cap). Python 3.10 has no tomllib: pip's
# vendored tomli reads the file there.
# (pillow 13 over fastembed's <13 cap).
RUN --mount=type=cache,target=/root/.cache/uv \
--mount=type=cache,target=/usr/local/cargo/registry \
--mount=type=cache,target=/usr/local/cargo/git \
+12 -17
View File
@@ -39,7 +39,7 @@ answers, fraction of the tokens. Compression runs on your machine; no prompt or
file content is sent anywhere to be compressed.
<p align="center">
<a href="https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=readme&utm_content=banner"><img src=".github/assets/enterprise-perk.svg" alt="Headroom Enterprise for teams: 3 months free with a 12-month plan. Claim the perk." width="880"></a>
<a href="https://headroomlabs.ai"><img src=".github/assets/enterprise-perk.svg" alt="Headroom Enterprise for teams. Talk to us: hello@headroomlabs.ai." width="880"></a>
</p>
<div align="center">
@@ -443,7 +443,7 @@ Granular extras: `[proxy]`, `[mcp]`, `[ml]` (Kompress-v2-base), `[code]`,
`[memory]`, `[vector]` (optional HNSW backend — needs a C++ toolchain, not in
`[all]`), `[relevance]`, `[image]`, `[agno]`, `[langchain]`, `[evals]`,
`[pytorch-mps]` (Apple-GPU memory-embedder offload — set
`HEADROOM_EMBEDDER_RUNTIME=pytorch_mps`). Requires **Python 3.10+**.
`HEADROOM_EMBEDDER_RUNTIME=pytorch_mps`). Requires **Python 3.11+**.
> `[all]` covers the core stack but not the framework adapters. Install those
> separately: `pip install "headroom-ai[langchain]"`, and likewise `[agno]`,
@@ -627,23 +627,18 @@ Turn it off with `HEADROOM_BEACON=off`, `HEADROOM_TELEMETRY=off`, the
## Headroom for teams
Headroom OSS is built for one developer on one laptop. Headroom Enterprise is built
for the whole team.
**Not seeing the savings you want?** Headroom OSS is a framework. It was built to show
what's possible on one developer's laptop, so it saves less than it could. We are
building bigger savings for teams and enterprises in Headroom Enterprise, which adds:
| | Open source | Enterprise |
|---|---|---|
| Savings | Compression, plus tool search on Anthropic requests (on by default) | Also: tool search for every model, harness tuning, extra compressors |
| Model routing | Rule-based and opt-in: turn it on with `HEADROOM_MODEL_ROUTER_ENABLED=1` and write the rules in `HEADROOM_MODEL_ROUTES` | Automatic: a cheaper model per request when quality allows, with a shadow mode that shows the savings before you turn it on |
| Security | — | Detect and strip prompt injection and risky tool calls |
| Rollout | Per laptop | Managed or in your VPC, SSO, central config, org-wide savings |
| Support | GitHub, Discord | Dedicated engineers, onboarding, priority response |
- More cost savings
- Tokens secured
- On premise deployment
- Deep attribution
- And many more features
**3 months of Enterprise free.** Companies that sign a 12-month Enterprise plan get
their first 3 months free. One perk per company.
**[Claim 3 months free →](https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=readme&utm_content=teams)**
· or email [hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) with your stack and
monthly LLM spend.
If you're a team or an enterprise, **talk to us**:
[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) · [headroomlabs.ai](https://headroomlabs.ai)
Everything in this repo stays open source under Apache 2.0.
+25 -2
View File
@@ -74,9 +74,9 @@ it necessarily handles both your traffic and your credentials. The
full — what is written to disk, with what permissions and retention, what leaves the
host, and what the local admin surface does and does not allow. The summary:
- **Provider API keys are forwarded, not persisted.** Keys supplied via environment
- **Interactive provider API keys are forwarded.** Keys supplied via environment
variables or request headers are used to authenticate the upstream call and are not
written to disk or emitted to logs. The one exception is deliberate: the
written to disk or emitted to logs. Settings can also persist keys: the
`ANTHROPIC_TARGET_API_HEADERS` / `OPENAI_TARGET_API_HEADERS` header maps, if you set
them through the dashboard settings GUI, are persisted to `~/.headroom/settings.json`
in plaintext — and a header map is where a gateway key typically goes.
@@ -84,6 +84,29 @@ host, and what the local admin surface does and does not allow. The summary:
GitHub Copilot OAuth refresh token to `~/.headroom/copilot_auth.json` (relocatable via
`$HEADROOM_COPILOT_AUTH_FILE`). It is plaintext JSON, written with `0600` permissions
on a best-effort basis.
- **Persistent installs store supplied environment values.** Every value passed
to `headroom install --env KEY=VALUE` is stored verbatim and in cleartext. Supervisors (launchd, systemd, Task
Scheduler, cron) start the proxy from a bare environment, so `--env` is the
only channel by which a provider API key reaches a supervised proxy — and
if you pass one, it is written to
`~/.headroom/deploy/<profile>/manifest.json` (mode `0600`) and `export`ed in
cleartext by the generated runner scripts `run-headroom.sh` /
`ensure-headroom.sh` (mode `0700`) in the same directory (mode `0700`).
The installer sets those modes and then reads them back: where the
platform enforces POSIX permission bits it refuses to leave behind a runner
script it could not restrict, and warns if the profile directory could not
be narrowed. On Windows the bits are advisory — access is governed by ACLs —
so the modes above are requested but not enforced there, and protection
comes from the account's own profile directory instead. Where they do hold,
the files are owner-only, not encrypted: anything that can read the account
— the user themself, root, a home-directory backup or sync — can read the
key, and rotating it requires re-running `headroom install apply`.
With the Docker runtime the same values are additionally passed as `docker
run --env NAME=VALUE`, which places them in the container process's command
line and so in `ps` output for other users on the host. Prefer a
supervisor-native secret facility (systemd `LoadCredential=` /
`EnvironmentFile=` on an owner-only file, a macOS keychain helper) over
`--env` for long-lived provider keys.
- **Cached request content is written to disk.** CCR stores pre-compression originals —
tool outputs, file contents, retrieved chunks — in a local SQLite database
(`~/.headroom/ccr_store.db`, `0600`) so they can be retrieved after compression. Entries
+101 -3
View File
@@ -45,6 +45,19 @@ use fastembed::{
use super::base::{RelevanceScore, RelevanceScorer};
/// Texts per ONNX Runtime run, matching Python's `EMBED_BATCH_SIZE`: fastembed's
/// default of 256 grows ORT's never-shrinking CPU arena to the biggest batch scored.
#[cfg(feature = "ml")]
const EMBED_BATCH_SIZE: usize = 16;
/// fastembed refuses to split a dynamic-quantized model's input across runs (each
/// run calibrates its own int8 range, so the embeddings would not compare): one batch.
#[cfg(feature = "ml")]
fn embed_batch_size(model_kind: &EmbeddingModel) -> Option<usize> {
(TextEmbedding::get_quantization_mode(model_kind) != QuantizationMode::Dynamic)
.then_some(EMBED_BATCH_SIZE)
}
/// Name the default model reports, the same string Python uses.
#[cfg(feature = "ml")]
const DEFAULT_MODEL_NAME: &str = "BAAI/bge-small-en-v1.5";
@@ -227,6 +240,8 @@ pub struct EmbeddingScorer {
/// dominant cost so contention is bounded by inference latency,
/// not lock latency.
model: Option<Mutex<TextEmbedding>>,
/// `embed` batch size for this model, from `embed_batch_size`.
batch_size: Option<usize>,
}
#[cfg(feature = "ml")]
@@ -246,6 +261,7 @@ impl Default for EmbeddingScorer {
EmbeddingScorer {
model_name: "BAAI/bge-small-en-v1.5".to_string(),
model: None,
batch_size: Some(EMBED_BATCH_SIZE),
}
}
}
@@ -288,9 +304,12 @@ impl EmbeddingScorer {
}
};
let model = load_snapshot(&snapshot)?;
// The snapshot is statically quantized (see `load_snapshot`), so it
// batches like fastembed's `BGESmallENV15Q`.
Ok(EmbeddingScorer {
model_name: DEFAULT_MODEL_NAME.to_string(),
model: Some(Mutex::new(model)),
batch_size: Some(EMBED_BATCH_SIZE),
})
}
@@ -309,11 +328,13 @@ impl EmbeddingScorer {
crate::offline::guard_egress("fastembed embedding model download", &name)
.map_err(|e| format!("EmbeddingScorer model load refused: {e}"))?;
onnx_runtime_ready()?;
let batch_size = embed_batch_size(&model_kind);
let model = TextEmbedding::try_new(InitOptions::new(model_kind))
.map_err(|e| format!("EmbeddingScorer model load failed: {}", e))?;
Ok(EmbeddingScorer {
model_name: name,
model: Some(Mutex::new(model)),
batch_size,
})
}
}
@@ -372,11 +393,11 @@ impl RelevanceScorer for EmbeddingScorer {
}
};
// Encode items + context in one batch — saves model dispatch
// overhead. Mirrors Python fastembed batch encoding.
// Encode items + context in one `embed` call, which runs them in
// `self.batch_size` chunks. Mirrors Python fastembed batch encoding.
let mut all_texts: Vec<String> = items.iter().map(|s| s.to_string()).collect();
all_texts.push(context.to_string());
let embeddings = match guard.embed(all_texts, None) {
let embeddings = match guard.embed(all_texts, self.batch_size) {
Ok(e) => e,
Err(e) => {
return items
@@ -693,6 +714,7 @@ mod tests {
EmbeddingScorer {
model_name: "test".to_string(),
model: None,
batch_size: Some(EMBED_BATCH_SIZE),
}
}
@@ -803,6 +825,15 @@ mod tests {
}
}
#[cfg(feature = "ml")]
#[test]
fn dynamic_quantized_models_embed_in_one_batch() {
let capped = Some(EMBED_BATCH_SIZE);
assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15), capped);
assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15Q), capped);
assert_eq!(embed_batch_size(&EmbeddingModel::AllMiniLML6V2Q), None);
}
// ---------- model-backed tests (gated on RUN_FASTEMBED_TESTS) ----------
#[cfg(feature = "ml")]
@@ -909,4 +940,71 @@ mod tests {
assert!((0.0..=1.0).contains(&sc.score));
}
}
#[cfg(feature = "ml")]
#[test]
fn fastembed_batches_past_the_cap_keep_each_score_with_its_item() {
if !fastembed_enabled() {
return;
}
// Three EMBED_BATCH_SIZE batches of mixed lengths, the one related item
// in the last: a misaligned batch boundary would hand its score away.
let s = EmbeddingScorer::try_new().expect("model loads");
let mut items: Vec<String> = (0..40)
.map(|i| {
format!(
"inventory row {i}: {}",
"pallets of garden furniture ".repeat(i % 5 + 1)
)
})
.collect();
items[37] = "authentication failed for user admin: invalid password".to_string();
let refs: Vec<&str> = items.iter().map(String::as_str).collect();
let batch = s.score_batch(&refs, "login error");
assert_eq!(batch.len(), 40);
let top = (0..40)
.max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
.unwrap();
assert_eq!(top, 37);
for (item, sc) in refs.iter().zip(&batch) {
let single = s.score(item, "login error").score;
assert!(
(sc.score - single).abs() < 1e-3,
"{item}: batched {} vs single {single}",
sc.score
);
}
}
#[cfg(feature = "ml")]
#[test]
fn fastembed_dynamic_quantized_model_scores_on_both_sides_of_the_cap() {
if !fastembed_enabled() {
return;
}
// 15 items plus the context fill exactly one EMBED_BATCH_SIZE run; 16
// and 40 go past it, where fastembed refuses to split this model.
let s = EmbeddingScorer::try_new_with_model(EmbeddingModel::AllMiniLML6V2Q)
.expect("model loads");
for n in [15, 16, 40] {
let mut items: Vec<String> = (0..n)
.map(|i| format!("inventory row {i}: pallets of garden furniture"))
.collect();
items[n - 3] = "authentication failed for user admin: invalid password".to_string();
let refs: Vec<&str> = items.iter().map(String::as_str).collect();
let batch = s.score_batch(&refs, "login error");
assert_eq!(batch.len(), n);
for sc in &batch {
assert!(
!sc.reason.contains("inference failed"),
"{n} items: {}",
sc.reason
);
}
let top = (0..n)
.max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
.unwrap();
assert_eq!(top, n - 3, "{n} items");
}
}
}
@@ -43753,5 +43753,763 @@
"thinking_always_on": true,
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
},
"openrouter/anthropic/claude-haiku-5.5": {
"input_cost_per_token": 1e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 1e-08,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"litellm_provider": "openrouter",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"source": "https://openrouter.ai/api/v1/models",
"supports_function_calling": true,
"supports_tool_choice": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_vision": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_web_search": true,
"supports_adaptive_thinking": true,
"prompt_cache_min_tokens": 512,
"supports_sampling_params": false
},
"claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens_batches": 2.5e-07,
"output_cost_per_token_above_100k_tokens_batches": 1.25e-06,
"cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07,
"cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08,
"supports_anthropic_compaction": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_creation_input_token_cost_batches": 6.25e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 5e-09,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
"litellm_provider": "anthropic",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"output_cost_per_token_batches": 2.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_native_structured_output": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"provider_specific_entry": {
"us": 1.1
},
"supports_output_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview",
"supports_web_search": true,
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08
},
"bedrock_mantle/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_mantle",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
"cache_read_input_token_cost": 1.2e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "bedrock_mantle",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
},
"anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"apac.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_creation_input_token_cost": 1.375e-07,
"input_cost_per_token": 1.1e-07,
"output_cost_per_token": 5.5e-07,
"cache_read_input_token_cost": 1.1e-08,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"au.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"azure_ai/claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_read_input_token_cost": 1e-08,
"deprecation_date": "2027-09-29",
"input_cost_per_token": 1e-07,
"litellm_provider": "azure_ai",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
},
"bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
"cache_read_input_token_cost": 1.2e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "bedrock",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
"cache_read_input_token_cost": 1.2e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "bedrock",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"eu.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"global.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_read_input_token_cost": 1e-08,
"input_cost_per_token": 1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"jp.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"perplexity/anthropic/claude-haiku-5-5": {
"litellm_provider": "perplexity",
"mode": "responses",
"supports_adaptive_thinking": true,
"supports_web_search": true,
"supports_function_calling": true,
"input_cost_per_token": 1e-07,
"output_cost_per_token": 5e-07,
"cache_read_input_token_cost": 1e-08,
"source": "https://docs.perplexity.ai/docs/agent-api/models"
},
"us-gov.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 6e-07,
"output_cost_per_token_above_100k_tokens": 3e-06,
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
"bedrock_converse_supports_strict_tools": false,
"bedrock_output_config_effort_ceiling": "xhigh",
"cache_creation_input_token_cost": 1.5e-07,
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
"cache_read_input_token_cost": 1.2e-08,
"input_cost_per_token": 1.2e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 6e-07,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_max_reasoning_effort": true,
"supports_mid_conversation_system": true,
"supports_native_structured_output": false,
"supports_output_config": true,
"supports_parallel_tool_use_config": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true
},
"us.anthropic.claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5.5e-07,
"output_cost_per_token_above_100k_tokens": 2.75e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
"bedrock_converse_supports_strict_tools": false,
"cache_creation_input_token_cost": 1.375e-07,
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
"cache_read_input_token_cost": 1.1e-08,
"input_cost_per_token": 1.1e-07,
"litellm_provider": "bedrock_converse",
"supports_tool_search": true,
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_mid_conversation_system": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_native_structured_output": false,
"supports_max_reasoning_effort": true,
"supports_output_config": true,
"bedrock_output_config_effort_ceiling": "xhigh",
"supports_parallel_tool_use_config": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
},
"vertex_ai/claude-haiku-5-5": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_creation_input_token_cost_batches": 6.25e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 5e-09,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"output_cost_per_token_batches": 2.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
},
"vertex_ai/claude-haiku-5-5@default": {
"input_cost_per_token_above_100k_tokens": 5e-07,
"output_cost_per_token_above_100k_tokens": 2.5e-06,
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
"regional_endpoint_uplift_multiplier": 1.1,
"supports_mid_conversation_system": true,
"cache_creation_input_token_cost": 1.25e-07,
"cache_creation_input_token_cost_above_1hr": 2e-07,
"cache_creation_input_token_cost_batches": 6.25e-08,
"cache_read_input_token_cost": 1e-08,
"cache_read_input_token_cost_batches": 5e-09,
"input_cost_per_token": 1e-07,
"input_cost_per_token_batches": 5e-08,
"litellm_provider": "vertex_ai-anthropic_models",
"max_input_tokens": 1000000,
"max_output_tokens": 128000,
"max_tokens": 128000,
"mode": "chat",
"output_cost_per_token": 5e-07,
"output_cost_per_token_batches": 2.5e-07,
"search_context_cost_per_query": {
"search_context_size_high": 0.01,
"search_context_size_low": 0.01,
"search_context_size_medium": 0.01
},
"supports_adaptive_thinking": true,
"supports_assistant_prefill": false,
"supports_computer_use": true,
"supports_function_calling": true,
"supports_pdf_input": true,
"supports_prompt_caching": true,
"supports_reasoning": true,
"supports_response_schema": true,
"supports_sampling_params": false,
"supports_tool_choice": true,
"supports_vision": true,
"supports_xhigh_reasoning_effort": true,
"supports_max_reasoning_effort": true,
"supports_forced_tool_use": true,
"thinking_always_on": false,
"prompt_cache_min_tokens": 512,
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
}
}
@@ -187,8 +187,11 @@ mod tests {
"claude-opus-5-5",
"claude-opus-5",
"claude-fable-5-1",
"claude-haiku-5-5",
"global.anthropic.claude-sonnet-5-5",
"vertex_ai/claude-sonnet-5-5",
"us.anthropic.claude-haiku-5-5",
"vertex_ai/claude-haiku-5-5",
] {
assert_eq!(context_window_for(model), 1_000_000, "{model}");
}
@@ -660,25 +660,44 @@ impl Ledger {
let usd = if ev.failed {
EventUsd::default()
} else {
match pricing::lookup(&ev.model) {
Some(p) => EventUsd {
compression: p.compression_savings_usd(ev.tokens_saved()),
cache: p
.cache_savings_usd(ev.usage.cache_read_tokens, ev.usage.cache_write_tokens),
input_cost: p.input_cost_usd(
// When the response carried no usage block,
// bill the post-compression estimate instead
// so passthrough lanes still show spend.
if ev.usage.total_input() > 0 {
ev.usage.input_tokens
} else {
ev.tokens_after
},
ev.usage.cache_read_tokens,
ev.usage.cache_write_tokens,
),
output_cost: p.output_cost_usd(ev.usage.output_tokens),
},
match pricing::lookup_card(&ev.model) {
Some(card) => {
// When the response carried no usage block, bill
// the post-compression estimate instead so
// passthrough lanes still show spend.
let uncached = if ev.usage.total_input() > 0 {
ev.usage.input_tokens
} else {
ev.tokens_after
};
// The billed prompt picks the rate card: past a
// model's long-context threshold every rate below
// switches to the higher tier.
let p = card.for_prompt(
uncached
.saturating_add(ev.usage.cache_read_tokens)
.saturating_add(ev.usage.cache_write_tokens),
);
EventUsd {
compression: card.compression_savings_usd(
uncached,
ev.usage.cache_read_tokens,
ev.usage.cache_write_tokens,
ev.usage.output_tokens,
ev.tokens_saved(),
),
cache: p.cache_savings_usd(
ev.usage.cache_read_tokens,
ev.usage.cache_write_tokens,
),
input_cost: p.input_cost_usd(
uncached,
ev.usage.cache_read_tokens,
ev.usage.cache_write_tokens,
),
output_cost: p.output_cost_usd(ev.usage.output_tokens),
}
}
None => EventUsd::default(),
}
};
@@ -1277,6 +1296,66 @@ mod tests {
assert!(total > input, "total spend must exceed input-only spend");
}
/// A Haiku 5.5 prompt over 100K bills every rate at the 5x card:
/// 150K uncached in + 5K out is $0.075 + $0.0125, not the base
/// card's $0.015 + $0.0025.
#[test]
fn long_prompt_bills_at_the_models_long_context_card() {
let ledger = Ledger::in_memory();
let mut ev = event("claude-haiku-5-5");
ev.usage = Usage {
input_tokens: 150_000,
output_tokens: 5_000,
cache_read_tokens: 0,
cache_write_tokens: 0,
};
ledger.record(ev);
let v = ledger.stats_payload(true);
let input = v["session"]["input_cost_usd"].as_f64().unwrap();
let output = v["session"]["output_cost_usd"].as_f64().unwrap();
assert!((input - 0.075).abs() < 1e-9, "input: {input}");
assert!((output - 0.0125).abs() < 1e-9, "output: {output}");
}
/// Devin's example on #4037: 110K uncached compressed to 90K saves
/// $0.055 - $0.009 = $0.046 of input, not 20K x $0.10/M = $0.002.
#[test]
fn compression_below_the_threshold_credits_the_avoided_long_card() {
let ledger = Ledger::in_memory();
let mut ev = event("claude-haiku-5-5");
ev.tokens_before = 110_000;
ev.tokens_after = 90_000;
ev.usage = Usage {
input_tokens: 90_000,
output_tokens: 0,
cache_read_tokens: 0,
cache_write_tokens: 0,
};
ledger.record(ev);
let v = ledger.stats_payload(true);
let saved = v["session"]["compression_savings_usd"].as_f64().unwrap();
assert!((saved - 0.046).abs() < 1e-9, "saved: {saved}");
}
/// Cache reads count toward the threshold: 60K uncached + 60K
/// read is a 120K prompt, so it bills at the long card too.
#[test]
fn cached_tokens_count_toward_the_long_context_threshold() {
let ledger = Ledger::in_memory();
let mut ev = event("claude-haiku-5-5");
ev.usage = Usage {
input_tokens: 60_000,
output_tokens: 0,
cache_read_tokens: 60_000,
cache_write_tokens: 0,
};
ledger.record(ev);
let v = ledger.stats_payload(true);
let input = v["session"]["input_cost_usd"].as_f64().unwrap();
// 60K x $0.50/M + 60K x $0.05/M.
assert!((input - 0.033).abs() < 1e-9, "input: {input}");
}
#[test]
fn hostile_token_counts_clamp_to_ceiling() {
let ledger = Ledger::in_memory();
@@ -33,6 +33,17 @@
//! while its Bedrock form is present). Those price at $0 with a WARN
//! until the table is refreshed — see the module note below.
//!
//! # Long-context tiers
//!
//! Some models publish a second, higher rate card for long prompts
//! (LiteLLM's `*_above_<N>k_tokens` fields: Claude Haiku 5.5 above
//! 100K, Sonnet 4 / 4.5 above 200K, GPT-5.x above 272K). Above the
//! threshold the provider re-prices the WHOLE request — input, output
//! and cache alike — not just the tokens past it, so [`lookup_card`]
//! keeps both cards and [`PriceCard::for_prompt`] picks one from the
//! billed prompt. A tier field a row leaves out falls back to that
//! rate's base price, the same rule the Python cost path follows.
//!
//! # Unknown models price at $0
//!
//! A miss returns `None`; the ledger then records $0 for every USD
@@ -121,7 +132,71 @@ impl ModelPrice {
}
}
static BOOK: OnceLock<HashMap<String, ModelPrice>> = OnceLock::new();
/// A model's rate cards: the base one, plus the long-context one for
/// models that publish it (see the module doc).
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct PriceCard {
pub base: ModelPrice,
pub long_context: Option<LongContextTier>,
}
/// The card a request bills at once its prompt passes `above_tokens`.
#[derive(Debug, Clone, Copy, PartialEq)]
pub struct LongContextTier {
/// Prompt size (uncached + cache read + cache write tokens) the
/// request must exceed for this card to apply.
pub above_tokens: u64,
pub price: ModelPrice,
}
impl PriceCard {
/// The rates a request with `prompt_tokens` of billed prompt pays.
pub fn for_prompt(&self, prompt_tokens: u64) -> ModelPrice {
match self.long_context {
Some(tier) if prompt_tokens > tier.above_tokens => tier.price,
_ => self.base,
}
}
/// USD compression saved on one request: what it would have cost
/// uncompressed minus what it did cost.
///
/// Inside one tier that is the removed tokens at that tier's input
/// rate. When the uncompressed prompt would have passed the
/// long-context threshold and the forwarded one does not, the
/// provider would have re-priced the WHOLE request — every input
/// bucket and the output — at the long card, so the saving is the
/// difference of the two full bills, not the removed tokens at the
/// cheap rate.
pub fn compression_savings_usd(
&self,
uncached_input_tokens: u64,
cache_read_tokens: u64,
cache_write_tokens: u64,
output_tokens: u64,
tokens_saved: u64,
) -> f64 {
let prompt = uncached_input_tokens
.saturating_add(cache_read_tokens)
.saturating_add(cache_write_tokens);
let sent = self.for_prompt(prompt);
let unsent = self.for_prompt(prompt.saturating_add(tokens_saved));
if unsent == sent {
return sent.compression_savings_usd(tokens_saved);
}
let without = unsent.input_cost_usd(
uncached_input_tokens.saturating_add(tokens_saved),
cache_read_tokens,
cache_write_tokens,
) + unsent.output_cost_usd(output_tokens);
let with =
sent.input_cost_usd(uncached_input_tokens, cache_read_tokens, cache_write_tokens)
+ sent.output_cost_usd(output_tokens);
(without - with).max(0.0)
}
}
static BOOK: OnceLock<HashMap<String, PriceCard>> = OnceLock::new();
/// Bounded once-per-model warn set so an unknown model logs one WARN,
/// not one per request. Capped so attacker-controlled model ids can't
@@ -140,11 +215,48 @@ pub fn warm() {
let _ = book();
}
fn book() -> &'static HashMap<String, ModelPrice> {
fn book() -> &'static HashMap<String, PriceCard> {
BOOK.get_or_init(parse_vendored)
}
fn parse_vendored() -> HashMap<String, ModelPrice> {
/// A finite, non-negative price field, or `None`.
fn price_field(spec: &serde_json::Map<String, serde_json::Value>, key: &str) -> Option<f64> {
spec.get(key)
.and_then(|v| v.as_f64())
.filter(|c| c.is_finite() && *c >= 0.0)
}
/// The row's long-context card, keyed off its
/// `input_cost_per_token_above_<N>k_tokens` field. Every row in the
/// vendored table publishes at most one such threshold.
fn parse_long_context_tier(
spec: &serde_json::Map<String, serde_json::Value>,
base: &ModelPrice,
) -> Option<LongContextTier> {
let (thousands, input) = spec.keys().find_map(|key| {
let n = key
.strip_prefix("input_cost_per_token_above_")?
.strip_suffix("k_tokens")?;
if n.is_empty() || !n.bytes().all(|b| b.is_ascii_digit()) {
return None;
}
Some((n.parse::<u64>().ok()?, price_field(spec, key)?))
})?;
let tiered = |field: &str, fallback: f64| {
price_field(spec, &format!("{field}_above_{thousands}k_tokens")).unwrap_or(fallback)
};
Some(LongContextTier {
above_tokens: thousands.saturating_mul(1000),
price: ModelPrice {
input,
output: tiered("output_cost_per_token", base.output),
cache_read: tiered("cache_read_input_token_cost", base.cache_read),
cache_write: tiered("cache_creation_input_token_cost", base.cache_write),
},
})
}
fn parse_vendored() -> HashMap<String, PriceCard> {
let raw: serde_json::Value = serde_json::from_str(VENDORED_JSON)
.expect("vendored LiteLLM JSON must parse — same invariant as model_limits");
let mut out = HashMap::new();
@@ -166,23 +278,17 @@ fn parse_vendored() -> HashMap<String, ModelPrice> {
if !input.is_finite() || !output.is_finite() || input < 0.0 || output < 0.0 {
continue;
}
let cache_read = spec
.get("cache_read_input_token_cost")
.and_then(|v| v.as_f64())
.filter(|c| c.is_finite() && *c >= 0.0)
.unwrap_or(input);
let cache_write = spec
.get("cache_creation_input_token_cost")
.and_then(|v| v.as_f64())
.filter(|c| c.is_finite() && *c >= 0.0)
.unwrap_or(input);
let base = ModelPrice {
input,
output,
cache_read: price_field(spec, "cache_read_input_token_cost").unwrap_or(input),
cache_write: price_field(spec, "cache_creation_input_token_cost").unwrap_or(input),
};
out.insert(
model_id.to_ascii_lowercase(),
ModelPrice {
input,
output,
cache_read,
cache_write,
PriceCard {
long_context: parse_long_context_tier(spec, &base),
base,
},
);
}
@@ -236,20 +342,26 @@ fn push_revision_trimmed(push: &mut impl FnMut(String), id: &str) {
}
}
/// Look up per-token pricing for a model id. `None` means "not in
/// the vendored table" — the caller records $0 for USD fields and
/// this module logs one WARN per distinct model id.
pub fn lookup(model: &str) -> Option<ModelPrice> {
/// Look up a model's rate cards. `None` means "not in the vendored
/// table" — the caller records $0 for USD fields and this module logs
/// one WARN per distinct model id.
pub fn lookup_card(model: &str) -> Option<PriceCard> {
let table = book();
for key in candidates(model) {
if let Some(p) = table.get(&key) {
return Some(*p);
if let Some(card) = table.get(&key) {
return Some(*card);
}
}
warn_once(model);
None
}
/// Look up a model's base per-token pricing — the card a prompt below
/// any long-context threshold pays. See [`lookup_card`].
pub fn lookup(model: &str) -> Option<ModelPrice> {
lookup_card(model).map(|card| card.base)
}
fn warn_once(model: &str) {
let set = WARNED.get_or_init(|| RwLock::new(std::collections::HashSet::new()));
{
@@ -288,6 +400,98 @@ mod tests {
assert!(p.cache_write > p.input, "cache writes carry a premium");
}
#[test]
fn claude_haiku_5_5_base_rate_card() {
let p = lookup("claude-haiku-5-5").expect("priced");
assert!((p.input - 1e-7).abs() < 1e-15, "input $0.10/MTok");
assert!((p.output - 5e-7).abs() < 1e-15, "output $0.50/MTok");
assert!((p.cache_read - 1e-8).abs() < 1e-15, "cache read $0.01/MTok");
assert!(
(p.cache_write - 1.25e-7).abs() < 1e-15,
"5m cache write $0.125/MTok"
);
}
/// Haiku 5.5 re-prices the whole request at 5x once the prompt
/// passes 100K; at exactly 100K it still pays the base card.
#[test]
fn claude_haiku_5_5_long_prompt_card_above_100k() {
let card = lookup_card("claude-haiku-5-5").expect("priced");
let tier = card.long_context.expect("Haiku 5.5 publishes a 100K tier");
assert_eq!(tier.above_tokens, 100_000);
assert_eq!(card.for_prompt(100_000), card.base);
let long = card.for_prompt(100_001);
assert!((long.input - 5e-7).abs() < 1e-15, "input $0.50/MTok");
assert!((long.output - 2.5e-6).abs() < 1e-15, "output $2.50/MTok");
assert!(
(long.cache_read - 5e-8).abs() < 1e-15,
"cache read $0.05/MTok"
);
assert!(
(long.cache_write - 6.25e-7).abs() < 1e-15,
"5m cache write $0.625/MTok"
);
}
/// The threshold comes from each row's own field name, not a
/// shared constant: Sonnet 4.5's card switches at 200K.
#[test]
fn long_context_threshold_is_read_per_model() {
let card = lookup_card("claude-sonnet-4-5").expect("priced");
let tier = card.long_context.expect("Sonnet 4.5 publishes a 200K tier");
assert_eq!(tier.above_tokens, 200_000);
assert_eq!(card.for_prompt(150_000), card.base);
assert!((card.for_prompt(200_001).input - 6e-6).abs() < 1e-15);
}
/// Compression that keeps a prompt under the threshold saves the
/// long card on the whole request, not just the removed tokens at
/// the base rate: 110K -> 90K uncached on Haiku 5.5, 1K output, is
/// (110K x $0.50 + 1K x $2.50) - (90K x $0.10 + 1K x $0.50) per MTok.
#[test]
fn compression_across_the_threshold_saves_the_whole_premium() {
let card = lookup_card("claude-haiku-5-5").expect("priced");
let usd = card.compression_savings_usd(90_000, 0, 0, 1_000, 20_000);
assert!((usd - (0.0575 - 0.0095)).abs() < 1e-9, "usd: {usd}");
// Inside one tier it stays removed tokens at that tier's rate.
let below = card.compression_savings_usd(50_000, 0, 0, 1_000, 20_000);
assert!((below - 20_000.0 * 1e-7).abs() < 1e-12, "below: {below}");
let above = card.compression_savings_usd(120_000, 0, 0, 1_000, 20_000);
assert!((above - 20_000.0 * 5e-7).abs() < 1e-12, "above: {above}");
}
/// A model without a published tier bills the base card at any size.
#[test]
fn flat_rated_model_has_no_long_context_tier() {
let card = lookup_card("claude-opus-4-6").expect("priced");
assert!(card.long_context.is_none());
assert_eq!(card.for_prompt(900_000), card.base);
}
/// A tier that omits a rate falls back to that rate's base price.
#[test]
fn missing_tier_field_falls_back_to_base_rate() {
let spec: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
r#"{"input_cost_per_token_above_128k_tokens": 2e-6,
"output_cost_per_token_above_128k_tokens_priority": 9e-6}"#,
)
.unwrap();
let base = ModelPrice {
input: 1e-6,
output: 4e-6,
cache_read: 1e-7,
cache_write: 1e-6,
};
let tier = parse_long_context_tier(&spec, &base).expect("tier parsed");
assert_eq!(tier.above_tokens, 128_000);
assert_eq!(tier.price.input, 2e-6);
assert_eq!(
tier.price.output, base.output,
"a _priority variant is not the tier"
);
assert_eq!(tier.price.cache_read, base.cache_read);
}
/// Bedrock cross-region pricing is region-specific, so an exact
/// regional entry must win over the geo-stripped base id.
#[test]
+9 -9
View File
@@ -10,7 +10,7 @@ target "runtime-default" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime"
args = {
HEADROOM_EXTRAS = "proxy,bedrock"
HEADROOM_EXTRAS = "proxy,bedrock,redis"
RUNTIME_USER = "nonroot"
}
}
@@ -19,7 +19,7 @@ target "runtime" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime"
args = {
HEADROOM_EXTRAS = "proxy,bedrock"
HEADROOM_EXTRAS = "proxy,bedrock,redis"
RUNTIME_USER = "root"
}
}
@@ -28,7 +28,7 @@ target "runtime-nonroot" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime"
args = {
HEADROOM_EXTRAS = "proxy,bedrock"
HEADROOM_EXTRAS = "proxy,bedrock,redis"
RUNTIME_USER = "nonroot"
}
}
@@ -37,7 +37,7 @@ target "runtime-code" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime"
args = {
HEADROOM_EXTRAS = "proxy,code,bedrock"
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
RUNTIME_USER = "root"
}
}
@@ -46,7 +46,7 @@ target "runtime-code-nonroot" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime"
args = {
HEADROOM_EXTRAS = "proxy,code,bedrock"
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
RUNTIME_USER = "nonroot"
}
}
@@ -55,7 +55,7 @@ target "runtime-slim" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime-slim"
args = {
HEADROOM_EXTRAS = "proxy,bedrock"
HEADROOM_EXTRAS = "proxy,bedrock,redis"
RUNTIME_USER = "root"
}
}
@@ -64,7 +64,7 @@ target "runtime-slim-nonroot" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime-slim"
args = {
HEADROOM_EXTRAS = "proxy,bedrock"
HEADROOM_EXTRAS = "proxy,bedrock,redis"
RUNTIME_USER = "nonroot"
}
}
@@ -73,7 +73,7 @@ target "runtime-code-slim" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime-slim"
args = {
HEADROOM_EXTRAS = "proxy,code,bedrock"
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
RUNTIME_USER = "root"
}
}
@@ -82,7 +82,7 @@ target "runtime-code-slim-nonroot" {
inherits = ["_common", "docker-metadata-action"]
target = "runtime-slim"
args = {
HEADROOM_EXTRAS = "proxy,code,bedrock"
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
RUNTIME_USER = "nonroot"
}
}
+3 -3
View File
@@ -51,12 +51,12 @@ export default function Layout({ children }: LayoutProps<'/'>) {
<body className="flex flex-col min-h-screen">
<RootProvider>
{/* Dismissible; Fumadocs remembers the dismissal per id in localStorage. */}
<Banner id="teams-perk-2026">
<Banner id="teams-contact-2026">
<a
href="https://headroom-perks.vercel.app/?source=github&utm_source=docs&utm_medium=banner"
href="https://headroomlabs.ai"
className="font-medium underline-offset-4 hover:underline"
>
Headroom for Teams: 3 months free with a 12-month plan →
Want more savings? Headroom for teams and enterprises: talk to us →
</a>
</Banner>
{children}
+9 -16
View File
@@ -41,26 +41,19 @@ Every call to `client.messages.create()` compresses messages first. The response
`withHeadroom()` returns a proxy around your Anthropic client that intercepts `messages.create()`:
1. Converts Anthropic-format messages to OpenAI format
2. Sends them to the Headroom proxy's `/v1/compress` endpoint
3. Converts the compressed messages back to Anthropic format
4. Forwards the request to Anthropic as normal
1. Sends your `messages` to the Headroom proxy's `/v1/compress` endpoint as they are, in Anthropic format
2. Forwards the request to Anthropic with the compressed messages in place
### Message format conversion
### Message format
The adapter handles the full Anthropic message format including content blocks:
Nothing is converted. `POST /v1/compress` compresses Anthropic content blocks natively — see [Message format](/docs/proxy#message-format) — and returns the same shape it received, so every block reaches Anthropic exactly as you built it:
| Anthropic format | OpenAI format |
|-----------------|---------------|
| `{ type: "text", text: "..." }` | `{ role: "user", content: "..." }` |
| `{ type: "tool_use", id, name, input }` | `{ tool_calls: [{ id, function: { name, arguments } }] }` |
| `{ type: "tool_result", tool_use_id, content }` | `{ role: "tool", tool_call_id, content }` |
- `thinking` and `redacted_thinking` blocks keep their `signature` and `data`, which Anthropic requires back unmodified within a tool-use turn
- `cache_control` markers stay on the blocks that carry them
- `tool_use` and `tool_result` blocks keep their ids; only the content of large `tool_result` blocks changes
- images, documents, citations and server-tool blocks pass through untouched
This conversion is lossless. Your request and response behave identically to an unwrapped client.
<Callout type="info" title="The conversion is an SDK choice, not an API requirement">
`POST /v1/compress` does no format conversion and compresses Anthropic content blocks natively — see [Message format](/docs/proxy#message-format). If you are calling the endpoint directly (from a gateway, or LiteLLM's `headroom` guardrail), send Anthropic-shaped messages as-is; you get the same shape back. Only this TypeScript adapter converts, because it normalises on OpenAI types internally.
</Callout>
If the proxy is unreachable and `fallback` is enabled, the original messages are forwarded unchanged.
## Options
+124
View File
@@ -0,0 +1,124 @@
---
title: FastAPI
description: Cut LLM input tokens in a FastAPI app. Compress tool outputs, logs and chat history with compress() before calling OpenAI or Anthropic. No API key needed to try it.
---
Headroom's `compress()` is a function over your message list, so in FastAPI it
is one call before your model call. This page builds a `POST /chat` endpoint
that returns the compressed messages and token counts, then shows the one-line
change that forwards them to a provider.
## Install
```bash
pip install fastapi "uvicorn[standard]" headroom-ai
```
No provider key, no model download. The base package is enough for
`compress()`; `[ml]` adds the Kompress text model later if you want more
savings on prose-heavy tool output.
## Endpoint
```python
# main.py
from typing import Any
from fastapi import FastAPI
from pydantic import BaseModel
from headroom import compress
app = FastAPI()
class ChatIn(BaseModel):
messages: list[dict[str, Any]] # OpenAI or Anthropic message dicts
model: str = "gpt-4o" # used for token counting and the context limit
@app.post("/chat")
def chat(body: ChatIn) -> dict[str, Any]:
result = compress(body.messages, model=body.model)
return {
"messages": result.messages,
"tokens_before": result.tokens_before,
"tokens_after": result.tokens_after,
"tokens_saved": result.tokens_saved,
"transforms": result.transforms_applied,
}
```
Save it as `main.py` and run it with no network at all:
```bash
HEADROOM_OFFLINE=1 uvicorn main:app --port 8000
```
Then post a 3,000-line tool output with one error in it:
```bash
python - <<'EOF_PY'
import json, urllib.request
log = "\n".join([f"INFO step {i} ok" for i in range(3000)] + ["FATAL disk full"])
msgs = [
{"role": "user", "content": "What failed?"},
{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "logs", "arguments": "{}"}}]},
{"role": "tool", "tool_call_id": "c1", "content": log},
]
req = urllib.request.Request("http://127.0.0.1:8000/chat", data=json.dumps({"messages": msgs}).encode(), headers={"content-type": "application/json"})
out = json.load(urllib.request.urlopen(req))
print(out["tokens_before"], "->", out["tokens_after"], "FATAL kept:", "FATAL" in json.dumps(out["messages"]))
EOF_PY
```
Expected output is on the order of `20000 -> 95 FATAL kept: True`: the
repeated `INFO` lines collapse, the `FATAL` line survives.
## Forward to the provider
Replace `messages=body.messages` with `messages=result.messages` in your
existing call. Nothing else changes:
```python
from openai import OpenAI
client = OpenAI()
@app.post("/answer")
def answer(body: ChatIn) -> dict[str, Any]:
result = compress(body.messages, model=body.model)
completion = client.chat.completions.create(model=body.model, messages=result.messages)
return {
"answer": completion.choices[0].message.content,
"tokens_saved": result.tokens_saved,
}
```
For Anthropic, pass `result.messages` to `client.messages.create` the same way.
## What it keeps
Error lines, stack traces, first and last items, and statistical outliers in
JSON arrays are kept. Repeated and low-information lines are dropped. Originals
are cached locally; the model can ask for them back when it has the retrieval
tool, which the [proxy](/docs/proxy) and the [MCP server](/docs/mcp) provide
([CCR](/docs/ccr)). The plain `compress()` call above does not expose it.
Compression applies to tool and assistant messages; user messages are left alone
unless you pass `compress_user_messages=True`. Short prompts and prose compress
very little; see [Limitations](/docs/limitations).
## Not this page
`headroom.integrations.asgi.CompressionMiddleware` is for FastAPI apps that
*serve* an LLM-shaped endpoint (`/v1/messages`, `/v1/chat/completions`), such
as a proxy you built. For an app that *calls* a provider, use `compress()` as
above.
## Settings you will reach for
- `HEADROOM_OFFLINE=1`: no update check, no model or pricing-table downloads, no beacon. Use it in CI and sandboxes. See [Security model](/docs/security-model).
- `compress(..., model_limit=N)`: tell Headroom the model's window is N tokens, so it compresses harder as the prompt approaches it. It never truncates history.
- `compress(..., protect_recent=K)`: leave source code in the last K messages uncompressed.
- [Configuration](/docs/configuration) has the rest.
+1 -1
View File
@@ -1,6 +1,6 @@
---
title: Introduction
description: Headroom is the context optimization layer for LLM applications. Compress tool outputs, DB results, file reads, and RAG results before they reach the model. Same answers, fraction of the tokens.
description: Headroom (headroom-ai) compresses tool outputs, logs, JSON, RAG results and chat history before each LLM call, to reduce input tokens and token cost and fit a context window. Python library (no API key), proxy and MCP server. Same answers, fewer tokens.
---
Headroom compresses everything your AI agent reads -- tool outputs, database results, file reads, RAG retrievals, API responses -- before it reaches the LLM. The model sees less noise, responds faster, and costs less.
+11 -10
View File
@@ -116,11 +116,12 @@ pip install "headroom-ai[all]"
| `autogen` | AutoGen AgentChat integration; **not in `[all]`** | `pip install "headroom-ai[autogen]"` |
| `anyllm` | any-llm multi-provider backend (Python 3.11+); **not in `[all]`** | `pip install "headroom-ai[anyllm]"` |
| `bedrock` | Native AWS Bedrock credentials/backend support; **not in `[all]`** | `pip install "headroom-ai[bedrock]"` |
| `redis` | Redis client for state shared across replicas (`HEADROOM_CCR_BACKEND=redis`, `HEADROOM_COMPRESS_SESSION_STORE`); **not in `[all]`**, included in every Docker image | `pip install "headroom-ai[redis]"` |
| `evals` | Evaluation framework (GSM8K, SQuAD, BFCL benchmarks) | `pip install "headroom-ai[evals]"` |
| `pytorch-mps` | Apple-GPU (MPS) memory-embedder offload — **macOS only**, not in `[all]` (torch + sentence-transformers); opt in with `HEADROOM_EMBEDDER_RUNTIME=pytorch_mps` | `pip install "headroom-ai[pytorch-mps]"` |
| `html` | HTML main-content extraction with trafilatura | `pip install "headroom-ai[html]"` |
| `spreadsheet` | `.xlsx` / `.xls` ingestion with openpyxl and xlrd | `pip install "headroom-ai[spreadsheet]"` |
| `sandbox` | Torch-free proxy bundle for constrained environments | `pip install "headroom-ai[sandbox]"` |
| `lean` | Torch-free proxy bundle for constrained environments (a dependency profile; it provides no isolation). `sandbox` is a deprecated alias | `pip install "headroom-ai[lean]"` |
| `all` | Runtime bundle: `proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,mcp,spreadsheet` | `pip install "headroom-ai[all]"` |
`[all]` is the full built-in runtime bundle, not every integration or optional
@@ -255,15 +256,15 @@ If you want a host `headroom` CLI that keeps Headroom itself inside a container
| Tag | Extras | Base image | Description |
|---|---|---|---|
| `latest` | `proxy,bedrock` | Debian slim | Default image, runs as non-root |
| `<version>` | `proxy,bedrock` | Debian slim | Pinned release, same default variant |
| `nonroot` | `proxy,bedrock` | Debian slim | Explicit non-root variant |
| `code` | `proxy,code,bedrock` | Debian slim | Includes tree-sitter for code compression |
| `code-nonroot` | `proxy,code,bedrock` | Debian slim | Code compression, non-root |
| `slim` | `proxy,bedrock` | Distroless | Minimal image, no shell |
| `slim-nonroot` | `proxy,bedrock` | Distroless | Minimal, non-root |
| `code-slim` | `proxy,code,bedrock` | Distroless | Code compression, minimal |
| `code-slim-nonroot` | `proxy,code,bedrock` | Distroless | Code compression, minimal, non-root |
| `latest` | `proxy,bedrock,redis` | Debian slim | Default image, runs as non-root |
| `<version>` | `proxy,bedrock,redis` | Debian slim | Pinned release, same default variant |
| `nonroot` | `proxy,bedrock,redis` | Debian slim | Explicit non-root variant |
| `code` | `proxy,code,bedrock,redis` | Debian slim | Includes tree-sitter for code compression |
| `code-nonroot` | `proxy,code,bedrock,redis` | Debian slim | Code compression, non-root |
| `slim` | `proxy,bedrock,redis` | Distroless | Minimal image, no shell |
| `slim-nonroot` | `proxy,bedrock,redis` | Distroless | Minimal, non-root |
| `code-slim` | `proxy,code,bedrock,redis` | Distroless | Code compression, minimal |
| `code-slim-nonroot` | `proxy,code,bedrock,redis` | Distroless | Code compression, minimal, non-root |
### Build from source
+1
View File
@@ -28,6 +28,7 @@
"corporate-networks",
"local-llm-prefill",
"---Integrations---",
"fastapi",
"vercel-ai-sdk",
"openai-sdk",
"anthropic-sdk",
+25 -12
View File
@@ -112,10 +112,10 @@ headroom proxy --mode cache
| Profile | Target savings | Mode | Notes |
|---------|----------------|------|-------|
| `coding` | emergent (~50%) | `cache` | **Default.** Delta-only compression at ~0 prefix-cache busts; never lossy-compresses file reads. |
| `balanced` | ~70% | `token` | Moderate compression with structural compaction. |
| `agent-90` | ~90% | `token` | Aggressive; pins a `0.10` keep-ratio and forces Kompress. |
| `general` | emergent (~60%) | `token` | Non-coding workloads. |
| `coding` | ~50% of new input tokens (emergent) | `cache` | **Default.** Delta-only compression at ~0 prefix-cache busts; never lossy-compresses file reads. Whole-request and lifetime ratios are much lower in cache mode. |
| `balanced` | ~70% of new input tokens | `token` | Moderate compression with structural compaction. |
| `agent-90` | ~90% of new input tokens | `token` | Aggressive; pins a `0.10` keep-ratio and forces Kompress. |
| `general` | ~60% of new input tokens (emergent) | `token` | Non-coding workloads. |
An unrecognized `HEADROOM_SAVINGS_PROFILE` value logs a warning and falls back to `coding` (the same default an unset variable resolves to) — the proxy never fails to start over a bad profile name. `balanced` is used only as a last-resort fallback when the runtime has no `coding` entry at all (old-runtime/new-client version skew). See `headroom/agent_savings.py` for each profile's full set of knobs.
@@ -194,10 +194,10 @@ HEADROOM_SAVINGS_PROFILE=agent-90 headroom proxy
| Profile | Target savings | proxy_mode | force_kompress | Best for |
|---------|---------------|------------|---------------|----------|
| `coding` (default) | ~50% (emergent) | `cache` | No | Coding agents — preserves Anthropic prefix-cache stability |
| `agent-90` | 90% | `token` | Yes | Non-coding, cost-sensitive, or high-volume workloads |
| `balanced` | 70% | `token` | No | General-purpose moderate compression |
| `general` | ~60% (emergent) | `token` | No | Non-coding chat, little code in context |
| `coding` (default) | ~50% of new input tokens (emergent) | `cache` | No | Coding agents — preserves Anthropic prefix-cache stability. Whole-request and lifetime ratios are much lower in cache mode. |
| `agent-90` | ~90% of new input tokens | `token` | Yes | Non-coding, cost-sensitive, or high-volume workloads |
| `balanced` | ~70% of new input tokens | `token` | No | General-purpose moderate compression |
| `general` | ~60% of new input tokens (emergent) | `token` | No | Non-coding chat, little code in context |
**`coding` (default)** — Optimizes for coding-agent workloads with Anthropic. Uses **cache mode** (`proxy_mode="cache"`): compresses only the newest delta in each turn so the provider's prefix-cache is never busted. User messages are compressed, system prompts preserved (hottest cache). `protect_recent` is `0` — in cache mode the delta already *is* the newest turn, so a turn-count guard would suppress the only thing cache mode compresses; byte-exact fidelity instead comes from `protect_reads=True` (file reads are never lossy-compressed). Lossless-first with lossy fallback; tool search and cross-turn dedup enabled. This is the profile that `headroom wrap` uses.
@@ -320,6 +320,7 @@ Fine-grained control over what gets compressed and how hard. Most users pick a [
| `--mode` / `HEADROOM_MODE` | `cache` | `cache` compresses only the newest delta (prefix-cache safe); `token` maximizes removal. |
| `--target-ratio` / `HEADROOM_TARGET_RATIO` | unset | Keep-ratio for ML text compression; lower = more aggressive (e.g. `0.10`). |
| `HEADROOM_MIN_TOKENS` | `500` | Minimum block size before a tool output is compressed. |
| `HEADROOM_SHELL_RESULT_MIN_CHARS` | `2000` | Shell tool results shorter than this many characters bypass lossy compression. For top-level text blocks, the threshold uses their combined text length. Byte-lossless folds such as search grouping may still apply. Applies to `bash`, `shell`, `local_shell`, and aliases such as `run_shell_command`/`execute_command`; set `0` to restore lossy compression. |
| `HEADROOM_COMPRESS_USER_MESSAGES` | profile | Compress text inside user-role messages. The `coding` profile turns this on; set `0` to keep user text verbatim under it. Anthropic `tool_result` blocks compress either way. No `--compress-user-messages` flag on `headroom proxy`; the direct module entry point (`python -m headroom.proxy.server`) does accept one. |
| `HEADROOM_COMPRESS_SYSTEM_MESSAGES` | unset | Compress system prompts. Off by default to keep the hottest cache prefix stable. |
| `HEADROOM_PROTECT_RECENT` | profile | Never compress the N most recent turns. |
@@ -652,7 +653,7 @@ Keep carrying both yourself and send them upstream unchanged. Two consequences w
| `transforms_applied` | array | Transform labels that ran. |
| `transforms_summary` | object | Per-transform counts. |
| `ccr_hashes` | array | Retrieval hashes for markers inserted (empty unless `mode: "ccr"`). |
| `session` | object | [Session mode](#session-mode) only: `{ "id", "frozen_message_count", "cached_prefix_replayed" }`. |
| `session` | object | [Session mode](#session-mode) only: `{ "id", "frozen_message_count", "cached_prefix_replayed" }`, described there. |
```json
{
@@ -741,15 +742,27 @@ Pass `config.session_id` and Headroom keeps the conversation's replay state itse
The response gains a `session` object:
```json
"session": { "id": "conv-8f1c", "frozen_message_count": 12, "cached_prefix_replayed": true }
"session": { "id": "conv-8f1c", "frozen_message_count": 12, "cached_prefix_replayed": false }
```
| Field | Type | Description |
| --- | --- | --- |
| `id` | string | The session id this turn ran under: `config.session_id` as sent, or the `x-headroom-session-id` header when `HEADROOM_COMPRESS_SESSION_FROM_HEADER=1` and `config.session_id` is absent. |
| `frozen_message_count` | integer | How many leading messages the pipeline was told not to recompress this turn. Headroom counts the leading run it can replay from the session's compression cache: plain messages always count, a tool result counts only once its content is in that cache, and the last message never counts. An explicit `config.frozen_message_count` wins when larger. It is derived from Headroom's own state, not from the provider, and it can be non-zero on a first turn. |
| `cached_prefix_replayed` | boolean | `true` when the pipeline's output for some leading messages differed from what this endpoint returned for the session last turn, and Headroom overwrote those messages with last turn's bytes. `false` when nothing was overwritten: when the serving process holds no previous turn for the session (a first turn, expired state, or a replica that never served or loaded it), when the pipeline already produced last turn's bytes (the usual case), or when the replay was skipped as unsafe (this turn's messages do not line up with last turn's, or the replay would be larger than the pipeline's output). So `false` does not mean the returned prefix matches last turn's. |
<Callout type="warning" title="cached_prefix_replayed is not a prompt-cache hit">
`cached_prefix_replayed` reports Headroom's own repair step. It says nothing about whether the provider served the prefix from its prompt cache. `false` is the normal value on a healthy session, and it is also what a turn that changed the prefix returns when there was nothing to replay. Read cache hits and misses from the provider's usage (`cache_read_input_tokens`, `prompt_tokens_details.cached_tokens`).
</Callout>
What differs from the stateless contract:
- **Forward the returned messages verbatim.** They are what the next turn replays.
- **`compress_user_messages` is refused** (`400`). User-message rewrites are not byte-replayable across turns, so they would bust the prefix later.
- **Timeouts and busy sessions are a `503`, never fail-open.** Handing back originals would leave the caller forwarding bytes the replay state does not know about, so a compression timeout answers `503` with `error.type = "compression_timeout"`; retry the turn and it lands on consistent state. Turns for one session are serialized under a per-session lock, and a concurrent turn for the same session that waits more than 10 seconds gets the same `503`.
- **State is in-process.** The replay cache lives for `HEADROOM_COMPRESSION_CACHE_TTL_SECONDS` of idle time (3900 by default, floor 600); the tracker's usage state expires after 10 idle minutes. Multi-process deployments (gunicorn workers, replicas) must pin a session to one process — hash the upstream on the session id — or replay silently misses.
- **State is in-process by default.** The replay cache lives for `HEADROOM_COMPRESSION_CACHE_TTL_SECONDS` of idle time (3900 by default, floor 600); the tracker's usage state expires after 10 idle minutes. Without a shared store, multi-process deployments (gunicorn workers, replicas) must pin a session to one process — hash the upstream on the session id — or replay silently misses.
- **`HEADROOM_COMPRESS_SESSION_STORE=redis://...` shares sessions across replicas.** Every replica also keeps each session's tracker in Redis, so any replica can serve the session's next turn and its `POST /v1/usage`. It needs the `redis` extra, which every Docker image includes. It also selects the Redis CCR store unless `HEADROOM_CCR_BACKEND` names another, so CCR markers resolve on any replica. Two limits remain. The per-session lock is per process: two concurrent turns of one session on two replicas both run, the first write wins (compare-and-set), and the next turn continues from the winner. Gateway-mode pending turns stay in-process, so `POST /v1/compress/response` must reach the replica that answered the request half; re-drive therefore still needs real affinity (`gateway.session_affinity`).
- **An unreachable Redis does not fail turns.** A Redis error is a miss: the replica serves from its own memory, logs one `WARNING` when a kind of call (read, write) starts failing and an `INFO` when it works again, and skips Redis for 5 seconds after a connection error (each command times out after 1 second). The cost is the sharing. A replica with no local state for the session recompresses the history, which is slower, and the prefix it returns can differ from last turn's, which is a provider prompt-cache miss. `cached_prefix_replayed` stays `false` on those turns. A Redis that comes back empty is refilled as each session's next turn writes it whole. Run Redis with high availability if prompt-cache hits matter.
- `HEADROOM_COMPRESS_SESSION_FROM_HEADER=1` lets the `x-headroom-session-id` header stand in for `config.session_id`. Off by default: gateways commonly stamp that header on all traffic, and a value shared across conversations (Claude Code subagents do this) would blend them into one replay state.
**Relaying provider usage: `POST /v1/usage`.** Session mode never sees the provider's response, so cache-hit attribution and savings accounting for the session come from this optional relay (same exposure rules as `/v1/compress`). Post the provider's `usage` block under the session id:
@@ -779,7 +792,7 @@ The rule behind it is the proxy's own **no shrink without reload**, moved to the
| --- | --- | --- | --- |
| `can_redrive` | boolean | `false` | The gateway can call the provider again with a request Headroom hands it and post the result back. Unlocks re-driving turn hooks and, with `config.mode: "ccr"`, in-band CCR retrieval. Send `false` for streaming requests. |
| `can_relay_response` | boolean | `false` | The gateway will post the provider's status and usage to `/v1/compress/response` after every turn. |
| `session_affinity` | boolean | `true` | The gateway pins this session to one Headroom replica. Pending turns are in-process state, so `false` disables re-driving for the turn even when `can_redrive` is set. |
| `session_affinity` | boolean | `true` | The gateway pins this session to one Headroom replica. Pending turns are in-process state, so `false` disables re-driving for the turn even when `can_redrive` is set. `HEADROOM_COMPRESS_SESSION_STORE` shares replay state, not pending turns, so it does not lift this. |
| `plugin_version` | string | unset | Diagnostics only. |
| `request_headers` | object | `{}` | The client's request headers the gateway is willing to hand over, as lowercase name → string value. Only `anthropic-beta` is read today: when a turn hook asks for an `anthropic-beta` token, the client's tokens go first in the returned `headers`; every other name is ignored, so a gateway may pass its whole header map. A non-object block or a non-string value is a `400`. Never forwarded anywhere by Headroom. |
+22 -8
View File
@@ -7,7 +7,7 @@ This guide gets you from zero to compressed LLM calls in under 5 minutes.
## 1. Install
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
<Tab value="TypeScript">
```bash
npm install headroom-ai
@@ -15,12 +15,15 @@ npm install headroom-ai
</Tab>
<Tab value="Python">
```bash
# CLI/proxy/wrap on your machine
uv tool install --python 3.13 "headroom-ai[all]"
# Python project or virtualenv: enough for compress(), no API key, no model download
pip install headroom-ai
# Python project or virtualenv
pip install "headroom-ai[all]"
# CLI, proxy and `headroom wrap` on your machine
uv tool install --python 3.13 "headroom-ai[all]"
```
`[all]` adds the proxy, the MCP server, the Kompress model and more; `compress()`
needs only the base package.
</Tab>
</Tabs>
@@ -38,7 +41,7 @@ The proxy runs the compression pipeline (Python) and exposes an HTTP API that th
## 2. Compress messages
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
<Tab value="TypeScript">
```ts twoslash
import { compress } from 'headroom-ai';
@@ -110,11 +113,22 @@ result = compress(messages, model="gpt-4o")
</Tab>
</Tabs>
<Callout type="info" title="No provider key needed to try it">
The Python call runs in-process. `HEADROOM_OFFLINE=1` makes it zero-network as
well (no update check, no model or pricing-table downloads):
```bash
HEADROOM_OFFLINE=1 python -c "from headroom import compress; log='\n'.join(f'INFO step {i} ok' for i in range(3000))+'\nFATAL disk full'; r=compress([{'role':'user','content':'What failed?'},{'role':'assistant','content':None,'tool_calls':[{'id':'c1','type':'function','function':{'name':'logs','arguments':'{}'}}]},{'role':'tool','tool_call_id':'c1','content':log}], model='gpt-4o'); print(r.tokens_before, '->', r.tokens_after, 'FATAL kept:', 'FATAL' in str(r.messages))"
```
A [FastAPI endpoint](/docs/fastapi) built on the same call is one page away.
</Callout>
## 3. Send to your LLM
Use the compressed messages exactly like the originals:
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
<Tab value="TypeScript">
```ts twoslash
import OpenAI from 'openai';
@@ -149,7 +163,7 @@ print(response.choices[0].message.content)
## 4. Check your savings
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
<Tab value="TypeScript">
```ts twoslash
const result = {
+1
View File
@@ -284,6 +284,7 @@ accept arbitrary environment variables — writes are restricted to a hardcoded
| `HEADROOM_INTERCEPT_READ_MIN_CHARS` | ast-grep read-interception threshold |
| `HEADROOM_VERIFY_TRUNCATION_ON_DISK` | Verify ast-grep Read truncation against disk when no client banner is found. Always returns `UNKNOWN` on Windows, where `dir_fd` / `O_NOFOLLOW` support is unavailable. |
| `HEADROOM_VERIFY_TRUNCATION_MAX_BYTES` | Max on-disk file size (bytes) the disk-verify fallback will read |
| `HEADROOM_DEFERRED_TOOL_AFFINITY` | Keep `defer_loading` tools out of the Anthropic prefix-tracker lineage key (default on; `0` counts them again) |
| `HEADROOM_WEB_SEARCH_HISTORY` | Drop `web_search_call` history from a Responses request that declares no web_search tool (default on; `0` disables) |
| `HEADROOM_BUFFERED_UPSTREAM_STREAM` | Send buffered CCR requests (and their continuations) upstream as `stream: true` and fold the reply (default on; `0` sends `stream: false`) |
+11 -19
View File
@@ -1,27 +1,19 @@
---
title: Headroom for Teams
description: Headroom Enterprise for engineering teams — more savings, automatic model routing, attack removal, SSO and an org-wide savings dashboard. 3 months free with a 12-month plan.
description: Not seeing the savings you want? Headroom Enterprise adds more cost savings, tokens secured, on premise deployment and deep attribution. Talk to us.
---
Headroom OSS is built for one developer on one laptop. Headroom Enterprise is built
for the whole team.
**Not seeing the savings you want?** Headroom OSS is a framework. It was built to show
what's possible on one developer's laptop, so it saves less than it could. We are
building bigger savings for teams and enterprises in Headroom Enterprise, which adds:
| | Open source | Enterprise |
|---|---|---|
| Savings | Compression, plus tool search on Anthropic requests (on by default) | Also: tool search for every model, harness tuning, extra compressors |
| Model routing | Rule-based and opt-in: turn it on with `HEADROOM_MODEL_ROUTER_ENABLED=1` and write the rules in [`HEADROOM_MODEL_ROUTES`](/docs/configuration#cost-aware-model-routing) | Automatic: a cheaper model per request when quality allows, with a shadow mode that shows the savings before you turn it on |
| Security | — | Detect and strip prompt injection and risky tool calls |
| Rollout | Per laptop | Managed or in your VPC, SSO, central config, org-wide savings |
| Support | GitHub, Discord | Dedicated engineers, onboarding, priority response |
- More cost savings
- Tokens secured
- On premise deployment
- Deep attribution
- And many more features
## 3 months free
Companies that sign a 12-month Enterprise plan get their first 3 months free. One
perk per company.
**[Claim 3 months free →](https://headroom-perks.vercel.app/?source=github&utm_source=docs&utm_medium=teams-page)**
Prefer to talk first? Email [hello@headroomlabs.ai](mailto:hello@headroomlabs.ai)
with your stack and rough monthly LLM spend.
If you're a team or an enterprise, **talk to us**:
[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) · [headroomlabs.ai](https://headroomlabs.ai)
Everything in the open-source repo stays open source under Apache 2.0.
+61 -8
View File
@@ -22,6 +22,8 @@ from ``e2e/init/Dockerfile``).
from __future__ import annotations
import json
import os
import subprocess
import sys
from pathlib import Path
@@ -149,6 +151,59 @@ def _verify_claude_local(ctx: CaseContext) -> None:
raise AssertionError(f"Unexpected Claude install commands: {claude_calls}")
def _expect_shell_exports(shell_file: Path, expected: dict[str, str]) -> None:
"""Check exported values in a child process after sourcing an rc file.
Args:
shell_file: Scratch shell configuration written by the init case.
expected: Environment variables that init must export verbatim.
Raises:
AssertionError: If sourcing fails or the exported values differ.
"""
# Clear inherited values so missing exports cannot accidentally pass. A
# separate Python child also distinguishes assignments from real exports.
env = {
name: value
for name, value in os.environ.items()
if name not in expected and name not in {"BASH_ENV", "ENV"}
}
probe = (
"import json, os, sys; "
"print(json.dumps({name: os.environ.get(name) for name in sys.argv[1:]}))"
)
try:
result = subprocess.run(
[
"bash",
"--noprofile",
"--norc",
"-c",
'. "$1" && "$2" -c "$3" "${@:4}"',
"headroom-env-check",
str(shell_file),
sys.executable,
probe,
*expected,
],
env=env,
capture_output=True,
text=True,
check=True,
timeout=10,
)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
raise AssertionError(f"{shell_file.name} could not be sourced successfully") from exc
try:
actual = json.loads(result.stdout)
except json.JSONDecodeError as exc:
raise AssertionError(
f"{shell_file.name} did not produce exported environment values"
) from exc
if actual != expected:
raise AssertionError(f"{shell_file.name} exported {actual!r}; expected {expected!r}")
def _verify_copilot_global(ctx: CaseContext) -> None:
config = json.loads((ctx.home / ".copilot" / "config.json").read_text(encoding="utf-8"))
if "SessionStart" not in config["hooks"]:
@@ -158,15 +213,13 @@ def _verify_copilot_global(ctx: CaseContext) -> None:
session_start = config["hooks"]["SessionStart"][0]["command"]
_expect_hook_command(session_start, "init-user")
expected_env = {
"COPILOT_PROVIDER_TYPE": "openai",
"COPILOT_PROVIDER_BASE_URL": "http://127.0.0.1:9005/v1",
"COPILOT_PROVIDER_WIRE_API": "completions",
}
for shell_file in (ctx.home / ".bashrc", ctx.home / ".zshrc", ctx.home / ".profile"):
content = shell_file.read_text(encoding="utf-8")
for literal in (
'export COPILOT_PROVIDER_TYPE="openai"',
'export COPILOT_PROVIDER_BASE_URL="http://127.0.0.1:9005/v1"',
'export COPILOT_PROVIDER_WIRE_API="completions"',
):
if literal not in content:
raise AssertionError(f"{shell_file.name} missing {literal!r}")
_expect_shell_exports(shell_file, expected_env)
copilot_calls = [
record["argv"] for record in _read_jsonl(ctx.shim_log) if record["tool"] == "copilot"
+4
View File
@@ -93,6 +93,10 @@ RUN npm install -g --no-fund --no-audit npm@11.16.0
COPY e2e/wrap/tools/package.json e2e/wrap/tools/package-lock.json /opt/wrap-tools/
RUN npm ci --prefix /opt/wrap-tools --no-fund --no-audit
# Exercise the installed dependency fixes in both existing wrap-E2E CI callers.
COPY e2e/wrap/tools/dependency-regression.test.cjs /opt/wrap-tools/
RUN npm test --prefix /opt/wrap-tools
WORKDIR /workspace
# Bring in the prebuilt manylinux wheel from stage 1.
+17 -2
View File
@@ -10,17 +10,25 @@ PATH. `npm ci` does not support global installs, so diagnostics must also use
`npm list --prefix /opt/wrap-tools`.
After editing a CLI version in `package.json`, regenerate the lockfile using
Node 22 and npm 11.16.0 on Linux:
Node 24.22.1+ within the Node 24 line and npm 11.16.0 on Linux (the pinned
OpenClaw version no longer supports Node 22):
```sh
cd e2e/wrap/tools
npm install --package-lock-only --no-fund --no-audit
npm ci --no-fund --no-audit
npm test
cd ../../..
docker build -f e2e/wrap/Dockerfile -t headroom-wrap-e2e .
docker run --rm headroom-wrap-e2e
```
`npm test` runs the Ajv URI and Express body-parser regression checks against
the installed locked dependencies. The Dockerfile runs the same command from
`/opt/wrap-tools` after `npm ci`, so failures stop the image build in both the
existing Wrap E2E workflow and CI's Docker-native wrap E2E job. Their path
filters include `e2e/**`, covering dependency and regression-test updates.
To intentionally refresh transitive versions without changing the pinned CLI
versions, use `npm update --package-lock-only --no-fund --no-audit` in this
directory instead of `npm install --package-lock-only`, then run the same
@@ -34,4 +42,11 @@ to make a build pass.
These are existing test tools, not new product dependencies. Codex is maintained
by OpenAI; OpenClaw and its transitive dependencies retain their upstream
maintainers. Their install surface includes registry downloads, native optional
packages and lifecycle scripts, as before. Top-level CLI versions are unchanged.
packages and lifecycle scripts, as before. OpenClaw 2026.10.6 and its fixed
Undici override follow the upstream #4453 security repair.
The raw audit still reports npm 12.2.1's bundled ip-address and Undici copies.
`npm-audit-exceptions.json` records the upstream-approved E2E-only risk
acceptance, restricted to exact advisory/path/version combinations and expiring
on 2026-11-15. Security CI rejects other findings and expired or stale entries;
these exceptions are not a claim that the contained packages are patched.
@@ -0,0 +1,85 @@
'use strict';
// From e2e/wrap/tools after installing its locked dependencies:
// node --test --test-concurrency=1 --test-reporter=tap dependency-regression.test.cjs
const path = require('node:path');
const { createRequire } = require('node:module');
const fromTools = createRequire(path.join(process.cwd(), 'package.json'));
if (process.argv[2] === '--serve') {
const express = fromTools('express');
const app = express();
app.get('/health', (_req, res) => res.json({ ok: true }));
app.post('/text', express.text({ verify() {} }), (req, res) => res.json(req.body));
app.use((error, _req, res, _next) => res.status(error.status || 500).json({ type: error.type }));
const server = app.listen(0, '127.0.0.1', () => process.send({ port: server.address().port }));
process.on('message', (message) => {
if (message === 'stop') { server.close(() => process.exit(0)); server.closeAllConnections(); }
});
} else {
const test = require('node:test');
const assert = require('node:assert/strict');
const http = require('node:http');
const { fork } = require('node:child_process');
test('Ajv rejects a percent-encoded scheme instead of aliasing an HTTPS schema', () => {
const Ajv = fromTools('ajv');
const ajv = new Ajv();
ajv.addSchema({ $id: 'https://example.invalid/defs.json', type: 'integer' });
const valid = ajv.compile({ $ref: 'https://example.invalid/defs.json' });
assert.equal(valid(42), true);
assert.equal(valid('42'), false);
assert.throws(
() => ajv.compile({ $ref: '%68ttps://example.invalid/defs.json' }),
{ message: 'URI scheme is malformed.' },
);
});
test('unsupported charset with invalid gzip leaves the Express process healthy', async (t) => {
const child = fork(__filename, ['--serve'], {
cwd: process.cwd(), env: process.env, execArgv: [], silent: true,
});
let stderr = '';
child.stdout.resume();
child.stderr.on('data', (chunk) => { stderr += chunk; });
t.after(async () => {
t.diagnostic(JSON.stringify({ exitCode: child.exitCode, signal: child.signalCode, stderr }));
if (child.exitCode !== null || child.signalCode !== null) return;
await new Promise((resolve) => {
const timer = setTimeout(() => { child.kill(); resolve(); }, 1000);
child.once('exit', () => { clearTimeout(timer); resolve(); });
if (child.connected) child.send('stop'); else child.kill();
});
});
const port = await new Promise((resolve, reject) => {
const timer = setTimeout(() => reject(new Error('server startup timeout')), 3000);
child.once('message', (msg) => { clearTimeout(timer); resolve(msg.port); });
child.once('error', (err) => { clearTimeout(timer); reject(err); });
child.once('exit', (code) => { clearTimeout(timer); reject(new Error('server exited: ' + code + stderr)); });
});
function request(route, body, headers = {}, method = 'POST') {
return new Promise((resolve, reject) => {
const req = http.request({ hostname: '127.0.0.1', port, path: route, method, agent: false,
headers: { ...headers, 'Content-Length': Buffer.byteLength(body) } }, (res) => {
const chunks = [];
res.on('data', (chunk) => chunks.push(chunk));
res.on('error', reject);
res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString() }));
});
req.setTimeout(2000, () => req.destroy(new Error('localhost request timeout')));
req.on('error', reject);
req.end(body);
});
}
const response = await request('/text', 'not-gzip', {
'Content-Type': 'text/plain; charset=not-a-real-charset', 'Content-Encoding': 'gzip',
});
assert.equal(response.status, 415);
assert.equal(JSON.parse(response.body).type, 'charset.unsupported');
// Let a pending zlib error surface before checking the child's health.
await new Promise((resolve) => setTimeout(resolve, 80));
assert.equal(child.exitCode, null, stderr);
const health = await request('/health', '', {}, 'GET');
assert.equal(health.status, 200);
assert.deepEqual(JSON.parse(health.body), { ok: true });
});
}
+10 -1
View File
@@ -1,6 +1,6 @@
{
"expires": "2026-11-15",
"reason": "E2E harness only, never shipped in the wheel or SDK. npm 12.2.1 bundles undici 6.29.0 and ip-address 10.7.2; bundled copies ignore overrides and no newer npm release is available. OpenClaw is updated to 2026.10.6 and uses fixed Undici 8.12.3, so no OpenClaw path is excepted. Recheck npm upstream and remove these exact-path exceptions by the existing expiry.",
"reason": "E2E harness only, never shipped in the wheel or SDK. npm 12.2.1 bundles undici 6.29.0 and ip-address 10.7.2; bundled copies ignore overrides and no newer npm release is available. OpenClaw is updated to 2026.10.6 and uses fixed Undici 8.12.3, so no OpenClaw path is excepted. Three further undici 6.x advisories (GHSA-2m5v-pch6-74hx, GHSA-qrxm-fqhp-jwvm, GHSA-9fm3-97px-rjf6) published after #4453 cover the same bundled copy. Recheck npm upstream and remove these exact-path exceptions by the existing expiry.",
"advisories": {
"GHSA-4qv8-2p9j-2685": {
"node_modules/npm/node_modules/undici": "6.29.0"
@@ -10,6 +10,15 @@
},
"GHSA-rxp7-6hxx-x8q5": {
"node_modules/npm/node_modules/ip-address": "10.7.2"
},
"GHSA-2m5v-pch6-74hx": {
"node_modules/npm/node_modules/undici": "6.29.0"
},
"GHSA-qrxm-fqhp-jwvm": {
"node_modules/npm/node_modules/undici": "6.29.0"
},
"GHSA-9fm3-97px-rjf6": {
"node_modules/npm/node_modules/undici": "6.29.0"
}
}
}
+3
View File
@@ -3,6 +3,9 @@
"version": "1.0.0",
"private": true,
"description": "Locked CLI dependencies for the Docker wrap E2E harness",
"scripts": {
"test": "node --test --test-concurrency=1 --test-reporter=tap dependency-regression.test.cjs"
},
"dependencies": {
"@openai/codex": "0.163.0",
"openclaw": "2026.10.6"
+217
View File
@@ -0,0 +1,217 @@
"""Actual SDK + loopback HTTP proof; no remote provider or model calls."""
import json
import os
import tempfile
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
def main():
scratch = tempfile.TemporaryDirectory(prefix="headroom-client-proof-")
for key in ["HEADROOM_WORKSPACE_DIR", "HEADROOM_CONFIG_DIR", "XDG_CACHE_HOME"]:
os.environ[key] = str(Path(scratch.name) / key.lower())
os.environ.update(HEADROOM_OFFLINE="1", HEADROOM_BEACON="off", DO_NOT_TRACK="1")
import httpx
from anthropic import Anthropic
from headroom import AnthropicProvider, HeadroomClient
from headroom.config import HeadroomConfig
requests = []
observed_styles = []
class Observer:
def on_pipeline_event(self, event):
if "api_style" in event.metadata:
observed_styles.append(event.metadata["api_style"])
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
data = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
requests.append(data)
if data.get("stream"):
events = [
(
"message_start",
{
"type": "message_start",
"message": {
"id": "msg_fixture",
"type": "message",
"role": "assistant",
"model": data["model"],
"content": [],
"stop_reason": None,
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 0},
},
},
),
(
"content_block_start",
{
"type": "content_block_start",
"index": 0,
"content_block": {"type": "text", "text": ""},
},
),
(
"content_block_delta",
{
"type": "content_block_delta",
"index": 0,
"delta": {"type": "text_delta", "text": "local fixture"},
},
),
("content_block_stop", {"type": "content_block_stop", "index": 0}),
(
"message_delta",
{
"type": "message_delta",
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
"usage": {"output_tokens": 2},
},
),
("message_stop", {"type": "message_stop"}),
]
payload = "".join(
f"event: {name}\ndata: {json.dumps(event)}\n\n" for name, event in events
).encode()
self.send_response(200)
self.send_header("Content-Type", "text/event-stream")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
return
payload = json.dumps(
{
"id": "msg_fixture",
"type": "message",
"role": "assistant",
"model": data["model"],
"content": [{"type": "text", "text": "local fixture"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 1, "output_tokens": 2},
}
).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
responses = []
http = httpx.Client(event_hooks={"response": [responses.append]})
original = Anthropic(
http_client=http,
api_key="local-fixture",
base_url=f"http://127.0.0.1:{server.server_port}",
max_retries=0,
)
wrapped = HeadroomClient(
original,
AnthropicProvider(warn=False),
store_url=f"sqlite:///{scratch.name}/metrics.db",
enable_cache_optimizer=False,
config=HeadroomConfig(
store_url=f"sqlite:///{scratch.name}/metrics.db",
discover_pipeline_extensions=False,
pipeline_extensions=[Observer()],
),
)
kwargs = {
"model": "claude-fixture",
"messages": [{"role": "user", "content": "hello"}],
"max_tokens": 8,
"stream": False,
}
try:
native = original.messages.create(**kwargs)
print(
json.dumps(
{"case": "native", "text": native.content[0].text, "requests": len(requests)}
)
)
try:
result = wrapped.messages.create(**kwargs)
print(
json.dumps(
{"case": "wrapped", "text": result.content[0].text, "requests": len(requests)}
)
)
kwargs["stream"] = True
native_stream = original.messages.create(**kwargs)
native_types = [event.type for event in native_stream]
native_stream.close()
wrapped_stream = wrapped.messages.create(**kwargs)
wrapped_types = [event.type for event in wrapped_stream]
wrapped_stream.close()
assert native_types == wrapped_types
print(
json.dumps(
{
"case": "create_stream",
"native_types": native_types,
"wrapped_types": wrapped_types,
"native_closed": native_stream.response.is_closed,
"requests": len(requests),
}
)
)
del kwargs["stream"]
with wrapped.messages.stream(**kwargs) as manager:
text = "".join(manager.text_stream)
assert text == "local fixture"
assert manager.response.is_closed
assert all(response.is_closed for response in responses)
assert observed_styles and set(observed_styles) == {"anthropic"}
print(
json.dumps(
{
"case": "resources",
"http_responses": len(responses),
"all_closed": all(response.is_closed for response in responses),
}
)
)
print(
json.dumps(
{
"case": "stream_manager",
"text": text,
"response_closed": manager.response.is_closed,
"requests": len(requests),
}
)
)
except Exception as exc:
print(
json.dumps(
{
"case": "wrapped",
"error": type(exc).__name__,
"message": str(exc),
"requests": len(requests),
}
)
)
raise
finally:
wrapped.close()
original.close()
server.shutdown()
server.server_close()
scratch.cleanup()
if __name__ == "__main__":
main()
+124
View File
@@ -0,0 +1,124 @@
"""Actual OpenAI SDK HTTP requests: typed cache opt-out must preserve caller spaces."""
import hashlib
import json
import os
import tempfile
import threading
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
def main():
with tempfile.TemporaryDirectory(prefix="headroom-cache-config-") as scratch:
for key in ["HEADROOM_WORKSPACE_DIR", "HEADROOM_CONFIG_DIR", "XDG_CACHE_HOME"]:
os.environ[key] = str(Path(scratch) / key.lower())
# This isolated proof needs no persistent, process-global CCR database.
# Keep the store in memory so Windows can remove the scratch directory.
os.environ.update(
HEADROOM_OFFLINE="1",
HEADROOM_BEACON="off",
DO_NOT_TRACK="1",
HEADROOM_CCR_BACKEND="memory",
)
from openai import OpenAI
from headroom import HeadroomClient, OpenAIProvider
from headroom.config import HeadroomConfig, HeadroomMode
requests = []
violations = []
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_POST(self):
raw = self.rfile.read(int(self.headers["Content-Length"]))
body = json.loads(raw)
wire_dir = os.environ.get("HEADROOM_PROOF_WIRE_DIR")
if wire_dir:
target = Path(wire_dir)
target.mkdir(parents=True, exist_ok=True)
(target / f"request-{len(requests)}.json").write_bytes(raw)
requests.append(body)
payload = json.dumps(
{
"id": "chatcmpl_fixture",
"object": "chat.completion",
"created": 0,
"model": body["model"],
"choices": [
{
"index": 0,
"message": {"role": "assistant", "content": "local fixture"},
"finish_reason": "stop",
}
],
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
}
).encode()
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
threading.Thread(target=server.serve_forever, daemon=True).start()
original = OpenAI(
api_key="local-fixture",
base_url=f"http://127.0.0.1:{server.server_port}",
max_retries=0,
)
try:
for enabled in [False, True]:
url = f"sqlite:///{scratch}/{enabled}.db"
config = HeadroomConfig(
store_url=url,
default_mode=HeadroomMode.OPTIMIZE,
discover_pipeline_extensions=False,
)
config.smart_crusher.enabled = False
config.cache_aligner.enabled = False
config.cache_optimizer.enabled = enabled
client = HeadroomClient(original, OpenAIProvider(), store_url=url, config=config)
try:
response = client.chat.completions.create(
model="gpt-4o",
messages=[
{"role": "system", "content": "Keep exact spaces.\n"},
{"role": "user", "content": "Hello."},
],
max_tokens=8,
)
wire_system = requests[-1]["messages"][0]["content"]
print(
json.dumps(
{
"configured_enabled": enabled,
"runtime_optimizer": client.get_stats()["config"][
"cache_optimizer"
],
"wire_system": wire_system,
"wire_system_sha256": hashlib.sha256(
wire_system.encode()
).hexdigest(),
"response": response.choices[0].message.content,
}
)
)
if not enabled and wire_system != "Keep exact spaces.\n":
violations.append("Disabled cache optimizer modified the system message")
finally:
client.close()
finally:
original.close()
server.shutdown()
server.server_close()
assert not violations, violations
if __name__ == "__main__":
main()
+68 -7
View File
@@ -76,12 +76,58 @@ def _extract_tool_result_content(msg: dict) -> str | None:
return None
def _can_swap_tool_result_content(msg: dict) -> bool:
"""Whether replacing extracted text preserves the OpenAI content shape.
A joined replacement cannot be mapped back to multiple blocks without
losing boundaries, metadata or media. All rewrite callers must check this
before extraction, transformation or savings accounting. Anthropic's
existing tool-result rewrite contract is unchanged.
"""
if msg.get("role") != "tool":
return True
content = msg.get("content")
if isinstance(content, str):
return True
return (
isinstance(content, list)
and len(content) == 1
and isinstance(content[0], dict)
and content[0].get("type") == "text"
and isinstance(content[0].get("text"), str)
)
def _carries_tool_result_text(msg: dict) -> bool:
"""Whether any tool result in ``msg`` holds text (both formats)."""
if msg.get("role") == "tool":
results = [msg.get("content")]
else:
results = [
b.get("content")
for b in msg.get("content") or []
if isinstance(b, dict) and b.get("type") == "tool_result"
]
return any(
isinstance(c, str) or (isinstance(c, list) and _extract_text_from_blocks(c) is not None)
for c in results
)
def _swap_tool_result_content(msg: dict, new_content: str) -> dict:
"""Deep copy msg and replace tool result content with new_content."""
"""Deep copy and rewrite supported content; preserve unsupported layouts."""
new_msg = copy.deepcopy(msg)
if not _can_swap_tool_result_content(msg):
return new_msg
# OpenAI format
if new_msg.get("role") == "tool":
new_msg["content"] = new_content
content = new_msg.get("content")
if isinstance(content, list):
# The shared guard permits only an unambiguous single text block.
# Keep its shape and metadata, including gateway cache breakpoints.
content[0]["text"] = new_content
else:
new_msg["content"] = new_content
return new_msg
# Anthropic format
content = new_msg.get("content")
@@ -294,9 +340,10 @@ class CompressionCache:
"""Count consecutive stable messages from the start.
A message is stable if it is a plain user/assistant/system message,
an assistant message with tool_use blocks, or a tool_result whose
content hash is already in the cache. The first unstable tool_result
(cache miss) stops the count.
an assistant message with tool_use blocks, an Anthropic tool_result with no text
(nothing to replay), or a tool_result whose cached text and shape can
be replayed. A cache miss or unsupported text shape stops the count.
A joined text hash alone does not prove shape-safe replay.
The trailing message is *always* excluded from the frozen prefix
(cap of ``len(messages) - 1``). The trailing message represents
@@ -314,14 +361,25 @@ class CompressionCache:
count = 0
for msg in messages:
if _is_tool_result_message(msg):
# A joined-text cache/stable hash cannot prove replay for
# shapes that apply_cached deliberately leaves untouched.
# Anthropic textless results retain their no-rewrite path.
if not _can_swap_tool_result_content(msg):
break
content = _extract_tool_result_content(msg)
if content is not None:
h = self.content_hash(content)
if h not in self._cache and h not in self._stable_hashes:
break
else:
# tool_result with non-string content; treat as unstable
elif _carries_tool_result_text(msg):
# Text after a textless first tool_result: the cache
# keys a message by its first one, so this text was
# never stored and cannot be byte-replayed.
break
# A tool_result with no text at all (a ToolSearch
# tool_reference list, an image) gives the cache nothing
# to replay, so it is stable. As a miss it stopped the
# count for the rest of the session.
# Regular user/assistant/system messages and assistant+tool_use
# are always stable — fall through.
count += 1
@@ -341,6 +399,9 @@ class CompressionCache:
with self._lock:
result: list[dict] = []
for msg in messages:
if not _can_swap_tool_result_content(msg):
result.append(msg)
continue
if _is_tool_result_message(msg):
content = _extract_tool_result_content(msg)
if content is not None:
+20 -2
View File
@@ -563,12 +563,30 @@ def classify_history_relation(
return _classify_history_canonical(current, previous)
def cache_key_tools(tools: Any) -> Any:
"""The Anthropic ``tools`` that count toward the prompt-cache key.
A ``defer_loading`` tool is kept out of the prompt until the model loads
it, so adding one to ``tools`` leaves the cached prefix readable. Claude
Code appends every tool its ToolSearch loads to ``tools``, still marked
deferred; counting those would start a new lineage at frozen=0 after each
load and re-run the whole history. Every other tool, and a tool whose
``defer_loading`` is cleared, still counts. The Anthropic handler decides
when to apply this (first-party upstream, ``HEADROOM_DEFERRED_TOOL_AFFINITY``).
"""
if not isinstance(tools, list):
return tools
return [t for t in tools if not (isinstance(t, dict) and t.get("defer_loading"))]
def segment_fingerprint(value: Any) -> str:
"""Stable hash for non-message provider cache-key segments.
Cache-control placement and transport annotations are deliberately ignored;
semantic tool/model/thinking changes remain visible. The hash is affinity
metadata only and is never used to reconstruct or forward request content.
semantic tool/model/thinking changes remain visible. It hashes what it is
given, so callers pass only what the provider keys its cache on (for
Anthropic tools, :func:`cache_key_tools`). The hash is affinity metadata
only and is never used to reconstruct or forward request content.
"""
canonical = _lineage_snapshot(_canonicalize_for_prefix_compare(value))
encoded = json.dumps(
+12 -5
View File
@@ -31,7 +31,14 @@ class CacheOptimizerRegistry:
"""
_optimizers: dict[str, type[BaseCacheOptimizer]] = {}
_instances: dict[str, BaseCacheOptimizer] = {}
_instances: dict[tuple[str, int | None], BaseCacheOptimizer] = {}
@classmethod
def _invalidate_instances(cls, name: str) -> None:
"""Discard every configuration cached for this exact registration."""
for key in list(cls._instances):
if key[0] == name:
del cls._instances[key]
@classmethod
def register(
@@ -57,8 +64,8 @@ class CacheOptimizerRegistry:
f"Optimizer '{name}' already registered. Use override=True to replace."
)
cls._optimizers[name] = optimizer_class
# Clear cached instance if exists
cls._instances.pop(name, None)
# Configured instances must not outlive their registered implementation.
cls._invalidate_instances(name)
@classmethod
def unregister(cls, name: str) -> None:
@@ -69,7 +76,7 @@ class CacheOptimizerRegistry:
name: Name to unregister
"""
cls._optimizers.pop(name, None)
cls._instances.pop(name, None)
cls._invalidate_instances(name)
@classmethod
def get(
@@ -109,7 +116,7 @@ class CacheOptimizerRegistry:
raise KeyError(f"No optimizer registered for '{key}'. Available: {available}")
# Return cached instance if requested
cache_key = f"{key}:{id(config)}" if config else key
cache_key = (key, id(config) if config is not None else None)
if cached and cache_key in cls._instances:
return cls._instances[cache_key]
+82 -19
View File
@@ -36,6 +36,7 @@ Usage:
from __future__ import annotations
import hashlib
import json
import time
from collections import OrderedDict
from collections.abc import Callable
@@ -74,6 +75,10 @@ class CacheEntry:
# Hash of the full messages for exact matching
messages_hash: str = ""
# Cache namespace (provider, model, metadata) so entries never replay
# across different provider/model/metadata contexts
namespace: str = ""
@dataclass
class SemanticCacheConfig:
@@ -122,7 +127,7 @@ class SemanticCache:
# LRU cache: key -> CacheEntry
self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
# Exact hash index: messages_hash -> key
# Exact hash index: namespace-scoped hash -> key
self._hash_index: dict[str, str] = {}
# Statistics
@@ -134,6 +139,7 @@ class SemanticCache:
self,
query: str,
messages_hash: str | None = None,
namespace: str = "",
) -> CacheEntry | None:
"""
Look up a cached entry.
@@ -141,6 +147,8 @@ class SemanticCache:
Args:
query: Query text to search for
messages_hash: Optional exact hash for fast lookup
namespace: Cache namespace (provider, model, metadata); entries are
never returned outside their own namespace
Returns:
CacheEntry if found, None otherwise
@@ -149,13 +157,16 @@ class SemanticCache:
# Try exact hash match first
if messages_hash and self.config.use_exact_matching:
key = self._hash_index.get(messages_hash)
index_key = self._combined_key(namespace, messages_hash)
key = self._hash_index.get(index_key)
if key and key in self._cache:
entry = self._cache[key]
# Verify the stored entry really belongs to this request. Guards
# against a stale index mapping ever pointing at an entry that was
# overwritten by a different conversation sharing the same key.
if entry.messages_hash == messages_hash:
# Verify the stored entry really belongs to this request and
# namespace. Guards against a stale index mapping ever pointing
# at an entry that was overwritten by a different conversation
# sharing the same key, and against replaying a cached response
# across provider/model/metadata contexts.
if entry.messages_hash == messages_hash and entry.namespace == namespace:
self._touch(key)
self._hits += 1
return entry
@@ -173,7 +184,7 @@ class SemanticCache:
# messages_hash above, which is context-complete and safe.
if self._embedding_fn and query.strip():
query_embedding = self._embedding_fn(query)
best_match, best_similarity = self._find_similar(query_embedding)
best_match, best_similarity = self._find_similar(query_embedding, namespace)
if best_similarity >= self.config.similarity_threshold:
self._touch(best_match)
@@ -188,6 +199,7 @@ class SemanticCache:
query: str,
response: Any,
messages_hash: str | None = None,
namespace: str = "",
) -> str:
"""
Store a response in the cache.
@@ -196,6 +208,7 @@ class SemanticCache:
query: Query text
response: Response to cache
messages_hash: Optional exact hash for fast lookup
namespace: Cache namespace (provider, model, metadata)
Returns:
Cache key for the entry
@@ -206,8 +219,11 @@ class SemanticCache:
# a trailing user message ("continue", "yes", "run the tests") but differ
# in earlier context must NOT collide on one query-derived slot and
# overwrite each other. Fall back to the query hash only when no
# messages_hash is supplied (e.g. embedding-only usage).
key = messages_hash or self._generate_key(query)
# messages_hash is supplied (e.g. embedding-only usage). The key always
# includes the namespace so entries from different provider/model/
# metadata contexts never share a slot.
base = messages_hash or self._generate_key(query)
key = self._combined_key(namespace, base)
# Evict if adding a NEW key would exceed capacity. Overwriting a key that
# is already present is an in-place update that does not grow the map, so
@@ -235,13 +251,14 @@ class SemanticCache:
created_at=now,
last_accessed=now,
messages_hash=messages_hash or "",
namespace=namespace,
)
self._cache[key] = entry
# Index by hash for fast exact matching
if messages_hash:
self._hash_index[messages_hash] = key
self._hash_index[self._combined_key(namespace, messages_hash)] = key
return key
@@ -250,7 +267,7 @@ class SemanticCache:
if key in self._cache:
entry = self._cache.pop(key)
if entry.messages_hash:
self._hash_index.pop(entry.messages_hash, None)
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
return True
return False
@@ -276,12 +293,15 @@ class SemanticCache:
def _find_similar(
self,
query_embedding: list[float],
namespace: str = "",
) -> tuple[str, float]:
"""Find the most similar cached entry."""
"""Find the most similar cached entry within the same namespace."""
best_key = ""
best_similarity = -1.0
for key, entry in self._cache.items():
if entry.namespace != namespace:
continue
if not entry.embedding:
continue
@@ -325,7 +345,7 @@ class SemanticCache:
if self._cache:
key, entry = self._cache.popitem(last=False)
if entry.messages_hash:
self._hash_index.pop(entry.messages_hash, None)
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
self._evictions += 1
def _cleanup_expired(self) -> None:
@@ -343,7 +363,15 @@ class SemanticCache:
for key in expired:
entry = self._cache.pop(key)
if entry.messages_hash:
self._hash_index.pop(entry.messages_hash, None)
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
def _combined_key(self, namespace: str, base: str) -> str:
"""Hash a namespace with an exact messages_hash or query-derived key.
Combining namespace + base (separated by a NUL so a namespace can't be
confused with a hash prefix) truncates like the existing hashes.
"""
return hashlib.sha256((namespace + "\0" + base).encode()).hexdigest()[:24]
def _generate_key(self, query: str) -> str:
"""Generate a cache key for a query."""
@@ -405,9 +433,15 @@ class SemanticCacheLayer:
# Extract query for semantic matching
query = context.query or self._extract_query(messages)
messages_hash = self._compute_messages_hash(messages)
namespace = self._compute_namespace(context)
# Unserializable context: never share cached responses across contexts,
# so skip the cache and just delegate to the provider optimizer.
if namespace is None:
return self.provider_optimizer.optimize(messages, context, config)
# Check semantic cache
cached = self.cache.get(query, messages_hash)
cached = self.cache.get(query, messages_hash, namespace)
if cached:
return CacheResult(
messages=messages,
@@ -440,15 +474,24 @@ class SemanticCacheLayer:
Args:
messages: Original messages
response: Response from LLM
context: Optional context with query
context: Optional context with query. Pass the same OptimizationContext
used for `process`; omitting it stores under the unscoped
namespace, which only matches lookups whose context also has
empty provider, model and metadata.
Returns:
Cache key
"""
query = (context.query if context else None) or self._extract_query(messages)
messages_hash = self._compute_messages_hash(messages)
namespace = self._compute_namespace(context)
return self.cache.put(query, response, messages_hash)
# Unserializable context: don't store anything that could be replayed
# for a different provider/model/metadata.
if namespace is None:
return ""
return self.cache.put(query, response, messages_hash, namespace)
def get_stats(self) -> dict[str, Any]:
"""Get combined statistics."""
@@ -457,6 +500,28 @@ class SemanticCacheLayer:
"provider_optimizer": self.provider_optimizer.name,
}
def _compute_namespace(self, context: OptimizationContext | None) -> str | None:
"""Compute the cache namespace from provider, model, and metadata.
request_id and timestamp are deliberately excluded so they never affect
matching. Returns None when the context isn't JSON-serializable (e.g.
unserializable metadata), so callers skip lookup/storage rather than
replaying entries across contexts.
"""
if context is None:
context = OptimizationContext()
try:
return json.dumps(
{
"provider": context.provider,
"model": context.model,
"metadata": context.metadata,
},
sort_keys=True,
)
except (TypeError, ValueError):
return None
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
"""Extract the last user query from messages."""
for msg in reversed(messages):
@@ -473,8 +538,6 @@ class SemanticCacheLayer:
def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:
"""Compute a hash of all messages."""
import json
try:
content = json.dumps(messages, sort_keys=True)
return hashlib.sha256(content.encode()).hexdigest()[:24]
+14 -5
View File
@@ -101,6 +101,8 @@ def resolve_learned_ttl(provider: str, model: str) -> int | None:
``"provider/model"`` first, then ``"provider"``. Cached by mtime so a hot-path
call is a dict lookup. Best-effort: returns None on any error, so the caller
falls back to its own default (config TTL for CC, static default elsewhere).
Invalid table shapes are ignored; invalid model values fall through to the
provider fallback. Only positive, finite TTLs (not booleans) are accepted.
"""
global _learned_cache, _learned_mtime
path = _learned_path()
@@ -108,17 +110,24 @@ def resolve_learned_ttl(provider: str, model: str) -> int | None:
mt = os.path.getmtime(path)
if _learned_cache is None or mt != _learned_mtime:
with open(path, encoding="utf-8") as f:
_learned_cache = json.load(f)
loaded = json.load(f)
_learned_cache = loaded if isinstance(loaded, dict) else {}
_learned_mtime = mt
except Exception:
return None
table = _learned_cache or {}
for key in (f"{provider}/{model}", provider):
v = table.get(key)
if isinstance(v, dict) and isinstance(v.get("ttl_seconds"), int | float):
return int(v["ttl_seconds"])
if isinstance(v, int | float):
return int(v)
if isinstance(v, dict):
v = v.get("ttl_seconds")
if isinstance(v, bool) or not isinstance(v, int | float):
continue
try:
ttl = int(v)
except (ValueError, OverflowError):
continue # JSON may decode NaN/Infinity; never fail a live request.
if ttl > 0:
return ttl
return None
+41 -5
View File
@@ -46,7 +46,7 @@ from headroom.providers.claude import (
)
from .main import get_version, main
from .port_discovery import find_live_proxy_elsewhere, is_headroom_livez
from .port_discovery import find_live_proxy_elsewhere, is_headroom_livez, loopback_port
from .wrap import _read_wrap_marker, _wrap_marker_is_stale
PASS = "pass"
@@ -59,6 +59,7 @@ DEFAULT_PROXY_PORT = 8787
_LOOPBACK_URL_RE = re.compile(r"https?://(?:127\.0\.0\.1|localhost):(\d+)")
_CODEX_BASE_URL_RE = re.compile(r'(?m)^[ \t]*base_url\s*=\s*"([^"\r\n]+)"')
_CODEX_MODEL_PROVIDER_RE = re.compile(r'(?m)^[ \t]*model_provider\s*=\s*"([^"\r\n]+)"')
_CODEX_OPENAI_BASE_URL_RE = re.compile(r'(?m)^[ \t]*openai_base_url\s*=\s*"([^"\r\n]+)"')
# Ollama's fixed default port. `ollama launch claude` writes
# ``ANTHROPIC_BASE_URL=http://127.0.0.1:11434`` into the launched Claude Code
@@ -573,7 +574,19 @@ def check_wrap_marker_staleness(settings_path: Path) -> CheckResult:
def codex_active_base_url(text: str) -> tuple[str, str | None]:
"""Return ``(provider_id, base_url)`` for the active Codex provider in ``text``."""
active_match = _CODEX_MODEL_PROVIDER_RE.search(text)
first_table = re.search(r"(?m)^[ \t]*\[", text)
root = text[: first_table.start()] if first_table is not None else text
# Only a root `model_provider` picks the default session's provider; one
# under `[profiles.*]` applies only when that profile is selected.
active_match = _CODEX_MODEL_PROVIDER_RE.search(root)
if active_match is None or active_match.group(1) == "openai":
# A root `openai_base_url` re-points Codex's built-in `openai` provider
# (the default when `model_provider` is unset), so it beats an inactive
# `[model_providers.headroom]` table. Headroom Desktop routes this way
# because the ChatGPT app hides features for any other provider.
root_url = _CODEX_OPENAI_BASE_URL_RE.search(root)
if root_url is not None:
return "openai", root_url.group(1)
provider_id = active_match.group(1) if active_match else "headroom"
return provider_id, _codex_provider_base_url(text, provider_id)
@@ -661,6 +674,24 @@ def _codex_provider_base_url(text: str, provider_id: str) -> str | None:
return base_url_match.group(1) if base_url_match else None
def _configured_client_ports(
claude_paths: Sequence[Path], codex_paths: Sequence[Path]
) -> list[int]:
"""Loopback ports the Claude Code and Codex configs (project and user) send traffic to.
Headroom Desktop (port 6767) and ``wrap --port N`` leave no deployment
manifest, so without these a bare ``doctor`` only ever finds 8787.
"""
urls = [_claude_base_url_in(path)[0] for path in claude_paths]
for path in codex_paths:
try:
text = path.read_text(encoding="utf-8", errors="replace")
except OSError:
continue
urls.append(codex_active_base_url(text)[1] or "")
return [port for port in map(loopback_port, urls) if port is not None]
def _codex_block_missing_openai_auth(
text: str, config_path: Path, provider_id: str = "headroom"
) -> bool:
@@ -1211,20 +1242,25 @@ def doctor(
health = probe_json(f"{base_url}/health", timeout=5.0) if livez else None
stats = probe_json(f"{base_url}/stats", timeout=5.0) if livez else None
installed = get_version()
project_claude_settings = Path.cwd() / ".claude" / "settings.json"
project_local_claude_settings = Path.cwd() / ".claude" / "settings.local.json"
live_elsewhere = None
if livez is None:
# Cheap, bounded: only ports Headroom recorded (HEADROOM_PORT, 8787,
# deployment manifests, the project's wrap marker), probed in parallel.
# deployment manifests, the project's wrap marker) and the ports the
# agents are configured for, probed in parallel.
live_elsewhere = find_live_proxy_elsewhere(
port,
manifests=manifests,
extra=_configured_client_ports(
[project_local_claude_settings, project_claude_settings, claude_settings_path()],
[codex_project_config_path(), codex_config_path()],
),
probe=lambda p: is_headroom_livez(
probe_json(f"http://127.0.0.1:{p}/livez", timeout=0.5)
),
)
project_claude_settings = Path.cwd() / ".claude" / "settings.json"
project_local_claude_settings = Path.cwd() / ".claude" / "settings.local.json"
checks = [
note_live_proxy_elsewhere(check_proxy_liveness(livez, base_url), live_elsewhere),
check_proxy_readiness(livez, readyz, base_url),
+21 -13
View File
@@ -58,6 +58,11 @@ from headroom.install.state import ManifestError, load_manifest, save_manifest
from headroom.install.supervisors import start_supervisor
from headroom.providers.claude import TOOL_SEARCH_DEFAULT, TOOL_SEARCH_ENV
from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT
from headroom.providers.codex.config_toml import (
loads_codex_config,
root_assignment_lines,
split_toml_lines,
)
from headroom.providers.codex.install import (
CodexAuthConfigError,
build_codex_auth_config,
@@ -91,6 +96,7 @@ _STARTUP_READY_TIMEOUT_SECONDS = 15
# report ready, and every session is permanently cold.
_HOOK_ENSURE_TIMEOUT_SECONDS = 60
_TOML_TABLE_HEADER_RE = re.compile(r"^[ \t]*(?:\[\[[^\]\r\n]+\]\]|\[[^\]\r\n]+\])[ \t]*(?:#.*)?$")
_ROOT_PROVIDER_KEY_RE = re.compile(r"[ \t]*(?:model_provider|openai_base_url)[ \t]*=")
_TOML_FEATURES_NAME_RE = r"(?:features|\"features\"|'features')"
_TOML_CODEX_HOOKS_NAME_RE = r"(?:codex_hooks|\"codex_hooks\"|'codex_hooks')"
_CODEX_FEATURES_TABLE_RE = re.compile(
@@ -298,7 +304,7 @@ def _replace_marker_block(
# The block carries top-level keys, so it must sit above the first table
# header; appended after a table (e.g. [features]) TOML scopes those keys
# into that table and Codex rejects the config (#260).
lines = content.splitlines()
lines = split_toml_lines(content)
for index, line in enumerate(lines):
if _TOML_TABLE_HEADER_RE.search(line):
head = "\n".join(lines[:index]).rstrip()
@@ -347,7 +353,7 @@ def _codex_init_provider_snapshot(content: str) -> str | None:
snapshot = json.loads(match.group(1))
if not isinstance(snapshot, str):
raise ValueError("Invalid Headroom init provider snapshot")
tomllib.loads(snapshot)
loads_codex_config(snapshot)
return snapshot
@@ -470,19 +476,21 @@ def _ensure_codex_provider(path: Path, port: int) -> None:
# first table header) -- these keys also appear legitimately inside
# [profiles.*] tables as per-profile overrides, and stripping them there
# silently reroutes the user's profiles to the injected "headroom" default.
_first_table = re.search(r"(?m)^[ \t]*\[", content)
_split = _first_table.start() if _first_table else len(content)
root, rest = content[:_split], content[_split:]
root = re.sub(r"(?m)^[ \t]*model_provider[ \t]*=.*\r?\n", "", root)
root = re.sub(r"(?m)^[ \t]*openai_base_url[ \t]*=.*\r?\n", "", root)
content = root + rest
# Lines nested in a multi-line inline table (e.g. ``profiles = {``) are
# profile overrides too, so only true root assignments are dropped.
_root_lines = root_assignment_lines(content)
content = "".join(
line
for index, line in enumerate(split_toml_lines(content, keepends=True))
if index not in _root_lines or not _ROOT_PROVIDER_KEY_RE.match(line)
)
# The provider block carries top-level keys (model_provider, openai_base_url),
# so it must land at the document root rather than after a trailing table (#260).
content = _replace_marker_block(
content, _CODEX_PROVIDER_MARKER_START, _CODEX_PROVIDER_MARKER_END, block, at_root=True
)
path.parent.mkdir(parents=True, exist_ok=True)
tomllib.loads(content)
loads_codex_config(content)
fsutil.write_text(path, content)
# Codex filters its history menu by the active model_provider, so existing
# native threads vanish once we switch to "headroom". Retag them to match the
@@ -511,7 +519,7 @@ def _codex_features(content: str) -> dict[str, Any] | None:
if not content.strip():
return None
try:
parsed = tomllib.loads(content)
parsed = loads_codex_config(content)
except tomllib.TOMLDecodeError:
return None
features = parsed.get("features")
@@ -523,7 +531,7 @@ def _codex_features_has_hooks(content: str) -> bool:
if features is None:
# Keep init resilient for already-invalid user configs; this fallback
# only needs to avoid adding a second obvious hooks line.
lines = content.splitlines()
lines = split_toml_lines(content)
features_index = _codex_features_table_index(lines)
if features_index is None:
return False
@@ -538,7 +546,7 @@ def _codex_features_has_hooks(content: str) -> bool:
def _strip_codex_legacy_feature_flag(content: str) -> str:
lines = content.splitlines(keepends=True)
lines = split_toml_lines(content, keepends=True)
retained: list[str] = []
in_features = False
in_root = True
@@ -588,7 +596,7 @@ def _ensure_codex_feature_flag(path: Path) -> None:
# the only work.
pass
else:
lines = content.splitlines()
lines = split_toml_lines(content)
features_index = _codex_features_table_index(lines)
if features_index is not None:
# Leading blank line matches the normalisation _replace_marker_block
+4 -1
View File
@@ -943,7 +943,10 @@ def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed pe
"--env HEADROOM_WORKSPACE_DIR=/path. Supervisors (launchd, systemd, cron) "
"start with a bare environment and do not inherit the interactive shell's "
"exports, so anything the runtime needs beyond the flags above must be set "
"here. Repeatable."
"here. Repeatable. Values are stored in cleartext in the profile's "
"manifest and runner scripts under ~/.headroom/deploy/<profile>/ "
"(owner-only, not encrypted); see SECURITY.md before passing a provider "
"API key this way."
),
)
def install_apply(
+26 -15
View File
@@ -41,10 +41,7 @@ from functools import wraps
from pathlib import Path
from typing import Any, NamedTuple, cast
try:
import tomllib
except ModuleNotFoundError: # pragma: no cover - Python < 3.11
import tomli as tomllib
import tomllib
from headroom._subprocess import identity_mismatch as _shared_identity_mismatch
from headroom._subprocess import pid_alive, proc_identity, run
@@ -113,6 +110,11 @@ from headroom.providers.claude import (
)
from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT
from headroom.providers.codex import build_launch_env as _build_codex_launch_env
from headroom.providers.codex.config_toml import (
loads_codex_config,
root_assignment_lines,
split_toml_lines,
)
from headroom.providers.codex.install import (
CodexAuthConfigError,
build_codex_auth_config,
@@ -3086,7 +3088,7 @@ def _codex_session_launch_settings(
"""Build process-local routing while preserving the selected provider id."""
config_file, _ = _codex_config_paths()
try:
config = tomllib.loads(_read_text(config_file)) if config_file.exists() else {}
config = loads_codex_config(_read_text(config_file)) if config_file.exists() else {}
except (OSError, tomllib.TOMLDecodeError) as exc:
raise click.ClickException(
f"could not read Codex config for session routing: {exc}"
@@ -3347,19 +3349,28 @@ def _redirect_existing_top_level_keys(content: str, port: int) -> str:
return _replace
redirected = content
# Only root assignments: the same keys inside [profiles.*] tables or a
# multi-line inline ``profiles = {`` table are the user's per-profile routing.
lines = split_toml_lines(content, keepends=True)
root_lines = sorted(root_assignment_lines(content))
for key in _REDIRECTABLE_KEYS:
pattern = re.compile(rf'(?m)^[ \t]*{re.escape(key)}[ \t]*=[ \t]*"(?P<value>[^"\n]*)"[^\n]*')
redirected = pattern.sub(_make_replacer(key, port), redirected, count=1)
return redirected
pattern = re.compile(rf'[ \t]*{re.escape(key)}[ \t]*=[ \t]*"(?P<value>[^"\n]*)"[^\n]*')
replace = _make_replacer(key, port)
for index in root_lines:
match = pattern.match(lines[index])
if match:
lines[index] = replace(match) + lines[index][match.end() :]
break
return "".join(lines)
def _has_redirectable_top_level_key(content: str, key: str) -> bool:
"""Return True if ``content`` declares ``key = "..."`` as a top-level key."""
import re # local import to match the module's existing convention
pattern = re.compile(rf'(?m)^[ \t]*{key}[ \t]*=[ \t]*"[^"\n]*"')
return pattern.search(content) is not None
pattern = re.compile(rf'[ \t]*{key}[ \t]*=[ \t]*"[^"\n]*"')
lines = split_toml_lines(content, keepends=True)
return any(pattern.match(lines[index]) for index in root_assignment_lines(content))
def _codex_config_has_headroom_markers(content: str) -> bool:
@@ -3718,10 +3729,10 @@ def _inject_codex_provider_config(port: int) -> str | None:
# name a line the user never wrote. Only the user's part has to
# parse: stripping a stale Headroom block may be what repairs it.
try:
tomllib.loads(content)
loads_codex_config(content)
except tomllib.TOMLDecodeError:
try:
tomllib.loads(original)
loads_codex_config(original)
reason = "it is no longer valid TOML once Headroom's block is removed"
except tomllib.TOMLDecodeError as exc:
reason = f"it is not valid TOML ({exc})"
@@ -3769,7 +3780,7 @@ def _inject_codex_provider_config(port: int) -> str | None:
)
# Never replace a usable config with a malformed generated candidate.
tomllib.loads(content)
loads_codex_config(content)
_write_text(config_file, content)
click.echo(f" Codex config: injected Headroom provider (WS + HTTP) into {config_file}")
if custom_upstream_base_url:
@@ -4066,7 +4077,7 @@ def _inject_memory_mcp_config(user_id: str) -> None:
content = mcp_section
# Never replace a usable config with a malformed generated candidate.
tomllib.loads(content)
loads_codex_config(content)
_write_text(config_file, content)
click.echo(f" Memory MCP: registered in {config_file}")
except Exception as e:
+86 -18
View File
@@ -3,7 +3,7 @@
from __future__ import annotations
import logging
from collections.abc import Iterator
from collections.abc import Callable, Iterable, Iterator
from datetime import datetime, timezone
from typing import Any
@@ -38,6 +38,74 @@ from .utils import (
logger = logging.getLogger(__name__)
class _OwnedStream(Iterator[Any]):
"""Iterator that owns and closes an upstream response stream."""
def __init__(self, stream: Iterable[Any], save_metrics: Callable[[], None]) -> None:
self._source: Iterable[Any] | None = stream
self._stream: Iterator[Any] | None = None
self._save_metrics: Callable[[], None] | None = save_metrics
self._finished = False
def __iter__(self) -> _OwnedStream:
return self
def __next__(self) -> Any:
if self._finished:
raise StopIteration
try:
if self._stream is None:
source = self._source
if source is None:
raise StopIteration
self._stream = iter(source)
return next(self._stream)
except BaseException:
self.close()
raise
def close(self) -> None:
"""Close the upstream stream and save metrics exactly once."""
if self._finished:
return
self._finished = True
source = self._source
stream = self._stream
save_metrics = self._save_metrics
try:
candidates = (source,) if source is stream else (source, stream)
for candidate in candidates:
if candidate is None:
continue
close = getattr(candidate, "close", None)
if close is not None:
try:
close()
except Exception:
logger.warning("Failed to close upstream response stream", exc_info=True)
finally:
self._source = None
self._stream = None
self._save_metrics = None
if save_metrics is not None:
save_metrics()
def __enter__(self) -> _OwnedStream:
return self
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
self.close()
def __del__(self) -> None:
# The previous generator wrapper ran its finally block when abandoned
# after iteration started. Preserve that cleanup for callers that do
# not explicitly close a stream.
try:
self.close()
except Exception:
logger.debug("Failed to finalize abandoned response stream", exc_info=True)
class ChatCompletions:
"""Wrapper for chat.completions API (OpenAI-style)."""
@@ -134,6 +202,7 @@ class Messages:
model: str,
messages: list[dict[str, Any]],
max_tokens: int = 1024,
stream: bool = False,
# Headroom-specific parameters
headroom_mode: str | None = None,
headroom_cache_prefix_tokens: int | None = None,
@@ -150,6 +219,7 @@ class Messages:
model: Model name.
messages: List of messages.
max_tokens: Maximum tokens in response.
stream: Return an iterator of raw message events when True.
headroom_mode: Override default mode ("audit" | "optimize").
headroom_cache_prefix_tokens: Target cache-aligned prefix size.
headroom_output_buffer_tokens: Reserve tokens for output.
@@ -163,13 +233,13 @@ class Messages:
return self._client._create(
model=model,
messages=messages,
stream=False,
stream=stream,
headroom_mode=headroom_mode,
headroom_cache_prefix_tokens=headroom_cache_prefix_tokens,
headroom_output_buffer_tokens=headroom_output_buffer_tokens,
headroom_keep_turns=headroom_keep_turns,
headroom_tool_profiles=headroom_tool_profiles,
api_style="anthropic",
api_style="anthropic_create",
max_tokens=max_tokens,
**kwargs,
)
@@ -307,11 +377,11 @@ class HeadroomClient:
# Use provided config or build from individual parameters
if config is not None:
self._config = config
# Override store_url and mode if explicitly provided in config
# Override store_url and mode if explicitly provided in config.
# The resolved URL is not written back: the caller may share this
# config with other clients, which would then read it as explicit.
if config.store_url:
self._store_url = config.store_url
else:
self._config.store_url = store_url
self._default_mode = config.default_mode
else:
# Build config from individual parameters
@@ -325,7 +395,7 @@ class HeadroomClient:
self._config.model_context_limits.update(model_context_limits)
# Initialize storage
self._storage = create_storage(store_url)
self._storage = create_storage(self._store_url)
# Initialize transform pipeline
self._pipeline = TransformPipeline(self._config, provider=self._provider)
@@ -338,7 +408,7 @@ class HeadroomClient:
self._cache_optimizer: BaseCacheOptimizer | None = None
self._semantic_cache_layer: SemanticCacheLayer | None = None
if enable_cache_optimizer:
if self._config.cache_optimizer.enabled:
if cache_optimizer is not None:
self._cache_optimizer = cache_optimizer
else:
@@ -372,7 +442,7 @@ class HeadroomClient:
provider=self._provider.name.lower(),
metadata={
"default_mode": self._default_mode.value,
"cache_optimizer_enabled": enable_cache_optimizer,
"cache_optimizer_enabled": self._config.cache_optimizer.enabled,
"semantic_cache_enabled": enable_semantic_cache,
},
)
@@ -406,6 +476,9 @@ class HeadroomClient:
**kwargs: Any,
) -> Any:
"""Internal implementation of create."""
transport_style = api_style
if api_style == "anthropic_create":
api_style = "anthropic"
request_id = generate_request_id()
timestamp = datetime.now(timezone.utc).replace(tzinfo=None)
mode = HeadroomMode(headroom_mode) if headroom_mode else self._default_mode
@@ -607,7 +680,7 @@ class HeadroomClient:
# Call underlying client based on API style
try:
response = call_client_transport(
api_style,
transport_style,
self,
model=model,
messages=optimized_messages,
@@ -644,16 +717,11 @@ class HeadroomClient:
def _wrap_stream(
self,
stream: Iterator[Any],
stream: Iterable[Any],
metrics: RequestMetrics,
) -> Iterator[Any]:
"""Wrap stream to pass through chunks and save metrics at end."""
try:
yield from stream
finally:
# Save metrics when stream completes
# Note: output tokens unknown for streams
self._storage.save(metrics)
"""Wrap stream, closing the upstream and saving metrics exactly once."""
return _OwnedStream(stream, lambda: self._storage.save(metrics))
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
"""Extract query from messages for semantic caching.
+3 -1
View File
@@ -923,7 +923,9 @@ class PrefixFreezeConfig:
class HeadroomConfig:
"""Main configuration for HeadroomClient."""
store_url: str = "sqlite:///headroom.db"
# None defers to HeadroomClient: its ``store_url`` argument, else a
# headroom.db in the system temp directory.
store_url: str | None = None
default_mode: HeadroomMode = HeadroomMode.AUDIT
model_context_limits: dict[str, int] = field(
default_factory=lambda: DEFAULT_MODEL_CONTEXT_LIMITS.copy()
+6 -6
View File
@@ -219,13 +219,13 @@
<p class="text-sm text-gray-300">
<span class="font-medium text-accent">Headroom for Teams</span>
<span x-text="(stats.summary?.cost?.total_saved_usd || 0) >= 1
? ' · You have saved $' + formatCurrency(stats.summary.cost.total_saved_usd) + ' on this machine. Get the same savings across your team, with 3 months free on a 12-month plan.'
: ' · Bring these savings to your whole team, with 3 months free on a 12-month plan.'"></span>
? ' · You have saved $' + formatCurrency(stats.summary.cost.total_saved_usd) + ' on this machine. Want more? Headroom Enterprise saves more, across your whole team. Talk to us at hello@headroomlabs.ai.'
: ' · Want more savings? Headroom Enterprise saves more, across your whole team. Talk to us at hello@headroomlabs.ai.'"></span>
</p>
<div class="flex items-center gap-3 shrink-0">
<a href="https://headroom-perks.vercel.app/?source=github&utm_source=headroom&utm_medium=dashboard&utm_content=banner"
<a href="https://headroomlabs.ai"
target="_blank" rel="noopener noreferrer"
class="px-3 py-1.5 text-sm font-medium rounded-md bg-accent text-black hover:opacity-90">Claim 3 months free →</a>
class="px-3 py-1.5 text-sm font-medium rounded-md bg-accent text-black hover:opacity-90">Talk to us →</a>
<button type="button" @click="dismissTeamsPerk()" aria-label="Dismiss Headroom for Teams offer"
class="text-sm text-gray-500 hover:text-gray-200">✕</button>
</div>
@@ -2056,12 +2056,12 @@
return {
stats: {},
teamsPerkDismissed: (() => {
try { return localStorage.getItem('headroom-teams-perk-dismissed') === '1'; }
try { return localStorage.getItem('headroom-teams-contact-dismissed') === '1'; }
catch (e) { return false; }
})(),
dismissTeamsPerk() {
this.teamsPerkDismissed = true;
try { localStorage.setItem('headroom-teams-perk-dismissed', '1'); } catch (e) {}
try { localStorage.setItem('headroom-teams-contact-dismissed', '1'); } catch (e) {}
},
lifetimeStats: {},
historyStats: {},
+17 -14
View File
@@ -46,7 +46,8 @@ _CCR_MARKER_RE = re.compile(r"Retrieve more: hash=|Retrieve original: hash=|<<cc
# A number with its immediate key context ("retry_limit: 3", "port=8787",
# JSON's '"latency_ms": 12'). Bare numbers are skipped: without context they
# are unverifiable noise.
_NUMERIC_RE = re.compile(r"[A-Za-z_][\w.-]{0,24}\"?[ =:]{1,3}\d+(?:\.\d+)?")
# Do not extract a decimal prefix from unsupported exponents or numeric suffixes.
_NUMERIC_RE = re.compile(r"[A-Za-z_][\w.-]{0,24}\"?[ =:]{1,3}\d+(?:\.\d+)?(?!\w|\.\d)")
_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+")
_PATH_RE = re.compile(r"(?:~/|\.{1,2}/|/)?(?:[\w.-]+/){2,}[\w.@-]+")
# Requires at least one a-f so bare decimal runs (timestamps, row counts) are
@@ -233,6 +234,21 @@ def _normalize(text: str) -> str:
def _target_survives(dimension: str, value: str, haystack: str, normalized_haystack: str) -> bool:
if dimension == "numerics":
# Check numbers before substring shortcuts: 80 is not retained in
# 8080, 80.5 or -80. A trailing sentence period is still punctuation.
# Format conversions (JSON -> table) may separate key from value.
match = _NUMERIC_SPLIT_RE.match(value)
if match:
key, number = match.groups()
normalized_key = _normalize(key)
if (
normalized_key
and normalized_key in normalized_haystack
and re.search(rf"(?<![\w.-]){re.escape(number)}(?!\w|\.\d)", normalized_haystack)
):
return True
return False
if value in haystack:
return True
normalized_value = _normalize(value)
@@ -244,19 +260,6 @@ def _target_survives(dimension: str, value: str, haystack: str, normalized_hayst
_, _, remainder = normalized_value.partition(" ")
if len(remainder) >= _MIN_TARGET_LEN and remainder in normalized_haystack:
return True
if dimension == "numerics":
# Format conversions (JSON -> table) separate key from value; count the
# probe as retained only when both still appear.
match = _NUMERIC_SPLIT_RE.match(value)
if match:
key, number = match.groups()
normalized_key = _normalize(key)
if (
normalized_key
and normalized_key in normalized_haystack
and re.search(rf"\b{re.escape(number)}\b", normalized_haystack)
):
return True
return False
+25 -6
View File
@@ -230,8 +230,8 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
opened world-readable, so a failure to create or narrow the file privately
raises :class:`PermissionError` rather than proceeding to open a wide
file. The existing-file path is symlink-race resistant — ``O_EXCL`` on the
create refuses to reuse a planted file or symlink; an existing file is
re-opened with ``O_NOFOLLOW`` and narrowed through that descriptor
create refuses to reuse a planted file or symlink; an existing file not
already 0600 is opened with ``O_NOFOLLOW`` and narrowed through it
(``fstat`` to confirm a regular file, ``fchmod`` to set the mode) rather
than by re-resolving the path, which closes the check-then-chmod TOCTOU a
``chmod(path)`` would leave open. On Windows POSIX mode bits and
@@ -249,15 +249,36 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
except FileExistsError:
pass
_narrow_existing(path, what=what, missing_ok=False)
def _narrow_existing(path: str | os.PathLike[str], *, what: str, missing_ok: bool) -> None:
"""Narrow an existing file to 0600 through an ``O_NOFOLLOW`` descriptor.
A regular file already 0600 and owned by the caller (any owner as root) is
only ``lstat``-ed: closing any descriptor on a sqlite file drops the
process's ``fcntl`` locks on it (sqlite.org/howtocorrupt.html, 2.2).
Non-regular files are refused; with *missing_ok*, a missing file is skipped.
"""
if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):
return
fd = -1
try:
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
st = os.lstat(path)
if stat.S_ISREG(st.st_mode):
euid = os.geteuid() # type: ignore[attr-defined]
if stat.S_IMODE(st.st_mode) == OWNER_ONLY_MODE and (st.st_uid == euid or euid == 0):
return
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
except OSError as exc:
if missing_ok and isinstance(exc, FileNotFoundError):
return
raise PermissionError(
f"refusing to open {what} at {path}: not a regular file (symlink or open error: {exc})"
) from exc
if fd < 0:
raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
try:
if not stat.S_ISREG(os.fstat(fd).st_mode):
raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
@@ -346,9 +367,7 @@ def connect_private_sqlite(
# them. Missing ones are left alone: sqlite creates them with the
# main file's mode, which is now 0600.
for suffix in _SQLITE_SIDECAR_SUFFIXES:
sidecar = file_path + suffix
if os.path.lexists(sidecar):
ensure_private_file(sidecar, what=f"{what} {suffix[1:]} file")
_narrow_existing(file_path + suffix, what=f"{what} {suffix[1:]} file", missing_ok=True)
# ``**connect_kwargs: Any`` makes mypy type the call as ``Any``; the
# annotation pins it back to the real return type.
conn: sqlite3.Connection = sqlite3.connect(os.fspath(path), **connect_kwargs)
+82
View File
@@ -2,8 +2,10 @@
from __future__ import annotations
import logging
import os
import re
import stat
import sys
from pathlib import Path
@@ -11,8 +13,88 @@ import click
from headroom import paths as _paths
logger = logging.getLogger(__name__)
_PROFILE_RE = re.compile(r"^[A-Za-z0-9._-]+$")
# A deployment profile directory holds secrets. `headroom install --env
# KEY=VALUE` is the supported way to give a supervised proxy a provider API key
# — supervisors (launchd, systemd, Task Scheduler, cron) start from a bare
# environment, so nothing else reaches the process — and every such value is
# persisted verbatim, both into `manifest.json` and into the generated runner
# scripts that `export` it before the exec. Those files are therefore written
# owner-only rather than at the process umask (which leaves them world-readable
# at the common 022). The supervisor always runs them as the installing user
# (user scope) or as root (system scope), and neither needs the group/other
# bits, so this is not a functional restriction.
#
# SECURITY.md documents these modes; `tests/test_packaging_extras_and_security_docs.py`
# reads the octal values back out of that document and compares them with what
# an install actually writes, so the policy and the code cannot drift apart.
# Named for what they ARE -- a permission mode -- not for what the files they
# protect contain. The previous SECRET_* spelling read as a credential to
# CodeQL's sensitive-data heuristic, so `logger.warning("... 0o%o", mode)`
# tripped py/clear-text-logging-sensitive-data on a diagnostic that logs a
# permission bitmask and no secret at all.
OWNER_ONLY_FILE_MODE = 0o600
OWNER_ONLY_SCRIPT_MODE = 0o700
OWNER_ONLY_DIR_MODE = 0o700
#: Whether this platform actually enforces POSIX permission bits. On Windows
#: access is governed by ACLs and ``chmod`` only toggles the read-only flag, so
#: it SUCCEEDS without establishing the requested mode — which is why the check
#: below reads the mode back rather than trusting the call not to raise.
POSIX_MODES_ENFORCED = os.name == "posix"
def chmod_owner_only(path: Path, mode: int) -> bool:
"""Narrow ``path`` to an owner-only ``mode``. Returns whether that took.
This used to swallow every ``OSError`` and return nothing, so a caller had
no way to tell a restricted file from an unrestricted one and continued as
if the documented mode had been established. That is the wrong default for
the two places it matters, because both NARROW A PRE-EXISTING INODE rather
than create one: a profile directory made before these modes existed is
0755, and a runner script rewritten through ``O_TRUNC`` keeps whatever mode
it already had. In both cases this chmod is the only thing standing between
a provider API key and every local user, and a silent failure left the
caller asserting a guarantee it did not have.
The result is verified with ``stat`` instead of inferred from ``chmod`` not
raising, because on Windows it does not raise and does not work either.
Returns ``True`` only when the mode is now exactly ``mode``. A caller
holding secret-bearing content must act on ``False``; see
:func:`headroom.install.supervisors._write_private_text`, which refuses to
leave the file behind.
"""
try:
path.chmod(mode)
actual = stat.S_IMODE(path.stat().st_mode)
except OSError as exc:
logger.warning(
"Could not restrict %s to mode 0o%o (%s); it may be readable by other local users.",
path,
mode,
exc,
)
return False
if actual != mode:
# Expected on Windows, where the bits are advisory -- noise there, a
# real finding on POSIX.
logger.log(
logging.WARNING if POSIX_MODES_ENFORCED else logging.DEBUG,
"%s is mode 0o%o after asking for 0o%o; this platform does not "
"enforce POSIX permission bits, so the file's protection comes "
"from its directory and the system ACLs instead.",
path,
actual,
mode,
)
return False
return True
def validate_profile_name(profile: str) -> str:
"""Validate and normalize a deployment profile name."""
+5 -2
View File
@@ -4,6 +4,7 @@ from __future__ import annotations
import os
import re
import shlex
import subprocess
from pathlib import Path
@@ -32,7 +33,8 @@ def _merge_marker_block(file_path: Path, block: str, pattern: re.Pattern[str], m
if file_path.exists():
existing = fsutil.read_text(file_path)
if marker in existing:
return pattern.sub(block, existing)
# A callable replacement: the block is literal text, not a template.
return pattern.sub(lambda _match: block, existing)
return existing.rstrip() + "\n\n" + block + "\n"
return block + "\n"
@@ -40,7 +42,8 @@ def _merge_marker_block(file_path: Path, block: str, pattern: re.Pattern[str], m
def _env_block(values: dict[str, str]) -> str:
lines = [_ENV_MARKER_START]
for name, value in values.items():
lines.append(f'export {name}="{value}"')
# shlex.quote so `$`, `"` and backticks reach the shell verbatim.
lines.append(f"export {name}={shlex.quote(value)}")
lines.append(_ENV_MARKER_END)
return "\n".join(lines)
+60 -4
View File
@@ -12,7 +12,16 @@ from pathlib import Path
from typing import Any
from .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_now
from .paths import deploy_root, manifest_path, profile_root, recovery_manifest_path
from .paths import (
OWNER_ONLY_DIR_MODE,
OWNER_ONLY_FILE_MODE,
POSIX_MODES_ENFORCED,
chmod_owner_only,
deploy_root,
manifest_path,
profile_root,
recovery_manifest_path,
)
logger = logging.getLogger(__name__)
@@ -29,6 +38,15 @@ def _atomic_write_text(path: Path, data: str) -> None:
both POSIX and Windows). A crash between truncate and full write therefore
leaves either the previous file or the complete new one on disk, never a
truncated manifest.
The file is owner-only. ``manifest.base_env`` holds whatever
``headroom install --env KEY=VALUE`` was given, and that is the supported
way to hand a provider API key to a supervised proxy (supervisors start
from a bare environment), so the manifest must be assumed to contain
secrets. :func:`tempfile.mkstemp` already creates the temporary file 0600
and :func:`os.replace` carries that mode across, but the chmod is explicit
so the guarantee survives a future rewrite of this function and is pinned
by a test rather than inherited by accident.
"""
directory = path.parent
fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=f".{path.name}.", suffix=".tmp")
@@ -38,6 +56,7 @@ def _atomic_write_text(path: Path, data: str) -> None:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
chmod_owner_only(tmp_path, OWNER_ONLY_FILE_MODE)
os.replace(tmp_path, path)
except BaseException:
tmp_path.unlink(missing_ok=True)
@@ -48,9 +67,39 @@ def _manifest_json(manifest: DeploymentManifest) -> str:
return json.dumps(asdict(manifest), indent=2) + "\n"
def _save_manifest_at(path: Path, manifest: DeploymentManifest, *, strict: bool) -> None:
def _narrow_profile_root(root: Path) -> None:
"""Make a deployment profile directory owner-only.
``exist_ok=True`` leaves a pre-existing directory's mode alone, and a
profile created before owner-only modes is 0755, so narrow it on every save
rather than only at creation.
Not fatal, unlike the runner script: the manifest itself is created through
``mkstemp``, which is 0600 from birth, so a directory that could not be
narrowed weakens the outer layer without exposing the file. Warn rather
than abandon a deployment over it.
"""
if not chmod_owner_only(root, OWNER_ONLY_DIR_MODE) and POSIX_MODES_ENFORCED:
logger.warning(
"Deployment profile directory %s is not owner-only; the "
"manifest inside it is still 0o%o, but other local users can "
"list the directory.",
root,
OWNER_ONLY_FILE_MODE,
)
def _save_manifest_at(
path: Path,
manifest: DeploymentManifest,
*,
strict: bool,
owner_only_dir: bool = False,
) -> None:
root = path.parent
root.mkdir(parents=True, exist_ok=True)
if owner_only_dir:
_narrow_profile_root(root)
manifest.updated_at = iso_utc_now()
try:
_atomic_write_text(path, _manifest_json(manifest))
@@ -66,9 +115,16 @@ def save_manifest(manifest: DeploymentManifest) -> None:
The write is atomic so an interrupted save (SIGKILL, system restart, OOM)
cannot leave a truncated ``manifest.json`` behind. Gracefully handles
read-only filesystems by logging a warning instead of crashing.
The profile directory and the manifest are owner-only: ``base_env`` carries
whatever ``headroom install --env`` was given, which is the supported way to
hand a provider API key to a supervised proxy. See
:data:`headroom.install.paths.OWNER_ONLY_DIR_MODE`.
"""
try:
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=False)
_save_manifest_at(
manifest_path(manifest.profile), manifest, strict=False, owner_only_dir=True
)
except OSError as e:
logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)
@@ -76,7 +132,7 @@ def save_manifest(manifest: DeploymentManifest) -> None:
def save_manifest_strict(manifest: DeploymentManifest) -> None:
"""Persist a manifest and propagate errors when recovery depends on it."""
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=True)
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=True, owner_only_dir=True)
def save_recovery_manifest(manifest: DeploymentManifest) -> None:
+81 -10
View File
@@ -21,6 +21,10 @@ from headroom._subprocess import run
from .models import ArtifactRecord, DeploymentManifest, SupervisorKind
from .paths import (
OWNER_ONLY_FILE_MODE,
OWNER_ONLY_SCRIPT_MODE,
POSIX_MODES_ENFORCED,
chmod_owner_only,
unix_ensure_script_path,
unix_run_script_path,
windows_ensure_cmd_path,
@@ -47,6 +51,32 @@ def _is_windows() -> bool:
return sys.platform.startswith("win")
def _write_private_text(path: Path, data: str, mode: int) -> None:
"""Atomically replace a secret-bearing runner with a private new inode.
Creating the temporary file exclusively avoids following a replacement
hard link. Restrict it before publishing; a failed write, restriction or
replacement leaves the previous runner and unrelated files untouched.
"""
path.parent.mkdir(parents=True, exist_ok=True)
fd, name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
temporary = Path(name)
try:
with os.fdopen(fd, "w", encoding="utf-8") as handle:
if hasattr(os, "fchmod"):
os.fchmod(handle.fileno(), mode)
handle.write(data)
if not chmod_owner_only(temporary, mode) and POSIX_MODES_ENFORCED:
raise click.ClickException(
f"Refusing to write {path}: permissions could not be restricted "
f"to {oct(mode)}, and the file carries any API key passed via "
f"`headroom install --env`. Fix directory permissions and re-run."
)
os.replace(temporary, path)
finally:
temporary.unlink(missing_ok=True)
def _validated_env_items(env: dict[str, str] | None) -> list[tuple[str, str]]:
items = list((env or {}).items())
for name, _value in items:
@@ -101,14 +131,19 @@ def _render_unix_runner(
export_lines = "".join(
f"export {name}={shlex.quote(value)}\n" for name, value in _validated_env_items(env)
)
path.write_text(
# 0700, not 0755: `export_lines` can contain a provider API key in
# cleartext (see `_write_private_text`). The supervisor runs this script as
# the installing user for a user-scope install and as root for a
# system-scope one, so dropping group/other loses nothing.
_write_private_text(
path,
"#!/usr/bin/env bash\nset -euo pipefail\n"
+ export_lines
+ "exec "
+ " ".join(shlex.quote(x) for x in command)
+ "\n"
+ "\n",
OWNER_ONLY_SCRIPT_MODE,
)
path.chmod(0o755)
return ArtifactRecord(kind="script", path=str(path))
@@ -124,9 +159,17 @@ def _render_windows_runner(
env_lines = "".join(
f"$env:{name} = {_powershell_literal(value)}\n" for name, value in _validated_env_items(env)
)
ps1_path.write_text(
f"$ErrorActionPreference = 'Stop'\n{env_lines}& {escaped}\nexit $LASTEXITCODE\n"
# The .ps1 carries `$env:` assignments that can include a provider API key,
# so it is owner-only like its POSIX counterpart. Windows resolves access
# by ACL rather than by these bits, but the mode still matters when the
# profile directory is read from WSL, a backup, or a synced home.
_write_private_text(
ps1_path,
f"$ErrorActionPreference = 'Stop'\n{env_lines}& {escaped}\nexit $LASTEXITCODE\n",
OWNER_ONLY_FILE_MODE,
)
# The .cmd shim holds no secrets — it only invokes the .ps1 — so it keeps a
# conventional mode.
cmd_path.write_text(
'@echo off\r\npowershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0'
+ ps1_path.name
@@ -651,6 +694,11 @@ def start_supervisor(manifest: DeploymentManifest) -> None:
subprocess.run(["schtasks", "/Run", "/TN", f"{manifest.service_name}-startup"], check=True)
# `/End` and `/Delete` report a missing task as "cannot find the file specified";
# `/Change` reports it as "does not exist in the system".
_SCHTASKS_ABSENT_TEXT = ("cannot find the file specified", "does not exist in the system")
def stop_supervisor(manifest: DeploymentManifest) -> None:
"""Stop the installed supervisor for a deployment."""
@@ -687,10 +735,32 @@ def stop_supervisor(manifest: DeploymentManifest) -> None:
)
return
if _is_windows():
# Session-scoped fallback installs never create the `-startup` task,
# and the health task may already be absent, so tolerate missing
# tasks (schtasks reports "cannot find the file specified") while
# still surfacing real failures such as access denied.
health = f"{manifest.service_name}-health"
subprocess.run(["schtasks", "/Change", "/TN", health, "/DISABLE"], check=True)
result = run(
["schtasks", "/Change", "/TN", health, "/DISABLE"],
capture_output=True,
text=True,
)
_require_removal_success(
result,
"schtasks health disable",
absent_text=_SCHTASKS_ABSENT_TEXT,
)
subprocess.run(["schtasks", "/End", "/TN", health], check=False)
subprocess.run(["schtasks", "/End", "/TN", f"{manifest.service_name}-startup"], check=True)
result = run(
["schtasks", "/End", "/TN", f"{manifest.service_name}-startup"],
capture_output=True,
text=True,
)
_require_removal_success(
result,
"schtasks startup end",
absent_text=_SCHTASKS_ABSENT_TEXT,
)
def remove_supervisor(manifest: DeploymentManifest) -> None:
@@ -785,7 +855,7 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:
_require_removal_success(
result,
f"schtasks {suffix} delete",
absent_text="cannot find the file specified",
absent_text=_SCHTASKS_ABSENT_TEXT,
)
@@ -800,7 +870,7 @@ def _require_removal_success(
operation: str,
*,
absent_codes: set[int] | None = None,
absent_text: str | None = None,
absent_text: str | tuple[str, ...] | None = None,
) -> None:
returncode = int(getattr(result, "returncode", 0))
detail = _result_text(result)
@@ -808,6 +878,7 @@ def _require_removal_success(
return
if absent_codes and returncode in absent_codes:
return
if absent_text and absent_text in detail.lower():
texts = (absent_text,) if isinstance(absent_text, str) else absent_text or ()
if any(text in detail.lower() for text in texts):
return
raise click.ClickException(f"{operation} failed: {detail}")
+5 -4
View File
@@ -18,6 +18,7 @@ from pathlib import Path
from typing import Any
from headroom import fsutil
from headroom.providers.codex.config_toml import loads_codex_config, split_toml_lines
from .base import (
MCPRegistrar,
@@ -84,7 +85,7 @@ def _evict_foreign_tables(content: str, server_name: str, start: str, end: str)
move, so a header-like line inside a multi-line string can split a table
wrongly; callers must check the final result with ``_only_server_changed``.
"""
lines = content.splitlines(keepends=True)
lines = split_toml_lines(content, keepends=True)
try:
i = next(n for n, line in enumerate(lines) if line.rstrip("\r\n") == start)
j = next(n for n in range(i + 1, len(lines)) if lines[n].rstrip("\r\n") == end)
@@ -121,7 +122,7 @@ def _only_server_changed(old: str, new: str, server_name: str) -> bool:
makes this False and the rewrite is refused instead of written.
"""
try:
before, after = tomllib.loads(old), tomllib.loads(new)
before, after = loads_codex_config(old), loads_codex_config(new)
except tomllib.TOMLDecodeError:
return False
return _without_server(before, server_name) == _without_server(after, server_name)
@@ -255,7 +256,7 @@ class _TomlMarkerRegistrar(MCPRegistrar):
# Read via fsutil (UTF-8 with locale fallback) so a config that a
# tool wrote in the system locale (e.g. GBK) still parses instead
# of failing tomllib's UTF-8 requirement. See #733.
data = tomllib.loads(fsutil.read_text(self._config_file))
data = loads_codex_config(fsutil.read_text(self._config_file))
except (tomllib.TOMLDecodeError, OSError):
return {}
return data if isinstance(data, dict) else {}
@@ -276,7 +277,7 @@ class _TomlMarkerRegistrar(MCPRegistrar):
if not raw.strip():
return None
try:
data = tomllib.loads(fsutil.read_text(self._config_file))
data = loads_codex_config(fsutil.read_text(self._config_file))
except (tomllib.TOMLDecodeError, OSError) as exc:
return f"{self._config_file} is not valid TOML ({exc})"
if not isinstance(data, dict):
+54 -14
View File
@@ -49,6 +49,20 @@ def _validate_metadata_key(key: str) -> bool:
return _SAFE_METADATA_KEY_PATTERN.match(key) is not None
def _utc_timestamp(value: str | None) -> str | None:
"""Comparable UTC text without rewriting stored offsets or rounding fractions.
Naive timestamps follow Memory's UTC defaults. SQLite date functions round
fractional seconds, so keep Python's full microsecond precision here.
"""
if value is None:
return None
timestamp = datetime.fromisoformat(value)
if timestamp.tzinfo is not None:
timestamp = timestamp.astimezone(timezone.utc).replace(tzinfo=None)
return timestamp.isoformat(timespec="microseconds")
class SQLiteMemoryStore:
"""SQLite-based memory store implementing the MemoryStore protocol.
@@ -67,6 +81,12 @@ class SQLiteMemoryStore:
Schema:
The memories table stores all Memory fields with appropriate indexes
for efficient querying by scope, category, importance, and time.
Metadata filters:
Numbers and booleans use native SQLite equality. String filters match only
JSON text values. None matches an explicit JSON null,
not a missing key. Arrays and objects compare as compact JSON text with
matching JSON types; array order and object member order are significant.
"""
def __init__(self, db_path: str | Path = "headroom_memory.db") -> None:
@@ -89,6 +109,7 @@ class SQLiteMemoryStore:
"""
conn = connect_private_sqlite(self.db_path, what="memory store")
conn.row_factory = sqlite3.Row
conn.create_function("headroom_utc_timestamp", 1, _utc_timestamp, deterministic=True)
try:
with conn:
yield conn
@@ -501,21 +522,23 @@ class SQLiteMemoryStore:
if scope_conditions:
conditions.append(f"({' OR '.join(scope_conditions)})")
# Normalize both sides: persisted rows may retain explicit UTC offsets.
# This favors correct instant comparisons over the raw TEXT time index.
# Temporal filtering
if filter.created_after is not None:
conditions.append("created_at >= ?")
params.append(filter.created_after.isoformat())
conditions.append("headroom_utc_timestamp(created_at) >= ?")
params.append(_utc_timestamp(filter.created_after.isoformat()))
if filter.created_before is not None:
conditions.append("created_at <= ?")
params.append(filter.created_before.isoformat())
conditions.append("headroom_utc_timestamp(created_at) <= ?")
params.append(_utc_timestamp(filter.created_before.isoformat()))
# Point-in-time query
if filter.valid_at is not None:
valid_at_str = filter.valid_at.isoformat()
conditions.append("valid_from <= ?")
valid_at_str = _utc_timestamp(filter.valid_at.isoformat())
conditions.append("headroom_utc_timestamp(valid_from) <= ?")
params.append(valid_at_str)
conditions.append("(valid_until IS NULL OR valid_until > ?)")
conditions.append("(valid_until IS NULL OR headroom_utc_timestamp(valid_until) > ?)")
params.append(valid_at_str)
# Superseded filtering
@@ -571,13 +594,22 @@ class SQLiteMemoryStore:
# equates ``5 = '5'`` — so an int/float/bool metadata filter matched
# nothing. ``bool`` is a subclass of ``int``, so it is covered here
# (a JSON ``true`` extracts to 1, and Python ``True`` binds to 1).
# A non-scalar value (dict/list) is not a bindable SQLite type, so it
# keeps the JSON-text comparison rather than raising.
conditions.append(f"json_extract(metadata, '$.{key}') = ?")
if isinstance(value, (str, int, float)):
if value is None:
# json_extract returns SQL NULL for both null and missing keys.
conditions.append(f"json_type(metadata, '$.{key}') = 'null'")
elif isinstance(value, (str, int, float)):
if isinstance(value, str):
# Extracted JSON arrays/objects are also SQL text.
conditions.append(f"json_type(metadata, '$.{key}') = 'text'")
conditions.append(f"json_extract(metadata, '$.{key}') = ?")
params.append(value)
else:
params.append(json.dumps(value))
# Normalize whitespace like json_extract does for arrays/objects,
# and do not match a string containing the same JSON text.
encoded = json.dumps(value)
conditions.append(f"json_type(metadata, '$.{key}') = json_type(?)")
conditions.append(f"json_extract(metadata, '$.{key}') = json(?)")
params.extend([encoded, encoded])
return conditions, params
@@ -605,6 +637,9 @@ class SQLiteMemoryStore:
):
order_column = "created_at"
if order_column in ("created_at", "last_accessed"):
order_column = f"headroom_utc_timestamp({order_column})"
order_direction = "DESC" if filter.order_desc else "ASC"
# Build full query
@@ -798,23 +833,28 @@ class SQLiteMemoryStore:
return []
history: list[Memory] = [current]
# Imported or reused IDs can form cycles. Visit each stored record
# only once across both directions instead of blocking indefinitely.
visited = {current.id}
# Follow chain backwards (supersedes)
back_id = current.supersedes
while back_id is not None:
while back_id is not None and back_id not in visited:
prev = await self.get(back_id)
if prev is None:
break
visited.add(prev.id)
history.insert(0, prev) # Add to beginning
back_id = prev.supersedes
# Follow chain forwards (superseded_by) if requested
if include_future:
forward_id = current.superseded_by
while forward_id is not None:
while forward_id is not None and forward_id not in visited:
next_mem = await self.get(forward_id)
if next_mem is None:
break
visited.add(next_mem.id)
history.append(next_mem)
forward_id = next_mem.superseded_by
+21 -1
View File
@@ -248,7 +248,9 @@ class SQLiteGraphStore:
async def add_entity(self, entity: Entity) -> None:
"""Add an entity to the graph store.
If an entity with the same ID already exists, it will be replaced.
If an entity with the same ID and owner already exists, its fields are
updated without deleting its relationships. Reassigning the owner
replaces the entity and deletes its old relationships.
Args:
entity: The entity to add.
@@ -257,6 +259,14 @@ class SQLiteGraphStore:
with self._lock:
with self._get_conn() as conn:
# Keep owner reassignment's replacement/cascade semantics. Both
# statements share a transaction, so a failed save restores the old graph.
conn.execute(
"DELETE FROM entities WHERE id = :id AND user_id != :user_id",
row,
)
# The ID upsert overrides REPLACE's delete. Keep REPLACE's existing
# NOT NULL/default handling for the incoming row.
conn.execute(
"""
INSERT OR REPLACE INTO entities (
@@ -266,6 +276,16 @@ class SQLiteGraphStore:
:id, :user_id, :name, :name_lower, :entity_type, :description,
:properties, :created_at, :updated_at, :metadata
)
ON CONFLICT(id) DO UPDATE SET
user_id = excluded.user_id,
name = excluded.name,
name_lower = excluded.name_lower,
entity_type = excluded.entity_type,
description = excluded.description,
properties = excluded.properties,
created_at = excluded.created_at,
updated_at = excluded.updated_at,
metadata = excluded.metadata
""",
row,
)
+10 -3
View File
@@ -91,7 +91,7 @@ class ResolvedScope:
"""
mode: MemoryStorageMode
db_path: Path
db_path: Path | None
display_name: str # human-readable label, e.g. project basename
project_key: str | None # stable hash, None for USER/GLOBAL
@@ -304,13 +304,20 @@ class BackendRouter:
self._backends: OrderedDict[Path, LocalBackend] = OrderedDict()
self._lock = threading.Lock()
def backend_for(self, ctx: RequestContext) -> tuple[LocalBackend, ResolvedScope]:
def backend_for(self, ctx: RequestContext) -> tuple[LocalBackend | None, ResolvedScope]:
"""Return the backend + scope metadata to use for this request."""
scope = self._resolve_scope(ctx)
if scope.db_path is None:
return None, scope
backend = self._get_or_create_backend(scope.db_path)
return backend, scope
def scope_for(self, ctx: RequestContext) -> ResolvedScope:
"""Resolve request scope without acquiring or creating a backend."""
return self._resolve_scope(ctx)
def _resolve_scope(self, ctx: RequestContext) -> ResolvedScope:
mode = self._config.mode
@@ -363,7 +370,7 @@ class BackendRouter:
)
return ResolvedScope(
mode=MemoryStorageMode.PROJECT,
db_path=self._config.global_db_path, # Unused — caller checks project_key.
db_path=None,
display_name="unresolved (no memory)",
project_key=None,
)
+18 -3
View File
@@ -320,6 +320,10 @@ def _paths_related_as_typo(failed: str, success: str) -> bool:
return False
if a == b:
return True
# Basenames come from tool traffic; no real file name comes near this cap,
# so skip the quadratic edit distance rather than run it on an oversized one.
if max(len(a), len(b)) > _RETRY_EDIT_DISTANCE_MAX_CHARS:
return False
threshold = max(2, max(len(a), len(b)) // 3)
return _levenshtein(a, b) <= threshold
@@ -375,13 +379,21 @@ def _bash_binaries_match(a: str, b: str) -> bool:
b_base = b.rsplit("/", 1)[-1]
if a_base == b_base:
return True
if (a_base.startswith(b_base) or b_base.startswith(a_base)) and _levenshtein(
a_base, b_base
# One name is a prefix of the other, so their edit distance is exactly the
# length difference: compare lengths instead of running _levenshtein (O(n*m))
# on an unbounded, traffic-derived token.
if (a_base.startswith(b_base) or b_base.startswith(a_base)) and abs(
len(a_base) - len(b_base)
) <= 2:
return True
return False
# _levenshtein is O(n*m) pure Python and runs on the request path; longer
# commands skip it and are judged by the linear token-overlap check below.
_RETRY_EDIT_DISTANCE_MAX_CHARS = 512
def _commands_related_as_retry(failed: str, success: str) -> bool:
"""Heuristic: is `success` plausibly a corrected retry of `failed`?
@@ -405,7 +417,10 @@ def _commands_related_as_retry(failed: str, success: str) -> bool:
return False
max_len = max(len(failed), len(success))
if max_len > 0 and _levenshtein(failed, success) / max_len <= 0.40:
if (
0 < max_len <= _RETRY_EDIT_DISTANCE_MAX_CHARS
and _levenshtein(failed, success) / max_len <= 0.40
):
return True
def _substantive(cmd: str, binary: str) -> set[str]:
+16 -4
View File
@@ -224,6 +224,7 @@ class PerfReport:
oldest_kept_ts: str | None = None
newest_kept_ts: str | None = None
records_filtered_out: int = 0
records_malformed: int = 0
# True when no time cutoff was applied (--hours 0, or a value so large it
# overflows datetime arithmetic). The header says "all data" instead of a
# misleading "last 0h".
@@ -386,9 +387,8 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport:
if not _within_window(ts):
report.records_filtered_out += 1
continue
_track_window(ts)
report.perf_records.append(
PerfRecord(
try:
record = PerfRecord(
timestamp=ts,
request_id=m.group("rid"),
model=kv.get("model", ""),
@@ -411,7 +411,13 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport:
from_response_cache=kv.get("cached", "0") == "1",
stages=stages_by_rid.get(m.group("rid"), {}),
)
)
except ValueError:
# A non-numeric field is not a PERF line Headroom wrote
# (e.g. text a client put into a logged model id).
report.records_malformed += 1
continue
_track_window(ts)
report.perf_records.append(record)
continue
# content_router summary lines
@@ -544,6 +550,11 @@ def format_report(report: PerfReport) -> str:
f"Records outside window: {report.records_filtered_out:,} "
"(filtered out — increase --hours to include them)"
)
if report.records_malformed > 0:
lines.append(
f"Malformed PERF lines: {report.records_malformed:,} "
"(skipped — a numeric field did not parse)"
)
lines.append("")
records = report.perf_records
@@ -1400,6 +1411,7 @@ def build_perf_summary(report: PerfReport) -> dict:
"newest": report.newest_kept_ts,
},
"records_filtered_out": report.records_filtered_out,
"records_malformed": report.records_malformed,
"total_requests": len(records),
"total_tokens_before": total_before,
"total_tokens_after": total_after,
+127 -22
View File
@@ -56,6 +56,7 @@ from __future__ import annotations
import logging
import math
import re
from dataclasses import dataclass
from enum import Enum
from functools import lru_cache
@@ -68,11 +69,16 @@ from headroom.pricing.cache_ttl import CACHE_WRITE_MULTIPLIERS
logger = logging.getLogger(__name__)
#: Context size at which the major catalogs publish a second, higher price tier
#: (LiteLLM spells it ``*_above_200k_tokens``). A request's billed prompt is
#: compared against this to pick which rate applies.
#: Prompt size at which most catalog rows publish a second, higher price tier
#: (LiteLLM spells it ``*_above_200k_tokens``). The default for
#: :func:`long_context_threshold`, which reads each model's own threshold from
#: its row: Claude Haiku 5.5's tier starts at 100K, GPT-5.x's at 272K.
LONG_CONTEXT_THRESHOLD_TOKENS = 200_000
#: A catalog row's long-context input rate; the number is the threshold in
#: thousands of tokens. Each row publishes at most one.
_LONG_CONTEXT_INPUT_FIELD = re.compile(r"^input_cost_per_token_above_(\d+)k_tokens$")
#: Provider-level cache discount ratios, as a fraction of the base input price.
#: FALLBACK ONLY — the per-model LiteLLM catalog is always preferred, because
#: these go stale per model and per context tier. The lightweight shared table
@@ -237,15 +243,18 @@ class CacheMix:
"""
return self.billed > 0
def is_long_context(self, *, local_tokens: int = 0) -> bool:
"""True when this request billed above the catalogs' 200k price tier.
def is_long_context(
self, *, local_tokens: int = 0, threshold: int = LONG_CONTEXT_THRESHOLD_TOKENS
) -> bool:
"""True when this request billed above ``threshold``, the model's long-context tier.
``local_tokens`` lets a caller contribute its own forwarded-token count
for requests where the provider reported no breakdown; the larger of the
two decides, so a long request is never priced at the cheap tier merely
because usage was missing.
because usage was missing. Pass :func:`long_context_threshold` for the
model so a tier that starts below 200K (Haiku 5.5: 100K) is honoured.
"""
return max(self.billed, _coerce_int(local_tokens)) > LONG_CONTEXT_THRESHOLD_TOKENS
return max(self.billed, _coerce_int(local_tokens)) > threshold
@dataclass(frozen=True)
@@ -358,6 +367,34 @@ def _litellm() -> Any | None:
_RATE_CACHE_MAXSIZE = 256
def long_context_tier(info: dict[str, Any]) -> tuple[int, str] | None:
"""The row's long-context tier as ``(threshold_tokens, field_suffix)``, if any."""
for key in info:
match = _LONG_CONTEXT_INPUT_FIELD.match(key)
if match:
thousands = int(match.group(1))
return thousands * 1000, f"_above_{thousands}k_tokens"
return None
@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)
def long_context_threshold(model: str) -> int:
"""Billed-prompt size above which ``model`` pays its long-context rates.
Read from the catalog row's ``*_above_<N>k_tokens`` fields: 100K for Claude
Haiku 5.5, 200K for Sonnet 4 / 4.5, 272K for GPT-5.x. A model with no tier,
or one the catalog does not list, gets :data:`LONG_CONTEXT_THRESHOLD_TOKENS`;
that is harmless, because :func:`resolve_rates` then has no tier to switch
to and returns the base rates either way.
Memoized on the same bounded LRU as :func:`resolve_rates`, for the same
reason; tests that swap the catalog clear both.
"""
row = _catalog_row(model)
tier = long_context_tier(row[2]) if row is not None else None
return tier[0] if tier else LONG_CONTEXT_THRESHOLD_TOKENS
def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context: bool) -> float:
"""Resolve an absent catalog slice through the model's billing calculator.
@@ -365,7 +402,7 @@ def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context:
zero, list price, or a model-specific discount. Never invent our own ratio.
The result is cached with the other resolved rates, not probed per request.
"""
tokens = LONG_CONTEXT_THRESHOLD_TOKENS + 1 if long_context else 1
tokens = long_context_threshold(model) + 1 if long_context else 1
try:
input_cost, _ = litellm.cost_per_token(
model=model, prompt_tokens=tokens, completion_tokens=0, **{field: tokens}
@@ -413,11 +450,12 @@ def resolve_rates(
Preference order, strongest first:
1. **Catalog** — LiteLLM's per-model published rates, including the
long-context tier and, where the row has it,
long-context tier (the row's own ``*_above_<N>k_tokens`` fields, see
:func:`long_context_threshold`) and, where the row has it,
``cache_creation_input_token_cost_above_1hr``. Anthropic publishes a real
1h write rate (Sonnet: 2.00x base), and reading it beats deriving it.
2. **Catalog + TTL ratio** — a catalog row that prices 5m writes but not 1h
ones, which is every row at the above-200k tier. The 1h rate is derived
ones, which is every row at its long-context tier. The 1h rate is derived
from the tier's base input price and the structural multiplier in
:mod:`headroom.pricing.cache_ttl`. Only applied when the row shows a real
write premium: a provider that does not bill for cache writes at all
@@ -446,14 +484,17 @@ def resolve_rates(
# savings on a model that costs nothing.
if base is None:
return None
if long_context:
base = info.get("input_cost_per_token_above_200k_tokens") or base
tier = long_context_tier(info) if long_context else None
if tier:
long_base = info.get(f"input_cost_per_token{tier[1]}")
if long_base is not None:
base = long_base
base = float(base)
def _tier(field: str, default: float) -> float:
"""Read ``field``, preferring its above-200k variant on long requests."""
if long_context:
hi = info.get(f"{field}_above_200k_tokens")
"""Read ``field``, preferring its long-context variant on long requests."""
if tier:
hi = info.get(f"{field}{tier[1]}")
if hi is not None:
return float(hi)
value = info.get(field)
@@ -468,10 +509,10 @@ def resolve_rates(
# billed provider-reported writes require their own catalog rate.
write_5m = _tier("cache_creation_input_token_cost", missing_cache_rate)
read_is_catalog = info.get("cache_read_input_token_cost") is not None or (
long_context and info.get("cache_read_input_token_cost_above_200k_tokens") is not None
tier is not None and info.get(f"cache_read_input_token_cost{tier[1]}") is not None
)
write_is_catalog = info.get("cache_creation_input_token_cost") is not None or (
long_context and info.get("cache_creation_input_token_cost_above_200k_tokens") is not None
tier is not None and info.get(f"cache_creation_input_token_cost{tier[1]}") is not None
)
if for_billing and not read_is_catalog:
read = _canonical_cache_rate(
@@ -484,10 +525,12 @@ def resolve_rates(
basis = BASIS_CATALOG
write_1h_raw = info.get("cache_creation_input_token_cost_above_1hr")
if long_context:
# No catalog publishes a combined 1h + above-200k rate, so the long
# tier always derives. Ratio basis, and labelled as such.
write_1h_raw = None
if tier:
# The long tier's 1h rate is its own field
# (``cache_creation_input_token_cost_above_1hr_above_100k_tokens``);
# most tiered rows omit it, and the rate then derives below. Ratio
# basis, and labelled as such.
write_1h_raw = info.get(f"cache_creation_input_token_cost_above_1hr{tier[1]}")
if write_1h_raw is not None:
write_1h = float(write_1h_raw)
elif write_5m > base:
@@ -540,6 +583,63 @@ def resolve_rates(
)
def long_context_premium_avoided_usd(
model: str,
mix: CacheMix,
*,
local_tokens: int = 0,
output_tokens: int = 0,
provider: str | None = None,
) -> float:
"""Long-context premium a request's forwarded input and output avoided, in USD.
For a request whose uncompressed prompt would have passed ``model``'s
long-context threshold but whose forwarded prompt did not: the provider
re-prices the WHOLE request above the threshold, so compression saved the
long card on every forwarded input token, on top of the removed tokens
themselves. This is that difference, forwarded input at the long card minus
the same input at the base card. When ``output_tokens`` is supplied, include
the premium avoided on that generated output too. These tokens were still
generated; this is compression savings, not output-shaping savings.
``local_tokens`` stands in as uncached input when the provider reported no
breakdown. Returns 0.0 for a model with no tier or no catalog row.
"""
try:
long_rates = resolve_rates(model, long_context=True, provider=provider)
base_rates = resolve_rates(model, long_context=False, provider=provider)
except Exception: # pragma: no cover - defensive; pricing must never raise
return 0.0
if long_rates is None or base_rates is None:
return 0.0
if mix.has_signal():
n = mix.normalized()
split = TokenSplit(
read=float(n.read),
write_5m=float(n.write_5m),
write_1h=float(n.write_1h),
uncached=float(n.uncached),
)
else:
split = TokenSplit(uncached=float(max(_coerce_int(local_tokens), 0)))
input_premium = max(0.0, long_rates.price(split) - base_rates.price(split))
output_premium = 0.0
if max(_coerce_int(output_tokens), 0):
try:
row = _catalog_row(model)
info = row[2] if row is not None else {}
tier = long_context_tier(info)
base_output = info.get("output_cost_per_token")
long_output = info.get(f"output_cost_per_token{tier[1]}") if tier else None
if base_output is not None and long_output is not None:
output_premium = max(0.0, float(long_output) - float(base_output)) * max(
_coerce_int(output_tokens), 0
)
except Exception: # Pricing metadata must not break telemetry.
pass
return input_premium + output_premium
def catalog_min_cacheable_tokens(model: str) -> int | None:
"""The catalog's smallest prompt the provider will cache for ``model``.
@@ -613,7 +713,9 @@ def price_savings(
mix = (mix or CacheMix()).normalized()
if long_context is None:
long_context = mix.is_long_context(local_tokens=local_tokens)
long_context = mix.is_long_context(
local_tokens=local_tokens, threshold=long_context_threshold(model)
)
try:
rates = resolve_rates(model, long_context=long_context, provider=provider)
@@ -664,6 +766,9 @@ __all__ = [
"PricedSavings",
"Region",
"TokenSplit",
"long_context_premium_avoided_usd",
"long_context_threshold",
"long_context_tier",
"catalog_min_cacheable_tokens",
"price_savings",
"resolve_rates",
+106 -26
View File
@@ -132,6 +132,8 @@ ANTHROPIC_CONTEXT_LIMITS: dict[str, int] = {
"claude-sonnet-4-6": 1000000,
# Claude Sonnet 4.5
"claude-sonnet-4-5": 200000,
# Claude Haiku 5.5 - 1M context
"claude-haiku-5-5": 1000000,
# Claude 4 (Sonnet 4, Haiku 4)
"claude-sonnet-4-20250514": 200000,
"claude-haiku-4-5-20251001": 200000,
@@ -155,9 +157,9 @@ ANTHROPIC_CONTEXT_LIMITS: dict[str, int] = {
# NOTE: These are ESTIMATES. Always verify against actual Anthropic billing.
# Last updated: 2026-09-29 (platform.claude.com/docs/en/about-claude/pricing)
#
# Newer ids come before their prefixes: `_get_pricing` falls back to substring
# matching in insertion order, so a dated or suffixed "claude-sonnet-5-5-..."
# must hit the Sonnet 5.5 row before it can hit "claude-sonnet-5".
# Newer ids still come before their prefixes, but order no longer decides a
# match: `_is_release_of` refuses to read "claude-sonnet-5-5-..." as a release
# of "claude-sonnet-5".
ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
# Claude Fable 5.1: $10 in / $50 out, cache read $0.25 (0.025x input).
"claude-fable-5-1": {"input": 10.00, "output": 50.00, "cached_input": 0.25},
@@ -183,6 +185,9 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
# Claude Sonnet 4.6 / 4.5: $3 in / $15 out, cache read $0.30
"claude-sonnet-4-6": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
"claude-sonnet-4-5": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
# Claude Haiku 5.5: $0.10 in / $0.50 out, cache read $0.01, for prompts up
# to 100K tokens; longer prompts use a 5x rate card (`_LONG_CONTEXT_TIERS`).
"claude-haiku-5-5": {"input": 0.10, "output": 0.50, "cached_input": 0.01},
# Claude 4 (Sonnet/Haiku tier pricing)
"claude-sonnet-4-20250514": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
"claude-haiku-4-5-20251001": {"input": 1.00, "output": 5.00, "cached_input": 0.10},
@@ -204,10 +209,10 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
# derived from LiteLLM's `*_above_200k_tokens` fields ($3->$6 in, $15->$22.50
# out, $0.30->$0.60 cache read).
#
# Only the Sonnet 4 / 4.5 family is tiered: Opus, and Sonnet 4.6 onward, are
# flat-rated across their whole window. This is the same population that needs
# the `[1m]` suffix to reach 1M at all, so a session that fills the window this
# unlocks is billed at these rates.
# Among the 200K-threshold models only the Sonnet 4 / 4.5 family is tiered:
# Opus, and Sonnet 4.6 onward, are flat-rated across their whole window. This
# is the same population that needs the `[1m]` suffix to reach 1M at all, so a
# session that fills the window this unlocks is billed at these rates.
_LONG_CONTEXT_THRESHOLD = 200_000
_LONG_CONTEXT_PREMIUM: dict[str, float] = {"input": 2.0, "output": 1.5, "cached_input": 2.0}
_LONG_CONTEXT_TIERED_MODELS = (
@@ -216,6 +221,55 @@ _LONG_CONTEXT_TIERED_MODELS = (
"claude-4-sonnet-20250514",
)
# Claude Haiku 5.5 has its own rate card for prompts above 100K tokens: every
# rate is 5x (LiteLLM's `*_above_100k_tokens` fields: $0.10->$0.50 in,
# $0.50->$2.50 out, $0.01->$0.05 cache read).
_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000
_HAIKU_5_5_LONG_PROMPT_PREMIUM: dict[str, float] = {
"input": 5.0,
"output": 5.0,
"cached_input": 5.0,
}
# (model ids, prompt-token threshold, per-rate multipliers above it). Ids match
# releases and provider-wrapped forms too (`_long_context_tier`).
_LONG_CONTEXT_TIERS: tuple[tuple[tuple[str, ...], int, dict[str, float]], ...] = (
(_LONG_CONTEXT_TIERED_MODELS, _LONG_CONTEXT_THRESHOLD, _LONG_CONTEXT_PREMIUM),
(("claude-haiku-5-5",), _HAIKU_5_5_LONG_PROMPT_THRESHOLD, _HAIKU_5_5_LONG_PROMPT_PREMIUM),
)
# A dated release (`-20250929`) or `-latest` alias of a model id.
_RELEASE_SUFFIX_RE = re.compile(r"-(?:\d{8}|latest)")
# A further version segment (the `-5` in `claude-haiku-5-5`): another model.
_VERSION_SEGMENT_RE = re.compile(r"-\d{1,2}(?!\d)")
def _is_release_of(model: str, known_model: str) -> bool:
"""True when ``model`` names ``known_model``, or a release or alias of it.
The fallback tables are keyed by a mix of undated and dated ids, so a
lookup accepts:
- a longer id containing ``known_model``: a dated release or a provider
wrapped id (``claude-sonnet-4-5-20250929``,
``anthropic.claude-haiku-5-5-v1:0``);
- an undated alias of a dated key (``claude-3-5-sonnet`` for
``claude-3-5-sonnet-20241022``), or its ``-latest`` form.
An id one version segment away is a different model and does not match:
``claude-haiku-5`` is not ``claude-haiku-5-5``, and ``claude-sonnet-5-5``
is not ``claude-sonnet-5``.
"""
if model == known_model:
return True
if known_model.startswith(model):
return bool(_RELEASE_SUFFIX_RE.fullmatch(known_model[len(model) :]))
at = model.find(known_model)
if at < 0:
return False
return not _VERSION_SEGMENT_RE.match(model, at + len(known_model))
def _apply_long_context_premium(
model: str, pricing: dict[str, float], input_tokens: int
@@ -226,11 +280,24 @@ def _apply_long_context_premium(
LiteLLM entry has no above-threshold rate (see
``_litellm_lacks_long_context_rate``).
"""
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
tier = _long_context_tier(model)
if tier is None or input_tokens <= tier[0]:
return pricing
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
return pricing
return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}
_threshold, premium = tier
return {key: rate * premium.get(key, 1.0) for key, rate in pricing.items()}
def _long_context_tier(model: str) -> tuple[int, dict[str, float]] | None:
"""Return the ``(threshold, premium)`` of the long-context tier ``model`` is in.
Matches with ``_is_release_of``, as the manual rate lookup does, so a
Bedrock id such as ``us.anthropic.claude-haiku-5-5-v1:0`` that the manual
table prices as Haiku 5.5 also gets Haiku 5.5's long-prompt card.
"""
for known_models, threshold, premium in _LONG_CONTEXT_TIERS:
if any(_is_release_of(model, known) for known in known_models):
return threshold, premium
return None
def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
@@ -240,17 +307,29 @@ def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
from the bare Anthropic ids before, and pricing such a request through it then
bills the base tier: about half the real cost of a long session. The manual
table still carries the premium, so the caller uses it instead.
Each tier is checked against its own field, named after its threshold
(``*_above_200k_tokens`` for Sonnet 4 / 4.5, ``*_above_100k_tokens`` for
Haiku 5.5).
The row is resolved as ``estimate_cost_from_tokens`` resolves it, so a
gateway id such as ``anthropic/claude-haiku-5-5`` is checked against the
bare row it is billed from. Only Anthropic's own rows count: a reseller row
without the field (``perplexity/anthropic/claude-haiku-5-5``) is
deliberately flat-rated, not a dropped rate.
"""
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
return False
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
return False
cost_data = get_litellm_model_cost()
for candidate in pricing_lookup_candidates(model):
info = cost_data.get(candidate)
if isinstance(info, dict):
return "input_cost_per_token_above_200k_tokens" not in info
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
row = next((c for c in pricing_lookup_candidates(model) if c in cost_data), None)
if row is None:
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
info = cost_data[row]
if not isinstance(info, dict) or info.get("litellm_provider") != "anthropic":
return False
tier = _long_context_tier(row)
if tier is None or input_tokens <= tier[0]:
return False
threshold, _premium = tier
return f"input_cost_per_token_above_{threshold // 1000}k_tokens" not in info
# Default limits for pattern-based inference
@@ -778,9 +857,10 @@ class AnthropicProvider(Provider):
if model in self._context_limits:
return self._context_limits[model]
# Check for partial matches (e.g., "claude-3-5-sonnet" matches "claude-3-5-sonnet-20241022")
# Releases and aliases of a known id (e.g. "claude-3-5-sonnet" matches
# "claude-3-5-sonnet-20241022"); see `_is_release_of`.
for known_model, limit in self._context_limits.items():
if model in known_model or known_model in model:
if _is_release_of(model, known_model):
return limit
# Try LiteLLM for context limit
@@ -853,8 +933,8 @@ class AnthropicProvider(Provider):
Tries LiteLLM first for up-to-date pricing, falls back to manual pricing.
Both paths apply Anthropic's long-context premium: on the Sonnet 4 / 4.5
family a prompt over 200K re-prices the whole request (see
``_LONG_CONTEXT_PREMIUM``).
family a prompt over 200K, and on Haiku 5.5 a prompt over 100K,
re-prices the whole request (see ``_LONG_CONTEXT_TIERS``).
``now`` selects a DeepSeek peak/off-peak tier (see
:mod:`headroom.pricing.deepseek_tiers`); ``None`` reads the wall clock.
@@ -898,9 +978,9 @@ class AnthropicProvider(Provider):
if model in self._pricing:
return self._pricing[model]
# Partial match
# Releases and aliases of a known id; see `_is_release_of`.
for known_model, prices in self._pricing.items():
if model in known_model or known_model in model:
if _is_release_of(model, known_model):
return prices
# Pattern-based inference
+102
View File
@@ -0,0 +1,102 @@
"""Parse Codex ``config.toml`` with the syntax Codex itself accepts."""
from __future__ import annotations
import sys
from typing import Any, cast
import tomlkit
from tomlkit.exceptions import TOMLKitError
if sys.version_info >= (3, 11):
import tomllib
else: # pragma: no cover - exercised only on Python 3.10
import tomli as tomllib # type: ignore[no-redef]
def loads_codex_config(content: str) -> dict[str, Any]:
"""Parse Codex config text, falling back to ``tomlkit`` for TOML 1.1 syntax.
Codex accepts TOML 1.1, where an inline table may span several lines and
end with a trailing comma (#4378). ``tomllib`` before Python 3.15 only
implements TOML 1.0 and rejects that, so retry with ``tomlkit`` (0.15+),
which parses it. Text neither parser accepts raises the original
``tomllib.TOMLDecodeError``, so callers keep their existing error handling.
"""
try:
return cast("dict[str, Any]", tomllib.loads(content))
except tomllib.TOMLDecodeError as exc:
try:
return cast("dict[str, Any]", tomlkit.parse(content).unwrap())
except TOMLKitError:
raise exc from None
def split_toml_lines(content: str, keepends: bool = False) -> list[str]:
"""Split ``content`` like ``str.splitlines``, but only at TOML newlines.
TOML ends a line at LF or CRLF only. ``str.splitlines`` also breaks at
characters such as U+2028 and U+0085, which TOML allows inside a
single-line string or comment, so its pieces are not always TOML lines.
"""
lines = [line + "\n" for line in content.split("\n")]
lines[-1] = lines[-1][:-1]
if not lines[-1]:
lines.pop()
if keepends:
return lines
return [line.removesuffix("\n").removesuffix("\r") for line in lines]
def root_assignment_lines(content: str) -> set[int]:
"""Return indexes of the lines that start a key/value pair at the document root.
Indexes refer to ``split_toml_lines(content)``. Only lines before
the first table header count, and a line inside a multi-line inline table,
array, or string is skipped, so a key nested in a multi-line inline table
(TOML 1.1, e.g. ``profiles = {\\n work = { model_provider = ... } }``) is
never mistaken for a root key.
"""
roots: set[int] = set()
depth = 0
open_string: str | None = None # delimiter of an open multi-line string
for index, line in enumerate(split_toml_lines(content, keepends=True)):
if depth == 0 and open_string is None:
stripped = line.lstrip(" \t")
if stripped.startswith("["):
break # first table header ends the root
if stripped and stripped[0] not in "#\r\n":
roots.add(index)
pos = 0
while pos < len(line):
if open_string is not None:
if line.startswith(open_string, pos):
pos += 3
# Up to two quotes may directly precede the closing delimiter.
for _ in range(2):
if pos < len(line) and line[pos] == open_string[0]:
pos += 1
open_string = None
elif open_string == '"""' and line[pos] == "\\":
pos += 2
else:
pos += 1
continue
char = line[pos]
if char == "#":
break
if line.startswith('"""', pos) or line.startswith("'''", pos):
open_string = line[pos : pos + 3]
pos += 3
elif char in "\"'":
pos += 1
while pos < len(line) and line[pos] not in (char, "\n"):
pos += 2 if char == '"' and line[pos] == "\\" else 1
pos += 1
else:
if char in "[{":
depth += 1
elif char in "]}" and depth:
depth -= 1
pos += 1
return roots
+17 -17
View File
@@ -21,6 +21,7 @@ from headroom._subprocess import run
from headroom.install.models import ConfigScope, DeploymentManifest, ManagedMutation, ToolTarget
from headroom.install.paths import codex_config_path
from .config_toml import loads_codex_config, root_assignment_lines, split_toml_lines
from .runtime import proxy_base_url
from .threads import retag_to_headroom, retag_to_native
@@ -65,7 +66,7 @@ print(key, end="")
def _codex_credential_store(config_dir: Path) -> str | None:
try:
config = tomllib.loads((config_dir / "config.toml").read_text(encoding="utf-8"))
config = loads_codex_config((config_dir / "config.toml").read_text(encoding="utf-8"))
except (OSError, tomllib.TOMLDecodeError):
return None
store = config.get("cli_auth_credentials_store")
@@ -261,7 +262,7 @@ def codex_auth_helper_is_referenced(content: str, helper_path: str) -> bool | No
the helper instead of risking deletion of a pre-existing user file.
"""
try:
document = tomllib.loads(content)
document = loads_codex_config(content)
except (tomllib.TOMLDecodeError, TypeError):
return None
@@ -358,7 +359,7 @@ def _insert_block_at_root(content: str, block: str) -> str:
root instead.
"""
block = block.strip()
lines = content.splitlines()
lines = split_toml_lines(content)
for index, line in enumerate(lines):
if _TOML_TABLE_HEADER_RE.search(line):
head = "\n".join(lines[:index]).rstrip()
@@ -369,19 +370,18 @@ def _insert_block_at_root(content: str, block: str) -> str:
def _strip_root_provider_assignments(content: str) -> str:
"""Remove root provider assignments without touching table-scoped settings."""
lines = content.splitlines(keepends=True)
kept: list[str] = []
in_root = True
for line in lines:
if in_root and _TOML_TABLE_HEADER_RE.search(line):
in_root = False
if in_root and (
_ROOT_MODEL_PROVIDER_RE.match(line) or _ROOT_OPENAI_BASE_URL_RE.match(line)
):
continue
kept.append(line)
return "".join(kept)
"""Remove root provider assignments without touching table-scoped settings.
Keys nested in a multi-line inline table (e.g. a ``profiles = {`` override)
are not root assignments and are kept.
"""
root_lines = root_assignment_lines(content)
return "".join(
line
for index, line in enumerate(split_toml_lines(content, keepends=True))
if index not in root_lines
or not (_ROOT_MODEL_PROVIDER_RE.match(line) or _ROOT_OPENAI_BASE_URL_RE.match(line))
)
def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None:
@@ -411,7 +411,7 @@ def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None
existing = _CODEX_PATTERN.sub("", existing)
existing = _strip_root_provider_assignments(existing)
merged = _insert_block_at_root(existing, section)
tomllib.loads(merged)
loads_codex_config(merged)
path.write_text(merged, encoding="utf-8")
# Pull existing native threads into the headroom-provider menu so Codex's
# history list stays whole once it routes through Headroom. Best-effort.
+3 -5
View File
@@ -14,12 +14,10 @@ from functools import lru_cache
from pathlib import Path
from typing import Any, Literal, cast
try:
import tomllib
except ModuleNotFoundError: # pragma: no cover - Python 3.10
import tomli as tomllib
import tomllib
from headroom.memory.storage_router import ProjectResolver, RequestContext
from headroom.providers.codex.config_toml import loads_codex_config
from headroom.providers.codex.threads import _codex_state_db_paths
logger = logging.getLogger(__name__)
@@ -298,7 +296,7 @@ class CodexProjectContextResolver:
return self._configured_sqlite_home
codex_home = Path(os.environ.get("CODEX_HOME", "").strip() or Path.home() / ".codex")
try:
config = tomllib.loads((codex_home / "config.toml").read_text(encoding="utf-8"))
config = loads_codex_config((codex_home / "config.toml").read_text(encoding="utf-8"))
configured_sqlite_home = config.get("sqlite_home")
if isinstance(configured_sqlite_home, str) and configured_sqlite_home.strip():
return Path(configured_sqlite_home).expanduser()
+1 -1
View File
@@ -646,7 +646,7 @@ class OpenAIProvider(Provider):
if model not in _UNKNOWN_MODEL_WARNINGS:
_UNKNOWN_MODEL_WARNINGS.add(model)
logger.warning(
f"Unknown OpenAI model '{model}': {reason} ({limit:,} tokens). "
f"Unknown OpenAI model {safe_id(model)}: {reason} ({limit:,} tokens). "
f"To configure explicitly, set HEADROOM_MODEL_LIMITS env var or "
f"add to ~/.headroom/models.json"
)
+29
View File
@@ -373,6 +373,34 @@ def _call_openai_transport(
return response
def _call_anthropic_create_transport(
client: Any,
*,
model: str,
messages: list[dict[str, Any]],
stream: bool,
metrics: Any,
**kwargs: Any,
) -> Any:
if stream:
response = client._original.messages.create(
model=model,
messages=messages,
stream=True,
**kwargs,
)
return client._wrap_stream(response, metrics)
return _call_anthropic_transport(
client,
model=model,
messages=messages,
stream=False,
metrics=metrics,
**kwargs,
)
def _call_anthropic_transport(
client: Any,
*,
@@ -409,6 +437,7 @@ def _call_anthropic_transport(
_ClientTransport = Callable[..., Any]
_CLIENT_TRANSPORTS: dict[str, _ClientTransport] = {
"anthropic": _call_anthropic_transport,
"anthropic_create": _call_anthropic_create_transport,
"openai": _call_openai_transport,
}
+19 -1
View File
@@ -50,6 +50,7 @@ import logging
import os
from collections.abc import Callable
from pathlib import Path
from urllib.parse import urlsplit
logger = logging.getLogger(__name__)
@@ -98,6 +99,23 @@ class CCSwitchReconciler:
self._task: asyncio.Task | None = None
self._last_mtime_ns: int | None = None
def _is_self(self, url: str) -> bool:
"""True if ``url`` names this proxy (``localhost`` or its bind address, any path).
Loopback families are distinct sockets: ``[::1]`` is not the proxy when it is
bound to ``127.0.0.1``, so another local provider on that address is captured.
"""
try:
want = urlsplit(self.proxy_url)
got = urlsplit(url)
return (
(got.hostname or "").lower() in {"localhost", (want.hostname or "").lower()}
and got.scheme == want.scheme
and got.port == want.port
)
except ValueError:
return False
async def start(self) -> None:
if self._task is not None:
return
@@ -178,7 +196,7 @@ class CCSwitchReconciler:
return False
# Already pointing at us: nothing to do (loop guard).
if url.rstrip("/") == self.proxy_url:
if self._is_self(url):
return False
# Third-party / custom endpoint: capture it as upstream, point Claude at us.
+70 -12
View File
@@ -133,10 +133,21 @@ class CostEntry(NamedTuple):
# (Anthropic's >200k rates differ), so they are never preferred over the catalog.
#: Context size at which the major catalogs publish a second, higher price
#: tier (LiteLLM spells it ``*_above_200k_tokens``). A request's billed prompt
#: is compared against this to pick which rate applies.
_LONG_CONTEXT_THRESHOLD_TOKENS = 200_000
def _long_context_threshold(model: str) -> int:
"""Billed-prompt size above which ``model`` pays its long-context rates.
Delegates to :func:`headroom.pricing.counterfactual.long_context_threshold`,
which reads the model's own tier from the catalog (Haiku 5.5: 100K, Sonnet
4.5: 200K). Imported at call time, like ``_get_cache_prices``: importing
``headroom.pricing`` pulls in litellm (~4s) and this module is on the
proxy's startup path. Falls back to 200K if the lookup fails.
"""
try:
from headroom.pricing.counterfactual import long_context_threshold
return long_context_threshold(model)
except Exception:
return 200_000
def _bucket_by_cache_mix(
@@ -906,6 +917,13 @@ class CostTracker:
self._saved_write_5m_by_tier: dict[tuple[str, bool], float] = {}
self._saved_write_1h_by_tier: dict[tuple[str, bool], float] = {}
self._saved_list_by_tier: dict[tuple[str, bool], float] = {}
# Forwarded tokens of requests that compression kept under the model's
# long-context threshold (their uncompressed prompt would have crossed
# it), as [read, write 5m, write 1h, uncached, output]. The provider
# re-prices the whole request above the threshold, so these tokens
# avoided the long card's premium; stats() prices that premium into
# the cache-aware savings.
self._tier_crossing_by_model: dict[str, list[float]] = {}
# Tool-schema deferral per model, DISJOINT from _tokens_saved_by_model
# (deferred schemas are never in the message counts). Tracked separately
# so the compression-only figure stays available; `stats()` reports both
@@ -964,6 +982,7 @@ class CostTracker:
self._saved_write_5m_by_tier.clear()
self._saved_write_1h_by_tier.clear()
self._saved_list_by_tier.clear()
self._tier_crossing_by_model.clear()
self._tool_saved_by_model.clear()
self._tool_saved_read_by_model.clear()
self._tool_saved_write_5m_by_model.clear()
@@ -1129,7 +1148,24 @@ class CostTracker:
write_5m_eff = 0 if cache_inferred else max(0, cache_write_5m_tokens)
write_1h_eff = 0 if cache_inferred else max(0, cache_write_1h_tokens)
billed_prompt = max(0, cache_read_tokens) + write_eff + max(0, uncached_tokens)
long_context = max(billed_prompt, tokens_sent) > _LONG_CONTEXT_THRESHOLD_TOKENS
threshold = _long_context_threshold(model)
forwarded_prompt = max(billed_prompt, tokens_sent)
long_context = forwarded_prompt > threshold
# Compression that kept the prompt under the threshold: the removed
# tokens would have billed at the long card, and so would everything
# forwarded (see `_tier_crossing_by_model`).
crossed = not long_context and forwarded_prompt + tokens_saved > threshold
if crossed:
w1h = min(write_1h_eff, write_eff)
row = self._tier_crossing_by_model.setdefault(key, [0.0] * 5)
if billed_prompt > 0:
row[0] += max(0, cache_read_tokens)
row[1] += write_eff - w1h
row[2] += w1h
row[3] += max(0, uncached_tokens)
else:
row[3] += max(0, tokens_sent)
row[4] += max(0, output_tokens)
if tokens_saved > 0:
# Message compression works the LIVE ZONE only: handlers freeze the
# cached prefix (system + prior turns) byte-for-byte for prefix-cache
@@ -1151,7 +1187,7 @@ class CostTracker:
uncached_tokens=uncached_tokens,
region="live_zone",
)
wkey = (key, long_context)
wkey = (key, long_context or crossed)
self._saved_write_5m_by_tier[wkey] = self._saved_write_5m_by_tier.get(wkey, 0.0) + c_w5m
self._saved_write_1h_by_tier[wkey] = self._saved_write_1h_by_tier.get(wkey, 0.0) + c_w1h
self._saved_list_by_tier[wkey] = self._saved_list_by_tier.get(wkey, 0.0) + c_list
@@ -1419,8 +1455,9 @@ class CostTracker:
def _get_output_price(self, model: str, *, long_context: bool = False) -> float | None:
"""Get the per-token completion price for a model, or None if unpriced.
``long_context`` selects the catalog's above-200k completion rate where
the model publishes one (Anthropic charges 1.5x there).
``long_context`` selects the completion rate of the model's
long-context tier where it publishes one (Sonnet 4.5 above 200K: 1.5x;
Haiku 5.5 above 100K: 5x).
"""
litellm = _get_litellm_module()
if litellm is None:
@@ -1432,7 +1469,11 @@ class CostTracker:
info = litellm.model_cost.get(resolved, {})
base = info.get("output_cost_per_token")
if long_context:
return info.get("output_cost_per_token_above_200k_tokens") or base or None
from headroom.pricing.counterfactual import long_context_tier
tier = long_context_tier(info)
if tier:
return info.get(f"output_cost_per_token{tier[1]}") or base or None
return base or None
except Exception as e:
_warn_pricing_once("output_price", "Failed to get output price", model, e)
@@ -1540,9 +1581,9 @@ class CostTracker:
``headroom.pricing.cache_ttl`` derives it. Previously every 1h write was
priced at the 5m rate.
``long_context`` picks the above-200k tier for each rate, falling back
per rate to the base one for a model that publishes no long-context
price.
``long_context`` picks the model's long-context tier for each rate,
falling back per rate to the base one for a model that publishes no
long-context price.
Imported at call time: ``headroom.pricing`` eagerly imports litellm
(~4s) and this module is on the proxy's startup path.
@@ -1721,6 +1762,23 @@ class CostTracker:
+ self._saved_list_by_tier.get(key, 0.0) * uncached_price
)
# The long card's premium that tier-crossing requests' forwarded
# tokens avoided (see `_tier_crossing_by_model`).
for model, crossing in self._tier_crossing_by_model.items():
long_prices = self._get_cache_prices(model, long_context=True)
base_prices = self._get_cache_prices(model)
if not long_prices or not base_prices:
continue
premium = sum(
tokens * max(0.0, hi - lo)
for tokens, hi, lo in zip(crossing[:4], long_prices, base_prices, strict=True)
)
long_out = self._get_output_price(model, long_context=True)
base_out = self._get_output_price(model)
if long_out and base_out:
premium += crossing[4] * max(0.0, long_out - base_out)
cache_aware_savings_usd += premium
# Tool-schema deferral, priced CACHE-AWARE rather than flat at list:
# had the schemas shipped they would have been part of the same prefix
# as the rest of the request — cold-written once at the provider's
+35 -9
View File
@@ -1544,7 +1544,9 @@ class AnthropicHandlerMixin:
# it was looked up by — the cache would never hit and would fill with
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
# (the same reason cache_key_fields is snapshotted here, #327).
cache_lookup_messages = messages
cache_lookup_messages = (
copy.deepcopy(messages) if self.cache and not stream else messages
)
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
@@ -1751,11 +1753,31 @@ class AnthropicHandlerMixin:
# parallel sub-calls commonly share model+system+history while
# carrying different tool sets. Sharing one tracker across those
# requests pins cache reads at the early tools segment (#2671).
from headroom.cache.prefix_tracker import segment_fingerprint
# Deferred tools are not in that key, so a ToolSearch load that
# appends one keeps the conversation's lineage, but only where they
# reach the provider still deferred. A third-party upstream gets
# them eagerly (the tool-search downgrade below uses this same
# predicate) and so does a translating backend; there a load does
# change the provider's key, and the key here counts them.
from headroom.cache.prefix_tracker import cache_key_tools, segment_fingerprint
from headroom.proxy.helpers import anthropic_first_party_tool_search_supported
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
_third_party_anthropic_upstream = provider_name == "anthropic" and (
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
)
affinity_tools = self._tools_for_forwarding(
body.get("tools"), preserve_order=preserve_tool_order
)
from headroom.proxy import runtime_env
if (
not _third_party_anthropic_upstream
and getattr(self, "anthropic_backend", None) is None
and runtime_env.getenv("HEADROOM_DEFERRED_TOOL_AFFINITY", "1").strip().lower()
not in ("0", "false", "no", "off")
):
affinity_tools = cache_key_tools(affinity_tools)
cache_affinity = segment_fingerprint(
{
"model": model,
@@ -2623,9 +2645,14 @@ class AnthropicHandlerMixin:
prefix_tracker.restored_maturation_state = None
maturation_mgr.restore_state(state)
_maturation_mgr = maturation_mgr
# Never below the floor finalize_turn just replayed: the
# pipeline's freeze is clamped to what the local compression
# cache can byte-replay, which stops at the first tool_result
# it never stored, so on its own it lets a cached Read mature
# and re-write the provider cache from that Read onward.
maturation = maturation_mgr.apply(
optimized_messages,
frozen_message_count=frozen_message_count,
frozen_message_count=max(frozen_message_count, _confirmed_frozen),
)
if maturation.replacements_applied or maturation.holding_msg_indices:
optimized_messages = maturation.messages
@@ -2910,7 +2937,10 @@ class AnthropicHandlerMixin:
)
# Traffic Learner: Extract patterns from inbound tool results
if self.traffic_learner:
if self.traffic_learner and not (
self.memory_handler
and self.memory_handler.is_project_unresolved(memory_request_ctx)
):
try:
# Wire backend on first use (lazy init after memory handler is ready)
if (
@@ -3324,14 +3354,10 @@ class AnthropicHandlerMixin:
tokens_saved = max(0, original_tokens - optimized_tokens)
from headroom.proxy.helpers import (
anthropic_first_party_tool_search_supported,
strip_first_party_tool_search_tools_for_third_party_upstream,
)
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
_third_party_anthropic_upstream = provider_name == "anthropic" and (
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
)
# _third_party_anthropic_upstream is decided before the lineage key.
if _third_party_anthropic_upstream:
_tools_before_strip = body.get("tools")
_tools_after_strip = strip_first_party_tool_search_tools_for_third_party_upstream(
+22 -21
View File
@@ -1551,28 +1551,29 @@ class GeminiHandlerMixin:
max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0
)
# Fallback denominator (see comment on the main gemini
# record_request site) — pre-comp request size.
# countTokens is a sizing helper; it never generates output
# tokens and never touches cache. The funnel handles the
# "nothing to report" shape with all-zero cache defaults.
await self._record_request_outcome(
RequestOutcome(
request_id=request_id,
provider=provider_name,
model=model,
status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=compressed_tokens,
output_tokens=0,
tokens_saved=tokens_saved,
attempted_input_tokens=compressed_tokens + tokens_saved,
total_latency_ms=total_latency,
transforms_applied=tuple(transforms_applied),
tags=tags,
client=client,
# countTokens is a sizing helper: the provider generates and bills
# nothing for it, so a successful call is logged below but never
# booked as a request or as savings. An upstream rejection (429,
# 5xx) still goes through the funnel, which records it under the
# rate-limit / failure counters and stops before any success stats.
if response.status_code >= 400:
await self._record_request_outcome(
RequestOutcome(
request_id=request_id,
provider=provider_name,
model=model,
status_code=response.status_code,
original_tokens=original_tokens,
optimized_tokens=compressed_tokens,
output_tokens=0,
tokens_saved=tokens_saved,
attempted_input_tokens=compressed_tokens + tokens_saved,
total_latency_ms=total_latency,
transforms_applied=tuple(transforms_applied),
tags=tags,
client=client,
)
)
)
if tokens_saved > 0:
logger.info(
+90 -53
View File
@@ -1965,6 +1965,7 @@ class OpenAIHandlerMixin:
body: dict[str, Any],
*,
request_id: str,
request_context: Any = None,
) -> None:
"""Feed one Responses HTTP request into the live traffic learner."""
traffic_learner = getattr(self, "traffic_learner", None)
@@ -1972,6 +1973,8 @@ class OpenAIHandlerMixin:
return
try:
memory_handler = getattr(self, "memory_handler", None)
if memory_handler and memory_handler.is_project_unresolved(request_context):
return
if (
traffic_learner._backend is None
and memory_handler
@@ -2001,6 +2004,7 @@ class OpenAIHandlerMixin:
messages: list[dict[str, Any]],
*,
request_id: str,
request_context: Any = None,
) -> None:
"""Feed one chat/completions request into the live traffic learner.
@@ -2017,6 +2021,8 @@ class OpenAIHandlerMixin:
return
try:
memory_handler = getattr(self, "memory_handler", None)
if memory_handler and memory_handler.is_project_unresolved(request_context):
return
if (
traffic_learner._backend is None
and memory_handler
@@ -2044,6 +2050,7 @@ class OpenAIHandlerMixin:
seen_call_ids: set[str],
baseline: bool,
request_id: str,
request_context: Any = None,
) -> None:
"""Feed one Codex WS ``response.create`` turn into the traffic learner.
@@ -2067,6 +2074,8 @@ class OpenAIHandlerMixin:
return
try:
memory_handler = getattr(self, "memory_handler", None)
if memory_handler and memory_handler.is_project_unresolved(request_context):
return
if (
traffic_learner._backend is None
and memory_handler
@@ -3454,8 +3463,6 @@ class OpenAIHandlerMixin:
# compression mutates it, mirroring the Responses and Anthropic
# ingestion paths. Without this, chat/completions traffic (Copilot CLI,
# opencode, OpenAI SDKs) fed nothing to the learner (part of #2060).
await self._observe_openai_chat_traffic(original_client_messages, request_id=request_id)
# Bypass: skip ALL compression for explicit opt-out
_bypass = self._headroom_bypass_enabled(request.headers)
if _bypass:
@@ -3605,6 +3612,12 @@ class OpenAIHandlerMixin:
),
)
await self._observe_openai_chat_traffic(
original_client_messages,
request_id=request_id,
request_context=memory_request_ctx,
)
# Canonical memory-injection gate (parallels Anthropic). Pre-
# PR-this the inline conjunction at the memory site silently
# ignored `x-headroom-bypass: true`, mutating request bytes
@@ -3688,7 +3701,7 @@ class OpenAIHandlerMixin:
# `messages`, but it runs before this snapshot, so its output is already
# captured — keep this snapshot after image compression, or a reorder
# silently reintroduces the drift.
cache_lookup_messages = messages
cache_lookup_messages = copy.deepcopy(messages) if self.cache and not stream else messages
# Response-cache partition: a cached response is only ever replayed to a
# caller presenting the same provider credentials and principal (01-F15).
# Snapshotted with the key fields so lookup and store agree. None means
@@ -4321,6 +4334,24 @@ class OpenAIHandlerMixin:
optimized_messages, original_client_messages
)
from headroom.proxy.helpers import enforce_cache_breakpoint_budget
# Bypass (x-headroom-bypass: true, _bypass computed at ~L3458) opts out
# of ALL compression: never mutate those requests here.
# client_messages=None: passing the client list runs
# normalize_message_cache_control, whose block-level legacy fallback can
# add a marker to a list-content user message on chat requests and then
# cost a real system marker. Without it only the oldest-first drop runs
# — stale replayed markers are oldest, so they go first.
if not _bypass:
_, optimized_messages, tools, _ = enforce_cache_breakpoint_budget(
None,
optimized_messages,
tools,
client_messages=None,
request_id=request_id,
)
# Memory: inject context and tools for OpenAI requests.
#
# PR-A3 follow-up to A2: memory context now routes exclusively to
@@ -6010,12 +6041,6 @@ class OpenAIHandlerMixin:
resolved_project_root_override = codex_project_root_override or (
str(codex_project.cwd) if codex_project and codex_project.cwd else None
)
# The shared learner cannot isolate project state. Scoped Codex turns
# skip learning until a project-scoped learner is available.
if not codex_project_scope_required and (
codex_project is None or codex_project.reason == "metadata_missing"
):
await self._observe_openai_responses_traffic(body, request_id=request_id)
memory_client = (
"codex"
if is_chatgpt_auth
@@ -6090,6 +6115,17 @@ class OpenAIHandlerMixin:
project_root_override=resolved_project_root_override,
)
# The shared learner cannot isolate project state. Scoped Codex turns
# skip learning until a project-scoped learner is available.
if not codex_project_scope_required and (
codex_project is None or codex_project.reason == "metadata_missing"
):
await self._observe_openai_responses_traffic(
body,
request_id=request_id,
request_context=memory_request_ctx,
)
# Rate limiting
if self.rate_limiter:
rate_key = rate_limit_identity(request, headers)
@@ -6998,13 +7034,13 @@ class OpenAIHandlerMixin:
except (json.JSONDecodeError, TypeError):
args = {}
await self.memory_handler._ensure_initialized()
if self.memory_handler._backend:
result = await self.memory_handler._execute_memory_tool(
name, args, memory_user_id, "openai"
)
else:
result = json.dumps({"error": "Memory backend not initialized"})
result = await self.memory_handler._execute_memory_tool(
name,
args,
memory_user_id,
"openai",
request_context=memory_request_ctx,
)
tool_outputs.append(
{
@@ -8131,21 +8167,37 @@ class OpenAIHandlerMixin:
resolved_project.project_key if ws_turn_project_features_allowed else None
)
if first_project_resolution and ws_turn_learning_allowed:
inner_payload = frame_body.get("response", frame_body)
if isinstance(inner_payload, dict):
await self._observe_openai_ws_response_create(
inner_payload,
seen_call_ids=ws_learner_seen_call_ids,
baseline=True,
request_id=request_id,
)
memory_user_id_candidate = (
resolve_memory_identity(websocket)
if self.memory_handler and ws_turn_project_features_allowed
else None
)
ws_response_body = frame_body.get("response", frame_body)
if memory_user_id_candidate is not None and isinstance(ws_response_body, dict):
from headroom.memory.storage_router import (
RequestContext as _MemRequestContext,
)
memory_request_ctx = _MemRequestContext(
headers=dict(ws_headers),
system_prompt=str(ws_response_body.get("instructions") or ""),
base_user_id=memory_user_id_candidate,
project_root_override=resolved_project_root_override,
)
if (
first_project_resolution
and ws_turn_learning_allowed
and isinstance(ws_response_body, dict)
):
await self._observe_openai_ws_response_create(
ws_response_body,
seen_call_ids=ws_learner_seen_call_ids,
baseline=True,
request_id=request_id,
request_context=memory_request_ctx,
)
memory_decision = MemoryDecision.decide(
headers=ws_headers,
memory_handler=(
@@ -8162,24 +8214,8 @@ class OpenAIHandlerMixin:
memory_user_id = memory_user_id_candidate
try:
# Unwrap response.create envelope to access the response body
ws_response_body = frame_body.get("response", frame_body)
if not isinstance(ws_response_body, dict):
return frame_raw
# Per-project memory routing (GH #462). For WS,
# ``ws_response_body`` carries ``instructions`` —
# that's the system-prompt-equivalent we feed to the
# resolver.
from headroom.memory.storage_router import (
RequestContext as _MemRequestContext,
)
memory_request_ctx = _MemRequestContext(
headers=dict(ws_headers),
system_prompt=str(ws_response_body.get("instructions") or ""),
base_user_id=memory_user_id,
project_root_override=resolved_project_root_override,
)
# Debug: log what Codex sends so we can see the full tool list
existing_tool_names = [
@@ -8761,6 +8797,7 @@ class OpenAIHandlerMixin:
seen_call_ids=ws_learner_seen_call_ids,
baseline=False,
request_id=request_id,
request_context=memory_request_ctx,
)
store_forced = _ensure_chatgpt_responses_store_false(
inner_payload,
@@ -9627,19 +9664,19 @@ class OpenAIHandlerMixin:
except (json.JSONDecodeError, TypeError):
fc_args = {}
await self.memory_handler._ensure_initialized()
if not self.memory_handler.is_project_unresolved(
response_memory_request_ctx
):
await self.memory_handler._ensure_initialized()
if response_memory_generation != ws_memory_generation:
break
if self.memory_handler._backend:
result = await self.memory_handler._execute_memory_tool(
fc_name,
fc_args,
response_memory_user_id,
"openai",
request_context=response_memory_request_ctx,
)
else:
result = json.dumps({"error": "backend not ready"})
result = await self.memory_handler._execute_memory_tool(
fc_name,
fc_args,
response_memory_user_id,
"openai",
request_context=response_memory_request_ctx,
)
tool_outputs.append(
{
+16
View File
@@ -458,6 +458,12 @@ def count_cache_breakpoints(
if not isinstance(msg, dict):
continue
found = 1 if "cache_control" in msg else 0
# OpenAI chat shape: markers on an assistant's tool_calls entries.
tool_calls = msg.get("tool_calls")
if isinstance(tool_calls, list):
found += sum(
1 for tc in tool_calls if isinstance(tc, dict) and "cache_control" in tc
)
content = msg.get("content")
if isinstance(content, list):
for block in content:
@@ -847,6 +853,16 @@ def enforce_cache_breakpoint_budget(
budget["left"] -= 1
new_blocks.append(block)
msg = {**msg, "content": new_blocks}
tool_calls = msg.get("tool_calls")
if isinstance(tool_calls, list) and budget["left"] > 0:
new_calls = []
for tc in tool_calls:
if budget["left"] > 0 and isinstance(tc, dict) and "cache_control" in tc:
tc = {k: v for k, v in tc.items() if k != "cache_control"}
budget["left"] -= 1
new_calls.append(tc)
if new_calls != tool_calls:
msg = {**msg, "tool_calls": new_calls}
if budget["left"] > 0 and "cache_control" in msg:
msg = {k: v for k, v in msg.items() if k != "cache_control"}
budget["left"] -= 1
+5
View File
@@ -9,6 +9,7 @@ from dataclasses import dataclass
from typing import Any, Protocol, runtime_checkable
from headroom.cache.compression_cache import (
_can_swap_tool_result_content,
_extract_tool_result_content,
_is_tool_result_message,
_swap_tool_result_content,
@@ -253,6 +254,10 @@ def apply_to_messages(
new_messages.append(msg)
continue
if not _can_swap_tool_result_content(msg):
new_messages.append(msg)
continue
original = _extract_tool_result_content(msg)
if not isinstance(original, str) or not original:
new_messages.append(msg)
+131 -69
View File
@@ -185,12 +185,18 @@ class MemoryHandler:
self.config = config
self.agent_type = agent_type
self._backend: LocalBackend | Any = None
# Per-project routing for the local backend. Built in
# ``_init_backend_locked`` so a single, shared resolver / LRU is
# kept on the handler. Qdrant deployments use composite user-id
# partitioning instead (see ``_compose_effective_user_id``) — the
# router stays None in that case.
self._router: BackendRouter | None = None
storage_root = (
Path(config.storage_root)
if config.storage_root
else (Path(config.db_path).resolve().parent / "memories")
)
self._router = BackendRouter(
BackendRouterConfig(
mode=config.storage_mode,
root_dir=storage_root,
global_db_path=Path(config.db_path).resolve(),
)
)
self._initialized = False
# Async singleflight guard for backend init. Ensures concurrent first
# callers land on one init (double-checked pattern inside
@@ -410,19 +416,8 @@ class MemoryHandler:
# remains the GLOBAL-mode fallback / legacy compatibility
# backend; callers that pass a ``RequestContext`` route
# through ``self._router`` instead.
storage_root = (
Path(self.config.storage_root)
if self.config.storage_root
else (Path(self.config.db_path).resolve().parent / "memories")
)
global_db_path = Path(self.config.db_path).resolve()
router_cfg = BackendRouterConfig(
mode=self.config.storage_mode,
root_dir=storage_root,
global_db_path=global_db_path,
backend_config_template=backend_config,
)
self._router = BackendRouter(router_cfg)
self._router._config.backend_config_template = backend_config
# Seed the router's LRU with the already-initialized
# legacy backend so GLOBAL-mode requests reuse it instead
# of opening a second handle to the same file.
@@ -431,7 +426,7 @@ class MemoryHandler:
logger.info(
"event=memory_router_initialized mode=%s root=%s global_db=%s",
self.config.storage_mode.value,
storage_root,
self._router._config.root_dir,
global_db_path,
)
@@ -670,7 +665,9 @@ class MemoryHandler:
# Non-local backends: derive scope but keep one shared backend
# and compose the user_id so the partition lives in the user_id
# column instead of in a separate file.
scope = self._router._resolve_scope(request_context)
scope = self._router.scope_for(request_context)
if scope.db_path is None:
return None, scope, base_user_id
composed = (
base_user_id
if scope.project_key is None or scope.mode is MemoryStorageMode.GLOBAL
@@ -678,6 +675,18 @@ class MemoryHandler:
)
return self._backend, scope, composed
def is_project_unresolved(self, request_context: RequestContext | None) -> bool:
"""Return whether fail-closed project routing has no usable target."""
if request_context is None or self._router is None:
return False
scope = self._router.scope_for(request_context)
return (
scope.mode is MemoryStorageMode.PROJECT
and scope.project_key is None
and scope.db_path is None
)
@staticmethod
def _unresolved_project_error(scope: ResolvedScope | None) -> str | None:
if (
@@ -771,6 +780,8 @@ class MemoryHandler:
)
return None
if self.is_project_unresolved(request_context):
return None
await self._ensure_initialized()
if not self._backend:
return None
@@ -1105,6 +1116,8 @@ your responses, not to drive new actions."""
"""
tool_calls = self._extract_tool_calls(response, provider)
results: list[dict[str, Any]] = []
project_unresolved = self.is_project_unresolved(request_context)
unresolved_result = json.dumps({"status": "skipped", "reason": "project_unresolved"})
for tc in tool_calls:
# `tc.get("function", {})` returns None for an explicit
@@ -1129,8 +1142,14 @@ your responses, not to drive new actions."""
input_data = {}
# Handle native memory tool
if tool_name == NATIVE_MEMORY_TOOL_NAME:
result_content = await self._execute_native_memory_tool(input_data, user_id)
if tool_name in ({NATIVE_MEMORY_TOOL_NAME} | MEMORY_TOOL_NAMES) and project_unresolved:
result_content = unresolved_result
elif tool_name == NATIVE_MEMORY_TOOL_NAME:
result_content = await self._execute_native_memory_tool(
input_data,
user_id,
request_context=request_context,
)
elif tool_name in MEMORY_TOOL_NAMES:
# Custom memory tools need backend
await self._ensure_initialized()
@@ -1178,7 +1197,12 @@ your responses, not to drive new actions."""
request_context: RequestContext | None = None,
) -> str:
"""Execute a memory tool and return result string."""
if self.is_project_unresolved(request_context):
return json.dumps({"status": "skipped", "reason": "project_unresolved"})
try:
await self._ensure_initialized()
if self._backend is None:
return json.dumps({"status": "error", "error": "Memory backend not initialized"})
if tool_name == "memory_save":
return await self._execute_save(input_data, user_id, provider, request_context)
elif tool_name == "memory_search":
@@ -1537,7 +1561,13 @@ your responses, not to drive new actions."""
# str_replace → Update memory content
# =========================================================================
async def _execute_native_memory_tool(self, input_data: dict[str, Any], user_id: str) -> str:
async def _execute_native_memory_tool(
self,
input_data: dict[str, Any],
user_id: str,
*,
request_context: RequestContext | None = None,
) -> str:
"""Execute Anthropic's native memory tool with semantic backend.
This is a TRANSLATION LAYER: Claude thinks it's doing file operations,
@@ -1551,24 +1581,40 @@ your responses, not to drive new actions."""
- delete: Remove from vector store
- rename: Update memory tags/path
"""
if self.is_project_unresolved(request_context):
return json.dumps({"status": "skipped", "reason": "project_unresolved"})
# Ensure our semantic backend is initialized
await self._ensure_initialized()
command = input_data.get("command", "")
try:
backend, _scope, effective_user_id = self._resolve_for_request(user_id, request_context)
if command == "view":
return await self._native_view_semantic(input_data, user_id)
return await self._native_view_semantic(
input_data, effective_user_id, backend=backend
)
elif command == "create":
return await self._native_create_semantic(input_data, user_id)
return await self._native_create_semantic(
input_data, effective_user_id, backend=backend
)
elif command == "str_replace":
return await self._native_update_semantic(input_data, user_id)
return await self._native_update_semantic(
input_data, effective_user_id, backend=backend
)
elif command == "insert":
return await self._native_append_semantic(input_data, user_id)
return await self._native_append_semantic(
input_data, effective_user_id, backend=backend
)
elif command == "delete":
return await self._native_delete_semantic(input_data, user_id)
return await self._native_delete_semantic(
input_data, effective_user_id, backend=backend
)
elif command == "rename":
return await self._native_rename_semantic(input_data, user_id)
return await self._native_rename_semantic(
input_data, effective_user_id, backend=backend
)
else:
return f"Error: Unknown command '{command}'"
except Exception as e:
@@ -1579,7 +1625,9 @@ your responses, not to drive new actions."""
# Semantic Translation Methods (Native Tool → Vector Store)
# =========================================================================
async def _native_view_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_view_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle VIEW command with semantic search capabilities.
Path patterns:
@@ -1602,31 +1650,35 @@ your responses, not to drive new actions."""
query = subpath[len("search/") :]
if not query:
return "Error: Please provide a search query. Example: view /memories/search/food preferences"
return await self._semantic_search(query, user_id)
return await self._semantic_search(query, user_id, backend=backend)
# CASE 2: /memories/recent → Recent memories
if subpath == "recent":
return await self._get_recent_memories(user_id, limit=10)
return await self._get_recent_memories(user_id, limit=10, backend=backend)
# CASE 3: /memories/all → List all (paginated)
if subpath == "all":
return await self._list_all_memories(user_id, limit=20)
return await self._list_all_memories(user_id, limit=20, backend=backend)
# CASE 4: /memories (root) → Overview with instructions
if not subpath or subpath == "":
return await self._get_memory_overview(user_id)
return await self._get_memory_overview(user_id, backend=backend)
# CASE 5: /memories/<something> → Search by topic
# Treat the path as a search query
return await self._semantic_search(subpath.replace("/", " ").replace("_", " "), user_id)
return await self._semantic_search(
subpath.replace("/", " ").replace("_", " "), user_id, backend=backend
)
async def _semantic_search(self, query: str, user_id: str, top_k: int = 5) -> str:
async def _semantic_search(
self, query: str, user_id: str, top_k: int = 5, *, backend: Any
) -> str:
"""Perform semantic search and format results."""
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
results = await self._backend.search_memories(
results = await backend.search_memories(
query=query,
user_id=user_id,
top_k=top_k,
@@ -1657,15 +1709,15 @@ your responses, not to drive new actions."""
logger.error(f"Memory: Semantic search failed: {e}")
return f"Error searching memories: {e}"
async def _get_recent_memories(self, user_id: str, limit: int = 10) -> str:
async def _get_recent_memories(self, user_id: str, limit: int = 10, *, backend: Any) -> str:
"""Get most recent memories."""
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
# Use a generic query to get recent items
# Most backends will return by recency when query is broad
results = await self._backend.search_memories(
results = await backend.search_memories(
query="recent memories",
user_id=user_id,
top_k=limit,
@@ -1692,14 +1744,14 @@ your responses, not to drive new actions."""
logger.error(f"Memory: Get recent failed: {e}")
return f"Error getting recent memories: {e}"
async def _list_all_memories(self, user_id: str, limit: int = 20) -> str:
async def _list_all_memories(self, user_id: str, limit: int = 20, *, backend: Any) -> str:
"""List all memories (paginated)."""
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
# Get all memories with a broad search
results = await self._backend.search_memories(
results = await backend.search_memories(
query="*", # Broad query
user_id=user_id,
top_k=limit,
@@ -1724,14 +1776,14 @@ your responses, not to drive new actions."""
logger.error(f"Memory: List all failed: {e}")
return f"Error listing memories: {e}"
async def _get_memory_overview(self, user_id: str) -> str:
async def _get_memory_overview(self, user_id: str, *, backend: Any) -> str:
"""Get memory directory overview with search instructions."""
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
# Get count of memories
results = await self._backend.search_memories(
results = await backend.search_memories(
query="*",
user_id=user_id,
top_k=100, # Just to get a count
@@ -1783,7 +1835,9 @@ To see RECENT: view /memories/recent
To SAVE: create /memories/<topic>.txt "content"
"""
async def _native_create_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_create_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle CREATE command - save to semantic vector store."""
path = input_data.get("path", "")
file_text = input_data.get("file_text", "")
@@ -1793,7 +1847,7 @@ To SAVE: create /memories/<topic>.txt "content"
if not file_text:
return "Error: file_text is required (the memory content)"
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
@@ -1806,7 +1860,7 @@ To SAVE: create /memories/<topic>.txt "content"
)
# Save to our semantic backend
memory = await self._backend.save_memory(
memory = await backend.save_memory(
content=file_text,
user_id=user_id,
importance=0.5,
@@ -1820,7 +1874,9 @@ To SAVE: create /memories/<topic>.txt "content"
logger.error(f"Memory: Semantic create failed: {e}")
return f"Error: {e}"
async def _native_update_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_update_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle STR_REPLACE command - update memory content."""
path = input_data.get("path", "")
old_str = input_data.get("old_str", "")
@@ -1831,12 +1887,12 @@ To SAVE: create /memories/<topic>.txt "content"
if not old_str:
return "Error: old_str is required"
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
# Search for memory containing old_str
results = await self._backend.search_memories(
results = await backend.search_memories(
query=old_str,
user_id=user_id,
top_k=5,
@@ -1860,15 +1916,15 @@ To SAVE: create /memories/<topic>.txt "content"
new_content = matching_memory.content.replace(old_str, new_str, 1)
# Update via delete + create (or update if backend supports it)
if hasattr(self._backend, "update_memory"):
await self._backend.update_memory(
if hasattr(backend, "update_memory"):
await backend.update_memory(
memory_id=matching_memory.id,
new_content=new_content,
user_id=user_id,
)
else:
await self._backend.delete_memory(matching_memory.id)
await self._backend.save_memory(
await backend.delete_memory(matching_memory.id)
await backend.save_memory(
content=new_content,
user_id=user_id,
importance=0.5,
@@ -1885,7 +1941,9 @@ To SAVE: create /memories/<topic>.txt "content"
logger.error(f"Memory: Semantic update failed: {e}")
return f"Error: {e}"
async def _native_append_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_append_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle INSERT command - append to memory or create new."""
path = input_data.get("path", "")
insert_text = input_data.get("insert_text", "")
@@ -1896,7 +1954,7 @@ To SAVE: create /memories/<topic>.txt "content"
if not insert_text:
return "Error: insert_text is required"
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
@@ -1904,7 +1962,7 @@ To SAVE: create /memories/<topic>.txt "content"
# with the additional context
topic = path.replace("/memories/", "").replace("/", "_").replace(".txt", "")
await self._backend.save_memory(
await backend.save_memory(
content=insert_text,
user_id=user_id,
importance=0.5,
@@ -1918,14 +1976,16 @@ To SAVE: create /memories/<topic>.txt "content"
logger.error(f"Memory: Semantic append failed: {e}")
return f"Error: {e}"
async def _native_delete_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_delete_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle DELETE command - remove from vector store."""
path = input_data.get("path", "")
if not path:
return "Error: path is required"
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
@@ -1937,7 +1997,7 @@ To SAVE: create /memories/<topic>.txt "content"
.replace(".txt", "")
)
results = await self._backend.search_memories(
results = await backend.search_memories(
query=topic,
user_id=user_id,
top_k=10,
@@ -1952,7 +2012,7 @@ To SAVE: create /memories/<topic>.txt "content"
# Check if metadata matches path
metadata = getattr(r.memory, "metadata", {}) or {}
if metadata.get("virtual_path") == path or r.score > 0.8:
await self._backend.delete_memory(r.memory.id)
await backend.delete_memory(r.memory.id)
deleted_count += 1
if deleted_count == 0:
@@ -1967,7 +2027,9 @@ To SAVE: create /memories/<topic>.txt "content"
logger.error(f"Memory: Semantic delete failed: {e}")
return f"Error: {e}"
async def _native_rename_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
async def _native_rename_semantic(
self, input_data: dict[str, Any], user_id: str, *, backend: Any
) -> str:
"""Handle RENAME command - update memory path/topic."""
old_path = input_data.get("old_path", "")
new_path = input_data.get("new_path", "")
@@ -1977,7 +2039,7 @@ To SAVE: create /memories/<topic>.txt "content"
if not new_path:
return "Error: new_path is required"
if not self._backend:
if not backend:
return "Error: Memory backend not initialized"
try:
@@ -1989,7 +2051,7 @@ To SAVE: create /memories/<topic>.txt "content"
.replace(".txt", "")
)
results = await self._backend.search_memories(
results = await backend.search_memories(
query=old_topic,
user_id=user_id,
top_k=10,
@@ -2006,8 +2068,8 @@ To SAVE: create /memories/<topic>.txt "content"
metadata = getattr(r.memory, "metadata", {}) or {}
if metadata.get("virtual_path") == old_path or r.score > 0.8:
# Delete old and create with new path
await self._backend.delete_memory(r.memory.id)
await self._backend.save_memory(
await backend.delete_memory(r.memory.id)
await backend.save_memory(
content=r.memory.content,
user_id=user_id,
importance=getattr(r.memory, "importance", 0.5),
+1
View File
@@ -932,6 +932,7 @@ class PrometheusMetrics:
compression_tokens_saved=tokens_saved,
tool_schema_tokens_saved=tool_search_saved,
output_tokens_saved=output_tokens_saved,
output_tokens=output_tokens,
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
cache_write_5m_tokens=cache_write_5m_tokens,
+6
View File
@@ -84,6 +84,12 @@ RUNTIME_ENV_KNOBS: tuple[Knob, ...] = (
"int",
"Max on-disk file size (bytes) the disk-verify fallback will read.",
),
Knob(
"HEADROOM_DEFERRED_TOOL_AFFINITY",
"bool",
"Keep defer_loading tools out of the Anthropic prefix-tracker lineage key "
"(default on; 0 counts them again).",
),
Knob(
"HEADROOM_WEB_SEARCH_HISTORY",
"bool",
+29 -6
View File
@@ -543,6 +543,7 @@ def estimate_request_savings_usd(
compression_tokens_saved: int = 0,
tool_schema_tokens_saved: int = 0,
output_tokens_saved: int = 0,
output_tokens: int = 0,
cache_read_tokens: int = 0,
cache_write_tokens: int = 0,
cache_write_5m_tokens: int = 0,
@@ -588,6 +589,8 @@ def estimate_request_savings_usd(
CacheMix,
PricedSavings,
Region,
long_context_premium_avoided_usd,
long_context_threshold,
price_savings,
weakest_basis,
)
@@ -600,26 +603,45 @@ def estimate_request_savings_usd(
uncached_input_tokens=uncached_input_tokens,
cache_inferred=cache_inferred,
)
long_context = mix.is_long_context(local_tokens=local_input_tokens)
threshold = long_context_threshold(model)
long_context = mix.is_long_context(local_tokens=local_input_tokens, threshold=threshold)
# Compression that kept the prompt under the threshold avoided the long
# card on the whole request: the removed tokens are priced at that card,
# and the forwarded input and generated output premiums are added on top.
forwarded = max(mix.billed, max(_coerce_int(local_input_tokens), 0))
crossed = (
not long_context and forwarded + max(_coerce_int(compression_tokens_saved), 0) > threshold
)
def _price(tokens: Any, region: Region) -> PricedSavings:
def _price(tokens: Any, region: Region, *, long_tier: bool) -> PricedSavings:
return price_savings(
max(_coerce_int(tokens), 0),
model=model,
mix=mix,
region=region,
long_context=long_context,
long_context=long_tier,
provider=provider,
# Matches the blended rate the flat estimators fall back to, so an
# unpriceable model reports the same dollars it always has.
fallback_rate_per_token=DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN,
)
compression = _price(compression_tokens_saved, Region.LIVE_ZONE)
tool_schema = _price(tool_schema_tokens_saved, Region.PREFIX)
compression = _price(
compression_tokens_saved, Region.LIVE_ZONE, long_tier=long_context or crossed
)
tool_schema = _price(tool_schema_tokens_saved, Region.PREFIX, long_tier=long_context)
compression_usd = compression.usd
if crossed:
compression_usd += long_context_premium_avoided_usd(
model,
mix,
local_tokens=local_input_tokens,
output_tokens=output_tokens,
provider=provider,
)
return {
"compression": compression.usd,
"compression": compression_usd,
"compression_list": compression.usd_list,
"tool_schema": tool_schema.usd,
"tool_schema_list": tool_schema.usd_list,
@@ -1461,6 +1483,7 @@ class SavingsTracker:
priced = estimate_request_savings_usd(
model,
compression_tokens_saved=_coerce_int(metrics.get("tokens_saved")),
output_tokens=_coerce_int(metrics.get("output_tokens")),
cache_read_tokens=cache_read_tokens,
cache_write_tokens=cache_write_tokens,
cache_write_5m_tokens=_coerce_int(metrics.get("cache_write_5m_tokens")),
+6 -1
View File
@@ -2004,7 +2004,12 @@ class HeadroomProxy(
if raw not in ("1", "true", "yes", "on"):
import platform
libc, version = platform.libc_ver()
try:
libc, version = platform.libc_ver()
except OSError:
# libc_ver() reads sys.executable; Microsoft Store Python's
# python.exe is an app-execution alias that can't be opened.
libc, version = "", ""
if libc == "glibc":
try:
major, minor = (int(part) for part in version.split(".")[:2])
+1 -4
View File
@@ -230,10 +230,7 @@ def compute_release_version(
def get_canonical_version(root: Path) -> str:
"""Read the canonical project version from pyproject.toml."""
try:
import tomllib
except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility
import tomli as tomllib
import tomllib
with open(root / "pyproject.toml", "rb") as file:
project = tomllib.load(file)["project"]
+7 -1
View File
@@ -41,6 +41,12 @@ from headroom.onnx_runtime import _resolve_revision, hf_hub_download_local_first
from .base import RelevanceScore, RelevanceScorer
# Texts per ONNX Runtime run. fastembed pads each batch to its longest text, and
# its default batch of 256 grows ORT's CPU arena (which never shrinks) to fit
# the biggest batch a proxy ever scores: 2.5 GB retained in a long-lived macOS
# proxy. 16 holds ~10x less and runs faster, with less padding per batch.
EMBED_BATCH_SIZE = 16
# numpy is an optional dependency - import lazily
_numpy = None
@@ -335,7 +341,7 @@ class EmbeddingScorer(RelevanceScorer):
"""
np = _get_numpy()
model = self._get_model()
embeddings = list(model.embed(texts))
embeddings = list(model.embed(texts, batch_size=EMBED_BATCH_SIZE))
return np.array(embeddings)
def score(self, item: str, context: str) -> RelevanceScore:
+29 -2
View File
@@ -999,13 +999,32 @@ def save(values: dict[str, Any]) -> None:
_atomic_write_text(paths.settings_path(), payload)
# The env vars Headroom itself inserted at startup, mapped to the value it
# inserted. This is provenance, not value equality: an operator export that
# happens to carry the same text as the settings file is still an operator
# export, and Headroom's own seed stays Headroom's own even after the file
# changes underneath it.
_SEEDED_ENV: dict[str, str] = {}
def seeded_environment() -> dict[str, str]:
"""Return the env vars Headroom inserted at startup, with their values."""
return dict(_SEEDED_ENV)
def apply_to_environ(values: dict[str, Any]) -> None:
"""``setdefault`` each stored value into ``os.environ`` (explicit export wins)."""
for key, value in values.items():
field = _BY_KEY.get(key)
if field is None or value is None:
continue
os.environ.setdefault(field.env, _serialize(field, value))
if field.env in os.environ:
# An explicit export is already there and keeps precedence, so
# Headroom did not insert this one.
continue
serialized = _serialize(field, value)
os.environ[field.env] = serialized
_SEEDED_ENV[field.env] = serialized
def _coerce_runtime_values(runtime_values: dict[str, Any] | None) -> dict[str, Any]:
@@ -1079,6 +1098,13 @@ def to_schema(
effective = {**environment_effective, **runtime}
fields: list[dict[str, Any]] = []
for field in SETTINGS:
# An env var is an operator export only when Headroom did not put it
# there. The startup seed is recorded when it happens, so the answer
# does not depend on how the value compares with the settings file:
# a value saved after the seed is still Headroom's own seed, and an
# operator export that matches the stored text is still an export.
env_raw = os.environ.get(field.env)
seeded_env = _SEEDED_ENV.get(field.env)
fields.append(
{
"key": field.key,
@@ -1094,7 +1120,8 @@ def to_schema(
"minimum": field.minimum,
"maximum": field.maximum,
"tier": field.tier,
"env_override": bool(os.environ.get(field.env))
"env_override": bool(env_raw)
and env_raw != seeded_env
and field.env not in env_override_exclusions,
"runtime_override": field.key in runtime
and runtime[field.key] != environment_effective[field.key],
+91 -27
View File
@@ -68,7 +68,7 @@ import threading
import time
import warnings
from collections.abc import Callable
from dataclasses import dataclass, field
from dataclasses import dataclass, field, fields
from typing import Any, Final
from .models import FieldSemantics, ToolSignature
@@ -409,9 +409,16 @@ class ToolPattern:
"last_updated",
}
filtered = {k: v for k, v in data.items() if k in valid_fields}
# A value of the wrong kind (a string where a list belongs) would be
# accepted as-is and then iterated or added to when patterns merge.
for name, value in filtered.items():
if not isinstance(value, _PATTERN_FIELD_KINDS.get(name, object)):
raise TypeError(f"{name} is a {type(value).__name__}")
# Handle seen_instance_hashes (serialized without underscore prefix)
seen_hashes = data.get("seen_instance_hashes", [])
if not isinstance(seen_hashes, list) or not all(isinstance(h, str) for h in seen_hashes):
raise TypeError("seen_instance_hashes is not a list of strings")
pattern = cls(**filtered)
pattern._seen_instance_hashes = seen_hashes[:100] # Limit on load
@@ -438,6 +445,38 @@ class ToolPattern:
return pattern
# What each stored ToolPattern field may hold, by the base of its annotation.
# Merges store some counts as floats, so numeric fields accept either.
_KIND_BY_ANNOTATION: Final[dict[str, type | tuple[type, ...]]] = {
"str": str,
"int": (int, float),
"float": (int, float),
"bool": bool,
"list": list,
"dict": dict,
}
_PATTERN_FIELD_KINDS: Final[dict[str, type | tuple[type, ...]]] = {
f.name: _KIND_BY_ANNOTATION.get(str(f.type).split("[")[0], object) for f in fields(ToolPattern)
}
_PATTERN_CONTAINER_FIELDS: Final = tuple(
f.name for f in fields(ToolPattern) if f.default_factory in (list, dict, set)
)
def _detached_copy(pattern: ToolPattern) -> ToolPattern:
"""Copy ``pattern`` so that merging into the copy leaves the original intact.
``_merge_patterns`` reassigns scalars and adds to the pattern's lists, dicts
and sets, so those containers are the parts that need their own copies.
"""
import copy
clone = copy.copy(pattern)
for name in _PATTERN_CONTAINER_FIELDS:
setattr(clone, name, getattr(pattern, name).copy())
return clone
@dataclass
class TOINConfig:
"""Configuration for the Tool Output Intelligence Network."""
@@ -1397,40 +1436,65 @@ class ToolIntelligenceNetwork:
patterns_data = data.get("patterns", {})
source_instance = data.get("instance_id", "unknown")
skipped = 0
first_error: Exception | None = None
with self._lock:
for serialized_key, pattern_dict in patterns_data.items():
key = _deserialize_pattern_key(serialized_key)
imported = ToolPattern.from_dict(pattern_dict)
self._sanitize_loaded_pattern(imported)
# Make sure dataclass fields agree with the dict key — pre-F3
# dumps don't carry tenant_key on the pattern, and pre-B5
# dumps don't carry auth_mode/model_family either; promote
# from the (possibly default) key.
imported.tenant_key = key[0]
imported.auth_mode = key[1]
imported.model_family = key[2]
# Each entry is parsed, tracked and merged into a pattern that is
# only stored once all of that succeeds. One unreadable entry must
# not abort the rest (the store that did load would be saved over
# the whole file) or leave a half-merged pattern behind.
try:
key = _deserialize_pattern_key(serialized_key)
imported = ToolPattern.from_dict(pattern_dict)
self._sanitize_loaded_pattern(imported)
# Make sure dataclass fields agree with the dict key — pre-F3
# dumps don't carry tenant_key on the pattern, and pre-B5
# dumps don't carry auth_mode/model_family either; promote
# from the (possibly default) key.
imported.tenant_key = key[0]
imported.auth_mode = key[1]
imported.model_family = key[2]
if key in self._patterns:
# Merge with existing
self._merge_patterns(self._patterns[key], imported)
else:
# Add new pattern - need to track source instance
self._patterns[key] = imported
existing = self._patterns.get(key)
if existing is not None:
# Merge with existing
pattern = _detached_copy(existing)
self._merge_patterns(pattern, imported)
else:
# Add new pattern - need to track source instance
pattern = imported
# For NEW patterns from another instance, track the source in
# _seen_instance_hashes so user_count reflects cross-user data
if source_instance != self._instance_id:
pattern = self._patterns[key]
if source_instance not in pattern._seen_instance_hashes:
# Limit storage to 100 unique instances to bound memory
if len(pattern._seen_instance_hashes) < 100:
pattern._seen_instance_hashes.append(source_instance)
# CRITICAL: Always increment user_count (even after cap)
pattern.user_count += 1
# For NEW patterns from another instance, track the source in
# _seen_instance_hashes so user_count reflects cross-user data
if source_instance != self._instance_id:
if source_instance not in pattern._seen_instance_hashes:
# Limit storage to 100 unique instances to bound memory
if len(pattern._seen_instance_hashes) < 100:
pattern._seen_instance_hashes.append(source_instance)
# CRITICAL: Always increment user_count (even after cap)
pattern.user_count += 1
except (AttributeError, TypeError, ValueError) as e:
skipped += 1
first_error = first_error or e
continue
self._patterns[key] = pattern
self._prune_patterns_locked()
self._dirty = True
if first_error is not None:
logger.warning(
"TOIN skipped malformed patterns on import",
extra={
"event": "toin_import_skipped",
"skipped": skipped,
"imported": len(patterns_data) - skipped,
"error_type": type(first_error).__name__,
"error": str(first_error),
},
)
def _merge_patterns(self, existing: ToolPattern, imported: ToolPattern) -> None:
"""Merge imported pattern into existing."""
total = existing.sample_size + imported.sample_size
+386 -10
View File
@@ -6,7 +6,11 @@ tokenizer backends must implement.
from __future__ import annotations
import base64
import binascii
import json
import re
import zlib
from abc import ABC, abstractmethod
from collections.abc import Callable
from typing import Any, Protocol, runtime_checkable
@@ -25,6 +29,323 @@ _COUNT_CACHE_MIN_CHARS = 256
_COUNT_CACHE_MAX_ENTRIES = 4096
_COUNT_CACHE_MAX_CHARS = 8_000_000
#: Document pricing (see ``BaseTokenizer._estimate_document_tokens``). Anthropic
#: documents ~1,500-3,000 text tokens per PDF page; the low end is used. Pages
#: come from the PDF's active page tree, read the way a PDF reader reads it: the
#: cross-reference data the last ``startxref`` chains to (tables and xref streams,
#: through ``/Prev``) gives ``/Root`` and where each object lives, and the catalog
#: and its ``/Pages`` node are read there (an ``N G obj`` at that offset, or entry
#: i of an object stream), each verified, never wherever their text happens to
#: appear in the file. Comments and string literals are cut from an object's
#: syntax, and each key comes from its own dictionary's top level, never from a
#: dict, array or hex string nested in it. A missing, damaged or unverifiable xref,
#: an absurd ``/Count``, or a spent budget gives ~3KB of file per page instead.
_DOCUMENT_TOKENS_PER_PAGE = 1500
_DOCUMENT_BYTES_PER_PAGE = 3000
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
# One budget per file of bytes scanned or inflated, and one of xref rows read
# (repeats included), so no chain of sections or streams can multiply the work.
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
_PDF_MAX_OBJECTS = 100_000
_PDF_MAX_XREF_SECTIONS = 1_000 # a longer /Prev chain is treated as malformed
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
# A stream's data starts after its keyword, any spaces and an EOL if present (a
# lone CR too): the spec requires CRLF or LF, but readers accept the others.
_PDF_SYNTAX_STOP = re.compile(
rb"(?=[%(se])(?:%[^\r\n]*|\(|(?<![A-Za-z/])(?:stream *(?:\r\n?|\n)?|endobj|startxref))"
)
_PDF_STRING_TOKEN = re.compile(rb"\\.|[()]", re.S)
_PDF_STREAM_LENGTH = re.compile(rb"/Length\s+(\d{1,10})(?!\d|\s+\d+\s+R)")
_PDF_STREAM_END = re.compile(rb"\s*endstream")
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+(\d{1,5})\s+R")
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+(\d{1,5})\s+R")
# A longer digit run never reaches int(): it is malformed, and the size estimate stands in.
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
_PDF_NESTING = re.compile(rb"<<|>>|<[^<>]*>|(\[[\d\s]*\])|\[|\]")
_PDF_LENGTH_REF = re.compile(rb"/Length\s+(\d{1,10})\s+(\d{1,5})\s+R")
_PDF_LENGTH_OBJECT = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\s*(\d{1,10})\s*endobj")
_PDF_OBJECT_HEADER = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\b")
_PDF_STARTXREF = re.compile(rb"startxref\s+(\d{1,10})")
_PDF_XREF_TABLE = re.compile(rb"\s*xref\b")
_PDF_XREF_SUBSECTION = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+")
_PDF_XREF_ROW = re.compile(rb"(\d{10}) (\d{5}) ([nf])")
_PDF_XREF_TRAILER = re.compile(rb"\s*trailer\b")
_PDF_XREF_WIDTHS = re.compile(rb"/W\s*\[\s*(\d)\s+(\d)\s+(\d)\s*\]")
def _pdf_syntax(data: bytes, pos: int = 0, end: int | None = None) -> tuple[bytes, int, bool]:
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref`` (or ``end``),
minus comments and string literals; also where it stopped and whether a stream
starts there."""
end = len(data) if end is None else min(end, len(data))
parts = []
while match := _PDF_SYNTAX_STOP.search(data, pos, end):
parts.append(data[pos : match.start()])
pos = match.end()
if match.group() == b"(":
depth = 0 # a string literal may nest balanced parentheses
pos = end
for token in _PDF_STRING_TOKEN.finditer(data, match.start(), end):
depth += {b"(": 1, b")": -1}.get(token.group(), 0)
if not depth:
pos = token.end()
break
elif not match.group().startswith(b"%"):
return b"".join(parts), pos, match.group().startswith(b"stream")
return b"".join(parts) + data[pos:end], end, False
def _pdf_own_keys(syntax: bytes, nested: list[bytes] | None = None) -> bytes:
"""The top level of the dictionary ``syntax`` opens with, each nested dict, array
and hex string cut to ``_<i>``: a key read from it is that dictionary's own. An
array of plain integers holds no key, so it stays (an xref stream's ``/W`` and
``/Index``). ``nested`` collects what was cut, ``_<i>`` being entry i."""
body = syntax.lstrip(b"\0\t\n\x0c\r ")
if not body.startswith(b"<<"):
return b""
cut: list[bytes] = [] if nested is None else nested
parts: list[bytes] = []
depth = pos = opened = 0
for token in _PDF_NESTING.finditer(body):
step = {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
if depth == 1:
parts.append(body[pos : token.start()])
if step < 0:
break # the dictionary's own end
if token.group(1):
parts.append(token.group(1))
else:
parts.append(b" _%d " % len(cut))
opened = token.start()
if not step:
cut.append(token.group()) # a hex string
depth += step
if step < 0 and depth == 1:
cut.append(body[opened : token.end()])
if not depth:
break
pos = token.end()
return b"".join(parts)
def _pdf_png_rows(data: bytes, columns: int) -> bytes:
"""``data`` decoded from PNG predictor rows: each a filter byte, then ``columns``
bytes, one per sample (an xref stream's /Colors 1 /BitsPerComponent 8)."""
out, prior = bytearray(), bytes(columns)
for at in range(0, len(data), columns + 1):
kind, row = data[at], bytearray(data[at + 1 : at + 1 + columns])
if kind == 2:
row = bytearray((a + b) & 255 for a, b in zip(row, prior))
elif kind:
for i in range(len(row)):
left, up, corner = row[i - 1] if i else 0, prior[i], prior[i - 1] if i else 0
if kind == 1:
row[i] = (row[i] + left) & 255
elif kind == 3:
row[i] = (row[i] + (left + up) // 2) & 255
elif kind == 4:
guess = left + up - corner # Paeth: the neighbour nearest the guess
pa, pb, pc = abs(guess - left), abs(guess - up), abs(guess - corner)
row[i] = (
row[i] + (left if pa <= pb and pa <= pc else up if pb <= pc else corner)
) & 255
else:
raise _PdfUnreadable
out += row
prior = bytes(row)
return bytes(out)
class _PdfUnreadable(Exception):
"""The PDF fails verification or a budget runs out: its size estimate stands in."""
class _PdfReader:
"""Reads objects at the locations the PDF's cross-reference data gives for them."""
def __init__(self, doc: bytes) -> None:
self.doc = doc
self.budget = _PDF_OBJECT_STREAM_MAX_BYTES
self.rows = _PDF_MAX_OBJECTS
self.entries: dict[int, tuple[int, int, int]] = {} # number: (type, field 2, field 3)
self.packed: dict[int, tuple[list[bytes], int, bytes]] = {} # header, /First, data
self.root: re.Match[bytes] | None = None
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
last = doc.rfind(b"startxref", 0, len(doc) if eof < 0 else eof)
start = _PDF_STARTXREF.match(doc, last) if last >= 0 else None
todo = [int(start.group(1))] if start else []
seen: set[int] = set()
while todo:
at = todo.pop(0)
if at in seen:
continue
seen.add(at)
if len(seen) > _PDF_MAX_XREF_SECTIONS:
raise _PdfUnreadable
self.spend(rows=1)
table = _PDF_XREF_TABLE.match(doc, at)
keys = self.read_table(table.end()) if table else self.read_xref_stream(at)
self.root = self.root or _PDF_ROOT.search(keys) # the newest trailer's
for key in (rb"/XRefStm", rb"/Prev"): # a table's stream entries, then older sections
if found := re.search(key + rb"\s+(\d{1,10})(?!\d)", keys):
todo.append(int(found.group(1)))
if not self.root:
raise _PdfUnreadable
def spend(self, scanned: int = 0, rows: int = 0) -> None:
self.budget -= scanned
self.rows -= rows
if self.budget < 0 or self.rows < 0:
raise _PdfUnreadable
def keys_at(self, pos: int, nested: list[bytes] | None = None) -> tuple[bytes, int, bool]:
"""Own keys of the dictionary at ``pos``, where its syntax stops, and whether a
stream starts there; the scan is charged to the budget."""
syntax, stop, is_stream = _pdf_syntax(self.doc, pos, pos + self.budget + 1)
self.spend(stop - pos) # a scan past the budget stops one byte beyond it
return _pdf_own_keys(syntax, nested), stop, is_stream
def read_table(self, pos: int) -> bytes:
while sub := _PDF_XREF_SUBSECTION.match(self.doc, pos):
self.spend(rows=int(sub.group(2)) + 1) # also caps the distinct counts compiled
rows = re.compile(rb"(?:\d{10} \d{5} [nf]\s{1,2}){%s}" % sub.group(2))
if not (block := rows.match(self.doc, sub.end())):
raise _PdfUnreadable
for number, (offset, gen, kind) in enumerate(
_PDF_XREF_ROW.findall(block.group()), int(sub.group(1))
):
if kind == b"n": # the newest section is read first, so its entry wins
self.entries.setdefault(number, (1, int(offset), int(gen)))
pos = block.end()
if not (trailer := _PDF_XREF_TRAILER.match(self.doc, pos)):
raise _PdfUnreadable
return self.keys_at(trailer.end())[0]
def read_xref_stream(self, at: int) -> bytes:
header = _PDF_OBJECT_HEADER.match(self.doc, at)
nested: list[bytes] = []
keys, data = self.stream(header.end(), nested) if header else (b"", b"")
widths = _PDF_XREF_WIDTHS.search(keys)
size = re.search(rb"/Size\s+(\d{1,10})", keys)
if not (widths and size and re.search(rb"/Type\s*/XRef\b", keys)):
raise _PdfUnreadable
kind, field, gen = map(int, widths.groups())
width = kind + field + gen
predicted = 0
if parms := re.search(rb"/DecodeParms\s+_(\d{1,4})(?!\d)", keys):
raw = nested[int(parms.group(1))] if int(parms.group(1)) < len(nested) else b""
if not raw.startswith(b"<<"):
raise _PdfUnreadable # an array, or "_<i>" typed into the dict itself
values = _pdf_own_keys(raw)
predictor = re.search(rb"/Predictor\s+(\d{1,2})(?!\d)", values)
columns = re.search(rb"/Columns\s+(\d{1,2})(?!\d)", values)
if re.search(rb"/Colors\s+(?!1(?!\d))|/BitsPerComponent\s+(?!8(?!\d))", values):
raise _PdfUnreadable
if predictor and int(predictor.group(1)) >= 10:
predicted = 1
if int(columns.group(1) if columns else 1) != width:
raise _PdfUnreadable
elif predictor and int(predictor.group(1)) != 1:
raise _PdfUnreadable # TIFF prediction: not written for xref streams
if not width:
raise _PdfUnreadable
index = re.search(rb"/Index\s*\[([\d\s]*)\]", keys)
bounds = (index.group(1) if index else b"0 " + size.group(1)).split()
if any(len(n) > 10 for n in bounds):
raise _PdfUnreadable
starts, lengths = [int(n) for n in bounds[::2]], [int(n) for n in bounds[1::2]]
count = min(sum(lengths), len(data) // (width + predicted))
self.spend(rows=count)
data = data[: count * (width + predicted)]
if predicted:
data = _pdf_png_rows(data, width)
numbers = (n for a, length in zip(starts, lengths) for n in range(a, a + length))
for number, at_row in zip(numbers, range(0, count * width, width)):
entry = data[at_row : at_row + width]
entry_type = int.from_bytes(entry[:kind] or b"\x01", "big")
if entry_type in (1, 2):
second = int.from_bytes(entry[kind : kind + field], "big")
self.entries.setdefault(
number, (entry_type, second, int.from_bytes(entry[kind + field :], "big"))
)
return keys
def stream(self, pos: int, nested: list[bytes]) -> tuple[bytes, bytes]:
"""Own keys and inflated data of the stream object whose dict starts at ``pos``.
Its end must be verified: ``endstream`` right after ``/Length`` bytes."""
keys, start, is_stream = self.keys_at(pos, nested)
length = _PDF_STREAM_LENGTH.search(keys)
size = int(length.group(1)) if length else self.integer(_PDF_LENGTH_REF.search(keys))
if not (is_stream and _PDF_STREAM_END.match(self.doc, start + size)):
raise _PdfUnreadable
data = self.doc[start : start + size]
if b"/Filter" not in keys:
if b"/DecodeParms" in keys:
raise _PdfUnreadable
return keys, data
if not re.search(rb"/Filter\s*/FlateDecode\b", keys) or self.budget <= 0:
raise _PdfUnreadable # (max_length 0 would mean no limit)
inflate = zlib.decompressobj()
try:
data = inflate.decompress(data, self.budget)
except zlib.error:
raise _PdfUnreadable from None
if inflate.unconsumed_tail:
raise _PdfUnreadable
self.spend(len(data))
return keys, data
def integer(self, ref: re.Match[bytes] | None) -> int:
"""The integer an indirect ``/Length`` names, at its xref offset."""
entry = self.entries.get(int(ref.group(1))) if ref else None
value = _PDF_LENGTH_OBJECT.match(self.doc, entry[1]) if entry and entry[0] == 1 else None
if not (ref and entry and value and value.group(1, 2) == ref.group(1, 2)):
raise _PdfUnreadable
if int(ref.group(2)) != entry[2]: # the generation the xref holds
raise _PdfUnreadable
return int(value.group(3))
def get(self, ref: re.Match[bytes] | None) -> bytes:
"""Own keys of the object ``ref`` names, read where the xref says it lives."""
number, gen = (int(ref.group(1)), int(ref.group(2))) if ref else (-1, -1)
kind, where, extra = self.entries.get(number, (0, 0, 0))
if kind == 1 and extra == gen:
header = _PDF_OBJECT_HEADER.match(self.doc, where)
if header and (int(header.group(1)), int(header.group(2))) == (number, gen):
return self.keys_at(header.end())[0]
elif kind == 2 and gen == 0:
head, first, data = self.packed.get(where) or self.unpack(where)
i = 2 * extra # entry i of the stream's header of (number, offset) pairs
if i + 1 < len(head) and int(head[i]) == number:
stop = first + int(head[i + 3]) if i + 3 < len(head) else len(data)
return _pdf_own_keys(_pdf_syntax(data[first + int(head[i + 1]) : stop])[0])
raise _PdfUnreadable
def unpack(self, number: int) -> tuple[list[bytes], int, bytes]:
"""An object stream's header numbers, its /First and its inflated data."""
kind, where, gen = self.entries.get(number, (0, 0, 0))
header = _PDF_OBJECT_HEADER.match(self.doc, where) if kind == 1 else None
if not (header and (int(header.group(1)), int(header.group(2))) == (number, gen)):
raise _PdfUnreadable
keys, data = self.stream(header.end(), [])
first = _PDF_FIRST.search(keys)
if not (first and re.search(rb"/Type\s*/ObjStm\b", keys)) or b"/DecodeParms" in keys:
raise _PdfUnreadable
head = re.findall(rb"\d{1,10}", data[: int(first.group(1))])
self.packed[number] = head, int(first.group(1)), data
return self.packed[number]
def _pdf_page_count(doc: bytes) -> int:
"""Pages in the PDF's active page tree, else 0."""
try:
reader = _PdfReader(doc)
count = _PDF_COUNT.search(reader.get(_PDF_PAGES.search(reader.get(reader.root))))
except _PdfUnreadable:
return 0
return int(count.group(1)) if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES else 0
class TokenCountCache:
"""Exact memo for ``count_text``.
@@ -78,12 +399,15 @@ class TokenCountCache:
must not run the eviction check (a dict hit would clear the whole
cache and force a full re-encode of every cached message on the
next miss — the storm this exists to prevent).
Oversized texts are never admitted, so warm entries survive.
"""
if text in self._counts:
self._counts[text] = count
return
if len(text) < self._min_chars:
return
if len(text) > self._max_chars:
return
if len(self._counts) >= self._max_entries or self._chars >= self._max_chars:
self._counts.clear()
self._chars = 0
@@ -211,16 +535,22 @@ class BaseTokenizer(ABC):
# Count role
role = message.get("role", "")
total += self.count_text(role)
total += self.count_text(coerce_countable_text(role))
# Count content
content = message.get("content")
if isinstance(content, dict):
# A single block sent without its list wrapper is priced like
# the wrapped block, so images keep their fixed estimate (#4273).
content = [content]
if content is not None:
if isinstance(content, str):
total += self.count_text(content)
elif isinstance(content, list):
# Multi-part content (images, tool results, etc.)
total += self._count_content_parts(content)
else:
total += self.count_text(coerce_countable_text(content))
# Count tool calls
tool_calls = message.get("tool_calls")
@@ -235,7 +565,7 @@ class BaseTokenizer(ABC):
# Count name field
name = message.get("name")
if name:
total += self.count_text(name)
total += self.count_text(coerce_countable_text(name))
total += 1 # Name field overhead
# Reply start overhead
@@ -255,7 +585,7 @@ class BaseTokenizer(ABC):
part_type = part.get("type", "")
if part_type == "text":
total += self.count_text(part.get("text", ""))
total += self.count_text(coerce_countable_text(part.get("text", "")))
elif part_type in ("image_url", "image", "input_image"):
# Images are NOT tokenized as text — they have a pixel-based cost.
# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.
@@ -264,6 +594,12 @@ class BaseTokenizer(ABC):
# This prevents the base64 blob from being json.dumps'd and counted
# as text tokens (1MB image = ~330K fake tokens without this).
total += 1600
elif part_type == "document":
# Anthropic document block. Its base64 PDF used to reach the
# JSON catch-all below and be priced as prose: a 1 MB PDF
# counted ~907K tokens, against the ~1,600 an image of the
# same bytes gets.
total += self._count_document_block(part)
elif part_type in ("input_audio", "audio"):
# Audio has fixed token cost, not tokenized as text
total += 200
@@ -281,15 +617,15 @@ class BaseTokenizer(ABC):
else:
total += self._count_serialized(content)
elif part_type == "tool_use":
total += self.count_text(part.get("name", ""))
total += self.count_text(coerce_countable_text(part.get("name", "")))
total += self._count_serialized(part.get("input", {}))
elif not part_type and "text" in part:
# Strands SDK format: {"text": "..."} without "type" field
total += self.count_text(part["text"])
total += self.count_text(coerce_countable_text(part["text"]))
elif not part_type and "toolUse" in part:
# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}
tool_use = part["toolUse"]
total += self.count_text(tool_use.get("name", ""))
total += self.count_text(coerce_countable_text(tool_use.get("name", "")))
total += self._count_serialized(tool_use.get("input", {}))
elif not part_type and "toolResult" in part:
# Strands SDK tool_result: {"toolResult": {"content": [...]}}
@@ -308,7 +644,9 @@ class BaseTokenizer(ABC):
reasoning = part["reasoningContent"]
reasoning_text = reasoning.get("reasoningText", {})
if isinstance(reasoning_text, dict):
total += self.count_text(reasoning_text.get("text", ""))
total += self.count_text(
coerce_countable_text(reasoning_text.get("text", ""))
)
elif isinstance(reasoning_text, str):
total += self.count_text(reasoning_text)
elif not part_type and "document" in part:
@@ -321,8 +659,7 @@ class BaseTokenizer(ABC):
source = doc.get("source", {})
doc_bytes = source.get("bytes", b"")
if isinstance(doc_bytes, bytes | bytearray):
estimated_pages = max(1, len(doc_bytes) // 3000)
total += estimated_pages * 1500
total += self._estimate_document_tokens(bytes(doc_bytes))
else:
total += self.count_text(str(doc_bytes))
elif not part_type and "image" in part:
@@ -358,7 +695,7 @@ class BaseTokenizer(ABC):
# ~4 bytes/token (len * 3/4 / 4), which lands that session
# near the provider figure and never prices an omitted
# block at zero.
total += self.count_text(part.get("thinking", "") or "")
total += self.count_text(coerce_countable_text(part.get("thinking", "") or ""))
total += len(part.get("signature") or "") * 3 // 16
else:
# Unknown type - estimate from JSON
@@ -393,6 +730,45 @@ class BaseTokenizer(ABC):
)
return int(self.count_text(sample) * len(s) / len(sample))
def _count_document_block(self, part: dict[str, Any]) -> int:
"""Price an Anthropic ``document`` block by its source type."""
source = part.get("source")
source = source if isinstance(source, dict) else {}
source_type = source.get("type")
if source_type == "base64":
try:
doc_bytes = base64.b64decode(source.get("data") or b"")
except (binascii.Error, TypeError, ValueError):
doc_bytes = b""
total = self._estimate_document_tokens(doc_bytes)
elif source_type == "content":
content = source.get("content")
if isinstance(content, list):
total = self._count_content_parts(content)
else:
total = self.count_text(str(content or ""))
elif source_type in ("url", "file"):
# The bytes live with the provider; one page as a nominal estimate.
total = _DOCUMENT_TOKENS_PER_PAGE
else:
# A plain-text source, or an untyped one carrying text.
total = self.count_text(str(source.get("data") or ""))
for key in ("title", "context"):
value = part.get(key)
if isinstance(value, str):
total += self.count_text(value)
return total
@staticmethod
def _estimate_document_tokens(doc_bytes: bytes) -> int:
"""Estimate a document file's tokens from its page count (see the constants)."""
if b"%PDF-" not in doc_bytes[:1024]:
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
# document) keep the size estimate even when their text quotes PDF syntax.
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
pages = _pdf_page_count(doc_bytes) or len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
return max(1, pages) * _DOCUMENT_TOKENS_PER_PAGE
@staticmethod
def _estimate_image_tokens(image_data: dict[str, Any]) -> int:
"""Estimate tokens for an image using Anthropic's formula: (w*h)/750.
+8 -1
View File
@@ -297,6 +297,9 @@ class TiktokenCounter(BaseTokenizer):
continue
if key == "content":
if isinstance(value, dict):
# Unwrapped single block: price it like the list form (#4273).
value = [value]
if isinstance(value, str):
total += self.count_text(value)
elif isinstance(value, list):
@@ -304,7 +307,9 @@ class TiktokenCounter(BaseTokenizer):
for part in value:
if isinstance(part, dict):
if part.get("type") == "text":
total += self.count_text(part.get("text", ""))
total += self.count_text(
coerce_countable_text(part.get("text", ""))
)
elif part.get("type") == "image_url":
# Image tokens vary by detail level
detail = part.get("image_url", {}).get("detail", "auto")
@@ -324,6 +329,8 @@ class TiktokenCounter(BaseTokenizer):
total += self._count_content_parts([part])
elif isinstance(part, str):
total += self.count_text(part)
else:
total += self.count_text(coerce_countable_text(value))
elif key == "role":
total += self.count_text(coerce_countable_text(value))
elif key == "name":
+8 -2
View File
@@ -383,9 +383,12 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
if not eq or not name.startswith("@"):
return None
try:
legends[name[1:]] = json.loads(payload)
result = json.loads(payload)
except (ValueError, TypeError):
return None
if not isinstance(result, list):
return None
legends[name[1:]] = result
out: list[dict[str, Any]] = []
for cells in _split_data_rows(rest, declared):
@@ -404,7 +407,10 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
_assign(record, col.path, None)
continue
try:
_assign(record, col.path, legends[col.name][int(value)])
idx_val = int(value)
if idx_val < 0:
return None
_assign(record, col.path, legends[col.name][idx_val])
except (ValueError, IndexError):
return None
continue
+16 -4
View File
@@ -16,7 +16,9 @@ from .compression_units import (
RoutedCompressionUnit,
TokenCounterLike,
UnitCompressionResult,
_categorize_reason,
_is_structured_shell_output,
check_unit_protection,
)
from .content_router import RouterCompressionResult
from .tag_protector import protect_tags, restore_tags
@@ -104,7 +106,12 @@ def build_compression_batches(
for entry in entries:
entry_bytes = _text_bytes(entry.routed.unit.text)
entry_key = _compatibility_key(entry)
if entry_bytes >= min_batch_bytes or entry_bytes > max_batch_bytes:
protection_reason = check_unit_protection(entry.routed.unit)
if (
protection_reason is not None
or entry_bytes >= min_batch_bytes
or entry_bytes > max_batch_bytes
):
flush()
skipped.append(entry)
continue
@@ -197,7 +204,7 @@ def _passthrough_batch_results(
batch: CompressionBatch,
*,
tokenizer: TokenCounterLike,
reason: str,
reason: str | None = None,
router_result: RouterCompressionResult | None = None,
) -> list[tuple[object, UnitCompressionResult]]:
strategy = (
@@ -217,11 +224,13 @@ def _passthrough_batch_results(
tokens_saved=0,
transforms_applied=[],
strategy=strategy,
reason=reason,
reason=reason or check_unit_protection(entry.routed.unit) or "batch_invalid",
router_result=router_result,
text_bytes=_text_bytes(entry.routed.unit.text),
min_bytes=entry.routed.unit.min_bytes,
reason_category=reason,
reason_category=_categorize_reason(
reason or check_unit_protection(entry.routed.unit) or "batch_invalid"
),
),
)
for entry in batch.entries
@@ -241,6 +250,9 @@ def compress_batch_with_router(
``deadline_started_at`` works as in ``compress_unit_with_router``.
"""
if any(check_unit_protection(entry.routed.unit) is not None for entry in batch.entries):
return _passthrough_batch_results(batch, tokenizer=tokenizer)
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")
+18 -10
View File
@@ -218,6 +218,21 @@ def _compress_marker_free_text(
)
def check_unit_protection(unit: CompressionUnit) -> str | None:
"""Return the early refusal reason if a unit is protected, else None."""
if not unit.mutable:
return "immutable"
if unit.role == "user" and unit.metadata.get("compress_user") != "true":
return "protected_user_message"
if unit.role in {"system", "developer"}:
return "protected_system_message"
if unit.role == "assistant" and unit.metadata.get("compress_assistant") != "true":
return "protected_assistant_message"
if unit.cache_zone != "live":
return f"cache_zone_{unit.cache_zone}"
return None
def compress_unit_with_router(
unit: CompressionUnit,
*,
@@ -263,16 +278,9 @@ def compress_unit_with_router(
kw["reason_category"] = _categorize_reason(reason_val)
return replace(base, **kw) # type: ignore[arg-type]
if not unit.mutable:
return _with_reason(reason="immutable")
if unit.role == "user" and unit.metadata.get("compress_user") != "true":
return _with_reason(reason="protected_user_message")
if unit.role in {"system", "developer"}:
return _with_reason(reason="protected_system_message")
if unit.role == "assistant" and unit.metadata.get("compress_assistant") != "true":
return _with_reason(reason="protected_assistant_message")
if unit.cache_zone != "live":
return _with_reason(reason=f"cache_zone_{unit.cache_zone}")
protection_reason = check_unit_protection(unit)
if protection_reason is not None:
return _with_reason(reason=protection_reason)
if text_bytes < unit.min_bytes:
return _with_reason(reason="below_unit_floor")
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
+6 -1
View File
@@ -53,7 +53,12 @@ _COMMENT_RES = {
# Content where whole-line elision is unsafe: a '#' line inside a YAML block
# scalar or a TOML multi-line string is data, not a comment. Detection is
# deliberately over-broad — when in doubt, Tier 2 stays off.
_YAML_BLOCK_SCALAR_RE = re.compile(r":\s*[|>][+-]?\d*\s*$", re.MULTILINE)
# Block scalars can be sequence items as well as mapping values; indentation
# and chomping indicators may appear in either order, and a header can carry
# a trailing comment. Conservatively skip elision for every such header.
_YAML_BLOCK_SCALAR_RE = re.compile(
r"(?:^|[ \t:])[|>][1-9+-]*[ \t]*(?:#[^\r\n]*)?\r?$", re.MULTILINE
)
_TOML_MULTILINE_RE = re.compile(r'"""|\'\'\'')
+42 -1
View File
@@ -770,11 +770,52 @@ def _try_detect_tabular(content: str) -> DetectionResult | None:
md_result = _try_detect_markdown_table(lines)
if md_result:
return md_result
# The tabular compressor replaces the whole payload with the table's
# rows, so a table with a heading, prose or other output around it is
# not tabular content: every line outside the table would be dropped.
if _is_whole_markdown_table(content):
return md_result
# It can still be CSV holding markdown in a quoted field. Its pipes
# belong to that markdown, so they are never the delimiter here.
delimited = _try_detect_delimited(lines)
if delimited and delimited.metadata.get("delimiter") != "|":
return delimited
return None
return _try_detect_delimited(lines)
def _is_whole_markdown_table(content: str) -> bool:
"""True when ``content`` is one markdown table and nothing else.
A blank line ends a markdown table, so every line after the header and
separator has to be a row: written in the header's leading-pipe style,
and neither another separator nor a list item. A note or a second table
straight after the rows means the content is not just the table.
Every row must also split into exactly the header's cells, the way the
tabular parser splits it, or the records built from it would truncate or
pad the row. The parser does not unescape ``\\|``, so a table that uses it
is left alone too.
"""
if "\\|" in content:
return False
rows = content.strip().split("\n")
if len(rows) < 3 or not _is_md_separator(rows[1]):
return False
cols = _md_cell_count(rows[0])
if cols < 2:
return False
leading_pipe = rows[0].lstrip().startswith("|")
return all(
_md_cell_count(row) == cols
and row.lstrip().startswith("|") == leading_pipe
and not _is_md_separator(row)
and not _FIXED_WIDTH_LIST_RE.match(row)
for row in rows[2:]
)
# Fixed-width (space-aligned) command output: `ls -l`, `ps aux`, `df -h`,
# `docker ps`, `kubectl get`. These rows have no delimiter, so without this
# check they fall through to PLAIN_TEXT and the prose compressor drops fields
+190 -30
View File
@@ -360,9 +360,18 @@ def _invoke_smart_crusher(router: ContentRouter, inp: CompressInput) -> str | No
crusher = router._get_smart_crusher()
if crusher is None:
return None
# F2.2: the router never calls ``crusher.apply()``, so the crusher's own
# policy field stays None. Forward the request's TOIN read-only gate per
# call instead (the crusher is shared across concurrent requests). The
# kwarg is only sent when gating, so crushers exposing the plain
# ``crush(content, query, bias)`` signature keep working.
policy = router._runtime_compression_policy
gate: dict[str, bool] = {}
if policy is not None and policy.toin_read_only:
gate["record_toin"] = False
# ``_get_*`` getters are typed ``Any``; pin the result to the contract type.
compressed: str = crusher.crush(
inp.content, query=inp.query, bias=_adapter_bias(inp)
inp.content, query=inp.query, bias=_adapter_bias(inp), **gate
).compressed
return compressed
@@ -596,6 +605,24 @@ def _external_compressor_matches(descriptor: CompressorDescriptor, content_mime:
return any(d in ("*", "*/*") or d == type_wildcard for d in declared)
def _malformed_tool_call_ids(messages: list[dict[str, Any]]) -> set[str]:
"""Ids of OpenAI tool calls whose ``function`` is not an object.
Such a call has no readable tool name, so its result cannot be classified;
it is passed through like a verbatim-excluded tool rather than compressed.
"""
ids: set[str] = set()
for msg in messages:
if msg.get("role") != "assistant":
continue
for tc in msg.get("tool_calls") or []:
if isinstance(tc, dict) and not isinstance(tc.get("function"), dict):
tc_id = tc.get("id")
if isinstance(tc_id, str) and tc_id:
ids.add(tc_id)
return ids
def _tool_call_args_text(raw: Any) -> str:
"""Compact, query-usable text from a tool call's args.
@@ -2113,6 +2140,30 @@ def _is_small_mcp_result(tool_name: Any, text: Any) -> bool:
)
def _shell_result_min_chars() -> int:
"""Below this many characters a shell tool result is never lossily compressed."""
try:
return max(0, int(os.environ.get("HEADROOM_SHELL_RESULT_MIN_CHARS", "2000")))
except ValueError:
return 2000
def _is_small_shell_result(tool_name: Any, text: Any, bash_tool_names: frozenset[str]) -> bool:
"""True when ``text`` is a short shell result that must not be lossily compressed.
A 200-token ``ls -la``, ``git status``, or JSON dump is cheap to keep and
expensive to get wrong: lossy compression can leave a numbers-only blob
that still looks structured, so the model treats it as data instead of
retrying the command (issue #3590). Large build/test logs still take the
lossy path. ``HEADROOM_SHELL_RESULT_MIN_CHARS=0`` restores the old
behaviour.
"""
if not isinstance(text, str) or not text:
return False
name = str(tool_name or "").lower()
return name in bash_tool_names and len(text) < _shell_result_min_chars()
@dataclass
class ContentRouterConfig:
"""Configuration for intelligent content routing.
@@ -2317,8 +2368,22 @@ class ContentRouterConfig:
# BUT a read-only *search* run through them (grep/rg/git grep) yields byte-
# losslessly foldable output — folded instead of lossy-compressed. See
# ``_bash_search_fold``. Config so new harness tool names / search programs
# can be added without code changes.
bash_tool_names: frozenset[str] = frozenset({"bash", "shell", "local_shell"})
# can be added without code changes. The aliases mirror
# ``headroom.learn._shared._TOOL_NAME_MAP`` so the same harnesses are
# covered here.
bash_tool_names: frozenset[str] = frozenset(
{
"bash",
"shell",
"local_shell",
"run_shell_command",
"execute_command",
"exec_command",
"terminal",
"run_command",
"run_terminal_command",
}
)
bash_search_commands: frozenset[str] = frozenset(
{"grep", "egrep", "fgrep", "rg", "ripgrep", "ag", "ack"}
)
@@ -2349,6 +2414,12 @@ class ContentRouterConfig:
# Default False; the proxy enables it in token mode.
search_group_by_file: bool = False
def __post_init__(self) -> None:
# Normalize once so the per-message shell check stays a set lookup.
self.bash_tool_names = frozenset(
name.lower() for name in self.bash_tool_names if isinstance(name, str)
)
@dataclass
class _PerRequestRuntimeState:
@@ -3716,6 +3787,31 @@ class ContentRouter(Transform):
return cand, f"lossless_{kind}"
@staticmethod
def _validate_provider_output(supplied: Any) -> tuple[str, str] | None:
"""Contract check shared by the general and excluded-tool provider paths.
Returns ``(candidate, safe_kind)`` or ``None`` ("ignore"). Never raises.
"""
if supplied is None:
return None
if not isinstance(supplied, tuple | list) or len(supplied) != 2:
logger.debug("lossless provider returned a malformed result; ignoring")
return None
if not isinstance(supplied[0], str) or not isinstance(supplied[1], str):
logger.debug("lossless provider returned non-str members; ignoring")
return None
# An empty / whitespace-only "fold" wins every length comparison and
# would silently delete the block's content. That is not lossless.
if not supplied[0].strip():
logger.debug("lossless provider returned an empty result; ignoring")
return None
# fullmatch, not match: Python's `$` also matches just BEFORE a
# trailing newline, so `re.match` would let "log\n" through and
# put a newline into a metric label.
kind = supplied[1] if _PROVIDER_KIND_RE.fullmatch(supplied[1]) else _PROVIDER_KIND_FALLBACK
return (supplied[0], kind)
def _lossless_provider_result(self, content: str) -> tuple[str, str] | None:
"""Validated + memoized ``(candidate, safe_kind)`` from the provider.
@@ -3746,29 +3842,7 @@ class ContentRouter(Transform):
result: tuple[str, str] | None = None
try:
supplied = provider(content)
if supplied is None:
result = None
# Validate defensively *inside* the try: unpacking a 3-tuple / a bare
# string / a non-sequence raises, and that exception used to escape
# all the way out of the request.
elif not isinstance(supplied, tuple | list) or len(supplied) != 2:
logger.debug("lossless provider returned a malformed result; ignoring")
elif not isinstance(supplied[0], str) or not isinstance(supplied[1], str):
logger.debug("lossless provider returned non-str members; ignoring")
# An empty / whitespace-only "fold" wins every length comparison and
# would silently delete the block's content. That is not lossless.
elif not supplied[0].strip():
logger.debug("lossless provider returned an empty result; ignoring")
else:
raw_kind = supplied[1]
# fullmatch, not match: Python's `$` also matches just BEFORE a
# trailing newline, so `re.match` would let "log\n" through and
# put a newline into a metric label.
kind = (
raw_kind if _PROVIDER_KIND_RE.fullmatch(raw_kind) else _PROVIDER_KIND_FALLBACK
)
result = (supplied[0], kind)
result = self._validate_provider_output(provider(content))
except Exception: # noqa: BLE001 - a broken provider must not break routing
logger.debug("lossless provider failed in _lossless_first", exc_info=True)
result = None
@@ -5788,7 +5862,9 @@ class ContentRouter(Transform):
for tc in msg.get("tool_calls") or []:
if isinstance(tc, dict):
tc_id = tc.get("id", "")
fn = tc.get("function", {})
fn = tc.get("function")
if not isinstance(fn, dict): # "function": null / non-object
continue
name = fn.get("name", "")
call_args = fn.get("arguments")
if name:
@@ -6140,6 +6216,9 @@ class ContentRouter(Transform):
for tool_id, name in tool_name_map.items()
if is_tool_excluded(name, exclude_tools)
}
# A call with no readable name keeps its result untouched, as it was
# before such calls stopped raising out of apply().
excluded_tool_ids |= _malformed_tool_call_ids(messages)
# CCR-retrieve tool IDs, precomputed once (mirrors excluded_tool_ids
# above, rather than calling is_tool_excluded() per message/block). A
@@ -6311,6 +6390,7 @@ class ContentRouter(Transform):
"ccr_retrieve": 0,
"user_msg": 0,
"small": 0,
"shell_small_result": 0,
"recent_code": 0,
"analysis_ctx": 0,
"ratio_too_high": 0,
@@ -6534,7 +6614,7 @@ class ContentRouter(Transform):
if role == "tool":
if tool_call_id in excluded_tool_ids:
tool_name = tool_name_map.get(tool_call_id, "")
if tool_name and is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
if not tool_name or is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
result_slots[i] = message
transforms_applied.append("router:excluded:tool")
route_counts["excluded_tool"] += 1
@@ -6600,6 +6680,36 @@ class ContentRouter(Transform):
_diag[i] = "compressed:bash_lossless_search"
continue
# Small shell results stay verbatim (#3590). A short `ls`,
# `git status`, or JSON dump is cheap to keep; lossy
# compression can leave a numbers-only blob that still looks
# structured, so the model treats it as data instead of
# retrying. Large build/test logs still take the lossy path.
if _is_small_shell_result(tool_name, content, self.config.bash_tool_names):
result_slots[i] = message
transforms_applied.append("router:shell_small_result_verbatim")
route_counts["shell_small_result"] = (
route_counts.get("shell_small_result", 0) + 1
)
if collect_diagnostics:
_diag[i] = "protected:shell_small_result"
continue
# Legacy OpenAI `role:"function"` carries the tool name on the
# message itself, not in a tool_call_id. Resolve it directly so the
# small-shell floor also covers this wire shape (#3590 review).
if role == "function":
function_name = message.get("name", "")
if _is_small_shell_result(function_name, content, self.config.bash_tool_names):
result_slots[i] = message
transforms_applied.append("router:shell_small_result_verbatim")
route_counts["shell_small_result"] = (
route_counts.get("shell_small_result", 0) + 1
)
if collect_diagnostics:
_diag[i] = "protected:shell_small_result"
continue
# Read protection (ROLE / SHAPE-AGNOSTIC). An observation produced by
# a file read command (cat/sed/head/…) is passed VERBATIM so the agent
# keeps exact bytes to patch — regardless of how THIS harness labels
@@ -7239,7 +7349,7 @@ class ContentRouter(Transform):
try:
# A registered provider is authoritative for excluded-tool
# compaction; fall back to the built-in folds only if it raises.
return provider(content)
return self._validate_provider_output(provider(content))
except Exception: # noqa: BLE001
logger.debug(
"lossless provider failed; using built-in compaction",
@@ -7569,6 +7679,21 @@ class ContentRouter(Transform):
any_compressed = False
role = message.get("role", "")
# Top-level text blocks are fragments of one tool result. Measure the
# whole result so splitting a large log cannot activate the small-result
# floor for every fragment. Sum lengths without copying the payload.
shell_result_chars = (
sum(
len(block["text"])
for block in content_blocks
if isinstance(block, dict)
and block.get("type") == "text"
and isinstance(block.get("text"), str)
)
if role in ("tool", "function")
else 0
)
# Role-based gate for `text` blocks. Tool/function roles are tool
# outputs and compress freely; assistant defaults to skip (cache
# safety) with explicit opt-in; unknown roles default to skip.
@@ -7663,7 +7788,7 @@ class ContentRouter(Transform):
continue
if tool_use_id in excluded_tool_ids:
tool_name = tool_name_map.get(tool_use_id, "") if tool_name_map else ""
if tool_name and is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
if not tool_name or is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
new_blocks.append(block)
transforms_applied.append("router:excluded:tool")
if route_counts is not None:
@@ -7767,6 +7892,18 @@ class ContentRouter(Transform):
any_compressed = True
continue
# Small shell results stay verbatim (#3590), same contract as
# the string-form path above. Build/test logs above the floor
# still go through lossy compression.
if _is_small_shell_result(tool_name, tool_text, self.config.bash_tool_names):
new_blocks.append(block)
transforms_applied.append("router:shell_small_result_verbatim")
if route_counts is not None:
route_counts["shell_small_result"] = (
route_counts.get("shell_small_result", 0) + 1
)
continue
# Protection: failed tool calls / error outputs stay verbatim
# (issue #847). `is_error` is Anthropic's explicit failure
# flag and suffices alone; the indicator scan catches error
@@ -7864,6 +8001,29 @@ class ContentRouter(Transform):
if route_counts is not None:
route_counts["ccr_retrieve"] = route_counts.get("ccr_retrieve", 0) + 1
continue
# A normalized tool/function message can carry its output
# as top-level text blocks rather than a tool_result
# wrapper. Apply the same small-shell floor here (#3590
# review).
_shell_name = (
(tool_name_map or {}).get(_msg_tool_id, "")
if role == "tool"
else message.get("name", "")
)
if (
_is_small_shell_result(
_shell_name, block.get("text", ""), self.config.bash_tool_names
)
and shell_result_chars < _shell_result_min_chars()
):
new_blocks.append(block)
transforms_applied.append("router:shell_small_result_verbatim")
if route_counts is not None:
route_counts["shell_small_result"] = (
route_counts.get("shell_small_result", 0) + 1
)
continue
text_content = block.get("text", "")
if isinstance(text_content, str) and (
len(text_content) > min_chars or self._has_lossless_fold(text_content)
+15 -8
View File
@@ -39,9 +39,9 @@ from ..log_safety import describe_exception, safe_id
logger = logging.getLogger(__name__)
# A run must be at least this many lines AND this many chars to be worth a
# pointer. Small dups are left alone (fragmenting context is not worth it) —
# and a larger floor keeps the pointer comfortably shorter than the span it
# replaces, so a fold is always a net byte win.
# pointer. Small dups are left alone (fragmenting context is not worth it).
# The floor alone does not make a fold a net byte win (the pointer can still be
# as long as a short span); dedup_blocks checks the UTF-8 sizes before folding.
DEFAULT_MIN_LINES = 3
DEFAULT_MIN_CHARS = 40
# Cap anchor candidates examined per line so a hot line (e.g. `` return``)
@@ -54,7 +54,7 @@ MAX_ANCHOR_CANDIDATES = 16
# line must start with the digit, so ``number + delta`` re-numbers byte-exactly
# without touching alignment padding, keeping renumbered folds strictly lossless.
#
# ``[1-9]\d*`` (not ``\d+``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
# The leading ``[1-9]`` (not ``\d``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
# a timestamped log, ``007:...``) is not an unpadded grep/sed line number, and
# recovery is ``str(int(number) + delta)``, which drops the zero pad — ``int("08")
# + 1`` renders ``"9"``, not ``"09"``. Matching those would fold them under a
@@ -63,7 +63,11 @@ MAX_ANCHOR_CANDIDATES = 16
# fold only on an EXACT match (delta 0) — the "prefer false negatives" posture.
# Real grep -n / sed -n / rg -n numbers never carry a leading zero, so the
# intended renumber-fold feature is unaffected.
_LINENO_RE = re.compile(r"^([1-9]\d*)(:|\t)(.*)$", re.DOTALL)
#
# ``\d{0,17}`` bounds the run at 18 digits: Python refuses ``int()`` on a run of
# 4,301+ digits (ValueError), which would abort dedup for the whole request. A
# longer run is not a real line number, so the line stays non-numbered.
_LINENO_RE = re.compile(r"^([1-9]\d{0,17})(:|\t)(.*)$", re.DOTALL)
def _num_and_key(line: str) -> tuple[int | None, str, str]:
@@ -264,9 +268,12 @@ def dedup_blocks(
if m is not None and m[0] >= min_lines:
span = lines[i : i + m[0]]
span_text = "\n".join(span)
if len(span_text) >= min_chars:
ref_turn = blocks[m[1]].turn
ptr = _pointer(span, ref_turn, m[3])
ptr = _pointer(span, blocks[m[1]].turn, m[3])
# Never fold when the pointer is not shorter than the span, in
# UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).
if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(
span_text.encode("utf-8")
):
out.append(ptr)
# Folded span is NOT verbatim in this block's output:
# mark None so it can't seed a later contiguous match,
+7
View File
@@ -7,6 +7,7 @@ import re
from dataclasses import dataclass
from .content_detector import ContentType, _is_search_result_line
from .lossless_compaction import _TIMESTAMP_ROW_RE
@dataclass
@@ -45,7 +46,13 @@ def _is_search_line(line: str) -> bool:
Sharing it keeps the splitter carving the same lines the detector claims
(#3580); otherwise code in context lines would strand in PLAIN_TEXT
sections and reach the prose compressor.
Timestamped log rows (``2026-09-23T10:05:07Z ...``) match the legacy
pattern but are never grep output; the detector rejects them (#3736), so
the splitter must too or it hands them to the lossy search compressor.
"""
if _TIMESTAMP_ROW_RE.match(line):
return False
return bool(_SEARCH_RESULT_PATTERN.match(line)) or _is_search_result_line(line)
+51 -4
View File
@@ -96,6 +96,16 @@ class ReadLifecycleResult:
ccr_hashes: list[str] = field(default_factory=list)
def _rejected_before_running(content: Any) -> bool:
"""True when a tool_result's text is a ``<tool_use_error>`` validation error."""
if isinstance(content, list):
content = next(
(b.get("text") for b in content if isinstance(b, dict) and b.get("type") == "text"),
None,
)
return isinstance(content, str) and content.lstrip().startswith("<tool_use_error>")
class ReadLifecycleManager:
"""Event-driven Read lifecycle management.
@@ -181,7 +191,9 @@ class ReadLifecycleManager:
if not isinstance(tc, dict):
continue
tc_id = tc.get("id", "")
func = tc.get("function", {})
func = tc.get("function")
if not isinstance(func, dict): # "function": null / non-object
continue
name = func.get("name", "")
if not tc_id or not name:
continue
@@ -191,11 +203,15 @@ class ReadLifecycleManager:
limit = None
try:
args = json.loads(func.get("arguments", "{}"))
except (json.JSONDecodeError, TypeError):
args = None
if isinstance(args, dict): # arguments may decode to a list/str/number
file_path = args.get("file_path") or args.get("path")
offset = args.get("offset")
limit = args.get("limit")
except (json.JSONDecodeError, TypeError):
pass
# file_path is a dict key downstream; an unhashable value must not reach it.
if not isinstance(file_path, str):
file_path = None
metadata[tc_id] = (name, file_path, offset, limit)
# Anthropic format: content blocks with type=tool_use
@@ -218,6 +234,8 @@ class ReadLifecycleManager:
file_path = inp.get("file_path") or inp.get("path")
offset = inp.get("offset")
limit = inp.get("limit")
if not isinstance(file_path, str):
file_path = None
metadata[tc_id] = (name, file_path, offset, limit)
return metadata
@@ -232,6 +250,7 @@ class ReadLifecycleManager:
Groups all Read/Edit/Write operations by file_path for lifecycle analysis.
"""
file_ops: dict[str, list[FileOperation]] = defaultdict(list)
failed_ids = self._failed_tool_use_ids(messages)
for tc_id, (name, file_path, offset, limit) in tool_metadata.items():
if not file_path:
@@ -239,7 +258,8 @@ class ReadLifecycleManager:
if name in _READ_TOOL_NAMES:
operation = "read"
elif name in _MUTATING_TOOL_NAMES:
elif name in _MUTATING_TOOL_NAMES and tc_id not in failed_ids:
# A failed Edit/Write left the file unchanged, so earlier Reads stay valid.
operation = "edit"
else:
continue
@@ -263,6 +283,33 @@ class ReadLifecycleManager:
return dict(file_ops)
@staticmethod
def _failed_tool_use_ids(messages: list[dict[str, Any]]) -> set[str]:
"""Ids of Anthropic tool_use blocks that were rejected before they ran.
Only an is_error tool_result whose text is a ``<tool_use_error>`` (the
harness's pre-execution validation, e.g. "String to replace not found")
proves the file was not touched. Any other error, such as a call that was
interrupted and "may have partially or fully completed", still counts as
an edit.
"""
failed: set[str] = set()
for msg in messages:
content = msg.get("content")
if not isinstance(content, list):
continue
for block in content:
if not (
isinstance(block, dict)
and block.get("type") == "tool_result"
and block.get("is_error")
):
continue
tool_use_id = block.get("tool_use_id")
if isinstance(tool_use_id, str) and _rejected_before_running(block.get("content")):
failed.add(tool_use_id)
return failed
def _find_tool_call_msg_index(
self, messages: list[dict[str, Any]], tool_call_id: str
) -> int | None:
+14 -4
View File
@@ -284,15 +284,21 @@ class ReadMaturationManager:
for tc in msg.get("tool_calls", []) or []:
if not isinstance(tc, dict):
continue
func = tc.get("function", {})
func = tc.get("function")
if not isinstance(func, dict): # "function": null / non-object
continue
name = func.get("name", "")
if name not in _TOUCH_TOOLS:
continue
try:
args = json.loads(func.get("arguments", "{}"))
except (ValueError, TypeError):
args = {}
except (ValueError, TypeError): # not JSON: unknown file
continue
if not isinstance(args, dict): # list/str/number: unknown file
continue
fp = args.get("file_path") or args.get("path") or ""
if not isinstance(fp, str): # unknown file: never mature this call
continue
if fp:
act.file_last_touch[fp] = turn
if name in _READ_TOOLS:
@@ -306,8 +312,12 @@ class ReadMaturationManager:
name = b.get("name", "")
if name not in _TOUCH_TOOLS:
continue
inp = b.get("input") or {}
inp = b.get("input")
if not isinstance(inp, dict): # unknown file: never mature this call
continue
fp = inp.get("file_path") or inp.get("path") or ""
if not isinstance(fp, str): # unknown file: never mature this call
continue
if fp:
act.file_last_touch[fp] = turn
if name in _READ_TOOLS:
+8 -1
View File
@@ -497,6 +497,7 @@ class SmartCrusher(Transform):
query: str = "",
bias: float = 1.0,
lossless_only: bool | None = None,
record_toin: bool = True,
) -> CrushResult:
"""Crush a single JSON content string.
@@ -510,6 +511,12 @@ class SmartCrusher(Transform):
but any path that would need a CCR marker (row-drop or
opaque-blob offload) leaves the content uncompacted instead.
`None` (default) uses the instance's configured value.
`record_toin=False` skips the TOIN write for this call only. The
ContentRouter passes it when the request's ``CompressionPolicy``
has ``toin_read_only=True`` (Subscription, F2.2): the crusher is
shared across concurrent requests, so the gate has to travel with
the call rather than through ``_runtime_compression_policy``.
"""
# Web search tools often return space-separated JSON objects
# (``{...} {...} {...}``) rather than a real array. The Rust crusher
@@ -548,7 +555,7 @@ class SmartCrusher(Transform):
# JSON re-canonicalization (whitespace normalization) without
# actually compressing — the strategy stays `"passthrough"` in
# that case, and there's no learning value in recording it.
if r.was_modified and r.strategy != "passthrough":
if record_toin and r.was_modified and r.strategy != "passthrough":
self._record_to_toin(
original=content,
compressed=r.compressed,
+3 -3
View File
@@ -64,10 +64,10 @@ The canonical, always-current documentation index lives at the docs site below.
## Headroom for Teams
For teams and companies that want Headroom across every developer rather than one laptop: Headroom Enterprise adds more savings (tool search for every model, harness tuning, extra compressors), automatic model routing (open source has opt-in, rule-based routing), prompt-injection and risky-tool-call removal, SSO, managed or VPC deployment, an org-wide savings dashboard, and support. Companies that sign a 12-month Enterprise plan get the first 3 months free.
Headroom OSS is a framework built to show what's possible on one developer's laptop, so it saves less than Headroom Enterprise. Enterprise adds more cost savings, tokens secured, on premise deployment, deep attribution and many more features. Teams and enterprises: contact hello@headroomlabs.ai or visit https://headroomlabs.ai.
- [Headroom for Teams](https://docs.headroomlabs.ai/docs/teams): Open source vs Enterprise, and the 3-months-free offer.
- [Claim 3 months free](https://headroom-perks.vercel.app/?source=github&utm_source=llms-txt): Perk sign-up form. One perk per company.
- [Headroom for Teams](https://docs.headroomlabs.ai/docs/teams): Open source vs Enterprise, and how to reach us.
- [headroomlabs.ai](https://headroomlabs.ai): Company site. Email hello@headroomlabs.ai.
## Licensing
+2 -1
View File
@@ -207,7 +207,8 @@ Compression is lossless via CCR (Compress-Cache-Retrieve): originals are stored
| `pythonPath` | auto-detected | Optional Python executable override for Python fallback launcher. |
| `autoStart` | `false` | Opt-in auto-start for a local `headroom proxy` if not already running (local URLs only; ignored for remote proxies). Keep `false` when systemd owns the proxy. |
| `startupTimeoutMs` | `20000` | Time to wait for auto-started proxy to become healthy |
| `requestTimeoutMs` | `30000` | Maximum milliseconds to wait for a single `compress()` call. If the proxy hangs or is slow, the call is cancelled after this deadline and the original uncompressed messages are used as a fallback. |
| `requestTimeoutMs` | `2000` | Maximum milliseconds to wait for a single `compress()` call. The SDK aborts the request at this deadline and the original messages are used as a fallback. |
| `minContextChars` | `800` | Skip proxy compression below this approximate 200-token context floor. Large single tool results still qualify; set to `0` to disable the gate. |
| `circuitBreakerThreshold` | `3` | Number of consecutive `assemble()` errors before the circuit breaker opens and all requests bypass the proxy. Prevents cascading failures when the proxy is unhealthy. |
| `circuitBreakerCooldownMs` | `60000` | How long (ms) the circuit breaker stays open after the threshold is reached. After the cool-down the breaker resets automatically and the next request re-probes the proxy via `/health`. |
| `routeCodexViaProxy` | `true` | Rewrite OpenClaw's built-in `openai-codex` provider to use the active Headroom proxy in memory so upstream Codex requests pass through Headroom. |
+10 -1
View File
@@ -26,6 +26,10 @@
"label": "Compression Request Timeout",
"help": "Maximum milliseconds to wait for one compression request before returning the original messages."
},
"minContextChars": {
"label": "Minimum Context Size",
"help": "Skip the proxy for smaller contexts that cannot produce meaningful savings. Set to 0 to disable."
},
"circuitBreakerThreshold": {
"label": "Circuit Breaker Threshold",
"help": "Consecutive compression failures before temporarily bypassing Headroom."
@@ -88,7 +92,12 @@
"requestTimeoutMs": {
"type": "integer",
"minimum": 1,
"default": 30000
"default": 2000
},
"minContextChars": {
"type": "integer",
"minimum": 0,
"default": 800
},
"circuitBreakerThreshold": {
"type": "integer",
+30 -20
View File
@@ -17,17 +17,6 @@ import {
type OpenClawCompactResult,
} from "./openclaw-compaction.js";
/** Race a promise against a timeout and always release the timer. */
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
let timerId: ReturnType<typeof setTimeout> | undefined;
const timer = new Promise<never>((_, reject) => {
timerId = setTimeout(() => reject(new Error(`headroom compress() timed out after ${ms}ms`)), ms);
});
return Promise.race([promise, timer]).finally(() => {
if (timerId !== undefined) clearTimeout(timerId);
});
}
// Must stay static — OpenClaw prepends it to the cached system prompt.
const HEADROOM_COMPRESSION_NOTICE =
"[Headroom is compressing tool outputs in this session. Use headroom_retrieve if you need the original, uncompressed content.]";
@@ -35,6 +24,7 @@ const HEADROOM_COMPRESSION_NOTICE =
export interface HeadroomEngineConfig extends ProxyManagerConfig {
enabled?: boolean;
requestTimeoutMs?: number;
minContextChars?: number;
circuitBreakerThreshold?: number;
circuitBreakerCooldownMs?: number;
/** Where to durably record committed turn-advancement keys (see
@@ -160,17 +150,26 @@ export class HeadroomContextEngine {
try {
// Convert AgentMessage → OpenAI format
const openaiMessages = agentToOpenAIIndexed(params.messages);
const minContextChars = this.config.minContextChars ?? 800;
if (minContextChars > 0 && countContextChars(openaiMessages) < minContextChars) {
this.logger.debug("[headroom] Context below compression threshold — using original messages");
return {
messages: normalizeAgentMessages(params.messages),
estimatedTokens: 0,
systemPromptAddition: this.sessionCompressionNotice(params.sessionId),
};
}
// Compress via proxy — pass tokenBudget so RollingWindow enforces it
const result = await withTimeout(
compress(openaiMessages, {
model: params.model ?? "claude-sonnet-4-5",
baseUrl: this.proxyUrl,
fallback: true,
tokenBudget: params.tokenBudget,
} as any),
this.config.requestTimeoutMs ?? 30_000,
);
const result = await compress(openaiMessages, {
model: params.model ?? "claude-sonnet-4-5",
baseUrl: this.proxyUrl,
fallback: true,
tokenBudget: params.tokenBudget,
// The SDK applies this to AbortSignal.timeout(), so a timed-out call is
// cancelled instead of continuing in the background after fail-open.
timeout: this.config.requestTimeoutMs ?? 2_000,
} as any);
const shouldAnnounceCompression =
(this.hasAnnouncedCompression(params.sessionId) || result.tokensSaved > 100) &&
@@ -400,3 +399,14 @@ export class HeadroomContextEngine {
}
}
}
function countContextChars(messages: ReturnType<typeof agentToOpenAIIndexed>): number {
let total = 0;
for (const message of messages) {
total += message.content?.length ?? 0;
for (const toolCall of message.tool_calls ?? []) {
total += toolCall.function?.arguments?.length ?? 0;
}
}
return total;
}
+81 -28
View File
@@ -277,40 +277,77 @@ describe("HeadroomContextEngine proxy startup helpers", () => {
expect(mocked.start).toHaveBeenCalledTimes(1);
});
it("clears the request timeout after successful compression", async () => {
vi.useFakeTimers();
try {
vi.mocked(compress).mockResolvedValue({
compressed: false,
messages: [{ role: "user", content: "hello" }],
tokensBefore: 5,
tokensAfter: 5,
tokensSaved: 0,
});
it("passes the configured request timeout to the aborting SDK client", async () => {
vi.mocked(compress).mockResolvedValue({
compressed: false,
messages: [{ role: "user", content: "hello" }],
tokensBefore: 5,
tokensAfter: 5,
tokensSaved: 0,
});
const engine = new HeadroomContextEngine({ requestTimeoutMs: 30_000 });
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
const engine = new HeadroomContextEngine({ requestTimeoutMs: 1_234, minContextChars: 0 });
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
await expect(
engine.assemble({
sessionId: "session-1",
messages: [{ role: "user", content: "hello" }],
}),
).resolves.toEqual({
messages: [{ role: "user", content: "hello" }],
estimatedTokens: 5,
});
await engine.assemble({
sessionId: "session-1",
messages: [{ role: "user", content: "hello" }],
});
expect(vi.getTimerCount()).toBe(0);
} finally {
vi.useRealTimers();
}
expect(compress).toHaveBeenCalledWith(
expect.any(Array),
expect.objectContaining({ timeout: 1_234 }),
);
});
it("skips proxy compression for short contexts with no meaningful savings", async () => {
const messages = [{ role: "user", content: "hello" }];
const engine = new HeadroomContextEngine();
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
await expect(engine.assemble({ sessionId: "session-1", messages })).resolves.toEqual({
messages,
estimatedTokens: 0,
});
expect(compress).not.toHaveBeenCalled();
});
it("counts a single large tool result as eligible context", async () => {
const messages = [{ role: "toolResult", toolCallId: "call-1", content: "x".repeat(800) }];
vi.mocked(compress).mockResolvedValue({
compressed: false,
messages: [{ role: "tool", tool_call_id: "call-1", content: "x".repeat(800) }],
tokensBefore: 200,
tokensAfter: 200,
tokensSaved: 0,
});
const engine = new HeadroomContextEngine();
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
await engine.assemble({ sessionId: "session-1", messages });
expect(compress).toHaveBeenCalledTimes(1);
});
it("allows the short-context gate to be disabled", async () => {
vi.mocked(compress).mockResolvedValue({
compressed: false,
messages: [{ role: "user", content: "hello" }],
tokensBefore: 2,
tokensAfter: 2,
tokensSaved: 0,
});
const engine = new HeadroomContextEngine({ minContextChars: 0 });
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
await engine.assemble({ sessionId: "session-1", messages: [{ role: "user", content: "hello" }] });
expect(compress).toHaveBeenCalledTimes(1);
});
it("opens the circuit after consecutive compression failures", async () => {
vi.mocked(compress).mockRejectedValue(new Error("proxy stalled"));
const messages = [{ role: "user", content: "hello" }];
const engine = new HeadroomContextEngine({
minContextChars: 0,
circuitBreakerThreshold: 2,
circuitBreakerCooldownMs: 60_000,
});
@@ -451,7 +488,7 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
}
function readyEngine(config?: HeadroomEngineConfig) {
const engine = new HeadroomContextEngine(config);
const engine = new HeadroomContextEngine({ minContextChars: 0, ...config });
(engine as unknown as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
return engine;
}
@@ -491,6 +528,18 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
expect(second.systemPromptAddition).toBe(HEADROOM_COMPRESSION_NOTICE);
});
it("keeps an earned notice on a later context below the compression threshold", async () => {
vi.mocked(compress).mockResolvedValueOnce(mockCompressResult({ tokensSaved: 150 }));
const engine = readyEngine({ minContextChars: 800 });
const first = await engine.assemble({
sessionId: "s1", messages: [{ role: "user", content: "x".repeat(1000) }],
});
const second = await engine.assemble({ sessionId: "s1", messages });
expect(first.systemPromptAddition).toBe(HEADROOM_COMPRESSION_NOTICE);
expect(second.systemPromptAddition).toBe(first.systemPromptAddition);
expect(compress).toHaveBeenCalledTimes(1);
});
it("never returns a notice when announceCompression is false", async () => {
vi.mocked(compress).mockResolvedValueOnce(mockCompressResult({ tokensSaved: 500 }));
@@ -571,7 +620,9 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
const firstNotice = turn1.systemPromptAddition;
expect(firstNotice).toBe(HEADROOM_COMPRESSION_NOTICE);
vi.mocked(compress).mockImplementationOnce(() => new Promise(() => {}));
vi.mocked(compress).mockImplementationOnce(() => new Promise((_, reject) => {
setTimeout(() => reject(new Error("SDK request timed out")), requestTimeoutMs);
}));
const turn2Promise = engine.assemble({ sessionId: "s1", messages });
await vi.advanceTimersByTimeAsync(requestTimeoutMs);
const turn2 = await turn2Promise;
@@ -619,7 +670,9 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
const turn1 = await engine.assemble({ sessionId: "s1", messages });
expect(turn1.systemPromptAddition).toBeUndefined();
vi.mocked(compress).mockImplementationOnce(() => new Promise(() => {}));
vi.mocked(compress).mockImplementationOnce(() => new Promise((_, reject) => {
setTimeout(() => reject(new Error("SDK request timed out")), requestTimeoutMs);
}));
const turn2Promise = engine.assemble({ sessionId: "s1", messages });
await vi.advanceTimersByTimeAsync(requestTimeoutMs);
const turn2 = await turn2Promise;
+2 -2
View File
@@ -47,8 +47,8 @@
},
"overrides": {
"esbuild": "^0.28.1",
"zod": "^4.6.6",
"toml": "^5.0.1"
"zod": "4.6.7",
"toml": "5.0.1"
},
"license": "Apache-2.0"
}
@@ -0,0 +1,62 @@
import { tool } from "@opencode-ai/plugin";
import { Effect, FileSystem, Path } from "effect";
import { Primitive } from "effect/unstable/cli";
import { describe, expect, it } from "vitest";
function parseConfig(content: string): Promise<unknown> {
const fs = FileSystem.makeNoop({
exists: () => Effect.succeed(true),
stat: () => Effect.succeed({ type: "File" } as FileSystem.File.Info),
readFileString: () => Effect.succeed(content),
});
return Effect.runPromise(
Primitive.fileParse({ format: "toml" }).parse("/headroom/config.toml").pipe(
Effect.provideService(FileSystem.FileSystem, fs),
Effect.provide(Path.layer),
),
);
}
describe("patched OpenCode dependency compatibility", () => {
it("builds and validates a tool with the upstream Zod schema interface", () => {
const schema = tool.schema.object({
hash: tool.schema.string().regex(/^[a-f0-9]{24}$/i),
});
const retrieve = tool({
description: "Retrieve an original tool result",
args: schema.shape,
async execute(args) {
return args.hash;
},
});
expect(schema.parse({ hash: "a".repeat(24) })).toEqual({ hash: "a".repeat(24) });
expect(schema.safeParse({ hash: "invalid" }).success).toBe(false);
expect(retrieve.args.hash).toBe(schema.shape.hash);
});
it("parses typed TOML through Effect's actual file parser", async () => {
const content = `
name = "headroom"
enabled = true
ports = [8787, 8788]
[proxy]
host = "127.0.0.1"
ratio = 0.5
[[models]]
name = "coding"
[[models]]
name = "review"
`;
await expect(parseConfig(content)).resolves.toEqual({
name: "headroom",
enabled: true,
ports: [8787, 8788],
proxy: { host: "127.0.0.1", ratio: 0.5 },
models: [{ name: "coding" }, { name: "review" }],
});
});
it("keeps malformed TOML on Effect's parse-error channel", async () => {
await expect(parseConfig("[unclosed")).rejects.toThrow("Failed to parse '.toml'");
});
});
+32 -19
View File
@@ -8,7 +8,7 @@ version = "0.42.0"
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
readme = "README.md"
license = "Apache-2.0"
requires-python = ">=3.10"
requires-python = ">=3.11"
authors = [
{ name = "Headroom Contributors" }
]
@@ -35,7 +35,6 @@ classifiers = [
"Intended Audience :: Developers",
"Operating System :: OS Independent",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
@@ -50,12 +49,12 @@ dependencies = [
"pydantic>=2.0.0", # Config and data models
# Model registry, pricing and non-core providers; all lazily imported and
# ImportError-guarded. 1.96.2 is the first litellm with prebuilt wheels for
# Python 3.10-3.14 on Linux, macOS and Windows: older releases either reject
# Python 3.11-3.14 on Linux, macOS and Windows: older releases either reject
# Python 3.14 (GH #956) or fall back to a Rust source build.
"litellm>=1.96.2,<2.0",
"click>=8.3.3", # CLI framework; PYSEC-2026-2132 fix (command injection in click.edit())
"rich>=13.0.0", # Rich terminal output
"opentelemetry-api>=1.24.0", # Safe no-op OTEL API for instrumentation
"opentelemetry-api>=1.45.1", # CVE-2026-114603: bounded, linear-time tracestate parsing
# AST-aware code slicing (CodeCompressor); binary wheel. 0.44.1 is excluded:
# that PyPI release was a compromised supply-chain build shipping an
# info-stealer `sg.exe` (Trojan:Win64/Lazy!MTB) alongside the real binary
@@ -63,8 +62,7 @@ dependencies = [
# future patched one.
"ast-grep-cli>=0.30.0,!=0.44.1",
"pyyaml>=6.0", # omp wrap: parse/merge omp's models.yml registry
"tomli>=2.0.0; python_version < '3.11'", # tomllib backport for helper scripts
"tomlkit>=0.13.0,<1.0", # Loss-minimizing Codex config recovery
"tomlkit>=0.15.0,<1.0", # Loss-minimizing Codex config recovery
# Verify upstream TLS through the OS trust store (macOS Keychain, Windows
# cert store), where IT installs corporate TLS-inspection roots (Zscaler,
# Netskope). Pure Python, no deps; the same library pip uses by default.
@@ -82,9 +80,8 @@ dependencies = [
# CVE-2026-112907/112913/112940 are fixed in 4.2.1, but litellm 1.106 caps
# filelock <4, so without this floor pip picks that litellm and 3.32.7;
# with it pip takes litellm 1.103.7 (uv.lock has 1.101.0). Declared here so
# the floor reaches pip installs. 4.2 needs Python 3.11, so pip on 3.10
# still gets an affected filelock.
"filelock>=4.2.1; python_version >= '3.11'",
# the floor reaches pip installs.
"filelock>=4.2.1",
]
[project.optional-dependencies]
@@ -103,6 +100,7 @@ proxy = [
"magika>=0.6.0; python_version<'3.14' or sys_platform != 'darwin' or platform_machine != 'x86_64'", # ML detection needs ORT wheels
"zstandard>=0.20.0", # Decompress zstd request bodies (Codex, etc.)
"websockets>=13.0", # WebSocket proxy for /v1/responses (Codex gpt-5.4+)
"python-socks[asyncio]>=2.7.1", # SOCKS support when websockets discovers a system proxy
# ORT <1.24 deadlocks in Rust fastembed instead of returning an init error.
# Python 3.10 keeps Python-only ORT and bypasses native detection. Intel
# macOS also needs older ORT because newer releases have no x86_64 wheels.
@@ -361,8 +359,13 @@ dev = [
all = [
"headroom-ai[proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,mcp,spreadsheet]",
]
# Sandbox: a lean proxy with ALL torch-free capability — for running Headroom in
# a locked-down/low-resource sandbox and offloading heavy ML elsewhere.
# Lean: a proxy with ALL torch-free capability — for running Headroom in a
# constrained or low-resource environment and offloading heavy ML elsewhere.
#
# This is a DEPENDENCY PROFILE. It provides no isolation, no confinement and
# no privilege reduction of any kind. It was previously named `sandbox`, which
# invited exactly that assumption; `sandbox` is retained below as a
# deprecated alias for one release so existing installs keep resolving.
#
# = [all] MINUS:
# - image (SigLIP/OCR — excluded by request)
@@ -373,13 +376,18 @@ all = [
# - memory + evals (both pull sentence-transformers -> torch, i.e. the very
# ML weight a sandbox avoids; evals is a dev/test harness, not a
# runtime feature). Opt back in explicitly if you accept torch:
# pip install headroom-ai[sandbox,memory]
# pip install headroom-ai[lean,memory]
#
# Everything kept here is torch-free: code-aware compression (tree-sitter),
# embedding relevance (fastembed), HTML/spreadsheet ingestion, reports, OTel.
sandbox = [
lean = [
"headroom-ai[proxy,code,relevance,reports,otel,html,mcp,spreadsheet]",
]
# Deprecated alias for `lean`, kept for one release. The name implied an
# isolation guarantee the extra never provided. Remove after 0.39.x.
sandbox = [
"headroom-ai[lean]",
]
[project.scripts]
headroom = "headroom.cli:main"
@@ -408,6 +416,8 @@ Changelog = "https://github.com/headroomlabs-ai/headroom/blob/main/CHANGELOG.md"
# transitively; the floor pins below ensure uv resolves to patched versions.
[tool.uv]
constraint-dependencies = [
# PYSEC-2026-4320 — remove distrusted root certificates from the CA bundle.
"certifi>=2026.10.19",
# GHSA-5239-wwwm-4pmq (Low) — transitive via rich; fix at 2.20.0
"pygments>=2.20.0",
# GHSA-4xgf-cpjx-pc3j (Medium) — transitive via mcp; fix at 2.14.2
@@ -423,8 +433,11 @@ constraint-dependencies = [
"lxml-html-clean>=0.4.5",
# CVE-2026-5241 (High) — direct optional dep for proxy/ml/voice; fix at 5.5.0
"transformers>=5.5.0",
# PYSEC-2026-3447 — transitive dependency; fix at 83.0.0
"setuptools>=83.0.0",
# GHSA-xmj3-9gf7-h52w — Windows wheel namespace path traversal.
"setuptools>=84.1.0",
# CVE-2026-112940/112913/112907. The patched series requires Python 3.11;
# Python 3.10 retains the legacy locked version; it is not remediated by this marker.
"filelock>=4.2.1; python_version >= '3.11'",
# PYSEC-2026-3545/3546/3547 — transitive HTTP/WebSocket parser fixes;
# CVE-2026-111063/111064, GHSA-2g87-pp6c-29x5; fix at 3.14.4
"aiohttp>=3.14.4",
@@ -448,10 +461,8 @@ constraint-dependencies = [
# `pip install headroom-ai[all]` still resolves against fastembed's cap; on
# that Pillow, headroom/image/decoder_guard.py forwards images undecoded.
# Drop this override once fastembed allows Pillow 13.
# Pillow 13 needs Python 3.11; on 3.10 no fixed release exists.
override-dependencies = [
"pillow>=13.0.0; python_version >= '3.11'",
"pillow>=12.3.0,<13; python_version < '3.11'",
"pillow>=13.0.0",
]
[[tool.uv.index]]
@@ -486,6 +497,8 @@ features = ["extension-module"]
bindings = "pyo3"
[tool.ruff]
# Kept at py310 until the py311 idiom migration lands (UP017/UP036/UP041/UP042
# touch ~200 sites); requires-python is 3.11.
target-version = "py310"
line-length = 100
@@ -513,7 +526,7 @@ quote-style = "double"
indent-style = "space"
[tool.mypy]
python_version = "3.10"
python_version = "3.11"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = true
+4 -4
View File
@@ -5,7 +5,7 @@
# to the Kompress-v2 Modal endpoint. Then confirm everything loaded.
#
# Plugins : lossless_guard, skill_search, observability, tier_router, tool_search
# Extra : headroom-ai[sandbox] (torch-free proxy; ML offloaded to Modal)
# Extra : headroom-ai[lean] (torch-free proxy; ML offloaded to Modal)
# Profile : coding (HEADROOM_SAVINGS_PROFILE) + cache mode (prefix-cache safe)
#
# Secrets are SOURCED from ~/env.txt and ~/.headroom/plugins.env — never inlined.
@@ -29,7 +29,7 @@ source "$VENV/bin/activate"
PY="$VENV/bin/python"
# ── 2. install (guarded) ──────────────────────────────────────────────────────
# headroom-ai[sandbox] pulls proxy,code,relevance,reports,otel,html,mcp,spreadsheet
# headroom-ai[lean] pulls proxy,code,relevance,reports,otel,html,mcp,spreadsheet
# (all torch-free — heavy ML is offloaded to the Modal Kompress endpoint below).
# The 5 plugins install --no-deps so pip won't drag PyPI's headroom-ai over the
# local editable one; headroom-license is their shared Ed25519 verifier.
@@ -44,14 +44,14 @@ if [ "$need_install" = "1" ]; then
if [ -n "$avail" ] && [ "$avail" -lt 2 ]; then
echo "!! <2Gi free — aborting before heavy install (free space, then re-run)"; exit 1
fi
echo "▶ installing pip+maturin, then headroom-ai[sandbox] + license + 5 plugins (editable)…"
echo "▶ installing pip+maturin, then headroom-ai[lean] + license + 5 plugins (editable)…"
"$PY" -m pip install -U pip maturin
# litellm >=1.92 ships an sdist-only Rust bridge whose AWS-SDK crates need rustc>=1.94.1;
# the default rustup toolchain here is older (pip builds litellm in a temp dir that misses
# the repo's 1.95 pin), so pin to the last pure-Python wheel line (1.91.4). Satisfies
# headroom's litellm>=1.86.2,<2.0 and skips the Rust build entirely.
"$PY" -m pip install "litellm<1.92"
"$PY" -m pip install -e "${HR}[sandbox]" "litellm<1.92"
"$PY" -m pip install -e "${HR}[lean]" "litellm<1.92"
"$PY" -m pip install -e /Users/tcms/demo/headroom-license
for p in lossless-guard skill-search observability tier-router tool-search; do
"$PY" -m pip install -e "/Users/tcms/demo/headroom-${p}" --no-deps

Some files were not shown because too many files have changed in this diff Show More