mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
Merge remote-tracking branch 'origin/main' into fix/fastembed-rust-pinned-snapshot
# Conflicts: # crates/headroom-core/src/relevance/embedding.rs
This commit is contained in:
@@ -1,96 +1,25 @@
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 240" width="880" height="240" role="img"
|
||||
aria-label="Headroom Enterprise for teams: 3 months free with a 12-month plan. Adds tool search, model routing, attack removal and org-wide savings. Claim the perk.">
|
||||
<title>Headroom Enterprise — 3 months free with a 12-month plan</title>
|
||||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 880 200" width="880" height="200" role="img"
|
||||
aria-label="Headroom for teams and enterprises. Want more savings? Headroom OSS shows what is possible; Enterprise saves more. More cost savings, tokens secured, on premise, deep attribution and many more features. Talk to us: hello@headroomlabs.ai.">
|
||||
<title>Headroom for teams and enterprises — talk to us</title>
|
||||
|
||||
<defs>
|
||||
<!-- Brand palette as in hero.svg: Mint #00F0B5, Obsidian #070A0F,
|
||||
Midnight #0C1118, Cyan #65D8FF, White, Slate #98A4B3.
|
||||
Dark in both GitHub themes, matching the hero. -->
|
||||
<pattern id="pgrid" width="44" height="44" patternUnits="userSpaceOnUse">
|
||||
<path d="M44 0H0V44" fill="none" stroke="#00F0B5" stroke-opacity="0.05" stroke-width="1"/>
|
||||
</pattern>
|
||||
|
||||
<radialGradient id="pglow" cx="12%" cy="0%" r="80%">
|
||||
<stop offset="0%" stop-color="#00F0B5" stop-opacity="0.16"/>
|
||||
<stop offset="100%" stop-color="#00F0B5" stop-opacity="0"/>
|
||||
</radialGradient>
|
||||
|
||||
<linearGradient id="pedge" x1="0" y1="0" x2="1" y2="0">
|
||||
<stop offset="0%" stop-color="#00F0B5" stop-opacity="0.65"/>
|
||||
<stop offset="55%" stop-color="#65D8FF" stop-opacity="0.35"/>
|
||||
<stop offset="100%" stop-color="#65D8FF" stop-opacity="0.10"/>
|
||||
</linearGradient>
|
||||
|
||||
<linearGradient id="pbtn" x1="0" y1="0" x2="1" y2="1">
|
||||
<stop offset="0%" stop-color="#00F0B5"/>
|
||||
<stop offset="100%" stop-color="#0FD9AE"/>
|
||||
</linearGradient>
|
||||
|
||||
<linearGradient id="pshine" gradientUnits="userSpaceOnUse" x1="0" y1="0" x2="70" y2="0">
|
||||
<stop offset="0%" stop-color="#FFFFFF" stop-opacity="0"/>
|
||||
<stop offset="50%" stop-color="#FFFFFF" stop-opacity="0.45"/>
|
||||
<stop offset="100%" stop-color="#FFFFFF" stop-opacity="0"/>
|
||||
</linearGradient>
|
||||
|
||||
<clipPath id="pbtnclip"><rect x="646" y="94" width="208" height="58" rx="29"/></clipPath>
|
||||
|
||||
<!-- Brand palette and type scale as in hero.svg: Mint #00F0B5, Obsidian #070A0F,
|
||||
Slate #98A4B3, White; title 42/700, lead 14.5, eyebrow 10.5 tracked. -->
|
||||
<style>
|
||||
.s { font-family: "Segoe UI", Inter, -apple-system, BlinkMacSystemFont, Helvetica, Arial, sans-serif; }
|
||||
|
||||
/* One light pass over the button per cycle draws the eye to the action. */
|
||||
.shine { animation: shine 5s ease-in-out infinite; }
|
||||
@keyframes shine { 0%, 55% { transform: translateX(-80px) } 85%, 100% { transform: translateX(240px) } }
|
||||
|
||||
.dot { animation: beat 2.4s ease-in-out infinite; }
|
||||
@keyframes beat { 0%, 100% { opacity: 1 } 50% { opacity: .3 } }
|
||||
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.shine, .dot { animation: none; }
|
||||
.shine { opacity: 0; }
|
||||
}
|
||||
</style>
|
||||
</defs>
|
||||
|
||||
<!-- Card. One font family, three sizes: 52 headline, 20 lead and button, 16 everything else. -->
|
||||
<rect x="1" y="1" width="878" height="238" rx="18" fill="#070A0F"/>
|
||||
<rect x="1" y="1" width="878" height="238" rx="18" fill="url(#pgrid)"/>
|
||||
<rect x="1" y="1" width="878" height="238" rx="18" fill="url(#pglow)"/>
|
||||
<rect x="1" y="1" width="878" height="238" rx="18" fill="none" stroke="url(#pedge)" stroke-width="1.5"/>
|
||||
<rect x="1" y="1" width="878" height="198" rx="18" fill="#070A0F" stroke="#00F0B5" stroke-opacity="0.35" stroke-width="1.5"/>
|
||||
|
||||
<g class="s">
|
||||
<!-- Eyebrow -->
|
||||
<circle class="dot" cx="46" cy="47" r="5" fill="#00F0B5"/>
|
||||
<text x="60" y="53" font-size="16" font-weight="600" letter-spacing="1.5" fill="#00F0B5">HEADROOM ENTERPRISE · FOR TEAMS</text>
|
||||
<text x="40" y="48" font-size="10.5" letter-spacing="1.6" fill="#00F0B5">HEADROOM FOR TEAMS & ENTERPRISES</text>
|
||||
<text x="40" y="96" font-size="42" font-weight="700" letter-spacing="-0.8" fill="#FFFFFF">Want more savings?</text>
|
||||
<text x="40" y="128" font-size="14.5" fill="#98A4B3">Headroom OSS shows what is possible. Enterprise saves more, and adds:</text>
|
||||
<text x="40" y="160" font-size="14.5" fill="#FFFFFF">More cost savings · Tokens secured · On premise · Deep attribution · and more</text>
|
||||
|
||||
<!-- Headline -->
|
||||
<text x="40" y="116" font-size="52" font-weight="700" fill="#FFFFFF">3 months free</text>
|
||||
<text x="40" y="152" font-size="20" fill="#98A4B3">with a 12-month plan · one perk per company</text>
|
||||
|
||||
<!-- What Enterprise adds -->
|
||||
<g font-size="16" fill="#E6EDF3">
|
||||
<rect x="40" y="178" width="118" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
|
||||
<text x="99" y="202" text-anchor="middle">Tool search</text>
|
||||
|
||||
<rect x="168" y="178" width="134" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
|
||||
<text x="235" y="202" text-anchor="middle">Model routing</text>
|
||||
|
||||
<rect x="312" y="178" width="140" height="36" rx="18" fill="#0C1118" stroke="#00F0B5" stroke-opacity="0.30"/>
|
||||
<text x="382" y="202" text-anchor="middle">Attack removal</text>
|
||||
|
||||
<rect x="462" y="178" width="156" height="36" rx="18" fill="#0C1118" stroke="#65D8FF" stroke-opacity="0.30"/>
|
||||
<text x="540" y="202" text-anchor="middle">Org-wide savings</text>
|
||||
</g>
|
||||
|
||||
<!-- Divider -->
|
||||
<line x1="634" y1="36" x2="634" y2="204" stroke="#98A4B3" stroke-opacity="0.18"/>
|
||||
|
||||
<!-- Call to action -->
|
||||
<text x="750" y="72" font-size="16" text-anchor="middle" fill="#98A4B3">SSO · VPC or managed</text>
|
||||
<g clip-path="url(#pbtnclip)">
|
||||
<rect x="646" y="94" width="208" height="58" rx="29" fill="url(#pbtn)"/>
|
||||
<rect class="shine" x="646" y="94" width="70" height="58" fill="url(#pshine)"/>
|
||||
</g>
|
||||
<text x="750" y="130" font-size="20" font-weight="700" text-anchor="middle" fill="#070A0F">Claim the perk →</text>
|
||||
<text x="750" y="190" font-size="16" text-anchor="middle" fill="#98A4B3">headroom-perks.vercel.app</text>
|
||||
<rect x="676" y="72" width="164" height="44" rx="22" fill="#00F0B5"/>
|
||||
<text x="758" y="99" font-size="14.5" font-weight="700" text-anchor="middle" fill="#070A0F">Talk to us →</text>
|
||||
<text x="758" y="140" font-size="13" text-anchor="middle" fill="#98A4B3">hello@headroomlabs.ai</text>
|
||||
</g>
|
||||
</svg>
|
||||
|
||||
|
Before Width: | Height: | Size: 4.8 KiB After Width: | Height: | Size: 1.7 KiB |
@@ -268,11 +268,11 @@ jobs:
|
||||
run: |
|
||||
python -m pip install --upgrade pip maturin
|
||||
maturin build --profile ci --out dist --interpreter python3.14
|
||||
- name: Install headroom-ai[sandbox] from prebuilt wheels only
|
||||
- name: Install headroom-ai[lean] from prebuilt wheels only
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
WHEEL="$(ls dist/*.whl)"
|
||||
python -m pip install --only-binary=:all: "${WHEEL}[sandbox]"
|
||||
python -m pip install --only-binary=:all: "${WHEEL}[lean]"
|
||||
- name: Proxy dependencies and LiteLLM pricing are available
|
||||
working-directory: ${{ runner.temp }}
|
||||
run: |
|
||||
|
||||
@@ -813,6 +813,11 @@ jobs:
|
||||
node-version: ">=24.16.0 <25"
|
||||
registry-url: ${{ env.NPM_REGISTRY_URL }}
|
||||
|
||||
- name: Verify npm publishing authentication
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: npm whoami --registry "${NPM_REGISTRY_URL}"
|
||||
|
||||
# No artifact download required: the publish steps below pack and
|
||||
# publish directly from the checked-out source tree (sdk/typescript
|
||||
# and plugins/openclaw) via `npm pack` + `npm publish`. The
|
||||
@@ -828,12 +833,15 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
version="${{ needs.detect-version.outputs.npm_version }}"
|
||||
if npm view "${NPM_SDK_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
|
||||
echo "${NPM_SDK_PACKAGE}@${version} is already published; continuing."
|
||||
exit 0
|
||||
fi
|
||||
cd sdk/typescript
|
||||
npm ci
|
||||
npm run build
|
||||
npm version "$version" --no-git-tag-version --allow-same-version
|
||||
npm publish --access public
|
||||
continue-on-error: true
|
||||
|
||||
- name: Publish ${{ env.NPM_OPENCLAW_PACKAGE }} to npmjs.org
|
||||
id: npm-openclaw-publish
|
||||
@@ -841,6 +849,10 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
version="${{ needs.detect-version.outputs.npm_version }}"
|
||||
if npm view "${NPM_OPENCLAW_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
|
||||
echo "${NPM_OPENCLAW_PACKAGE}@${version} is already published; continuing."
|
||||
exit 0
|
||||
fi
|
||||
cd plugins/openclaw
|
||||
npm ci
|
||||
npm run build
|
||||
@@ -854,7 +866,6 @@ jobs:
|
||||
EOF
|
||||
node prepare-dist.mjs
|
||||
npm publish --access public
|
||||
continue-on-error: true
|
||||
|
||||
- name: Publish ${{ env.NPM_OPENCODE_PACKAGE }} to npmjs.org
|
||||
id: npm-opencode-publish
|
||||
@@ -862,6 +873,10 @@ jobs:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
version="${{ needs.detect-version.outputs.npm_version }}"
|
||||
if npm view "${NPM_OPENCODE_PACKAGE}@${version}" version --registry "${NPM_REGISTRY_URL}" >/dev/null 2>&1; then
|
||||
echo "${NPM_OPENCODE_PACKAGE}@${version} is already published; continuing."
|
||||
exit 0
|
||||
fi
|
||||
cd plugins/opencode
|
||||
npm ci
|
||||
npm run build
|
||||
@@ -874,12 +889,6 @@ jobs:
|
||||
fs.writeFileSync("package.json", `${JSON.stringify(pkg, null, 2)}\n`);
|
||||
EOF
|
||||
npm publish --access public
|
||||
continue-on-error: true
|
||||
|
||||
- name: npm publish notice
|
||||
if: steps.npm-sdk-publish.outcome == 'failure' || steps.npm-openclaw-publish.outcome == 'failure' || steps.npm-opencode-publish.outcome == 'failure'
|
||||
run: |
|
||||
echo "::notice::One or more npm publishes failed. Set NPM_SKIP=true in repo Variables to skip npm publishes if tokens are not configured."
|
||||
|
||||
publish-github-packages:
|
||||
needs: [detect-version, build]
|
||||
@@ -1067,7 +1076,7 @@ jobs:
|
||||
|
||||
# Appends once: a re-run finds this step's own marker and leaves the notes
|
||||
# alone. Keyed on the marker, not the link, so a changelog entry that
|
||||
# happens to mention the perks page does not suppress the footer.
|
||||
# happens to mention the contact link does not suppress the footer.
|
||||
# Release notes stay the changelog; this is one line under it.
|
||||
- name: Add Headroom for Teams footer to release notes
|
||||
env:
|
||||
@@ -1079,8 +1088,8 @@ jobs:
|
||||
if ! grep -qF "$MARKER" release-notes.md; then
|
||||
{
|
||||
printf '\n%s\n\n---\n\n' "$MARKER"
|
||||
printf '**Headroom for Teams:** 3 months of Headroom Enterprise free with a 12-month plan. '
|
||||
printf '[Claim the perk →](https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=release-notes&utm_content=%s)\n' "$TAG"
|
||||
printf '**Want more savings?** Headroom Enterprise saves more for teams and enterprises. Talk to us at '
|
||||
printf '[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) or [headroomlabs.ai](https://headroomlabs.ai).\n'
|
||||
} >> release-notes.md
|
||||
gh release edit "$TAG" --notes-file release-notes.md
|
||||
fi
|
||||
|
||||
+1
-1
@@ -122,7 +122,7 @@ Enable or disable automatic Copilot review in **Settings → Rules → Rulesets
|
||||
- [Ruff](https://github.com/astral-sh/ruff) for lint + format, line length 100, PEP 8.
|
||||
- Type hints on public functions; Google-style docstrings.
|
||||
- Cover new behavior + edge cases; aim >80% coverage on new code.
|
||||
- Python 3.10+. Optional features go behind extras.
|
||||
- Python 3.11+. Optional features go behind extras.
|
||||
- **Headroom writes LF and normalizes on read, on every platform.** Pass
|
||||
`newline="\n"` to every `write_text`/`open` that writes a context, memory,
|
||||
or state file, and normalize `\r\n`/`\r` when you read one back. Without
|
||||
|
||||
+4
-3
@@ -50,11 +50,12 @@ COPY headroom/ headroom/
|
||||
# The standalone Dockerfile must support every backend advertised by
|
||||
# `headroom proxy --backend`, including Bedrock temporary/SSO credentials.
|
||||
# Those credentials require botocore (GH #1551), supplied by [bedrock].
|
||||
ARG HEADROOM_EXTRAS=proxy,code,bedrock
|
||||
# [redis] backs the replica-shared stores: without it HEADROOM_COMPRESS_SESSION_STORE
|
||||
# stops startup and HEADROOM_CCR_BACKEND=redis falls back to a per-process store.
|
||||
ARG HEADROOM_EXTRAS=proxy,code,bedrock,redis
|
||||
# `uv pip install` ignores [tool.uv] override-dependencies (uv.lock only), so
|
||||
# pass them explicitly: the image gets the security overrides the lock has
|
||||
# (pillow 13 over fastembed's <13 cap). Python 3.10 has no tomllib: pip's
|
||||
# vendored tomli reads the file there.
|
||||
# (pillow 13 over fastembed's <13 cap).
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=cache,target=/usr/local/cargo/registry \
|
||||
--mount=type=cache,target=/usr/local/cargo/git \
|
||||
|
||||
@@ -39,7 +39,7 @@ answers, fraction of the tokens. Compression runs on your machine; no prompt or
|
||||
file content is sent anywhere to be compressed.
|
||||
|
||||
<p align="center">
|
||||
<a href="https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=readme&utm_content=banner"><img src=".github/assets/enterprise-perk.svg" alt="Headroom Enterprise for teams: 3 months free with a 12-month plan. Claim the perk." width="880"></a>
|
||||
<a href="https://headroomlabs.ai"><img src=".github/assets/enterprise-perk.svg" alt="Headroom Enterprise for teams. Talk to us: hello@headroomlabs.ai." width="880"></a>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
@@ -443,7 +443,7 @@ Granular extras: `[proxy]`, `[mcp]`, `[ml]` (Kompress-v2-base), `[code]`,
|
||||
`[memory]`, `[vector]` (optional HNSW backend — needs a C++ toolchain, not in
|
||||
`[all]`), `[relevance]`, `[image]`, `[agno]`, `[langchain]`, `[evals]`,
|
||||
`[pytorch-mps]` (Apple-GPU memory-embedder offload — set
|
||||
`HEADROOM_EMBEDDER_RUNTIME=pytorch_mps`). Requires **Python 3.10+**.
|
||||
`HEADROOM_EMBEDDER_RUNTIME=pytorch_mps`). Requires **Python 3.11+**.
|
||||
|
||||
> `[all]` covers the core stack but not the framework adapters. Install those
|
||||
> separately: `pip install "headroom-ai[langchain]"`, and likewise `[agno]`,
|
||||
@@ -627,23 +627,18 @@ Turn it off with `HEADROOM_BEACON=off`, `HEADROOM_TELEMETRY=off`, the
|
||||
|
||||
## Headroom for teams
|
||||
|
||||
Headroom OSS is built for one developer on one laptop. Headroom Enterprise is built
|
||||
for the whole team.
|
||||
**Not seeing the savings you want?** Headroom OSS is a framework. It was built to show
|
||||
what's possible on one developer's laptop, so it saves less than it could. We are
|
||||
building bigger savings for teams and enterprises in Headroom Enterprise, which adds:
|
||||
|
||||
| | Open source | Enterprise |
|
||||
|---|---|---|
|
||||
| Savings | Compression, plus tool search on Anthropic requests (on by default) | Also: tool search for every model, harness tuning, extra compressors |
|
||||
| Model routing | Rule-based and opt-in: turn it on with `HEADROOM_MODEL_ROUTER_ENABLED=1` and write the rules in `HEADROOM_MODEL_ROUTES` | Automatic: a cheaper model per request when quality allows, with a shadow mode that shows the savings before you turn it on |
|
||||
| Security | — | Detect and strip prompt injection and risky tool calls |
|
||||
| Rollout | Per laptop | Managed or in your VPC, SSO, central config, org-wide savings |
|
||||
| Support | GitHub, Discord | Dedicated engineers, onboarding, priority response |
|
||||
- More cost savings
|
||||
- Tokens secured
|
||||
- On premise deployment
|
||||
- Deep attribution
|
||||
- And many more features
|
||||
|
||||
**3 months of Enterprise free.** Companies that sign a 12-month Enterprise plan get
|
||||
their first 3 months free. One perk per company.
|
||||
|
||||
**[Claim 3 months free →](https://headroom-perks.vercel.app/?source=github&utm_source=github&utm_medium=readme&utm_content=teams)**
|
||||
· or email [hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) with your stack and
|
||||
monthly LLM spend.
|
||||
If you're a team or an enterprise, **talk to us**:
|
||||
[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) · [headroomlabs.ai](https://headroomlabs.ai)
|
||||
|
||||
Everything in this repo stays open source under Apache 2.0.
|
||||
|
||||
|
||||
+25
-2
@@ -74,9 +74,9 @@ it necessarily handles both your traffic and your credentials. The
|
||||
full — what is written to disk, with what permissions and retention, what leaves the
|
||||
host, and what the local admin surface does and does not allow. The summary:
|
||||
|
||||
- **Provider API keys are forwarded, not persisted.** Keys supplied via environment
|
||||
- **Interactive provider API keys are forwarded.** Keys supplied via environment
|
||||
variables or request headers are used to authenticate the upstream call and are not
|
||||
written to disk or emitted to logs. The one exception is deliberate: the
|
||||
written to disk or emitted to logs. Settings can also persist keys: the
|
||||
`ANTHROPIC_TARGET_API_HEADERS` / `OPENAI_TARGET_API_HEADERS` header maps, if you set
|
||||
them through the dashboard settings GUI, are persisted to `~/.headroom/settings.json`
|
||||
in plaintext — and a header map is where a gateway key typically goes.
|
||||
@@ -84,6 +84,29 @@ host, and what the local admin surface does and does not allow. The summary:
|
||||
GitHub Copilot OAuth refresh token to `~/.headroom/copilot_auth.json` (relocatable via
|
||||
`$HEADROOM_COPILOT_AUTH_FILE`). It is plaintext JSON, written with `0600` permissions
|
||||
on a best-effort basis.
|
||||
- **Persistent installs store supplied environment values.** Every value passed
|
||||
to `headroom install --env KEY=VALUE` is stored verbatim and in cleartext. Supervisors (launchd, systemd, Task
|
||||
Scheduler, cron) start the proxy from a bare environment, so `--env` is the
|
||||
only channel by which a provider API key reaches a supervised proxy — and
|
||||
if you pass one, it is written to
|
||||
`~/.headroom/deploy/<profile>/manifest.json` (mode `0600`) and `export`ed in
|
||||
cleartext by the generated runner scripts `run-headroom.sh` /
|
||||
`ensure-headroom.sh` (mode `0700`) in the same directory (mode `0700`).
|
||||
The installer sets those modes and then reads them back: where the
|
||||
platform enforces POSIX permission bits it refuses to leave behind a runner
|
||||
script it could not restrict, and warns if the profile directory could not
|
||||
be narrowed. On Windows the bits are advisory — access is governed by ACLs —
|
||||
so the modes above are requested but not enforced there, and protection
|
||||
comes from the account's own profile directory instead. Where they do hold,
|
||||
the files are owner-only, not encrypted: anything that can read the account
|
||||
— the user themself, root, a home-directory backup or sync — can read the
|
||||
key, and rotating it requires re-running `headroom install apply`.
|
||||
With the Docker runtime the same values are additionally passed as `docker
|
||||
run --env NAME=VALUE`, which places them in the container process's command
|
||||
line and so in `ps` output for other users on the host. Prefer a
|
||||
supervisor-native secret facility (systemd `LoadCredential=` /
|
||||
`EnvironmentFile=` on an owner-only file, a macOS keychain helper) over
|
||||
`--env` for long-lived provider keys.
|
||||
- **Cached request content is written to disk.** CCR stores pre-compression originals —
|
||||
tool outputs, file contents, retrieved chunks — in a local SQLite database
|
||||
(`~/.headroom/ccr_store.db`, `0600`) so they can be retrieved after compression. Entries
|
||||
|
||||
@@ -45,6 +45,19 @@ use fastembed::{
|
||||
|
||||
use super::base::{RelevanceScore, RelevanceScorer};
|
||||
|
||||
/// Texts per ONNX Runtime run, matching Python's `EMBED_BATCH_SIZE`: fastembed's
|
||||
/// default of 256 grows ORT's never-shrinking CPU arena to the biggest batch scored.
|
||||
#[cfg(feature = "ml")]
|
||||
const EMBED_BATCH_SIZE: usize = 16;
|
||||
|
||||
/// fastembed refuses to split a dynamic-quantized model's input across runs (each
|
||||
/// run calibrates its own int8 range, so the embeddings would not compare): one batch.
|
||||
#[cfg(feature = "ml")]
|
||||
fn embed_batch_size(model_kind: &EmbeddingModel) -> Option<usize> {
|
||||
(TextEmbedding::get_quantization_mode(model_kind) != QuantizationMode::Dynamic)
|
||||
.then_some(EMBED_BATCH_SIZE)
|
||||
}
|
||||
|
||||
/// Name the default model reports, the same string Python uses.
|
||||
#[cfg(feature = "ml")]
|
||||
const DEFAULT_MODEL_NAME: &str = "BAAI/bge-small-en-v1.5";
|
||||
@@ -443,6 +456,8 @@ pub struct EmbeddingScorer {
|
||||
/// dominant cost so contention is bounded by inference latency,
|
||||
/// not lock latency.
|
||||
model: Option<Mutex<TextEmbedding>>,
|
||||
/// `embed` batch size for this model, from `embed_batch_size`.
|
||||
batch_size: Option<usize>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "ml")]
|
||||
@@ -462,6 +477,7 @@ impl Default for EmbeddingScorer {
|
||||
EmbeddingScorer {
|
||||
model_name: "BAAI/bge-small-en-v1.5".to_string(),
|
||||
model: None,
|
||||
batch_size: Some(EMBED_BATCH_SIZE),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -504,9 +520,12 @@ impl EmbeddingScorer {
|
||||
}
|
||||
};
|
||||
let model = load_snapshot(&snapshot)?;
|
||||
// The snapshot is statically quantized (see `load_snapshot`), so it
|
||||
// batches like fastembed's `BGESmallENV15Q`.
|
||||
Ok(EmbeddingScorer {
|
||||
model_name: DEFAULT_MODEL_NAME.to_string(),
|
||||
model: Some(Mutex::new(model)),
|
||||
batch_size: Some(EMBED_BATCH_SIZE),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -525,11 +544,13 @@ impl EmbeddingScorer {
|
||||
crate::offline::guard_egress("fastembed embedding model download", &name)
|
||||
.map_err(|e| format!("EmbeddingScorer model load refused: {e}"))?;
|
||||
onnx_runtime_ready()?;
|
||||
let batch_size = embed_batch_size(&model_kind);
|
||||
let model = TextEmbedding::try_new(InitOptions::new(model_kind))
|
||||
.map_err(|e| format!("EmbeddingScorer model load failed: {}", e))?;
|
||||
Ok(EmbeddingScorer {
|
||||
model_name: name,
|
||||
model: Some(Mutex::new(model)),
|
||||
batch_size,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -588,11 +609,11 @@ impl RelevanceScorer for EmbeddingScorer {
|
||||
}
|
||||
};
|
||||
|
||||
// Encode items + context in one batch — saves model dispatch
|
||||
// overhead. Mirrors Python fastembed batch encoding.
|
||||
// Encode items + context in one `embed` call, which runs them in
|
||||
// `self.batch_size` chunks. Mirrors Python fastembed batch encoding.
|
||||
let mut all_texts: Vec<String> = items.iter().map(|s| s.to_string()).collect();
|
||||
all_texts.push(context.to_string());
|
||||
let embeddings = match guard.embed(all_texts, None) {
|
||||
let embeddings = match guard.embed(all_texts, self.batch_size) {
|
||||
Ok(e) => e,
|
||||
Err(e) => {
|
||||
return items
|
||||
@@ -1104,6 +1125,7 @@ mod tests {
|
||||
EmbeddingScorer {
|
||||
model_name: "test".to_string(),
|
||||
model: None,
|
||||
batch_size: Some(EMBED_BATCH_SIZE),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1214,6 +1236,15 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ml")]
|
||||
#[test]
|
||||
fn dynamic_quantized_models_embed_in_one_batch() {
|
||||
let capped = Some(EMBED_BATCH_SIZE);
|
||||
assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15), capped);
|
||||
assert_eq!(embed_batch_size(&EmbeddingModel::BGESmallENV15Q), capped);
|
||||
assert_eq!(embed_batch_size(&EmbeddingModel::AllMiniLML6V2Q), None);
|
||||
}
|
||||
|
||||
// ---------- model-backed tests (gated on RUN_FASTEMBED_TESTS) ----------
|
||||
|
||||
#[cfg(feature = "ml")]
|
||||
@@ -1320,4 +1351,71 @@ mod tests {
|
||||
assert!((0.0..=1.0).contains(&sc.score));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ml")]
|
||||
#[test]
|
||||
fn fastembed_batches_past_the_cap_keep_each_score_with_its_item() {
|
||||
if !fastembed_enabled() {
|
||||
return;
|
||||
}
|
||||
// Three EMBED_BATCH_SIZE batches of mixed lengths, the one related item
|
||||
// in the last: a misaligned batch boundary would hand its score away.
|
||||
let s = EmbeddingScorer::try_new().expect("model loads");
|
||||
let mut items: Vec<String> = (0..40)
|
||||
.map(|i| {
|
||||
format!(
|
||||
"inventory row {i}: {}",
|
||||
"pallets of garden furniture ".repeat(i % 5 + 1)
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
items[37] = "authentication failed for user admin: invalid password".to_string();
|
||||
let refs: Vec<&str> = items.iter().map(String::as_str).collect();
|
||||
let batch = s.score_batch(&refs, "login error");
|
||||
assert_eq!(batch.len(), 40);
|
||||
let top = (0..40)
|
||||
.max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
|
||||
.unwrap();
|
||||
assert_eq!(top, 37);
|
||||
for (item, sc) in refs.iter().zip(&batch) {
|
||||
let single = s.score(item, "login error").score;
|
||||
assert!(
|
||||
(sc.score - single).abs() < 1e-3,
|
||||
"{item}: batched {} vs single {single}",
|
||||
sc.score
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "ml")]
|
||||
#[test]
|
||||
fn fastembed_dynamic_quantized_model_scores_on_both_sides_of_the_cap() {
|
||||
if !fastembed_enabled() {
|
||||
return;
|
||||
}
|
||||
// 15 items plus the context fill exactly one EMBED_BATCH_SIZE run; 16
|
||||
// and 40 go past it, where fastembed refuses to split this model.
|
||||
let s = EmbeddingScorer::try_new_with_model(EmbeddingModel::AllMiniLML6V2Q)
|
||||
.expect("model loads");
|
||||
for n in [15, 16, 40] {
|
||||
let mut items: Vec<String> = (0..n)
|
||||
.map(|i| format!("inventory row {i}: pallets of garden furniture"))
|
||||
.collect();
|
||||
items[n - 3] = "authentication failed for user admin: invalid password".to_string();
|
||||
let refs: Vec<&str> = items.iter().map(String::as_str).collect();
|
||||
let batch = s.score_batch(&refs, "login error");
|
||||
assert_eq!(batch.len(), n);
|
||||
for sc in &batch {
|
||||
assert!(
|
||||
!sc.reason.contains("inference failed"),
|
||||
"{n} items: {}",
|
||||
sc.reason
|
||||
);
|
||||
}
|
||||
let top = (0..n)
|
||||
.max_by(|&a, &b| batch[a].score.total_cmp(&batch[b].score))
|
||||
.unwrap();
|
||||
assert_eq!(top, n - 3, "{n} items");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43753,5 +43753,763 @@
|
||||
"thinking_always_on": true,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing"
|
||||
},
|
||||
"openrouter/anthropic/claude-haiku-5.5": {
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"litellm_provider": "openrouter",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"source": "https://openrouter.ai/api/v1/models",
|
||||
"supports_function_calling": true,
|
||||
"supports_tool_choice": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_vision": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_web_search": true,
|
||||
"supports_adaptive_thinking": true,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"supports_sampling_params": false
|
||||
},
|
||||
"claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens_batches": 2.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens_batches": 1.25e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens_batches": 3.125e-07,
|
||||
"cache_read_input_token_cost_above_100k_tokens_batches": 2.5e-08,
|
||||
"supports_anthropic_compaction": true,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_creation_input_token_cost_batches": 6.25e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-09,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_batches": 5e-08,
|
||||
"litellm_provider": "anthropic",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"output_cost_per_token_batches": 2.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_native_structured_output": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"provider_specific_entry": {
|
||||
"us": 1.1
|
||||
},
|
||||
"supports_output_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/overview",
|
||||
"supports_web_search": true,
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08
|
||||
},
|
||||
"bedrock_mantle/anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
|
||||
},
|
||||
"bedrock_mantle/us-gov-west-1/anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 6e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 3e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"cache_creation_input_token_cost": 1.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
|
||||
"cache_read_input_token_cost": 1.2e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "bedrock_mantle",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"source": "https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-haiku-5-5.html"
|
||||
},
|
||||
"anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"apac.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"au.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"azure_ai/claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"supports_mid_conversation_system": true,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"deprecation_date": "2027-09-29",
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "azure_ai",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://platform.claude.com/docs/en/models/haiku-5-5/migration-guide"
|
||||
},
|
||||
"bedrock/us-gov-east-1/anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 6e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 3e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"cache_creation_input_token_cost": 1.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
|
||||
"cache_read_input_token_cost": 1.2e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"bedrock/us-gov-west-1/anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 6e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 3e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"cache_creation_input_token_cost": 1.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
|
||||
"cache_read_input_token_cost": 1.2e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "bedrock",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"eu.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"global.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"jp.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"perplexity/anthropic/claude-haiku-5-5": {
|
||||
"litellm_provider": "perplexity",
|
||||
"mode": "responses",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_web_search": true,
|
||||
"supports_function_calling": true,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"output_cost_per_token": 5e-07,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"source": "https://docs.perplexity.ai/docs/agent-api/models"
|
||||
},
|
||||
"us-gov.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 6e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 3e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 7.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.2e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 6e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"cache_creation_input_token_cost": 1.5e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.4e-07,
|
||||
"cache_read_input_token_cost": 1.2e-08,
|
||||
"input_cost_per_token": 1.2e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 6e-07,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json",
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_output_config": true,
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true
|
||||
},
|
||||
"us.anthropic.claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5.5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.75e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.875e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1.1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5.5e-08,
|
||||
"bedrock_converse_supports_strict_tools": false,
|
||||
"cache_creation_input_token_cost": 1.375e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2.2e-07,
|
||||
"cache_read_input_token_cost": 1.1e-08,
|
||||
"input_cost_per_token": 1.1e-07,
|
||||
"litellm_provider": "bedrock_converse",
|
||||
"supports_tool_search": true,
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_mid_conversation_system": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_native_structured_output": false,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_output_config": true,
|
||||
"bedrock_output_config_effort_ceiling": "xhigh",
|
||||
"supports_parallel_tool_use_config": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonBedrockFoundationModels/current/index.json"
|
||||
},
|
||||
"vertex_ai/claude-haiku-5-5": {
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
"supports_mid_conversation_system": true,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_creation_input_token_cost_batches": 6.25e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-09,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_batches": 5e-08,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"output_cost_per_token_batches": 2.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
|
||||
},
|
||||
"vertex_ai/claude-haiku-5-5@default": {
|
||||
"input_cost_per_token_above_100k_tokens": 5e-07,
|
||||
"output_cost_per_token_above_100k_tokens": 2.5e-06,
|
||||
"cache_creation_input_token_cost_above_100k_tokens": 6.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr_above_100k_tokens": 1e-06,
|
||||
"cache_read_input_token_cost_above_100k_tokens": 5e-08,
|
||||
"regional_endpoint_uplift_multiplier": 1.1,
|
||||
"supports_mid_conversation_system": true,
|
||||
"cache_creation_input_token_cost": 1.25e-07,
|
||||
"cache_creation_input_token_cost_above_1hr": 2e-07,
|
||||
"cache_creation_input_token_cost_batches": 6.25e-08,
|
||||
"cache_read_input_token_cost": 1e-08,
|
||||
"cache_read_input_token_cost_batches": 5e-09,
|
||||
"input_cost_per_token": 1e-07,
|
||||
"input_cost_per_token_batches": 5e-08,
|
||||
"litellm_provider": "vertex_ai-anthropic_models",
|
||||
"max_input_tokens": 1000000,
|
||||
"max_output_tokens": 128000,
|
||||
"max_tokens": 128000,
|
||||
"mode": "chat",
|
||||
"output_cost_per_token": 5e-07,
|
||||
"output_cost_per_token_batches": 2.5e-07,
|
||||
"search_context_cost_per_query": {
|
||||
"search_context_size_high": 0.01,
|
||||
"search_context_size_low": 0.01,
|
||||
"search_context_size_medium": 0.01
|
||||
},
|
||||
"supports_adaptive_thinking": true,
|
||||
"supports_assistant_prefill": false,
|
||||
"supports_computer_use": true,
|
||||
"supports_function_calling": true,
|
||||
"supports_pdf_input": true,
|
||||
"supports_prompt_caching": true,
|
||||
"supports_reasoning": true,
|
||||
"supports_response_schema": true,
|
||||
"supports_sampling_params": false,
|
||||
"supports_tool_choice": true,
|
||||
"supports_vision": true,
|
||||
"supports_xhigh_reasoning_effort": true,
|
||||
"supports_max_reasoning_effort": true,
|
||||
"supports_forced_tool_use": true,
|
||||
"thinking_always_on": false,
|
||||
"prompt_cache_min_tokens": 512,
|
||||
"source": "https://cloud.google.com/vertex-ai/generative-ai/pricing"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -187,8 +187,11 @@ mod tests {
|
||||
"claude-opus-5-5",
|
||||
"claude-opus-5",
|
||||
"claude-fable-5-1",
|
||||
"claude-haiku-5-5",
|
||||
"global.anthropic.claude-sonnet-5-5",
|
||||
"vertex_ai/claude-sonnet-5-5",
|
||||
"us.anthropic.claude-haiku-5-5",
|
||||
"vertex_ai/claude-haiku-5-5",
|
||||
] {
|
||||
assert_eq!(context_window_for(model), 1_000_000, "{model}");
|
||||
}
|
||||
|
||||
@@ -660,25 +660,44 @@ impl Ledger {
|
||||
let usd = if ev.failed {
|
||||
EventUsd::default()
|
||||
} else {
|
||||
match pricing::lookup(&ev.model) {
|
||||
Some(p) => EventUsd {
|
||||
compression: p.compression_savings_usd(ev.tokens_saved()),
|
||||
cache: p
|
||||
.cache_savings_usd(ev.usage.cache_read_tokens, ev.usage.cache_write_tokens),
|
||||
input_cost: p.input_cost_usd(
|
||||
// When the response carried no usage block,
|
||||
// bill the post-compression estimate instead
|
||||
// so passthrough lanes still show spend.
|
||||
if ev.usage.total_input() > 0 {
|
||||
ev.usage.input_tokens
|
||||
} else {
|
||||
ev.tokens_after
|
||||
},
|
||||
ev.usage.cache_read_tokens,
|
||||
ev.usage.cache_write_tokens,
|
||||
),
|
||||
output_cost: p.output_cost_usd(ev.usage.output_tokens),
|
||||
},
|
||||
match pricing::lookup_card(&ev.model) {
|
||||
Some(card) => {
|
||||
// When the response carried no usage block, bill
|
||||
// the post-compression estimate instead so
|
||||
// passthrough lanes still show spend.
|
||||
let uncached = if ev.usage.total_input() > 0 {
|
||||
ev.usage.input_tokens
|
||||
} else {
|
||||
ev.tokens_after
|
||||
};
|
||||
// The billed prompt picks the rate card: past a
|
||||
// model's long-context threshold every rate below
|
||||
// switches to the higher tier.
|
||||
let p = card.for_prompt(
|
||||
uncached
|
||||
.saturating_add(ev.usage.cache_read_tokens)
|
||||
.saturating_add(ev.usage.cache_write_tokens),
|
||||
);
|
||||
EventUsd {
|
||||
compression: card.compression_savings_usd(
|
||||
uncached,
|
||||
ev.usage.cache_read_tokens,
|
||||
ev.usage.cache_write_tokens,
|
||||
ev.usage.output_tokens,
|
||||
ev.tokens_saved(),
|
||||
),
|
||||
cache: p.cache_savings_usd(
|
||||
ev.usage.cache_read_tokens,
|
||||
ev.usage.cache_write_tokens,
|
||||
),
|
||||
input_cost: p.input_cost_usd(
|
||||
uncached,
|
||||
ev.usage.cache_read_tokens,
|
||||
ev.usage.cache_write_tokens,
|
||||
),
|
||||
output_cost: p.output_cost_usd(ev.usage.output_tokens),
|
||||
}
|
||||
}
|
||||
None => EventUsd::default(),
|
||||
}
|
||||
};
|
||||
@@ -1277,6 +1296,66 @@ mod tests {
|
||||
assert!(total > input, "total spend must exceed input-only spend");
|
||||
}
|
||||
|
||||
/// A Haiku 5.5 prompt over 100K bills every rate at the 5x card:
|
||||
/// 150K uncached in + 5K out is $0.075 + $0.0125, not the base
|
||||
/// card's $0.015 + $0.0025.
|
||||
#[test]
|
||||
fn long_prompt_bills_at_the_models_long_context_card() {
|
||||
let ledger = Ledger::in_memory();
|
||||
let mut ev = event("claude-haiku-5-5");
|
||||
ev.usage = Usage {
|
||||
input_tokens: 150_000,
|
||||
output_tokens: 5_000,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
};
|
||||
ledger.record(ev);
|
||||
let v = ledger.stats_payload(true);
|
||||
let input = v["session"]["input_cost_usd"].as_f64().unwrap();
|
||||
let output = v["session"]["output_cost_usd"].as_f64().unwrap();
|
||||
assert!((input - 0.075).abs() < 1e-9, "input: {input}");
|
||||
assert!((output - 0.0125).abs() < 1e-9, "output: {output}");
|
||||
}
|
||||
|
||||
/// Devin's example on #4037: 110K uncached compressed to 90K saves
|
||||
/// $0.055 - $0.009 = $0.046 of input, not 20K x $0.10/M = $0.002.
|
||||
#[test]
|
||||
fn compression_below_the_threshold_credits_the_avoided_long_card() {
|
||||
let ledger = Ledger::in_memory();
|
||||
let mut ev = event("claude-haiku-5-5");
|
||||
ev.tokens_before = 110_000;
|
||||
ev.tokens_after = 90_000;
|
||||
ev.usage = Usage {
|
||||
input_tokens: 90_000,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
};
|
||||
ledger.record(ev);
|
||||
let v = ledger.stats_payload(true);
|
||||
let saved = v["session"]["compression_savings_usd"].as_f64().unwrap();
|
||||
assert!((saved - 0.046).abs() < 1e-9, "saved: {saved}");
|
||||
}
|
||||
|
||||
/// Cache reads count toward the threshold: 60K uncached + 60K
|
||||
/// read is a 120K prompt, so it bills at the long card too.
|
||||
#[test]
|
||||
fn cached_tokens_count_toward_the_long_context_threshold() {
|
||||
let ledger = Ledger::in_memory();
|
||||
let mut ev = event("claude-haiku-5-5");
|
||||
ev.usage = Usage {
|
||||
input_tokens: 60_000,
|
||||
output_tokens: 0,
|
||||
cache_read_tokens: 60_000,
|
||||
cache_write_tokens: 0,
|
||||
};
|
||||
ledger.record(ev);
|
||||
let v = ledger.stats_payload(true);
|
||||
let input = v["session"]["input_cost_usd"].as_f64().unwrap();
|
||||
// 60K x $0.50/M + 60K x $0.05/M.
|
||||
assert!((input - 0.033).abs() < 1e-9, "input: {input}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_token_counts_clamp_to_ceiling() {
|
||||
let ledger = Ledger::in_memory();
|
||||
|
||||
@@ -33,6 +33,17 @@
|
||||
//! while its Bedrock form is present). Those price at $0 with a WARN
|
||||
//! until the table is refreshed — see the module note below.
|
||||
//!
|
||||
//! # Long-context tiers
|
||||
//!
|
||||
//! Some models publish a second, higher rate card for long prompts
|
||||
//! (LiteLLM's `*_above_<N>k_tokens` fields: Claude Haiku 5.5 above
|
||||
//! 100K, Sonnet 4 / 4.5 above 200K, GPT-5.x above 272K). Above the
|
||||
//! threshold the provider re-prices the WHOLE request — input, output
|
||||
//! and cache alike — not just the tokens past it, so [`lookup_card`]
|
||||
//! keeps both cards and [`PriceCard::for_prompt`] picks one from the
|
||||
//! billed prompt. A tier field a row leaves out falls back to that
|
||||
//! rate's base price, the same rule the Python cost path follows.
|
||||
//!
|
||||
//! # Unknown models price at $0
|
||||
//!
|
||||
//! A miss returns `None`; the ledger then records $0 for every USD
|
||||
@@ -121,7 +132,71 @@ impl ModelPrice {
|
||||
}
|
||||
}
|
||||
|
||||
static BOOK: OnceLock<HashMap<String, ModelPrice>> = OnceLock::new();
|
||||
/// A model's rate cards: the base one, plus the long-context one for
|
||||
/// models that publish it (see the module doc).
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct PriceCard {
|
||||
pub base: ModelPrice,
|
||||
pub long_context: Option<LongContextTier>,
|
||||
}
|
||||
|
||||
/// The card a request bills at once its prompt passes `above_tokens`.
|
||||
#[derive(Debug, Clone, Copy, PartialEq)]
|
||||
pub struct LongContextTier {
|
||||
/// Prompt size (uncached + cache read + cache write tokens) the
|
||||
/// request must exceed for this card to apply.
|
||||
pub above_tokens: u64,
|
||||
pub price: ModelPrice,
|
||||
}
|
||||
|
||||
impl PriceCard {
|
||||
/// The rates a request with `prompt_tokens` of billed prompt pays.
|
||||
pub fn for_prompt(&self, prompt_tokens: u64) -> ModelPrice {
|
||||
match self.long_context {
|
||||
Some(tier) if prompt_tokens > tier.above_tokens => tier.price,
|
||||
_ => self.base,
|
||||
}
|
||||
}
|
||||
|
||||
/// USD compression saved on one request: what it would have cost
|
||||
/// uncompressed minus what it did cost.
|
||||
///
|
||||
/// Inside one tier that is the removed tokens at that tier's input
|
||||
/// rate. When the uncompressed prompt would have passed the
|
||||
/// long-context threshold and the forwarded one does not, the
|
||||
/// provider would have re-priced the WHOLE request — every input
|
||||
/// bucket and the output — at the long card, so the saving is the
|
||||
/// difference of the two full bills, not the removed tokens at the
|
||||
/// cheap rate.
|
||||
pub fn compression_savings_usd(
|
||||
&self,
|
||||
uncached_input_tokens: u64,
|
||||
cache_read_tokens: u64,
|
||||
cache_write_tokens: u64,
|
||||
output_tokens: u64,
|
||||
tokens_saved: u64,
|
||||
) -> f64 {
|
||||
let prompt = uncached_input_tokens
|
||||
.saturating_add(cache_read_tokens)
|
||||
.saturating_add(cache_write_tokens);
|
||||
let sent = self.for_prompt(prompt);
|
||||
let unsent = self.for_prompt(prompt.saturating_add(tokens_saved));
|
||||
if unsent == sent {
|
||||
return sent.compression_savings_usd(tokens_saved);
|
||||
}
|
||||
let without = unsent.input_cost_usd(
|
||||
uncached_input_tokens.saturating_add(tokens_saved),
|
||||
cache_read_tokens,
|
||||
cache_write_tokens,
|
||||
) + unsent.output_cost_usd(output_tokens);
|
||||
let with =
|
||||
sent.input_cost_usd(uncached_input_tokens, cache_read_tokens, cache_write_tokens)
|
||||
+ sent.output_cost_usd(output_tokens);
|
||||
(without - with).max(0.0)
|
||||
}
|
||||
}
|
||||
|
||||
static BOOK: OnceLock<HashMap<String, PriceCard>> = OnceLock::new();
|
||||
|
||||
/// Bounded once-per-model warn set so an unknown model logs one WARN,
|
||||
/// not one per request. Capped so attacker-controlled model ids can't
|
||||
@@ -140,11 +215,48 @@ pub fn warm() {
|
||||
let _ = book();
|
||||
}
|
||||
|
||||
fn book() -> &'static HashMap<String, ModelPrice> {
|
||||
fn book() -> &'static HashMap<String, PriceCard> {
|
||||
BOOK.get_or_init(parse_vendored)
|
||||
}
|
||||
|
||||
fn parse_vendored() -> HashMap<String, ModelPrice> {
|
||||
/// A finite, non-negative price field, or `None`.
|
||||
fn price_field(spec: &serde_json::Map<String, serde_json::Value>, key: &str) -> Option<f64> {
|
||||
spec.get(key)
|
||||
.and_then(|v| v.as_f64())
|
||||
.filter(|c| c.is_finite() && *c >= 0.0)
|
||||
}
|
||||
|
||||
/// The row's long-context card, keyed off its
|
||||
/// `input_cost_per_token_above_<N>k_tokens` field. Every row in the
|
||||
/// vendored table publishes at most one such threshold.
|
||||
fn parse_long_context_tier(
|
||||
spec: &serde_json::Map<String, serde_json::Value>,
|
||||
base: &ModelPrice,
|
||||
) -> Option<LongContextTier> {
|
||||
let (thousands, input) = spec.keys().find_map(|key| {
|
||||
let n = key
|
||||
.strip_prefix("input_cost_per_token_above_")?
|
||||
.strip_suffix("k_tokens")?;
|
||||
if n.is_empty() || !n.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return None;
|
||||
}
|
||||
Some((n.parse::<u64>().ok()?, price_field(spec, key)?))
|
||||
})?;
|
||||
let tiered = |field: &str, fallback: f64| {
|
||||
price_field(spec, &format!("{field}_above_{thousands}k_tokens")).unwrap_or(fallback)
|
||||
};
|
||||
Some(LongContextTier {
|
||||
above_tokens: thousands.saturating_mul(1000),
|
||||
price: ModelPrice {
|
||||
input,
|
||||
output: tiered("output_cost_per_token", base.output),
|
||||
cache_read: tiered("cache_read_input_token_cost", base.cache_read),
|
||||
cache_write: tiered("cache_creation_input_token_cost", base.cache_write),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_vendored() -> HashMap<String, PriceCard> {
|
||||
let raw: serde_json::Value = serde_json::from_str(VENDORED_JSON)
|
||||
.expect("vendored LiteLLM JSON must parse — same invariant as model_limits");
|
||||
let mut out = HashMap::new();
|
||||
@@ -166,23 +278,17 @@ fn parse_vendored() -> HashMap<String, ModelPrice> {
|
||||
if !input.is_finite() || !output.is_finite() || input < 0.0 || output < 0.0 {
|
||||
continue;
|
||||
}
|
||||
let cache_read = spec
|
||||
.get("cache_read_input_token_cost")
|
||||
.and_then(|v| v.as_f64())
|
||||
.filter(|c| c.is_finite() && *c >= 0.0)
|
||||
.unwrap_or(input);
|
||||
let cache_write = spec
|
||||
.get("cache_creation_input_token_cost")
|
||||
.and_then(|v| v.as_f64())
|
||||
.filter(|c| c.is_finite() && *c >= 0.0)
|
||||
.unwrap_or(input);
|
||||
let base = ModelPrice {
|
||||
input,
|
||||
output,
|
||||
cache_read: price_field(spec, "cache_read_input_token_cost").unwrap_or(input),
|
||||
cache_write: price_field(spec, "cache_creation_input_token_cost").unwrap_or(input),
|
||||
};
|
||||
out.insert(
|
||||
model_id.to_ascii_lowercase(),
|
||||
ModelPrice {
|
||||
input,
|
||||
output,
|
||||
cache_read,
|
||||
cache_write,
|
||||
PriceCard {
|
||||
long_context: parse_long_context_tier(spec, &base),
|
||||
base,
|
||||
},
|
||||
);
|
||||
}
|
||||
@@ -236,20 +342,26 @@ fn push_revision_trimmed(push: &mut impl FnMut(String), id: &str) {
|
||||
}
|
||||
}
|
||||
|
||||
/// Look up per-token pricing for a model id. `None` means "not in
|
||||
/// the vendored table" — the caller records $0 for USD fields and
|
||||
/// this module logs one WARN per distinct model id.
|
||||
pub fn lookup(model: &str) -> Option<ModelPrice> {
|
||||
/// Look up a model's rate cards. `None` means "not in the vendored
|
||||
/// table" — the caller records $0 for USD fields and this module logs
|
||||
/// one WARN per distinct model id.
|
||||
pub fn lookup_card(model: &str) -> Option<PriceCard> {
|
||||
let table = book();
|
||||
for key in candidates(model) {
|
||||
if let Some(p) = table.get(&key) {
|
||||
return Some(*p);
|
||||
if let Some(card) = table.get(&key) {
|
||||
return Some(*card);
|
||||
}
|
||||
}
|
||||
warn_once(model);
|
||||
None
|
||||
}
|
||||
|
||||
/// Look up a model's base per-token pricing — the card a prompt below
|
||||
/// any long-context threshold pays. See [`lookup_card`].
|
||||
pub fn lookup(model: &str) -> Option<ModelPrice> {
|
||||
lookup_card(model).map(|card| card.base)
|
||||
}
|
||||
|
||||
fn warn_once(model: &str) {
|
||||
let set = WARNED.get_or_init(|| RwLock::new(std::collections::HashSet::new()));
|
||||
{
|
||||
@@ -288,6 +400,98 @@ mod tests {
|
||||
assert!(p.cache_write > p.input, "cache writes carry a premium");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn claude_haiku_5_5_base_rate_card() {
|
||||
let p = lookup("claude-haiku-5-5").expect("priced");
|
||||
assert!((p.input - 1e-7).abs() < 1e-15, "input $0.10/MTok");
|
||||
assert!((p.output - 5e-7).abs() < 1e-15, "output $0.50/MTok");
|
||||
assert!((p.cache_read - 1e-8).abs() < 1e-15, "cache read $0.01/MTok");
|
||||
assert!(
|
||||
(p.cache_write - 1.25e-7).abs() < 1e-15,
|
||||
"5m cache write $0.125/MTok"
|
||||
);
|
||||
}
|
||||
|
||||
/// Haiku 5.5 re-prices the whole request at 5x once the prompt
|
||||
/// passes 100K; at exactly 100K it still pays the base card.
|
||||
#[test]
|
||||
fn claude_haiku_5_5_long_prompt_card_above_100k() {
|
||||
let card = lookup_card("claude-haiku-5-5").expect("priced");
|
||||
let tier = card.long_context.expect("Haiku 5.5 publishes a 100K tier");
|
||||
assert_eq!(tier.above_tokens, 100_000);
|
||||
assert_eq!(card.for_prompt(100_000), card.base);
|
||||
let long = card.for_prompt(100_001);
|
||||
assert!((long.input - 5e-7).abs() < 1e-15, "input $0.50/MTok");
|
||||
assert!((long.output - 2.5e-6).abs() < 1e-15, "output $2.50/MTok");
|
||||
assert!(
|
||||
(long.cache_read - 5e-8).abs() < 1e-15,
|
||||
"cache read $0.05/MTok"
|
||||
);
|
||||
assert!(
|
||||
(long.cache_write - 6.25e-7).abs() < 1e-15,
|
||||
"5m cache write $0.625/MTok"
|
||||
);
|
||||
}
|
||||
|
||||
/// The threshold comes from each row's own field name, not a
|
||||
/// shared constant: Sonnet 4.5's card switches at 200K.
|
||||
#[test]
|
||||
fn long_context_threshold_is_read_per_model() {
|
||||
let card = lookup_card("claude-sonnet-4-5").expect("priced");
|
||||
let tier = card.long_context.expect("Sonnet 4.5 publishes a 200K tier");
|
||||
assert_eq!(tier.above_tokens, 200_000);
|
||||
assert_eq!(card.for_prompt(150_000), card.base);
|
||||
assert!((card.for_prompt(200_001).input - 6e-6).abs() < 1e-15);
|
||||
}
|
||||
|
||||
/// Compression that keeps a prompt under the threshold saves the
|
||||
/// long card on the whole request, not just the removed tokens at
|
||||
/// the base rate: 110K -> 90K uncached on Haiku 5.5, 1K output, is
|
||||
/// (110K x $0.50 + 1K x $2.50) - (90K x $0.10 + 1K x $0.50) per MTok.
|
||||
#[test]
|
||||
fn compression_across_the_threshold_saves_the_whole_premium() {
|
||||
let card = lookup_card("claude-haiku-5-5").expect("priced");
|
||||
let usd = card.compression_savings_usd(90_000, 0, 0, 1_000, 20_000);
|
||||
assert!((usd - (0.0575 - 0.0095)).abs() < 1e-9, "usd: {usd}");
|
||||
// Inside one tier it stays removed tokens at that tier's rate.
|
||||
let below = card.compression_savings_usd(50_000, 0, 0, 1_000, 20_000);
|
||||
assert!((below - 20_000.0 * 1e-7).abs() < 1e-12, "below: {below}");
|
||||
let above = card.compression_savings_usd(120_000, 0, 0, 1_000, 20_000);
|
||||
assert!((above - 20_000.0 * 5e-7).abs() < 1e-12, "above: {above}");
|
||||
}
|
||||
|
||||
/// A model without a published tier bills the base card at any size.
|
||||
#[test]
|
||||
fn flat_rated_model_has_no_long_context_tier() {
|
||||
let card = lookup_card("claude-opus-4-6").expect("priced");
|
||||
assert!(card.long_context.is_none());
|
||||
assert_eq!(card.for_prompt(900_000), card.base);
|
||||
}
|
||||
|
||||
/// A tier that omits a rate falls back to that rate's base price.
|
||||
#[test]
|
||||
fn missing_tier_field_falls_back_to_base_rate() {
|
||||
let spec: serde_json::Map<String, serde_json::Value> = serde_json::from_str(
|
||||
r#"{"input_cost_per_token_above_128k_tokens": 2e-6,
|
||||
"output_cost_per_token_above_128k_tokens_priority": 9e-6}"#,
|
||||
)
|
||||
.unwrap();
|
||||
let base = ModelPrice {
|
||||
input: 1e-6,
|
||||
output: 4e-6,
|
||||
cache_read: 1e-7,
|
||||
cache_write: 1e-6,
|
||||
};
|
||||
let tier = parse_long_context_tier(&spec, &base).expect("tier parsed");
|
||||
assert_eq!(tier.above_tokens, 128_000);
|
||||
assert_eq!(tier.price.input, 2e-6);
|
||||
assert_eq!(
|
||||
tier.price.output, base.output,
|
||||
"a _priority variant is not the tier"
|
||||
);
|
||||
assert_eq!(tier.price.cache_read, base.cache_read);
|
||||
}
|
||||
|
||||
/// Bedrock cross-region pricing is region-specific, so an exact
|
||||
/// regional entry must win over the geo-stripped base id.
|
||||
#[test]
|
||||
|
||||
+9
-9
@@ -10,7 +10,7 @@ target "runtime-default" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,bedrock,redis"
|
||||
RUNTIME_USER = "nonroot"
|
||||
}
|
||||
}
|
||||
@@ -19,7 +19,7 @@ target "runtime" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,bedrock,redis"
|
||||
RUNTIME_USER = "root"
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ target "runtime-nonroot" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,bedrock,redis"
|
||||
RUNTIME_USER = "nonroot"
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,7 @@ target "runtime-code" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
|
||||
RUNTIME_USER = "root"
|
||||
}
|
||||
}
|
||||
@@ -46,7 +46,7 @@ target "runtime-code-nonroot" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
|
||||
RUNTIME_USER = "nonroot"
|
||||
}
|
||||
}
|
||||
@@ -55,7 +55,7 @@ target "runtime-slim" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime-slim"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,bedrock,redis"
|
||||
RUNTIME_USER = "root"
|
||||
}
|
||||
}
|
||||
@@ -64,7 +64,7 @@ target "runtime-slim-nonroot" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime-slim"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,bedrock,redis"
|
||||
RUNTIME_USER = "nonroot"
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ target "runtime-code-slim" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime-slim"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
|
||||
RUNTIME_USER = "root"
|
||||
}
|
||||
}
|
||||
@@ -82,7 +82,7 @@ target "runtime-code-slim-nonroot" {
|
||||
inherits = ["_common", "docker-metadata-action"]
|
||||
target = "runtime-slim"
|
||||
args = {
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock"
|
||||
HEADROOM_EXTRAS = "proxy,code,bedrock,redis"
|
||||
RUNTIME_USER = "nonroot"
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -51,12 +51,12 @@ export default function Layout({ children }: LayoutProps<'/'>) {
|
||||
<body className="flex flex-col min-h-screen">
|
||||
<RootProvider>
|
||||
{/* Dismissible; Fumadocs remembers the dismissal per id in localStorage. */}
|
||||
<Banner id="teams-perk-2026">
|
||||
<Banner id="teams-contact-2026">
|
||||
<a
|
||||
href="https://headroom-perks.vercel.app/?source=github&utm_source=docs&utm_medium=banner"
|
||||
href="https://headroomlabs.ai"
|
||||
className="font-medium underline-offset-4 hover:underline"
|
||||
>
|
||||
Headroom for Teams: 3 months free with a 12-month plan →
|
||||
Want more savings? Headroom for teams and enterprises: talk to us →
|
||||
</a>
|
||||
</Banner>
|
||||
{children}
|
||||
|
||||
@@ -41,26 +41,19 @@ Every call to `client.messages.create()` compresses messages first. The response
|
||||
|
||||
`withHeadroom()` returns a proxy around your Anthropic client that intercepts `messages.create()`:
|
||||
|
||||
1. Converts Anthropic-format messages to OpenAI format
|
||||
2. Sends them to the Headroom proxy's `/v1/compress` endpoint
|
||||
3. Converts the compressed messages back to Anthropic format
|
||||
4. Forwards the request to Anthropic as normal
|
||||
1. Sends your `messages` to the Headroom proxy's `/v1/compress` endpoint as they are, in Anthropic format
|
||||
2. Forwards the request to Anthropic with the compressed messages in place
|
||||
|
||||
### Message format conversion
|
||||
### Message format
|
||||
|
||||
The adapter handles the full Anthropic message format including content blocks:
|
||||
Nothing is converted. `POST /v1/compress` compresses Anthropic content blocks natively — see [Message format](/docs/proxy#message-format) — and returns the same shape it received, so every block reaches Anthropic exactly as you built it:
|
||||
|
||||
| Anthropic format | OpenAI format |
|
||||
|-----------------|---------------|
|
||||
| `{ type: "text", text: "..." }` | `{ role: "user", content: "..." }` |
|
||||
| `{ type: "tool_use", id, name, input }` | `{ tool_calls: [{ id, function: { name, arguments } }] }` |
|
||||
| `{ type: "tool_result", tool_use_id, content }` | `{ role: "tool", tool_call_id, content }` |
|
||||
- `thinking` and `redacted_thinking` blocks keep their `signature` and `data`, which Anthropic requires back unmodified within a tool-use turn
|
||||
- `cache_control` markers stay on the blocks that carry them
|
||||
- `tool_use` and `tool_result` blocks keep their ids; only the content of large `tool_result` blocks changes
|
||||
- images, documents, citations and server-tool blocks pass through untouched
|
||||
|
||||
This conversion is lossless. Your request and response behave identically to an unwrapped client.
|
||||
|
||||
<Callout type="info" title="The conversion is an SDK choice, not an API requirement">
|
||||
`POST /v1/compress` does no format conversion and compresses Anthropic content blocks natively — see [Message format](/docs/proxy#message-format). If you are calling the endpoint directly (from a gateway, or LiteLLM's `headroom` guardrail), send Anthropic-shaped messages as-is; you get the same shape back. Only this TypeScript adapter converts, because it normalises on OpenAI types internally.
|
||||
</Callout>
|
||||
If the proxy is unreachable and `fallback` is enabled, the original messages are forwarded unchanged.
|
||||
|
||||
## Options
|
||||
|
||||
|
||||
@@ -0,0 +1,124 @@
|
||||
---
|
||||
title: FastAPI
|
||||
description: Cut LLM input tokens in a FastAPI app. Compress tool outputs, logs and chat history with compress() before calling OpenAI or Anthropic. No API key needed to try it.
|
||||
---
|
||||
|
||||
Headroom's `compress()` is a function over your message list, so in FastAPI it
|
||||
is one call before your model call. This page builds a `POST /chat` endpoint
|
||||
that returns the compressed messages and token counts, then shows the one-line
|
||||
change that forwards them to a provider.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
pip install fastapi "uvicorn[standard]" headroom-ai
|
||||
```
|
||||
|
||||
No provider key, no model download. The base package is enough for
|
||||
`compress()`; `[ml]` adds the Kompress text model later if you want more
|
||||
savings on prose-heavy tool output.
|
||||
|
||||
## Endpoint
|
||||
|
||||
```python
|
||||
# main.py
|
||||
from typing import Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
from pydantic import BaseModel
|
||||
|
||||
from headroom import compress
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
|
||||
class ChatIn(BaseModel):
|
||||
messages: list[dict[str, Any]] # OpenAI or Anthropic message dicts
|
||||
model: str = "gpt-4o" # used for token counting and the context limit
|
||||
|
||||
|
||||
@app.post("/chat")
|
||||
def chat(body: ChatIn) -> dict[str, Any]:
|
||||
result = compress(body.messages, model=body.model)
|
||||
return {
|
||||
"messages": result.messages,
|
||||
"tokens_before": result.tokens_before,
|
||||
"tokens_after": result.tokens_after,
|
||||
"tokens_saved": result.tokens_saved,
|
||||
"transforms": result.transforms_applied,
|
||||
}
|
||||
```
|
||||
|
||||
Save it as `main.py` and run it with no network at all:
|
||||
|
||||
```bash
|
||||
HEADROOM_OFFLINE=1 uvicorn main:app --port 8000
|
||||
```
|
||||
|
||||
Then post a 3,000-line tool output with one error in it:
|
||||
|
||||
```bash
|
||||
python - <<'EOF_PY'
|
||||
import json, urllib.request
|
||||
log = "\n".join([f"INFO step {i} ok" for i in range(3000)] + ["FATAL disk full"])
|
||||
msgs = [
|
||||
{"role": "user", "content": "What failed?"},
|
||||
{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "logs", "arguments": "{}"}}]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": log},
|
||||
]
|
||||
req = urllib.request.Request("http://127.0.0.1:8000/chat", data=json.dumps({"messages": msgs}).encode(), headers={"content-type": "application/json"})
|
||||
out = json.load(urllib.request.urlopen(req))
|
||||
print(out["tokens_before"], "->", out["tokens_after"], "FATAL kept:", "FATAL" in json.dumps(out["messages"]))
|
||||
EOF_PY
|
||||
```
|
||||
|
||||
Expected output is on the order of `20000 -> 95 FATAL kept: True`: the
|
||||
repeated `INFO` lines collapse, the `FATAL` line survives.
|
||||
|
||||
## Forward to the provider
|
||||
|
||||
Replace `messages=body.messages` with `messages=result.messages` in your
|
||||
existing call. Nothing else changes:
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI()
|
||||
|
||||
|
||||
@app.post("/answer")
|
||||
def answer(body: ChatIn) -> dict[str, Any]:
|
||||
result = compress(body.messages, model=body.model)
|
||||
completion = client.chat.completions.create(model=body.model, messages=result.messages)
|
||||
return {
|
||||
"answer": completion.choices[0].message.content,
|
||||
"tokens_saved": result.tokens_saved,
|
||||
}
|
||||
```
|
||||
|
||||
For Anthropic, pass `result.messages` to `client.messages.create` the same way.
|
||||
|
||||
## What it keeps
|
||||
|
||||
Error lines, stack traces, first and last items, and statistical outliers in
|
||||
JSON arrays are kept. Repeated and low-information lines are dropped. Originals
|
||||
are cached locally; the model can ask for them back when it has the retrieval
|
||||
tool, which the [proxy](/docs/proxy) and the [MCP server](/docs/mcp) provide
|
||||
([CCR](/docs/ccr)). The plain `compress()` call above does not expose it.
|
||||
Compression applies to tool and assistant messages; user messages are left alone
|
||||
unless you pass `compress_user_messages=True`. Short prompts and prose compress
|
||||
very little; see [Limitations](/docs/limitations).
|
||||
|
||||
## Not this page
|
||||
|
||||
`headroom.integrations.asgi.CompressionMiddleware` is for FastAPI apps that
|
||||
*serve* an LLM-shaped endpoint (`/v1/messages`, `/v1/chat/completions`), such
|
||||
as a proxy you built. For an app that *calls* a provider, use `compress()` as
|
||||
above.
|
||||
|
||||
## Settings you will reach for
|
||||
|
||||
- `HEADROOM_OFFLINE=1`: no update check, no model or pricing-table downloads, no beacon. Use it in CI and sandboxes. See [Security model](/docs/security-model).
|
||||
- `compress(..., model_limit=N)`: tell Headroom the model's window is N tokens, so it compresses harder as the prompt approaches it. It never truncates history.
|
||||
- `compress(..., protect_recent=K)`: leave source code in the last K messages uncompressed.
|
||||
- [Configuration](/docs/configuration) has the rest.
|
||||
@@ -1,6 +1,6 @@
|
||||
---
|
||||
title: Introduction
|
||||
description: Headroom is the context optimization layer for LLM applications. Compress tool outputs, DB results, file reads, and RAG results before they reach the model. Same answers, fraction of the tokens.
|
||||
description: Headroom (headroom-ai) compresses tool outputs, logs, JSON, RAG results and chat history before each LLM call, to reduce input tokens and token cost and fit a context window. Python library (no API key), proxy and MCP server. Same answers, fewer tokens.
|
||||
---
|
||||
|
||||
Headroom compresses everything your AI agent reads -- tool outputs, database results, file reads, RAG retrievals, API responses -- before it reaches the LLM. The model sees less noise, responds faster, and costs less.
|
||||
|
||||
@@ -116,11 +116,12 @@ pip install "headroom-ai[all]"
|
||||
| `autogen` | AutoGen AgentChat integration; **not in `[all]`** | `pip install "headroom-ai[autogen]"` |
|
||||
| `anyllm` | any-llm multi-provider backend (Python 3.11+); **not in `[all]`** | `pip install "headroom-ai[anyllm]"` |
|
||||
| `bedrock` | Native AWS Bedrock credentials/backend support; **not in `[all]`** | `pip install "headroom-ai[bedrock]"` |
|
||||
| `redis` | Redis client for state shared across replicas (`HEADROOM_CCR_BACKEND=redis`, `HEADROOM_COMPRESS_SESSION_STORE`); **not in `[all]`**, included in every Docker image | `pip install "headroom-ai[redis]"` |
|
||||
| `evals` | Evaluation framework (GSM8K, SQuAD, BFCL benchmarks) | `pip install "headroom-ai[evals]"` |
|
||||
| `pytorch-mps` | Apple-GPU (MPS) memory-embedder offload — **macOS only**, not in `[all]` (torch + sentence-transformers); opt in with `HEADROOM_EMBEDDER_RUNTIME=pytorch_mps` | `pip install "headroom-ai[pytorch-mps]"` |
|
||||
| `html` | HTML main-content extraction with trafilatura | `pip install "headroom-ai[html]"` |
|
||||
| `spreadsheet` | `.xlsx` / `.xls` ingestion with openpyxl and xlrd | `pip install "headroom-ai[spreadsheet]"` |
|
||||
| `sandbox` | Torch-free proxy bundle for constrained environments | `pip install "headroom-ai[sandbox]"` |
|
||||
| `lean` | Torch-free proxy bundle for constrained environments (a dependency profile; it provides no isolation). `sandbox` is a deprecated alias | `pip install "headroom-ai[lean]"` |
|
||||
| `all` | Runtime bundle: `proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,mcp,spreadsheet` | `pip install "headroom-ai[all]"` |
|
||||
|
||||
`[all]` is the full built-in runtime bundle, not every integration or optional
|
||||
@@ -255,15 +256,15 @@ If you want a host `headroom` CLI that keeps Headroom itself inside a container
|
||||
|
||||
| Tag | Extras | Base image | Description |
|
||||
|---|---|---|---|
|
||||
| `latest` | `proxy,bedrock` | Debian slim | Default image, runs as non-root |
|
||||
| `<version>` | `proxy,bedrock` | Debian slim | Pinned release, same default variant |
|
||||
| `nonroot` | `proxy,bedrock` | Debian slim | Explicit non-root variant |
|
||||
| `code` | `proxy,code,bedrock` | Debian slim | Includes tree-sitter for code compression |
|
||||
| `code-nonroot` | `proxy,code,bedrock` | Debian slim | Code compression, non-root |
|
||||
| `slim` | `proxy,bedrock` | Distroless | Minimal image, no shell |
|
||||
| `slim-nonroot` | `proxy,bedrock` | Distroless | Minimal, non-root |
|
||||
| `code-slim` | `proxy,code,bedrock` | Distroless | Code compression, minimal |
|
||||
| `code-slim-nonroot` | `proxy,code,bedrock` | Distroless | Code compression, minimal, non-root |
|
||||
| `latest` | `proxy,bedrock,redis` | Debian slim | Default image, runs as non-root |
|
||||
| `<version>` | `proxy,bedrock,redis` | Debian slim | Pinned release, same default variant |
|
||||
| `nonroot` | `proxy,bedrock,redis` | Debian slim | Explicit non-root variant |
|
||||
| `code` | `proxy,code,bedrock,redis` | Debian slim | Includes tree-sitter for code compression |
|
||||
| `code-nonroot` | `proxy,code,bedrock,redis` | Debian slim | Code compression, non-root |
|
||||
| `slim` | `proxy,bedrock,redis` | Distroless | Minimal image, no shell |
|
||||
| `slim-nonroot` | `proxy,bedrock,redis` | Distroless | Minimal, non-root |
|
||||
| `code-slim` | `proxy,code,bedrock,redis` | Distroless | Code compression, minimal |
|
||||
| `code-slim-nonroot` | `proxy,code,bedrock,redis` | Distroless | Code compression, minimal, non-root |
|
||||
|
||||
### Build from source
|
||||
|
||||
|
||||
@@ -28,6 +28,7 @@
|
||||
"corporate-networks",
|
||||
"local-llm-prefill",
|
||||
"---Integrations---",
|
||||
"fastapi",
|
||||
"vercel-ai-sdk",
|
||||
"openai-sdk",
|
||||
"anthropic-sdk",
|
||||
|
||||
+25
-12
@@ -112,10 +112,10 @@ headroom proxy --mode cache
|
||||
|
||||
| Profile | Target savings | Mode | Notes |
|
||||
|---------|----------------|------|-------|
|
||||
| `coding` | emergent (~50%) | `cache` | **Default.** Delta-only compression at ~0 prefix-cache busts; never lossy-compresses file reads. |
|
||||
| `balanced` | ~70% | `token` | Moderate compression with structural compaction. |
|
||||
| `agent-90` | ~90% | `token` | Aggressive; pins a `0.10` keep-ratio and forces Kompress. |
|
||||
| `general` | emergent (~60%) | `token` | Non-coding workloads. |
|
||||
| `coding` | ~50% of new input tokens (emergent) | `cache` | **Default.** Delta-only compression at ~0 prefix-cache busts; never lossy-compresses file reads. Whole-request and lifetime ratios are much lower in cache mode. |
|
||||
| `balanced` | ~70% of new input tokens | `token` | Moderate compression with structural compaction. |
|
||||
| `agent-90` | ~90% of new input tokens | `token` | Aggressive; pins a `0.10` keep-ratio and forces Kompress. |
|
||||
| `general` | ~60% of new input tokens (emergent) | `token` | Non-coding workloads. |
|
||||
|
||||
An unrecognized `HEADROOM_SAVINGS_PROFILE` value logs a warning and falls back to `coding` (the same default an unset variable resolves to) — the proxy never fails to start over a bad profile name. `balanced` is used only as a last-resort fallback when the runtime has no `coding` entry at all (old-runtime/new-client version skew). See `headroom/agent_savings.py` for each profile's full set of knobs.
|
||||
|
||||
@@ -194,10 +194,10 @@ HEADROOM_SAVINGS_PROFILE=agent-90 headroom proxy
|
||||
|
||||
| Profile | Target savings | proxy_mode | force_kompress | Best for |
|
||||
|---------|---------------|------------|---------------|----------|
|
||||
| `coding` (default) | ~50% (emergent) | `cache` | No | Coding agents — preserves Anthropic prefix-cache stability |
|
||||
| `agent-90` | 90% | `token` | Yes | Non-coding, cost-sensitive, or high-volume workloads |
|
||||
| `balanced` | 70% | `token` | No | General-purpose moderate compression |
|
||||
| `general` | ~60% (emergent) | `token` | No | Non-coding chat, little code in context |
|
||||
| `coding` (default) | ~50% of new input tokens (emergent) | `cache` | No | Coding agents — preserves Anthropic prefix-cache stability. Whole-request and lifetime ratios are much lower in cache mode. |
|
||||
| `agent-90` | ~90% of new input tokens | `token` | Yes | Non-coding, cost-sensitive, or high-volume workloads |
|
||||
| `balanced` | ~70% of new input tokens | `token` | No | General-purpose moderate compression |
|
||||
| `general` | ~60% of new input tokens (emergent) | `token` | No | Non-coding chat, little code in context |
|
||||
|
||||
**`coding` (default)** — Optimizes for coding-agent workloads with Anthropic. Uses **cache mode** (`proxy_mode="cache"`): compresses only the newest delta in each turn so the provider's prefix-cache is never busted. User messages are compressed, system prompts preserved (hottest cache). `protect_recent` is `0` — in cache mode the delta already *is* the newest turn, so a turn-count guard would suppress the only thing cache mode compresses; byte-exact fidelity instead comes from `protect_reads=True` (file reads are never lossy-compressed). Lossless-first with lossy fallback; tool search and cross-turn dedup enabled. This is the profile that `headroom wrap` uses.
|
||||
|
||||
@@ -320,6 +320,7 @@ Fine-grained control over what gets compressed and how hard. Most users pick a [
|
||||
| `--mode` / `HEADROOM_MODE` | `cache` | `cache` compresses only the newest delta (prefix-cache safe); `token` maximizes removal. |
|
||||
| `--target-ratio` / `HEADROOM_TARGET_RATIO` | unset | Keep-ratio for ML text compression; lower = more aggressive (e.g. `0.10`). |
|
||||
| `HEADROOM_MIN_TOKENS` | `500` | Minimum block size before a tool output is compressed. |
|
||||
| `HEADROOM_SHELL_RESULT_MIN_CHARS` | `2000` | Shell tool results shorter than this many characters bypass lossy compression. For top-level text blocks, the threshold uses their combined text length. Byte-lossless folds such as search grouping may still apply. Applies to `bash`, `shell`, `local_shell`, and aliases such as `run_shell_command`/`execute_command`; set `0` to restore lossy compression. |
|
||||
| `HEADROOM_COMPRESS_USER_MESSAGES` | profile | Compress text inside user-role messages. The `coding` profile turns this on; set `0` to keep user text verbatim under it. Anthropic `tool_result` blocks compress either way. No `--compress-user-messages` flag on `headroom proxy`; the direct module entry point (`python -m headroom.proxy.server`) does accept one. |
|
||||
| `HEADROOM_COMPRESS_SYSTEM_MESSAGES` | unset | Compress system prompts. Off by default to keep the hottest cache prefix stable. |
|
||||
| `HEADROOM_PROTECT_RECENT` | profile | Never compress the N most recent turns. |
|
||||
@@ -652,7 +653,7 @@ Keep carrying both yourself and send them upstream unchanged. Two consequences w
|
||||
| `transforms_applied` | array | Transform labels that ran. |
|
||||
| `transforms_summary` | object | Per-transform counts. |
|
||||
| `ccr_hashes` | array | Retrieval hashes for markers inserted (empty unless `mode: "ccr"`). |
|
||||
| `session` | object | [Session mode](#session-mode) only: `{ "id", "frozen_message_count", "cached_prefix_replayed" }`. |
|
||||
| `session` | object | [Session mode](#session-mode) only: `{ "id", "frozen_message_count", "cached_prefix_replayed" }`, described there. |
|
||||
|
||||
```json
|
||||
{
|
||||
@@ -741,15 +742,27 @@ Pass `config.session_id` and Headroom keeps the conversation's replay state itse
|
||||
The response gains a `session` object:
|
||||
|
||||
```json
|
||||
"session": { "id": "conv-8f1c", "frozen_message_count": 12, "cached_prefix_replayed": true }
|
||||
"session": { "id": "conv-8f1c", "frozen_message_count": 12, "cached_prefix_replayed": false }
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
| --- | --- | --- |
|
||||
| `id` | string | The session id this turn ran under: `config.session_id` as sent, or the `x-headroom-session-id` header when `HEADROOM_COMPRESS_SESSION_FROM_HEADER=1` and `config.session_id` is absent. |
|
||||
| `frozen_message_count` | integer | How many leading messages the pipeline was told not to recompress this turn. Headroom counts the leading run it can replay from the session's compression cache: plain messages always count, a tool result counts only once its content is in that cache, and the last message never counts. An explicit `config.frozen_message_count` wins when larger. It is derived from Headroom's own state, not from the provider, and it can be non-zero on a first turn. |
|
||||
| `cached_prefix_replayed` | boolean | `true` when the pipeline's output for some leading messages differed from what this endpoint returned for the session last turn, and Headroom overwrote those messages with last turn's bytes. `false` when nothing was overwritten: when the serving process holds no previous turn for the session (a first turn, expired state, or a replica that never served or loaded it), when the pipeline already produced last turn's bytes (the usual case), or when the replay was skipped as unsafe (this turn's messages do not line up with last turn's, or the replay would be larger than the pipeline's output). So `false` does not mean the returned prefix matches last turn's. |
|
||||
|
||||
<Callout type="warning" title="cached_prefix_replayed is not a prompt-cache hit">
|
||||
`cached_prefix_replayed` reports Headroom's own repair step. It says nothing about whether the provider served the prefix from its prompt cache. `false` is the normal value on a healthy session, and it is also what a turn that changed the prefix returns when there was nothing to replay. Read cache hits and misses from the provider's usage (`cache_read_input_tokens`, `prompt_tokens_details.cached_tokens`).
|
||||
</Callout>
|
||||
|
||||
What differs from the stateless contract:
|
||||
|
||||
- **Forward the returned messages verbatim.** They are what the next turn replays.
|
||||
- **`compress_user_messages` is refused** (`400`). User-message rewrites are not byte-replayable across turns, so they would bust the prefix later.
|
||||
- **Timeouts and busy sessions are a `503`, never fail-open.** Handing back originals would leave the caller forwarding bytes the replay state does not know about, so a compression timeout answers `503` with `error.type = "compression_timeout"`; retry the turn and it lands on consistent state. Turns for one session are serialized under a per-session lock, and a concurrent turn for the same session that waits more than 10 seconds gets the same `503`.
|
||||
- **State is in-process.** The replay cache lives for `HEADROOM_COMPRESSION_CACHE_TTL_SECONDS` of idle time (3900 by default, floor 600); the tracker's usage state expires after 10 idle minutes. Multi-process deployments (gunicorn workers, replicas) must pin a session to one process — hash the upstream on the session id — or replay silently misses.
|
||||
- **State is in-process by default.** The replay cache lives for `HEADROOM_COMPRESSION_CACHE_TTL_SECONDS` of idle time (3900 by default, floor 600); the tracker's usage state expires after 10 idle minutes. Without a shared store, multi-process deployments (gunicorn workers, replicas) must pin a session to one process — hash the upstream on the session id — or replay silently misses.
|
||||
- **`HEADROOM_COMPRESS_SESSION_STORE=redis://...` shares sessions across replicas.** Every replica also keeps each session's tracker in Redis, so any replica can serve the session's next turn and its `POST /v1/usage`. It needs the `redis` extra, which every Docker image includes. It also selects the Redis CCR store unless `HEADROOM_CCR_BACKEND` names another, so CCR markers resolve on any replica. Two limits remain. The per-session lock is per process: two concurrent turns of one session on two replicas both run, the first write wins (compare-and-set), and the next turn continues from the winner. Gateway-mode pending turns stay in-process, so `POST /v1/compress/response` must reach the replica that answered the request half; re-drive therefore still needs real affinity (`gateway.session_affinity`).
|
||||
- **An unreachable Redis does not fail turns.** A Redis error is a miss: the replica serves from its own memory, logs one `WARNING` when a kind of call (read, write) starts failing and an `INFO` when it works again, and skips Redis for 5 seconds after a connection error (each command times out after 1 second). The cost is the sharing. A replica with no local state for the session recompresses the history, which is slower, and the prefix it returns can differ from last turn's, which is a provider prompt-cache miss. `cached_prefix_replayed` stays `false` on those turns. A Redis that comes back empty is refilled as each session's next turn writes it whole. Run Redis with high availability if prompt-cache hits matter.
|
||||
- `HEADROOM_COMPRESS_SESSION_FROM_HEADER=1` lets the `x-headroom-session-id` header stand in for `config.session_id`. Off by default: gateways commonly stamp that header on all traffic, and a value shared across conversations (Claude Code subagents do this) would blend them into one replay state.
|
||||
|
||||
**Relaying provider usage: `POST /v1/usage`.** Session mode never sees the provider's response, so cache-hit attribution and savings accounting for the session come from this optional relay (same exposure rules as `/v1/compress`). Post the provider's `usage` block under the session id:
|
||||
@@ -779,7 +792,7 @@ The rule behind it is the proxy's own **no shrink without reload**, moved to the
|
||||
| --- | --- | --- | --- |
|
||||
| `can_redrive` | boolean | `false` | The gateway can call the provider again with a request Headroom hands it and post the result back. Unlocks re-driving turn hooks and, with `config.mode: "ccr"`, in-band CCR retrieval. Send `false` for streaming requests. |
|
||||
| `can_relay_response` | boolean | `false` | The gateway will post the provider's status and usage to `/v1/compress/response` after every turn. |
|
||||
| `session_affinity` | boolean | `true` | The gateway pins this session to one Headroom replica. Pending turns are in-process state, so `false` disables re-driving for the turn even when `can_redrive` is set. |
|
||||
| `session_affinity` | boolean | `true` | The gateway pins this session to one Headroom replica. Pending turns are in-process state, so `false` disables re-driving for the turn even when `can_redrive` is set. `HEADROOM_COMPRESS_SESSION_STORE` shares replay state, not pending turns, so it does not lift this. |
|
||||
| `plugin_version` | string | unset | Diagnostics only. |
|
||||
| `request_headers` | object | `{}` | The client's request headers the gateway is willing to hand over, as lowercase name → string value. Only `anthropic-beta` is read today: when a turn hook asks for an `anthropic-beta` token, the client's tokens go first in the returned `headers`; every other name is ignored, so a gateway may pass its whole header map. A non-object block or a non-string value is a `400`. Never forwarded anywhere by Headroom. |
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ This guide gets you from zero to compressed LLM calls in under 5 minutes.
|
||||
|
||||
## 1. Install
|
||||
|
||||
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
|
||||
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
|
||||
<Tab value="TypeScript">
|
||||
```bash
|
||||
npm install headroom-ai
|
||||
@@ -15,12 +15,15 @@ npm install headroom-ai
|
||||
</Tab>
|
||||
<Tab value="Python">
|
||||
```bash
|
||||
# CLI/proxy/wrap on your machine
|
||||
uv tool install --python 3.13 "headroom-ai[all]"
|
||||
# Python project or virtualenv: enough for compress(), no API key, no model download
|
||||
pip install headroom-ai
|
||||
|
||||
# Python project or virtualenv
|
||||
pip install "headroom-ai[all]"
|
||||
# CLI, proxy and `headroom wrap` on your machine
|
||||
uv tool install --python 3.13 "headroom-ai[all]"
|
||||
```
|
||||
|
||||
`[all]` adds the proxy, the MCP server, the Kompress model and more; `compress()`
|
||||
needs only the base package.
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
@@ -38,7 +41,7 @@ The proxy runs the compression pipeline (Python) and exposes an HTTP API that th
|
||||
|
||||
## 2. Compress messages
|
||||
|
||||
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
|
||||
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
|
||||
<Tab value="TypeScript">
|
||||
```ts twoslash
|
||||
import { compress } from 'headroom-ai';
|
||||
@@ -110,11 +113,22 @@ result = compress(messages, model="gpt-4o")
|
||||
</Tab>
|
||||
</Tabs>
|
||||
|
||||
<Callout type="info" title="No provider key needed to try it">
|
||||
The Python call runs in-process. `HEADROOM_OFFLINE=1` makes it zero-network as
|
||||
well (no update check, no model or pricing-table downloads):
|
||||
|
||||
```bash
|
||||
HEADROOM_OFFLINE=1 python -c "from headroom import compress; log='\n'.join(f'INFO step {i} ok' for i in range(3000))+'\nFATAL disk full'; r=compress([{'role':'user','content':'What failed?'},{'role':'assistant','content':None,'tool_calls':[{'id':'c1','type':'function','function':{'name':'logs','arguments':'{}'}}]},{'role':'tool','tool_call_id':'c1','content':log}], model='gpt-4o'); print(r.tokens_before, '->', r.tokens_after, 'FATAL kept:', 'FATAL' in str(r.messages))"
|
||||
```
|
||||
|
||||
A [FastAPI endpoint](/docs/fastapi) built on the same call is one page away.
|
||||
</Callout>
|
||||
|
||||
## 3. Send to your LLM
|
||||
|
||||
Use the compressed messages exactly like the originals:
|
||||
|
||||
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
|
||||
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
|
||||
<Tab value="TypeScript">
|
||||
```ts twoslash
|
||||
import OpenAI from 'openai';
|
||||
@@ -149,7 +163,7 @@ print(response.choices[0].message.content)
|
||||
|
||||
## 4. Check your savings
|
||||
|
||||
<Tabs groupId="lang" items={['TypeScript', 'Python']}>
|
||||
<Tabs groupId="lang" items={['Python', 'TypeScript']}>
|
||||
<Tab value="TypeScript">
|
||||
```ts twoslash
|
||||
const result = {
|
||||
|
||||
@@ -284,6 +284,7 @@ accept arbitrary environment variables — writes are restricted to a hardcoded
|
||||
| `HEADROOM_INTERCEPT_READ_MIN_CHARS` | ast-grep read-interception threshold |
|
||||
| `HEADROOM_VERIFY_TRUNCATION_ON_DISK` | Verify ast-grep Read truncation against disk when no client banner is found. Always returns `UNKNOWN` on Windows, where `dir_fd` / `O_NOFOLLOW` support is unavailable. |
|
||||
| `HEADROOM_VERIFY_TRUNCATION_MAX_BYTES` | Max on-disk file size (bytes) the disk-verify fallback will read |
|
||||
| `HEADROOM_DEFERRED_TOOL_AFFINITY` | Keep `defer_loading` tools out of the Anthropic prefix-tracker lineage key (default on; `0` counts them again) |
|
||||
| `HEADROOM_WEB_SEARCH_HISTORY` | Drop `web_search_call` history from a Responses request that declares no web_search tool (default on; `0` disables) |
|
||||
| `HEADROOM_BUFFERED_UPSTREAM_STREAM` | Send buffered CCR requests (and their continuations) upstream as `stream: true` and fold the reply (default on; `0` sends `stream: false`) |
|
||||
|
||||
|
||||
+11
-19
@@ -1,27 +1,19 @@
|
||||
---
|
||||
title: Headroom for Teams
|
||||
description: Headroom Enterprise for engineering teams — more savings, automatic model routing, attack removal, SSO and an org-wide savings dashboard. 3 months free with a 12-month plan.
|
||||
description: Not seeing the savings you want? Headroom Enterprise adds more cost savings, tokens secured, on premise deployment and deep attribution. Talk to us.
|
||||
---
|
||||
|
||||
Headroom OSS is built for one developer on one laptop. Headroom Enterprise is built
|
||||
for the whole team.
|
||||
**Not seeing the savings you want?** Headroom OSS is a framework. It was built to show
|
||||
what's possible on one developer's laptop, so it saves less than it could. We are
|
||||
building bigger savings for teams and enterprises in Headroom Enterprise, which adds:
|
||||
|
||||
| | Open source | Enterprise |
|
||||
|---|---|---|
|
||||
| Savings | Compression, plus tool search on Anthropic requests (on by default) | Also: tool search for every model, harness tuning, extra compressors |
|
||||
| Model routing | Rule-based and opt-in: turn it on with `HEADROOM_MODEL_ROUTER_ENABLED=1` and write the rules in [`HEADROOM_MODEL_ROUTES`](/docs/configuration#cost-aware-model-routing) | Automatic: a cheaper model per request when quality allows, with a shadow mode that shows the savings before you turn it on |
|
||||
| Security | — | Detect and strip prompt injection and risky tool calls |
|
||||
| Rollout | Per laptop | Managed or in your VPC, SSO, central config, org-wide savings |
|
||||
| Support | GitHub, Discord | Dedicated engineers, onboarding, priority response |
|
||||
- More cost savings
|
||||
- Tokens secured
|
||||
- On premise deployment
|
||||
- Deep attribution
|
||||
- And many more features
|
||||
|
||||
## 3 months free
|
||||
|
||||
Companies that sign a 12-month Enterprise plan get their first 3 months free. One
|
||||
perk per company.
|
||||
|
||||
**[Claim 3 months free →](https://headroom-perks.vercel.app/?source=github&utm_source=docs&utm_medium=teams-page)**
|
||||
|
||||
Prefer to talk first? Email [hello@headroomlabs.ai](mailto:hello@headroomlabs.ai)
|
||||
with your stack and rough monthly LLM spend.
|
||||
If you're a team or an enterprise, **talk to us**:
|
||||
[hello@headroomlabs.ai](mailto:hello@headroomlabs.ai) · [headroomlabs.ai](https://headroomlabs.ai)
|
||||
|
||||
Everything in the open-source repo stays open source under Apache 2.0.
|
||||
|
||||
+61
-8
@@ -22,6 +22,8 @@ from ``e2e/init/Dockerfile``).
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -149,6 +151,59 @@ def _verify_claude_local(ctx: CaseContext) -> None:
|
||||
raise AssertionError(f"Unexpected Claude install commands: {claude_calls}")
|
||||
|
||||
|
||||
def _expect_shell_exports(shell_file: Path, expected: dict[str, str]) -> None:
|
||||
"""Check exported values in a child process after sourcing an rc file.
|
||||
|
||||
Args:
|
||||
shell_file: Scratch shell configuration written by the init case.
|
||||
expected: Environment variables that init must export verbatim.
|
||||
|
||||
Raises:
|
||||
AssertionError: If sourcing fails or the exported values differ.
|
||||
"""
|
||||
# Clear inherited values so missing exports cannot accidentally pass. A
|
||||
# separate Python child also distinguishes assignments from real exports.
|
||||
env = {
|
||||
name: value
|
||||
for name, value in os.environ.items()
|
||||
if name not in expected and name not in {"BASH_ENV", "ENV"}
|
||||
}
|
||||
probe = (
|
||||
"import json, os, sys; "
|
||||
"print(json.dumps({name: os.environ.get(name) for name in sys.argv[1:]}))"
|
||||
)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
"--noprofile",
|
||||
"--norc",
|
||||
"-c",
|
||||
'. "$1" && "$2" -c "$3" "${@:4}"',
|
||||
"headroom-env-check",
|
||||
str(shell_file),
|
||||
sys.executable,
|
||||
probe,
|
||||
*expected,
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
timeout=10,
|
||||
)
|
||||
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as exc:
|
||||
raise AssertionError(f"{shell_file.name} could not be sourced successfully") from exc
|
||||
try:
|
||||
actual = json.loads(result.stdout)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise AssertionError(
|
||||
f"{shell_file.name} did not produce exported environment values"
|
||||
) from exc
|
||||
if actual != expected:
|
||||
raise AssertionError(f"{shell_file.name} exported {actual!r}; expected {expected!r}")
|
||||
|
||||
|
||||
def _verify_copilot_global(ctx: CaseContext) -> None:
|
||||
config = json.loads((ctx.home / ".copilot" / "config.json").read_text(encoding="utf-8"))
|
||||
if "SessionStart" not in config["hooks"]:
|
||||
@@ -158,15 +213,13 @@ def _verify_copilot_global(ctx: CaseContext) -> None:
|
||||
session_start = config["hooks"]["SessionStart"][0]["command"]
|
||||
_expect_hook_command(session_start, "init-user")
|
||||
|
||||
expected_env = {
|
||||
"COPILOT_PROVIDER_TYPE": "openai",
|
||||
"COPILOT_PROVIDER_BASE_URL": "http://127.0.0.1:9005/v1",
|
||||
"COPILOT_PROVIDER_WIRE_API": "completions",
|
||||
}
|
||||
for shell_file in (ctx.home / ".bashrc", ctx.home / ".zshrc", ctx.home / ".profile"):
|
||||
content = shell_file.read_text(encoding="utf-8")
|
||||
for literal in (
|
||||
'export COPILOT_PROVIDER_TYPE="openai"',
|
||||
'export COPILOT_PROVIDER_BASE_URL="http://127.0.0.1:9005/v1"',
|
||||
'export COPILOT_PROVIDER_WIRE_API="completions"',
|
||||
):
|
||||
if literal not in content:
|
||||
raise AssertionError(f"{shell_file.name} missing {literal!r}")
|
||||
_expect_shell_exports(shell_file, expected_env)
|
||||
|
||||
copilot_calls = [
|
||||
record["argv"] for record in _read_jsonl(ctx.shim_log) if record["tool"] == "copilot"
|
||||
|
||||
@@ -93,6 +93,10 @@ RUN npm install -g --no-fund --no-audit npm@11.16.0
|
||||
COPY e2e/wrap/tools/package.json e2e/wrap/tools/package-lock.json /opt/wrap-tools/
|
||||
RUN npm ci --prefix /opt/wrap-tools --no-fund --no-audit
|
||||
|
||||
# Exercise the installed dependency fixes in both existing wrap-E2E CI callers.
|
||||
COPY e2e/wrap/tools/dependency-regression.test.cjs /opt/wrap-tools/
|
||||
RUN npm test --prefix /opt/wrap-tools
|
||||
|
||||
WORKDIR /workspace
|
||||
|
||||
# Bring in the prebuilt manylinux wheel from stage 1.
|
||||
|
||||
@@ -10,17 +10,25 @@ PATH. `npm ci` does not support global installs, so diagnostics must also use
|
||||
`npm list --prefix /opt/wrap-tools`.
|
||||
|
||||
After editing a CLI version in `package.json`, regenerate the lockfile using
|
||||
Node 22 and npm 11.16.0 on Linux:
|
||||
Node 24.22.1+ within the Node 24 line and npm 11.16.0 on Linux (the pinned
|
||||
OpenClaw version no longer supports Node 22):
|
||||
|
||||
```sh
|
||||
cd e2e/wrap/tools
|
||||
npm install --package-lock-only --no-fund --no-audit
|
||||
npm ci --no-fund --no-audit
|
||||
npm test
|
||||
cd ../../..
|
||||
docker build -f e2e/wrap/Dockerfile -t headroom-wrap-e2e .
|
||||
docker run --rm headroom-wrap-e2e
|
||||
```
|
||||
|
||||
`npm test` runs the Ajv URI and Express body-parser regression checks against
|
||||
the installed locked dependencies. The Dockerfile runs the same command from
|
||||
`/opt/wrap-tools` after `npm ci`, so failures stop the image build in both the
|
||||
existing Wrap E2E workflow and CI's Docker-native wrap E2E job. Their path
|
||||
filters include `e2e/**`, covering dependency and regression-test updates.
|
||||
|
||||
To intentionally refresh transitive versions without changing the pinned CLI
|
||||
versions, use `npm update --package-lock-only --no-fund --no-audit` in this
|
||||
directory instead of `npm install --package-lock-only`, then run the same
|
||||
@@ -34,4 +42,11 @@ to make a build pass.
|
||||
These are existing test tools, not new product dependencies. Codex is maintained
|
||||
by OpenAI; OpenClaw and its transitive dependencies retain their upstream
|
||||
maintainers. Their install surface includes registry downloads, native optional
|
||||
packages and lifecycle scripts, as before. Top-level CLI versions are unchanged.
|
||||
packages and lifecycle scripts, as before. OpenClaw 2026.10.6 and its fixed
|
||||
Undici override follow the upstream #4453 security repair.
|
||||
|
||||
The raw audit still reports npm 12.2.1's bundled ip-address and Undici copies.
|
||||
`npm-audit-exceptions.json` records the upstream-approved E2E-only risk
|
||||
acceptance, restricted to exact advisory/path/version combinations and expiring
|
||||
on 2026-11-15. Security CI rejects other findings and expired or stale entries;
|
||||
these exceptions are not a claim that the contained packages are patched.
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
'use strict';
|
||||
// From e2e/wrap/tools after installing its locked dependencies:
|
||||
// node --test --test-concurrency=1 --test-reporter=tap dependency-regression.test.cjs
|
||||
const path = require('node:path');
|
||||
const { createRequire } = require('node:module');
|
||||
const fromTools = createRequire(path.join(process.cwd(), 'package.json'));
|
||||
|
||||
if (process.argv[2] === '--serve') {
|
||||
const express = fromTools('express');
|
||||
const app = express();
|
||||
app.get('/health', (_req, res) => res.json({ ok: true }));
|
||||
app.post('/text', express.text({ verify() {} }), (req, res) => res.json(req.body));
|
||||
app.use((error, _req, res, _next) => res.status(error.status || 500).json({ type: error.type }));
|
||||
const server = app.listen(0, '127.0.0.1', () => process.send({ port: server.address().port }));
|
||||
process.on('message', (message) => {
|
||||
if (message === 'stop') { server.close(() => process.exit(0)); server.closeAllConnections(); }
|
||||
});
|
||||
} else {
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const http = require('node:http');
|
||||
const { fork } = require('node:child_process');
|
||||
|
||||
test('Ajv rejects a percent-encoded scheme instead of aliasing an HTTPS schema', () => {
|
||||
const Ajv = fromTools('ajv');
|
||||
const ajv = new Ajv();
|
||||
ajv.addSchema({ $id: 'https://example.invalid/defs.json', type: 'integer' });
|
||||
const valid = ajv.compile({ $ref: 'https://example.invalid/defs.json' });
|
||||
assert.equal(valid(42), true);
|
||||
assert.equal(valid('42'), false);
|
||||
assert.throws(
|
||||
() => ajv.compile({ $ref: '%68ttps://example.invalid/defs.json' }),
|
||||
{ message: 'URI scheme is malformed.' },
|
||||
);
|
||||
});
|
||||
|
||||
test('unsupported charset with invalid gzip leaves the Express process healthy', async (t) => {
|
||||
const child = fork(__filename, ['--serve'], {
|
||||
cwd: process.cwd(), env: process.env, execArgv: [], silent: true,
|
||||
});
|
||||
let stderr = '';
|
||||
child.stdout.resume();
|
||||
child.stderr.on('data', (chunk) => { stderr += chunk; });
|
||||
t.after(async () => {
|
||||
t.diagnostic(JSON.stringify({ exitCode: child.exitCode, signal: child.signalCode, stderr }));
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
await new Promise((resolve) => {
|
||||
const timer = setTimeout(() => { child.kill(); resolve(); }, 1000);
|
||||
child.once('exit', () => { clearTimeout(timer); resolve(); });
|
||||
if (child.connected) child.send('stop'); else child.kill();
|
||||
});
|
||||
});
|
||||
const port = await new Promise((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error('server startup timeout')), 3000);
|
||||
child.once('message', (msg) => { clearTimeout(timer); resolve(msg.port); });
|
||||
child.once('error', (err) => { clearTimeout(timer); reject(err); });
|
||||
child.once('exit', (code) => { clearTimeout(timer); reject(new Error('server exited: ' + code + stderr)); });
|
||||
});
|
||||
function request(route, body, headers = {}, method = 'POST') {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = http.request({ hostname: '127.0.0.1', port, path: route, method, agent: false,
|
||||
headers: { ...headers, 'Content-Length': Buffer.byteLength(body) } }, (res) => {
|
||||
const chunks = [];
|
||||
res.on('data', (chunk) => chunks.push(chunk));
|
||||
res.on('error', reject);
|
||||
res.on('end', () => resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString() }));
|
||||
});
|
||||
req.setTimeout(2000, () => req.destroy(new Error('localhost request timeout')));
|
||||
req.on('error', reject);
|
||||
req.end(body);
|
||||
});
|
||||
}
|
||||
const response = await request('/text', 'not-gzip', {
|
||||
'Content-Type': 'text/plain; charset=not-a-real-charset', 'Content-Encoding': 'gzip',
|
||||
});
|
||||
assert.equal(response.status, 415);
|
||||
assert.equal(JSON.parse(response.body).type, 'charset.unsupported');
|
||||
// Let a pending zlib error surface before checking the child's health.
|
||||
await new Promise((resolve) => setTimeout(resolve, 80));
|
||||
assert.equal(child.exitCode, null, stderr);
|
||||
const health = await request('/health', '', {}, 'GET');
|
||||
assert.equal(health.status, 200);
|
||||
assert.deepEqual(JSON.parse(health.body), { ok: true });
|
||||
});
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"expires": "2026-11-15",
|
||||
"reason": "E2E harness only, never shipped in the wheel or SDK. npm 12.2.1 bundles undici 6.29.0 and ip-address 10.7.2; bundled copies ignore overrides and no newer npm release is available. OpenClaw is updated to 2026.10.6 and uses fixed Undici 8.12.3, so no OpenClaw path is excepted. Recheck npm upstream and remove these exact-path exceptions by the existing expiry.",
|
||||
"reason": "E2E harness only, never shipped in the wheel or SDK. npm 12.2.1 bundles undici 6.29.0 and ip-address 10.7.2; bundled copies ignore overrides and no newer npm release is available. OpenClaw is updated to 2026.10.6 and uses fixed Undici 8.12.3, so no OpenClaw path is excepted. Three further undici 6.x advisories (GHSA-2m5v-pch6-74hx, GHSA-qrxm-fqhp-jwvm, GHSA-9fm3-97px-rjf6) published after #4453 cover the same bundled copy. Recheck npm upstream and remove these exact-path exceptions by the existing expiry.",
|
||||
"advisories": {
|
||||
"GHSA-4qv8-2p9j-2685": {
|
||||
"node_modules/npm/node_modules/undici": "6.29.0"
|
||||
@@ -10,6 +10,15 @@
|
||||
},
|
||||
"GHSA-rxp7-6hxx-x8q5": {
|
||||
"node_modules/npm/node_modules/ip-address": "10.7.2"
|
||||
},
|
||||
"GHSA-2m5v-pch6-74hx": {
|
||||
"node_modules/npm/node_modules/undici": "6.29.0"
|
||||
},
|
||||
"GHSA-qrxm-fqhp-jwvm": {
|
||||
"node_modules/npm/node_modules/undici": "6.29.0"
|
||||
},
|
||||
"GHSA-9fm3-97px-rjf6": {
|
||||
"node_modules/npm/node_modules/undici": "6.29.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
"version": "1.0.0",
|
||||
"private": true,
|
||||
"description": "Locked CLI dependencies for the Docker wrap E2E harness",
|
||||
"scripts": {
|
||||
"test": "node --test --test-concurrency=1 --test-reporter=tap dependency-regression.test.cjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"@openai/codex": "0.163.0",
|
||||
"openclaw": "2026.10.6"
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
"""Actual SDK + loopback HTTP proof; no remote provider or model calls."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
scratch = tempfile.TemporaryDirectory(prefix="headroom-client-proof-")
|
||||
for key in ["HEADROOM_WORKSPACE_DIR", "HEADROOM_CONFIG_DIR", "XDG_CACHE_HOME"]:
|
||||
os.environ[key] = str(Path(scratch.name) / key.lower())
|
||||
os.environ.update(HEADROOM_OFFLINE="1", HEADROOM_BEACON="off", DO_NOT_TRACK="1")
|
||||
|
||||
import httpx
|
||||
from anthropic import Anthropic
|
||||
|
||||
from headroom import AnthropicProvider, HeadroomClient
|
||||
from headroom.config import HeadroomConfig
|
||||
|
||||
requests = []
|
||||
observed_styles = []
|
||||
|
||||
class Observer:
|
||||
def on_pipeline_event(self, event):
|
||||
if "api_style" in event.metadata:
|
||||
observed_styles.append(event.metadata["api_style"])
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
data = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
requests.append(data)
|
||||
if data.get("stream"):
|
||||
events = [
|
||||
(
|
||||
"message_start",
|
||||
{
|
||||
"type": "message_start",
|
||||
"message": {
|
||||
"id": "msg_fixture",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": data["model"],
|
||||
"content": [],
|
||||
"stop_reason": None,
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 0},
|
||||
},
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_start",
|
||||
{
|
||||
"type": "content_block_start",
|
||||
"index": 0,
|
||||
"content_block": {"type": "text", "text": ""},
|
||||
},
|
||||
),
|
||||
(
|
||||
"content_block_delta",
|
||||
{
|
||||
"type": "content_block_delta",
|
||||
"index": 0,
|
||||
"delta": {"type": "text_delta", "text": "local fixture"},
|
||||
},
|
||||
),
|
||||
("content_block_stop", {"type": "content_block_stop", "index": 0}),
|
||||
(
|
||||
"message_delta",
|
||||
{
|
||||
"type": "message_delta",
|
||||
"delta": {"stop_reason": "end_turn", "stop_sequence": None},
|
||||
"usage": {"output_tokens": 2},
|
||||
},
|
||||
),
|
||||
("message_stop", {"type": "message_stop"}),
|
||||
]
|
||||
payload = "".join(
|
||||
f"event: {name}\ndata: {json.dumps(event)}\n\n" for name, event in events
|
||||
).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/event-stream")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
return
|
||||
payload = json.dumps(
|
||||
{
|
||||
"id": "msg_fixture",
|
||||
"type": "message",
|
||||
"role": "assistant",
|
||||
"model": data["model"],
|
||||
"content": [{"type": "text", "text": "local fixture"}],
|
||||
"stop_reason": "end_turn",
|
||||
"stop_sequence": None,
|
||||
"usage": {"input_tokens": 1, "output_tokens": 2},
|
||||
}
|
||||
).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
responses = []
|
||||
http = httpx.Client(event_hooks={"response": [responses.append]})
|
||||
original = Anthropic(
|
||||
http_client=http,
|
||||
api_key="local-fixture",
|
||||
base_url=f"http://127.0.0.1:{server.server_port}",
|
||||
max_retries=0,
|
||||
)
|
||||
wrapped = HeadroomClient(
|
||||
original,
|
||||
AnthropicProvider(warn=False),
|
||||
store_url=f"sqlite:///{scratch.name}/metrics.db",
|
||||
enable_cache_optimizer=False,
|
||||
config=HeadroomConfig(
|
||||
store_url=f"sqlite:///{scratch.name}/metrics.db",
|
||||
discover_pipeline_extensions=False,
|
||||
pipeline_extensions=[Observer()],
|
||||
),
|
||||
)
|
||||
kwargs = {
|
||||
"model": "claude-fixture",
|
||||
"messages": [{"role": "user", "content": "hello"}],
|
||||
"max_tokens": 8,
|
||||
"stream": False,
|
||||
}
|
||||
try:
|
||||
native = original.messages.create(**kwargs)
|
||||
print(
|
||||
json.dumps(
|
||||
{"case": "native", "text": native.content[0].text, "requests": len(requests)}
|
||||
)
|
||||
)
|
||||
try:
|
||||
result = wrapped.messages.create(**kwargs)
|
||||
print(
|
||||
json.dumps(
|
||||
{"case": "wrapped", "text": result.content[0].text, "requests": len(requests)}
|
||||
)
|
||||
)
|
||||
kwargs["stream"] = True
|
||||
native_stream = original.messages.create(**kwargs)
|
||||
native_types = [event.type for event in native_stream]
|
||||
native_stream.close()
|
||||
wrapped_stream = wrapped.messages.create(**kwargs)
|
||||
wrapped_types = [event.type for event in wrapped_stream]
|
||||
wrapped_stream.close()
|
||||
assert native_types == wrapped_types
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"case": "create_stream",
|
||||
"native_types": native_types,
|
||||
"wrapped_types": wrapped_types,
|
||||
"native_closed": native_stream.response.is_closed,
|
||||
"requests": len(requests),
|
||||
}
|
||||
)
|
||||
)
|
||||
del kwargs["stream"]
|
||||
with wrapped.messages.stream(**kwargs) as manager:
|
||||
text = "".join(manager.text_stream)
|
||||
assert text == "local fixture"
|
||||
assert manager.response.is_closed
|
||||
assert all(response.is_closed for response in responses)
|
||||
assert observed_styles and set(observed_styles) == {"anthropic"}
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"case": "resources",
|
||||
"http_responses": len(responses),
|
||||
"all_closed": all(response.is_closed for response in responses),
|
||||
}
|
||||
)
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"case": "stream_manager",
|
||||
"text": text,
|
||||
"response_closed": manager.response.is_closed,
|
||||
"requests": len(requests),
|
||||
}
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"case": "wrapped",
|
||||
"error": type(exc).__name__,
|
||||
"message": str(exc),
|
||||
"requests": len(requests),
|
||||
}
|
||||
)
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
wrapped.close()
|
||||
original.close()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
scratch.cleanup()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,124 @@
|
||||
"""Actual OpenAI SDK HTTP requests: typed cache opt-out must preserve caller spaces."""
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main():
|
||||
with tempfile.TemporaryDirectory(prefix="headroom-cache-config-") as scratch:
|
||||
for key in ["HEADROOM_WORKSPACE_DIR", "HEADROOM_CONFIG_DIR", "XDG_CACHE_HOME"]:
|
||||
os.environ[key] = str(Path(scratch) / key.lower())
|
||||
# This isolated proof needs no persistent, process-global CCR database.
|
||||
# Keep the store in memory so Windows can remove the scratch directory.
|
||||
os.environ.update(
|
||||
HEADROOM_OFFLINE="1",
|
||||
HEADROOM_BEACON="off",
|
||||
DO_NOT_TRACK="1",
|
||||
HEADROOM_CCR_BACKEND="memory",
|
||||
)
|
||||
from openai import OpenAI
|
||||
|
||||
from headroom import HeadroomClient, OpenAIProvider
|
||||
from headroom.config import HeadroomConfig, HeadroomMode
|
||||
|
||||
requests = []
|
||||
violations = []
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_POST(self):
|
||||
raw = self.rfile.read(int(self.headers["Content-Length"]))
|
||||
body = json.loads(raw)
|
||||
wire_dir = os.environ.get("HEADROOM_PROOF_WIRE_DIR")
|
||||
if wire_dir:
|
||||
target = Path(wire_dir)
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
(target / f"request-{len(requests)}.json").write_bytes(raw)
|
||||
requests.append(body)
|
||||
payload = json.dumps(
|
||||
{
|
||||
"id": "chatcmpl_fixture",
|
||||
"object": "chat.completion",
|
||||
"created": 0,
|
||||
"model": body["model"],
|
||||
"choices": [
|
||||
{
|
||||
"index": 0,
|
||||
"message": {"role": "assistant", "content": "local fixture"},
|
||||
"finish_reason": "stop",
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 1, "completion_tokens": 2, "total_tokens": 3},
|
||||
}
|
||||
).encode()
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
|
||||
threading.Thread(target=server.serve_forever, daemon=True).start()
|
||||
original = OpenAI(
|
||||
api_key="local-fixture",
|
||||
base_url=f"http://127.0.0.1:{server.server_port}",
|
||||
max_retries=0,
|
||||
)
|
||||
try:
|
||||
for enabled in [False, True]:
|
||||
url = f"sqlite:///{scratch}/{enabled}.db"
|
||||
config = HeadroomConfig(
|
||||
store_url=url,
|
||||
default_mode=HeadroomMode.OPTIMIZE,
|
||||
discover_pipeline_extensions=False,
|
||||
)
|
||||
config.smart_crusher.enabled = False
|
||||
config.cache_aligner.enabled = False
|
||||
config.cache_optimizer.enabled = enabled
|
||||
client = HeadroomClient(original, OpenAIProvider(), store_url=url, config=config)
|
||||
try:
|
||||
response = client.chat.completions.create(
|
||||
model="gpt-4o",
|
||||
messages=[
|
||||
{"role": "system", "content": "Keep exact spaces.\n"},
|
||||
{"role": "user", "content": "Hello."},
|
||||
],
|
||||
max_tokens=8,
|
||||
)
|
||||
wire_system = requests[-1]["messages"][0]["content"]
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"configured_enabled": enabled,
|
||||
"runtime_optimizer": client.get_stats()["config"][
|
||||
"cache_optimizer"
|
||||
],
|
||||
"wire_system": wire_system,
|
||||
"wire_system_sha256": hashlib.sha256(
|
||||
wire_system.encode()
|
||||
).hexdigest(),
|
||||
"response": response.choices[0].message.content,
|
||||
}
|
||||
)
|
||||
)
|
||||
if not enabled and wire_system != "Keep exact spaces.\n":
|
||||
violations.append("Disabled cache optimizer modified the system message")
|
||||
finally:
|
||||
client.close()
|
||||
finally:
|
||||
original.close()
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
|
||||
assert not violations, violations
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Vendored
+68
-7
@@ -76,12 +76,58 @@ def _extract_tool_result_content(msg: dict) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _can_swap_tool_result_content(msg: dict) -> bool:
|
||||
"""Whether replacing extracted text preserves the OpenAI content shape.
|
||||
|
||||
A joined replacement cannot be mapped back to multiple blocks without
|
||||
losing boundaries, metadata or media. All rewrite callers must check this
|
||||
before extraction, transformation or savings accounting. Anthropic's
|
||||
existing tool-result rewrite contract is unchanged.
|
||||
"""
|
||||
if msg.get("role") != "tool":
|
||||
return True
|
||||
content = msg.get("content")
|
||||
if isinstance(content, str):
|
||||
return True
|
||||
return (
|
||||
isinstance(content, list)
|
||||
and len(content) == 1
|
||||
and isinstance(content[0], dict)
|
||||
and content[0].get("type") == "text"
|
||||
and isinstance(content[0].get("text"), str)
|
||||
)
|
||||
|
||||
|
||||
def _carries_tool_result_text(msg: dict) -> bool:
|
||||
"""Whether any tool result in ``msg`` holds text (both formats)."""
|
||||
if msg.get("role") == "tool":
|
||||
results = [msg.get("content")]
|
||||
else:
|
||||
results = [
|
||||
b.get("content")
|
||||
for b in msg.get("content") or []
|
||||
if isinstance(b, dict) and b.get("type") == "tool_result"
|
||||
]
|
||||
return any(
|
||||
isinstance(c, str) or (isinstance(c, list) and _extract_text_from_blocks(c) is not None)
|
||||
for c in results
|
||||
)
|
||||
|
||||
|
||||
def _swap_tool_result_content(msg: dict, new_content: str) -> dict:
|
||||
"""Deep copy msg and replace tool result content with new_content."""
|
||||
"""Deep copy and rewrite supported content; preserve unsupported layouts."""
|
||||
new_msg = copy.deepcopy(msg)
|
||||
if not _can_swap_tool_result_content(msg):
|
||||
return new_msg
|
||||
# OpenAI format
|
||||
if new_msg.get("role") == "tool":
|
||||
new_msg["content"] = new_content
|
||||
content = new_msg.get("content")
|
||||
if isinstance(content, list):
|
||||
# The shared guard permits only an unambiguous single text block.
|
||||
# Keep its shape and metadata, including gateway cache breakpoints.
|
||||
content[0]["text"] = new_content
|
||||
else:
|
||||
new_msg["content"] = new_content
|
||||
return new_msg
|
||||
# Anthropic format
|
||||
content = new_msg.get("content")
|
||||
@@ -294,9 +340,10 @@ class CompressionCache:
|
||||
"""Count consecutive stable messages from the start.
|
||||
|
||||
A message is stable if it is a plain user/assistant/system message,
|
||||
an assistant message with tool_use blocks, or a tool_result whose
|
||||
content hash is already in the cache. The first unstable tool_result
|
||||
(cache miss) stops the count.
|
||||
an assistant message with tool_use blocks, an Anthropic tool_result with no text
|
||||
(nothing to replay), or a tool_result whose cached text and shape can
|
||||
be replayed. A cache miss or unsupported text shape stops the count.
|
||||
A joined text hash alone does not prove shape-safe replay.
|
||||
|
||||
The trailing message is *always* excluded from the frozen prefix
|
||||
(cap of ``len(messages) - 1``). The trailing message represents
|
||||
@@ -314,14 +361,25 @@ class CompressionCache:
|
||||
count = 0
|
||||
for msg in messages:
|
||||
if _is_tool_result_message(msg):
|
||||
# A joined-text cache/stable hash cannot prove replay for
|
||||
# shapes that apply_cached deliberately leaves untouched.
|
||||
# Anthropic textless results retain their no-rewrite path.
|
||||
if not _can_swap_tool_result_content(msg):
|
||||
break
|
||||
content = _extract_tool_result_content(msg)
|
||||
if content is not None:
|
||||
h = self.content_hash(content)
|
||||
if h not in self._cache and h not in self._stable_hashes:
|
||||
break
|
||||
else:
|
||||
# tool_result with non-string content; treat as unstable
|
||||
elif _carries_tool_result_text(msg):
|
||||
# Text after a textless first tool_result: the cache
|
||||
# keys a message by its first one, so this text was
|
||||
# never stored and cannot be byte-replayed.
|
||||
break
|
||||
# A tool_result with no text at all (a ToolSearch
|
||||
# tool_reference list, an image) gives the cache nothing
|
||||
# to replay, so it is stable. As a miss it stopped the
|
||||
# count for the rest of the session.
|
||||
# Regular user/assistant/system messages and assistant+tool_use
|
||||
# are always stable — fall through.
|
||||
count += 1
|
||||
@@ -341,6 +399,9 @@ class CompressionCache:
|
||||
with self._lock:
|
||||
result: list[dict] = []
|
||||
for msg in messages:
|
||||
if not _can_swap_tool_result_content(msg):
|
||||
result.append(msg)
|
||||
continue
|
||||
if _is_tool_result_message(msg):
|
||||
content = _extract_tool_result_content(msg)
|
||||
if content is not None:
|
||||
|
||||
Vendored
+20
-2
@@ -563,12 +563,30 @@ def classify_history_relation(
|
||||
return _classify_history_canonical(current, previous)
|
||||
|
||||
|
||||
def cache_key_tools(tools: Any) -> Any:
|
||||
"""The Anthropic ``tools`` that count toward the prompt-cache key.
|
||||
|
||||
A ``defer_loading`` tool is kept out of the prompt until the model loads
|
||||
it, so adding one to ``tools`` leaves the cached prefix readable. Claude
|
||||
Code appends every tool its ToolSearch loads to ``tools``, still marked
|
||||
deferred; counting those would start a new lineage at frozen=0 after each
|
||||
load and re-run the whole history. Every other tool, and a tool whose
|
||||
``defer_loading`` is cleared, still counts. The Anthropic handler decides
|
||||
when to apply this (first-party upstream, ``HEADROOM_DEFERRED_TOOL_AFFINITY``).
|
||||
"""
|
||||
if not isinstance(tools, list):
|
||||
return tools
|
||||
return [t for t in tools if not (isinstance(t, dict) and t.get("defer_loading"))]
|
||||
|
||||
|
||||
def segment_fingerprint(value: Any) -> str:
|
||||
"""Stable hash for non-message provider cache-key segments.
|
||||
|
||||
Cache-control placement and transport annotations are deliberately ignored;
|
||||
semantic tool/model/thinking changes remain visible. The hash is affinity
|
||||
metadata only and is never used to reconstruct or forward request content.
|
||||
semantic tool/model/thinking changes remain visible. It hashes what it is
|
||||
given, so callers pass only what the provider keys its cache on (for
|
||||
Anthropic tools, :func:`cache_key_tools`). The hash is affinity metadata
|
||||
only and is never used to reconstruct or forward request content.
|
||||
"""
|
||||
canonical = _lineage_snapshot(_canonicalize_for_prefix_compare(value))
|
||||
encoded = json.dumps(
|
||||
|
||||
Vendored
+12
-5
@@ -31,7 +31,14 @@ class CacheOptimizerRegistry:
|
||||
"""
|
||||
|
||||
_optimizers: dict[str, type[BaseCacheOptimizer]] = {}
|
||||
_instances: dict[str, BaseCacheOptimizer] = {}
|
||||
_instances: dict[tuple[str, int | None], BaseCacheOptimizer] = {}
|
||||
|
||||
@classmethod
|
||||
def _invalidate_instances(cls, name: str) -> None:
|
||||
"""Discard every configuration cached for this exact registration."""
|
||||
for key in list(cls._instances):
|
||||
if key[0] == name:
|
||||
del cls._instances[key]
|
||||
|
||||
@classmethod
|
||||
def register(
|
||||
@@ -57,8 +64,8 @@ class CacheOptimizerRegistry:
|
||||
f"Optimizer '{name}' already registered. Use override=True to replace."
|
||||
)
|
||||
cls._optimizers[name] = optimizer_class
|
||||
# Clear cached instance if exists
|
||||
cls._instances.pop(name, None)
|
||||
# Configured instances must not outlive their registered implementation.
|
||||
cls._invalidate_instances(name)
|
||||
|
||||
@classmethod
|
||||
def unregister(cls, name: str) -> None:
|
||||
@@ -69,7 +76,7 @@ class CacheOptimizerRegistry:
|
||||
name: Name to unregister
|
||||
"""
|
||||
cls._optimizers.pop(name, None)
|
||||
cls._instances.pop(name, None)
|
||||
cls._invalidate_instances(name)
|
||||
|
||||
@classmethod
|
||||
def get(
|
||||
@@ -109,7 +116,7 @@ class CacheOptimizerRegistry:
|
||||
raise KeyError(f"No optimizer registered for '{key}'. Available: {available}")
|
||||
|
||||
# Return cached instance if requested
|
||||
cache_key = f"{key}:{id(config)}" if config else key
|
||||
cache_key = (key, id(config) if config is not None else None)
|
||||
if cached and cache_key in cls._instances:
|
||||
return cls._instances[cache_key]
|
||||
|
||||
|
||||
Vendored
+82
-19
@@ -36,6 +36,7 @@ Usage:
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import time
|
||||
from collections import OrderedDict
|
||||
from collections.abc import Callable
|
||||
@@ -74,6 +75,10 @@ class CacheEntry:
|
||||
# Hash of the full messages for exact matching
|
||||
messages_hash: str = ""
|
||||
|
||||
# Cache namespace (provider, model, metadata) so entries never replay
|
||||
# across different provider/model/metadata contexts
|
||||
namespace: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SemanticCacheConfig:
|
||||
@@ -122,7 +127,7 @@ class SemanticCache:
|
||||
# LRU cache: key -> CacheEntry
|
||||
self._cache: OrderedDict[str, CacheEntry] = OrderedDict()
|
||||
|
||||
# Exact hash index: messages_hash -> key
|
||||
# Exact hash index: namespace-scoped hash -> key
|
||||
self._hash_index: dict[str, str] = {}
|
||||
|
||||
# Statistics
|
||||
@@ -134,6 +139,7 @@ class SemanticCache:
|
||||
self,
|
||||
query: str,
|
||||
messages_hash: str | None = None,
|
||||
namespace: str = "",
|
||||
) -> CacheEntry | None:
|
||||
"""
|
||||
Look up a cached entry.
|
||||
@@ -141,6 +147,8 @@ class SemanticCache:
|
||||
Args:
|
||||
query: Query text to search for
|
||||
messages_hash: Optional exact hash for fast lookup
|
||||
namespace: Cache namespace (provider, model, metadata); entries are
|
||||
never returned outside their own namespace
|
||||
|
||||
Returns:
|
||||
CacheEntry if found, None otherwise
|
||||
@@ -149,13 +157,16 @@ class SemanticCache:
|
||||
|
||||
# Try exact hash match first
|
||||
if messages_hash and self.config.use_exact_matching:
|
||||
key = self._hash_index.get(messages_hash)
|
||||
index_key = self._combined_key(namespace, messages_hash)
|
||||
key = self._hash_index.get(index_key)
|
||||
if key and key in self._cache:
|
||||
entry = self._cache[key]
|
||||
# Verify the stored entry really belongs to this request. Guards
|
||||
# against a stale index mapping ever pointing at an entry that was
|
||||
# overwritten by a different conversation sharing the same key.
|
||||
if entry.messages_hash == messages_hash:
|
||||
# Verify the stored entry really belongs to this request and
|
||||
# namespace. Guards against a stale index mapping ever pointing
|
||||
# at an entry that was overwritten by a different conversation
|
||||
# sharing the same key, and against replaying a cached response
|
||||
# across provider/model/metadata contexts.
|
||||
if entry.messages_hash == messages_hash and entry.namespace == namespace:
|
||||
self._touch(key)
|
||||
self._hits += 1
|
||||
return entry
|
||||
@@ -173,7 +184,7 @@ class SemanticCache:
|
||||
# messages_hash above, which is context-complete and safe.
|
||||
if self._embedding_fn and query.strip():
|
||||
query_embedding = self._embedding_fn(query)
|
||||
best_match, best_similarity = self._find_similar(query_embedding)
|
||||
best_match, best_similarity = self._find_similar(query_embedding, namespace)
|
||||
|
||||
if best_similarity >= self.config.similarity_threshold:
|
||||
self._touch(best_match)
|
||||
@@ -188,6 +199,7 @@ class SemanticCache:
|
||||
query: str,
|
||||
response: Any,
|
||||
messages_hash: str | None = None,
|
||||
namespace: str = "",
|
||||
) -> str:
|
||||
"""
|
||||
Store a response in the cache.
|
||||
@@ -196,6 +208,7 @@ class SemanticCache:
|
||||
query: Query text
|
||||
response: Response to cache
|
||||
messages_hash: Optional exact hash for fast lookup
|
||||
namespace: Cache namespace (provider, model, metadata)
|
||||
|
||||
Returns:
|
||||
Cache key for the entry
|
||||
@@ -206,8 +219,11 @@ class SemanticCache:
|
||||
# a trailing user message ("continue", "yes", "run the tests") but differ
|
||||
# in earlier context must NOT collide on one query-derived slot and
|
||||
# overwrite each other. Fall back to the query hash only when no
|
||||
# messages_hash is supplied (e.g. embedding-only usage).
|
||||
key = messages_hash or self._generate_key(query)
|
||||
# messages_hash is supplied (e.g. embedding-only usage). The key always
|
||||
# includes the namespace so entries from different provider/model/
|
||||
# metadata contexts never share a slot.
|
||||
base = messages_hash or self._generate_key(query)
|
||||
key = self._combined_key(namespace, base)
|
||||
|
||||
# Evict if adding a NEW key would exceed capacity. Overwriting a key that
|
||||
# is already present is an in-place update that does not grow the map, so
|
||||
@@ -235,13 +251,14 @@ class SemanticCache:
|
||||
created_at=now,
|
||||
last_accessed=now,
|
||||
messages_hash=messages_hash or "",
|
||||
namespace=namespace,
|
||||
)
|
||||
|
||||
self._cache[key] = entry
|
||||
|
||||
# Index by hash for fast exact matching
|
||||
if messages_hash:
|
||||
self._hash_index[messages_hash] = key
|
||||
self._hash_index[self._combined_key(namespace, messages_hash)] = key
|
||||
|
||||
return key
|
||||
|
||||
@@ -250,7 +267,7 @@ class SemanticCache:
|
||||
if key in self._cache:
|
||||
entry = self._cache.pop(key)
|
||||
if entry.messages_hash:
|
||||
self._hash_index.pop(entry.messages_hash, None)
|
||||
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
|
||||
return True
|
||||
return False
|
||||
|
||||
@@ -276,12 +293,15 @@ class SemanticCache:
|
||||
def _find_similar(
|
||||
self,
|
||||
query_embedding: list[float],
|
||||
namespace: str = "",
|
||||
) -> tuple[str, float]:
|
||||
"""Find the most similar cached entry."""
|
||||
"""Find the most similar cached entry within the same namespace."""
|
||||
best_key = ""
|
||||
best_similarity = -1.0
|
||||
|
||||
for key, entry in self._cache.items():
|
||||
if entry.namespace != namespace:
|
||||
continue
|
||||
if not entry.embedding:
|
||||
continue
|
||||
|
||||
@@ -325,7 +345,7 @@ class SemanticCache:
|
||||
if self._cache:
|
||||
key, entry = self._cache.popitem(last=False)
|
||||
if entry.messages_hash:
|
||||
self._hash_index.pop(entry.messages_hash, None)
|
||||
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
|
||||
self._evictions += 1
|
||||
|
||||
def _cleanup_expired(self) -> None:
|
||||
@@ -343,7 +363,15 @@ class SemanticCache:
|
||||
for key in expired:
|
||||
entry = self._cache.pop(key)
|
||||
if entry.messages_hash:
|
||||
self._hash_index.pop(entry.messages_hash, None)
|
||||
self._hash_index.pop(self._combined_key(entry.namespace, entry.messages_hash), None)
|
||||
|
||||
def _combined_key(self, namespace: str, base: str) -> str:
|
||||
"""Hash a namespace with an exact messages_hash or query-derived key.
|
||||
|
||||
Combining namespace + base (separated by a NUL so a namespace can't be
|
||||
confused with a hash prefix) truncates like the existing hashes.
|
||||
"""
|
||||
return hashlib.sha256((namespace + "\0" + base).encode()).hexdigest()[:24]
|
||||
|
||||
def _generate_key(self, query: str) -> str:
|
||||
"""Generate a cache key for a query."""
|
||||
@@ -405,9 +433,15 @@ class SemanticCacheLayer:
|
||||
# Extract query for semantic matching
|
||||
query = context.query or self._extract_query(messages)
|
||||
messages_hash = self._compute_messages_hash(messages)
|
||||
namespace = self._compute_namespace(context)
|
||||
|
||||
# Unserializable context: never share cached responses across contexts,
|
||||
# so skip the cache and just delegate to the provider optimizer.
|
||||
if namespace is None:
|
||||
return self.provider_optimizer.optimize(messages, context, config)
|
||||
|
||||
# Check semantic cache
|
||||
cached = self.cache.get(query, messages_hash)
|
||||
cached = self.cache.get(query, messages_hash, namespace)
|
||||
if cached:
|
||||
return CacheResult(
|
||||
messages=messages,
|
||||
@@ -440,15 +474,24 @@ class SemanticCacheLayer:
|
||||
Args:
|
||||
messages: Original messages
|
||||
response: Response from LLM
|
||||
context: Optional context with query
|
||||
context: Optional context with query. Pass the same OptimizationContext
|
||||
used for `process`; omitting it stores under the unscoped
|
||||
namespace, which only matches lookups whose context also has
|
||||
empty provider, model and metadata.
|
||||
|
||||
Returns:
|
||||
Cache key
|
||||
"""
|
||||
query = (context.query if context else None) or self._extract_query(messages)
|
||||
messages_hash = self._compute_messages_hash(messages)
|
||||
namespace = self._compute_namespace(context)
|
||||
|
||||
return self.cache.put(query, response, messages_hash)
|
||||
# Unserializable context: don't store anything that could be replayed
|
||||
# for a different provider/model/metadata.
|
||||
if namespace is None:
|
||||
return ""
|
||||
|
||||
return self.cache.put(query, response, messages_hash, namespace)
|
||||
|
||||
def get_stats(self) -> dict[str, Any]:
|
||||
"""Get combined statistics."""
|
||||
@@ -457,6 +500,28 @@ class SemanticCacheLayer:
|
||||
"provider_optimizer": self.provider_optimizer.name,
|
||||
}
|
||||
|
||||
def _compute_namespace(self, context: OptimizationContext | None) -> str | None:
|
||||
"""Compute the cache namespace from provider, model, and metadata.
|
||||
|
||||
request_id and timestamp are deliberately excluded so they never affect
|
||||
matching. Returns None when the context isn't JSON-serializable (e.g.
|
||||
unserializable metadata), so callers skip lookup/storage rather than
|
||||
replaying entries across contexts.
|
||||
"""
|
||||
if context is None:
|
||||
context = OptimizationContext()
|
||||
try:
|
||||
return json.dumps(
|
||||
{
|
||||
"provider": context.provider,
|
||||
"model": context.model,
|
||||
"metadata": context.metadata,
|
||||
},
|
||||
sort_keys=True,
|
||||
)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
|
||||
"""Extract the last user query from messages."""
|
||||
for msg in reversed(messages):
|
||||
@@ -473,8 +538,6 @@ class SemanticCacheLayer:
|
||||
|
||||
def _compute_messages_hash(self, messages: list[dict[str, Any]]) -> str:
|
||||
"""Compute a hash of all messages."""
|
||||
import json
|
||||
|
||||
try:
|
||||
content = json.dumps(messages, sort_keys=True)
|
||||
return hashlib.sha256(content.encode()).hexdigest()[:24]
|
||||
|
||||
Vendored
+14
-5
@@ -101,6 +101,8 @@ def resolve_learned_ttl(provider: str, model: str) -> int | None:
|
||||
``"provider/model"`` first, then ``"provider"``. Cached by mtime so a hot-path
|
||||
call is a dict lookup. Best-effort: returns None on any error, so the caller
|
||||
falls back to its own default (config TTL for CC, static default elsewhere).
|
||||
Invalid table shapes are ignored; invalid model values fall through to the
|
||||
provider fallback. Only positive, finite TTLs (not booleans) are accepted.
|
||||
"""
|
||||
global _learned_cache, _learned_mtime
|
||||
path = _learned_path()
|
||||
@@ -108,17 +110,24 @@ def resolve_learned_ttl(provider: str, model: str) -> int | None:
|
||||
mt = os.path.getmtime(path)
|
||||
if _learned_cache is None or mt != _learned_mtime:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
_learned_cache = json.load(f)
|
||||
loaded = json.load(f)
|
||||
_learned_cache = loaded if isinstance(loaded, dict) else {}
|
||||
_learned_mtime = mt
|
||||
except Exception:
|
||||
return None
|
||||
table = _learned_cache or {}
|
||||
for key in (f"{provider}/{model}", provider):
|
||||
v = table.get(key)
|
||||
if isinstance(v, dict) and isinstance(v.get("ttl_seconds"), int | float):
|
||||
return int(v["ttl_seconds"])
|
||||
if isinstance(v, int | float):
|
||||
return int(v)
|
||||
if isinstance(v, dict):
|
||||
v = v.get("ttl_seconds")
|
||||
if isinstance(v, bool) or not isinstance(v, int | float):
|
||||
continue
|
||||
try:
|
||||
ttl = int(v)
|
||||
except (ValueError, OverflowError):
|
||||
continue # JSON may decode NaN/Infinity; never fail a live request.
|
||||
if ttl > 0:
|
||||
return ttl
|
||||
return None
|
||||
|
||||
|
||||
|
||||
+41
-5
@@ -46,7 +46,7 @@ from headroom.providers.claude import (
|
||||
)
|
||||
|
||||
from .main import get_version, main
|
||||
from .port_discovery import find_live_proxy_elsewhere, is_headroom_livez
|
||||
from .port_discovery import find_live_proxy_elsewhere, is_headroom_livez, loopback_port
|
||||
from .wrap import _read_wrap_marker, _wrap_marker_is_stale
|
||||
|
||||
PASS = "pass"
|
||||
@@ -59,6 +59,7 @@ DEFAULT_PROXY_PORT = 8787
|
||||
_LOOPBACK_URL_RE = re.compile(r"https?://(?:127\.0\.0\.1|localhost):(\d+)")
|
||||
_CODEX_BASE_URL_RE = re.compile(r'(?m)^[ \t]*base_url\s*=\s*"([^"\r\n]+)"')
|
||||
_CODEX_MODEL_PROVIDER_RE = re.compile(r'(?m)^[ \t]*model_provider\s*=\s*"([^"\r\n]+)"')
|
||||
_CODEX_OPENAI_BASE_URL_RE = re.compile(r'(?m)^[ \t]*openai_base_url\s*=\s*"([^"\r\n]+)"')
|
||||
|
||||
# Ollama's fixed default port. `ollama launch claude` writes
|
||||
# ``ANTHROPIC_BASE_URL=http://127.0.0.1:11434`` into the launched Claude Code
|
||||
@@ -573,7 +574,19 @@ def check_wrap_marker_staleness(settings_path: Path) -> CheckResult:
|
||||
|
||||
def codex_active_base_url(text: str) -> tuple[str, str | None]:
|
||||
"""Return ``(provider_id, base_url)`` for the active Codex provider in ``text``."""
|
||||
active_match = _CODEX_MODEL_PROVIDER_RE.search(text)
|
||||
first_table = re.search(r"(?m)^[ \t]*\[", text)
|
||||
root = text[: first_table.start()] if first_table is not None else text
|
||||
# Only a root `model_provider` picks the default session's provider; one
|
||||
# under `[profiles.*]` applies only when that profile is selected.
|
||||
active_match = _CODEX_MODEL_PROVIDER_RE.search(root)
|
||||
if active_match is None or active_match.group(1) == "openai":
|
||||
# A root `openai_base_url` re-points Codex's built-in `openai` provider
|
||||
# (the default when `model_provider` is unset), so it beats an inactive
|
||||
# `[model_providers.headroom]` table. Headroom Desktop routes this way
|
||||
# because the ChatGPT app hides features for any other provider.
|
||||
root_url = _CODEX_OPENAI_BASE_URL_RE.search(root)
|
||||
if root_url is not None:
|
||||
return "openai", root_url.group(1)
|
||||
provider_id = active_match.group(1) if active_match else "headroom"
|
||||
return provider_id, _codex_provider_base_url(text, provider_id)
|
||||
|
||||
@@ -661,6 +674,24 @@ def _codex_provider_base_url(text: str, provider_id: str) -> str | None:
|
||||
return base_url_match.group(1) if base_url_match else None
|
||||
|
||||
|
||||
def _configured_client_ports(
|
||||
claude_paths: Sequence[Path], codex_paths: Sequence[Path]
|
||||
) -> list[int]:
|
||||
"""Loopback ports the Claude Code and Codex configs (project and user) send traffic to.
|
||||
|
||||
Headroom Desktop (port 6767) and ``wrap --port N`` leave no deployment
|
||||
manifest, so without these a bare ``doctor`` only ever finds 8787.
|
||||
"""
|
||||
urls = [_claude_base_url_in(path)[0] for path in claude_paths]
|
||||
for path in codex_paths:
|
||||
try:
|
||||
text = path.read_text(encoding="utf-8", errors="replace")
|
||||
except OSError:
|
||||
continue
|
||||
urls.append(codex_active_base_url(text)[1] or "")
|
||||
return [port for port in map(loopback_port, urls) if port is not None]
|
||||
|
||||
|
||||
def _codex_block_missing_openai_auth(
|
||||
text: str, config_path: Path, provider_id: str = "headroom"
|
||||
) -> bool:
|
||||
@@ -1211,20 +1242,25 @@ def doctor(
|
||||
health = probe_json(f"{base_url}/health", timeout=5.0) if livez else None
|
||||
stats = probe_json(f"{base_url}/stats", timeout=5.0) if livez else None
|
||||
installed = get_version()
|
||||
project_claude_settings = Path.cwd() / ".claude" / "settings.json"
|
||||
project_local_claude_settings = Path.cwd() / ".claude" / "settings.local.json"
|
||||
live_elsewhere = None
|
||||
if livez is None:
|
||||
# Cheap, bounded: only ports Headroom recorded (HEADROOM_PORT, 8787,
|
||||
# deployment manifests, the project's wrap marker), probed in parallel.
|
||||
# deployment manifests, the project's wrap marker) and the ports the
|
||||
# agents are configured for, probed in parallel.
|
||||
live_elsewhere = find_live_proxy_elsewhere(
|
||||
port,
|
||||
manifests=manifests,
|
||||
extra=_configured_client_ports(
|
||||
[project_local_claude_settings, project_claude_settings, claude_settings_path()],
|
||||
[codex_project_config_path(), codex_config_path()],
|
||||
),
|
||||
probe=lambda p: is_headroom_livez(
|
||||
probe_json(f"http://127.0.0.1:{p}/livez", timeout=0.5)
|
||||
),
|
||||
)
|
||||
|
||||
project_claude_settings = Path.cwd() / ".claude" / "settings.json"
|
||||
project_local_claude_settings = Path.cwd() / ".claude" / "settings.local.json"
|
||||
checks = [
|
||||
note_live_proxy_elsewhere(check_proxy_liveness(livez, base_url), live_elsewhere),
|
||||
check_proxy_readiness(livez, readyz, base_url),
|
||||
|
||||
+21
-13
@@ -58,6 +58,11 @@ from headroom.install.state import ManifestError, load_manifest, save_manifest
|
||||
from headroom.install.supervisors import start_supervisor
|
||||
from headroom.providers.claude import TOOL_SEARCH_DEFAULT, TOOL_SEARCH_ENV
|
||||
from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT
|
||||
from headroom.providers.codex.config_toml import (
|
||||
loads_codex_config,
|
||||
root_assignment_lines,
|
||||
split_toml_lines,
|
||||
)
|
||||
from headroom.providers.codex.install import (
|
||||
CodexAuthConfigError,
|
||||
build_codex_auth_config,
|
||||
@@ -91,6 +96,7 @@ _STARTUP_READY_TIMEOUT_SECONDS = 15
|
||||
# report ready, and every session is permanently cold.
|
||||
_HOOK_ENSURE_TIMEOUT_SECONDS = 60
|
||||
_TOML_TABLE_HEADER_RE = re.compile(r"^[ \t]*(?:\[\[[^\]\r\n]+\]\]|\[[^\]\r\n]+\])[ \t]*(?:#.*)?$")
|
||||
_ROOT_PROVIDER_KEY_RE = re.compile(r"[ \t]*(?:model_provider|openai_base_url)[ \t]*=")
|
||||
_TOML_FEATURES_NAME_RE = r"(?:features|\"features\"|'features')"
|
||||
_TOML_CODEX_HOOKS_NAME_RE = r"(?:codex_hooks|\"codex_hooks\"|'codex_hooks')"
|
||||
_CODEX_FEATURES_TABLE_RE = re.compile(
|
||||
@@ -298,7 +304,7 @@ def _replace_marker_block(
|
||||
# The block carries top-level keys, so it must sit above the first table
|
||||
# header; appended after a table (e.g. [features]) TOML scopes those keys
|
||||
# into that table and Codex rejects the config (#260).
|
||||
lines = content.splitlines()
|
||||
lines = split_toml_lines(content)
|
||||
for index, line in enumerate(lines):
|
||||
if _TOML_TABLE_HEADER_RE.search(line):
|
||||
head = "\n".join(lines[:index]).rstrip()
|
||||
@@ -347,7 +353,7 @@ def _codex_init_provider_snapshot(content: str) -> str | None:
|
||||
snapshot = json.loads(match.group(1))
|
||||
if not isinstance(snapshot, str):
|
||||
raise ValueError("Invalid Headroom init provider snapshot")
|
||||
tomllib.loads(snapshot)
|
||||
loads_codex_config(snapshot)
|
||||
return snapshot
|
||||
|
||||
|
||||
@@ -470,19 +476,21 @@ def _ensure_codex_provider(path: Path, port: int) -> None:
|
||||
# first table header) -- these keys also appear legitimately inside
|
||||
# [profiles.*] tables as per-profile overrides, and stripping them there
|
||||
# silently reroutes the user's profiles to the injected "headroom" default.
|
||||
_first_table = re.search(r"(?m)^[ \t]*\[", content)
|
||||
_split = _first_table.start() if _first_table else len(content)
|
||||
root, rest = content[:_split], content[_split:]
|
||||
root = re.sub(r"(?m)^[ \t]*model_provider[ \t]*=.*\r?\n", "", root)
|
||||
root = re.sub(r"(?m)^[ \t]*openai_base_url[ \t]*=.*\r?\n", "", root)
|
||||
content = root + rest
|
||||
# Lines nested in a multi-line inline table (e.g. ``profiles = {``) are
|
||||
# profile overrides too, so only true root assignments are dropped.
|
||||
_root_lines = root_assignment_lines(content)
|
||||
content = "".join(
|
||||
line
|
||||
for index, line in enumerate(split_toml_lines(content, keepends=True))
|
||||
if index not in _root_lines or not _ROOT_PROVIDER_KEY_RE.match(line)
|
||||
)
|
||||
# The provider block carries top-level keys (model_provider, openai_base_url),
|
||||
# so it must land at the document root rather than after a trailing table (#260).
|
||||
content = _replace_marker_block(
|
||||
content, _CODEX_PROVIDER_MARKER_START, _CODEX_PROVIDER_MARKER_END, block, at_root=True
|
||||
)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tomllib.loads(content)
|
||||
loads_codex_config(content)
|
||||
fsutil.write_text(path, content)
|
||||
# Codex filters its history menu by the active model_provider, so existing
|
||||
# native threads vanish once we switch to "headroom". Retag them to match the
|
||||
@@ -511,7 +519,7 @@ def _codex_features(content: str) -> dict[str, Any] | None:
|
||||
if not content.strip():
|
||||
return None
|
||||
try:
|
||||
parsed = tomllib.loads(content)
|
||||
parsed = loads_codex_config(content)
|
||||
except tomllib.TOMLDecodeError:
|
||||
return None
|
||||
features = parsed.get("features")
|
||||
@@ -523,7 +531,7 @@ def _codex_features_has_hooks(content: str) -> bool:
|
||||
if features is None:
|
||||
# Keep init resilient for already-invalid user configs; this fallback
|
||||
# only needs to avoid adding a second obvious hooks line.
|
||||
lines = content.splitlines()
|
||||
lines = split_toml_lines(content)
|
||||
features_index = _codex_features_table_index(lines)
|
||||
if features_index is None:
|
||||
return False
|
||||
@@ -538,7 +546,7 @@ def _codex_features_has_hooks(content: str) -> bool:
|
||||
|
||||
|
||||
def _strip_codex_legacy_feature_flag(content: str) -> str:
|
||||
lines = content.splitlines(keepends=True)
|
||||
lines = split_toml_lines(content, keepends=True)
|
||||
retained: list[str] = []
|
||||
in_features = False
|
||||
in_root = True
|
||||
@@ -588,7 +596,7 @@ def _ensure_codex_feature_flag(path: Path) -> None:
|
||||
# the only work.
|
||||
pass
|
||||
else:
|
||||
lines = content.splitlines()
|
||||
lines = split_toml_lines(content)
|
||||
features_index = _codex_features_table_index(lines)
|
||||
if features_index is not None:
|
||||
# Leading blank line matches the normalisation _replace_marker_block
|
||||
|
||||
@@ -943,7 +943,10 @@ def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed pe
|
||||
"--env HEADROOM_WORKSPACE_DIR=/path. Supervisors (launchd, systemd, cron) "
|
||||
"start with a bare environment and do not inherit the interactive shell's "
|
||||
"exports, so anything the runtime needs beyond the flags above must be set "
|
||||
"here. Repeatable."
|
||||
"here. Repeatable. Values are stored in cleartext in the profile's "
|
||||
"manifest and runner scripts under ~/.headroom/deploy/<profile>/ "
|
||||
"(owner-only, not encrypted); see SECURITY.md before passing a provider "
|
||||
"API key this way."
|
||||
),
|
||||
)
|
||||
def install_apply(
|
||||
|
||||
+26
-15
@@ -41,10 +41,7 @@ from functools import wraps
|
||||
from pathlib import Path
|
||||
from typing import Any, NamedTuple, cast
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - Python < 3.11
|
||||
import tomli as tomllib
|
||||
import tomllib
|
||||
|
||||
from headroom._subprocess import identity_mismatch as _shared_identity_mismatch
|
||||
from headroom._subprocess import pid_alive, proc_identity, run
|
||||
@@ -113,6 +110,11 @@ from headroom.providers.claude import (
|
||||
)
|
||||
from headroom.providers.claude.runtime import TOOL_SEARCH_FOUNDRY_DEFAULT
|
||||
from headroom.providers.codex import build_launch_env as _build_codex_launch_env
|
||||
from headroom.providers.codex.config_toml import (
|
||||
loads_codex_config,
|
||||
root_assignment_lines,
|
||||
split_toml_lines,
|
||||
)
|
||||
from headroom.providers.codex.install import (
|
||||
CodexAuthConfigError,
|
||||
build_codex_auth_config,
|
||||
@@ -3086,7 +3088,7 @@ def _codex_session_launch_settings(
|
||||
"""Build process-local routing while preserving the selected provider id."""
|
||||
config_file, _ = _codex_config_paths()
|
||||
try:
|
||||
config = tomllib.loads(_read_text(config_file)) if config_file.exists() else {}
|
||||
config = loads_codex_config(_read_text(config_file)) if config_file.exists() else {}
|
||||
except (OSError, tomllib.TOMLDecodeError) as exc:
|
||||
raise click.ClickException(
|
||||
f"could not read Codex config for session routing: {exc}"
|
||||
@@ -3347,19 +3349,28 @@ def _redirect_existing_top_level_keys(content: str, port: int) -> str:
|
||||
|
||||
return _replace
|
||||
|
||||
redirected = content
|
||||
# Only root assignments: the same keys inside [profiles.*] tables or a
|
||||
# multi-line inline ``profiles = {`` table are the user's per-profile routing.
|
||||
lines = split_toml_lines(content, keepends=True)
|
||||
root_lines = sorted(root_assignment_lines(content))
|
||||
for key in _REDIRECTABLE_KEYS:
|
||||
pattern = re.compile(rf'(?m)^[ \t]*{re.escape(key)}[ \t]*=[ \t]*"(?P<value>[^"\n]*)"[^\n]*')
|
||||
redirected = pattern.sub(_make_replacer(key, port), redirected, count=1)
|
||||
return redirected
|
||||
pattern = re.compile(rf'[ \t]*{re.escape(key)}[ \t]*=[ \t]*"(?P<value>[^"\n]*)"[^\n]*')
|
||||
replace = _make_replacer(key, port)
|
||||
for index in root_lines:
|
||||
match = pattern.match(lines[index])
|
||||
if match:
|
||||
lines[index] = replace(match) + lines[index][match.end() :]
|
||||
break
|
||||
return "".join(lines)
|
||||
|
||||
|
||||
def _has_redirectable_top_level_key(content: str, key: str) -> bool:
|
||||
"""Return True if ``content`` declares ``key = "..."`` as a top-level key."""
|
||||
import re # local import to match the module's existing convention
|
||||
|
||||
pattern = re.compile(rf'(?m)^[ \t]*{key}[ \t]*=[ \t]*"[^"\n]*"')
|
||||
return pattern.search(content) is not None
|
||||
pattern = re.compile(rf'[ \t]*{key}[ \t]*=[ \t]*"[^"\n]*"')
|
||||
lines = split_toml_lines(content, keepends=True)
|
||||
return any(pattern.match(lines[index]) for index in root_assignment_lines(content))
|
||||
|
||||
|
||||
def _codex_config_has_headroom_markers(content: str) -> bool:
|
||||
@@ -3718,10 +3729,10 @@ def _inject_codex_provider_config(port: int) -> str | None:
|
||||
# name a line the user never wrote. Only the user's part has to
|
||||
# parse: stripping a stale Headroom block may be what repairs it.
|
||||
try:
|
||||
tomllib.loads(content)
|
||||
loads_codex_config(content)
|
||||
except tomllib.TOMLDecodeError:
|
||||
try:
|
||||
tomllib.loads(original)
|
||||
loads_codex_config(original)
|
||||
reason = "it is no longer valid TOML once Headroom's block is removed"
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
reason = f"it is not valid TOML ({exc})"
|
||||
@@ -3769,7 +3780,7 @@ def _inject_codex_provider_config(port: int) -> str | None:
|
||||
)
|
||||
|
||||
# Never replace a usable config with a malformed generated candidate.
|
||||
tomllib.loads(content)
|
||||
loads_codex_config(content)
|
||||
_write_text(config_file, content)
|
||||
click.echo(f" Codex config: injected Headroom provider (WS + HTTP) into {config_file}")
|
||||
if custom_upstream_base_url:
|
||||
@@ -4066,7 +4077,7 @@ def _inject_memory_mcp_config(user_id: str) -> None:
|
||||
content = mcp_section
|
||||
|
||||
# Never replace a usable config with a malformed generated candidate.
|
||||
tomllib.loads(content)
|
||||
loads_codex_config(content)
|
||||
_write_text(config_file, content)
|
||||
click.echo(f" Memory MCP: registered in {config_file}")
|
||||
except Exception as e:
|
||||
|
||||
+86
-18
@@ -3,7 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Iterator
|
||||
from collections.abc import Callable, Iterable, Iterator
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
@@ -38,6 +38,74 @@ from .utils import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _OwnedStream(Iterator[Any]):
|
||||
"""Iterator that owns and closes an upstream response stream."""
|
||||
|
||||
def __init__(self, stream: Iterable[Any], save_metrics: Callable[[], None]) -> None:
|
||||
self._source: Iterable[Any] | None = stream
|
||||
self._stream: Iterator[Any] | None = None
|
||||
self._save_metrics: Callable[[], None] | None = save_metrics
|
||||
self._finished = False
|
||||
|
||||
def __iter__(self) -> _OwnedStream:
|
||||
return self
|
||||
|
||||
def __next__(self) -> Any:
|
||||
if self._finished:
|
||||
raise StopIteration
|
||||
try:
|
||||
if self._stream is None:
|
||||
source = self._source
|
||||
if source is None:
|
||||
raise StopIteration
|
||||
self._stream = iter(source)
|
||||
return next(self._stream)
|
||||
except BaseException:
|
||||
self.close()
|
||||
raise
|
||||
|
||||
def close(self) -> None:
|
||||
"""Close the upstream stream and save metrics exactly once."""
|
||||
if self._finished:
|
||||
return
|
||||
self._finished = True
|
||||
source = self._source
|
||||
stream = self._stream
|
||||
save_metrics = self._save_metrics
|
||||
try:
|
||||
candidates = (source,) if source is stream else (source, stream)
|
||||
for candidate in candidates:
|
||||
if candidate is None:
|
||||
continue
|
||||
close = getattr(candidate, "close", None)
|
||||
if close is not None:
|
||||
try:
|
||||
close()
|
||||
except Exception:
|
||||
logger.warning("Failed to close upstream response stream", exc_info=True)
|
||||
finally:
|
||||
self._source = None
|
||||
self._stream = None
|
||||
self._save_metrics = None
|
||||
if save_metrics is not None:
|
||||
save_metrics()
|
||||
|
||||
def __enter__(self) -> _OwnedStream:
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
|
||||
self.close()
|
||||
|
||||
def __del__(self) -> None:
|
||||
# The previous generator wrapper ran its finally block when abandoned
|
||||
# after iteration started. Preserve that cleanup for callers that do
|
||||
# not explicitly close a stream.
|
||||
try:
|
||||
self.close()
|
||||
except Exception:
|
||||
logger.debug("Failed to finalize abandoned response stream", exc_info=True)
|
||||
|
||||
|
||||
class ChatCompletions:
|
||||
"""Wrapper for chat.completions API (OpenAI-style)."""
|
||||
|
||||
@@ -134,6 +202,7 @@ class Messages:
|
||||
model: str,
|
||||
messages: list[dict[str, Any]],
|
||||
max_tokens: int = 1024,
|
||||
stream: bool = False,
|
||||
# Headroom-specific parameters
|
||||
headroom_mode: str | None = None,
|
||||
headroom_cache_prefix_tokens: int | None = None,
|
||||
@@ -150,6 +219,7 @@ class Messages:
|
||||
model: Model name.
|
||||
messages: List of messages.
|
||||
max_tokens: Maximum tokens in response.
|
||||
stream: Return an iterator of raw message events when True.
|
||||
headroom_mode: Override default mode ("audit" | "optimize").
|
||||
headroom_cache_prefix_tokens: Target cache-aligned prefix size.
|
||||
headroom_output_buffer_tokens: Reserve tokens for output.
|
||||
@@ -163,13 +233,13 @@ class Messages:
|
||||
return self._client._create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
stream=False,
|
||||
stream=stream,
|
||||
headroom_mode=headroom_mode,
|
||||
headroom_cache_prefix_tokens=headroom_cache_prefix_tokens,
|
||||
headroom_output_buffer_tokens=headroom_output_buffer_tokens,
|
||||
headroom_keep_turns=headroom_keep_turns,
|
||||
headroom_tool_profiles=headroom_tool_profiles,
|
||||
api_style="anthropic",
|
||||
api_style="anthropic_create",
|
||||
max_tokens=max_tokens,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -307,11 +377,11 @@ class HeadroomClient:
|
||||
# Use provided config or build from individual parameters
|
||||
if config is not None:
|
||||
self._config = config
|
||||
# Override store_url and mode if explicitly provided in config
|
||||
# Override store_url and mode if explicitly provided in config.
|
||||
# The resolved URL is not written back: the caller may share this
|
||||
# config with other clients, which would then read it as explicit.
|
||||
if config.store_url:
|
||||
self._store_url = config.store_url
|
||||
else:
|
||||
self._config.store_url = store_url
|
||||
self._default_mode = config.default_mode
|
||||
else:
|
||||
# Build config from individual parameters
|
||||
@@ -325,7 +395,7 @@ class HeadroomClient:
|
||||
self._config.model_context_limits.update(model_context_limits)
|
||||
|
||||
# Initialize storage
|
||||
self._storage = create_storage(store_url)
|
||||
self._storage = create_storage(self._store_url)
|
||||
|
||||
# Initialize transform pipeline
|
||||
self._pipeline = TransformPipeline(self._config, provider=self._provider)
|
||||
@@ -338,7 +408,7 @@ class HeadroomClient:
|
||||
self._cache_optimizer: BaseCacheOptimizer | None = None
|
||||
self._semantic_cache_layer: SemanticCacheLayer | None = None
|
||||
|
||||
if enable_cache_optimizer:
|
||||
if self._config.cache_optimizer.enabled:
|
||||
if cache_optimizer is not None:
|
||||
self._cache_optimizer = cache_optimizer
|
||||
else:
|
||||
@@ -372,7 +442,7 @@ class HeadroomClient:
|
||||
provider=self._provider.name.lower(),
|
||||
metadata={
|
||||
"default_mode": self._default_mode.value,
|
||||
"cache_optimizer_enabled": enable_cache_optimizer,
|
||||
"cache_optimizer_enabled": self._config.cache_optimizer.enabled,
|
||||
"semantic_cache_enabled": enable_semantic_cache,
|
||||
},
|
||||
)
|
||||
@@ -406,6 +476,9 @@ class HeadroomClient:
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
"""Internal implementation of create."""
|
||||
transport_style = api_style
|
||||
if api_style == "anthropic_create":
|
||||
api_style = "anthropic"
|
||||
request_id = generate_request_id()
|
||||
timestamp = datetime.now(timezone.utc).replace(tzinfo=None)
|
||||
mode = HeadroomMode(headroom_mode) if headroom_mode else self._default_mode
|
||||
@@ -607,7 +680,7 @@ class HeadroomClient:
|
||||
# Call underlying client based on API style
|
||||
try:
|
||||
response = call_client_transport(
|
||||
api_style,
|
||||
transport_style,
|
||||
self,
|
||||
model=model,
|
||||
messages=optimized_messages,
|
||||
@@ -644,16 +717,11 @@ class HeadroomClient:
|
||||
|
||||
def _wrap_stream(
|
||||
self,
|
||||
stream: Iterator[Any],
|
||||
stream: Iterable[Any],
|
||||
metrics: RequestMetrics,
|
||||
) -> Iterator[Any]:
|
||||
"""Wrap stream to pass through chunks and save metrics at end."""
|
||||
try:
|
||||
yield from stream
|
||||
finally:
|
||||
# Save metrics when stream completes
|
||||
# Note: output tokens unknown for streams
|
||||
self._storage.save(metrics)
|
||||
"""Wrap stream, closing the upstream and saving metrics exactly once."""
|
||||
return _OwnedStream(stream, lambda: self._storage.save(metrics))
|
||||
|
||||
def _extract_query(self, messages: list[dict[str, Any]]) -> str:
|
||||
"""Extract query from messages for semantic caching.
|
||||
|
||||
+3
-1
@@ -923,7 +923,9 @@ class PrefixFreezeConfig:
|
||||
class HeadroomConfig:
|
||||
"""Main configuration for HeadroomClient."""
|
||||
|
||||
store_url: str = "sqlite:///headroom.db"
|
||||
# None defers to HeadroomClient: its ``store_url`` argument, else a
|
||||
# headroom.db in the system temp directory.
|
||||
store_url: str | None = None
|
||||
default_mode: HeadroomMode = HeadroomMode.AUDIT
|
||||
model_context_limits: dict[str, int] = field(
|
||||
default_factory=lambda: DEFAULT_MODEL_CONTEXT_LIMITS.copy()
|
||||
|
||||
@@ -219,13 +219,13 @@
|
||||
<p class="text-sm text-gray-300">
|
||||
<span class="font-medium text-accent">Headroom for Teams</span>
|
||||
<span x-text="(stats.summary?.cost?.total_saved_usd || 0) >= 1
|
||||
? ' · You have saved $' + formatCurrency(stats.summary.cost.total_saved_usd) + ' on this machine. Get the same savings across your team, with 3 months free on a 12-month plan.'
|
||||
: ' · Bring these savings to your whole team, with 3 months free on a 12-month plan.'"></span>
|
||||
? ' · You have saved $' + formatCurrency(stats.summary.cost.total_saved_usd) + ' on this machine. Want more? Headroom Enterprise saves more, across your whole team. Talk to us at hello@headroomlabs.ai.'
|
||||
: ' · Want more savings? Headroom Enterprise saves more, across your whole team. Talk to us at hello@headroomlabs.ai.'"></span>
|
||||
</p>
|
||||
<div class="flex items-center gap-3 shrink-0">
|
||||
<a href="https://headroom-perks.vercel.app/?source=github&utm_source=headroom&utm_medium=dashboard&utm_content=banner"
|
||||
<a href="https://headroomlabs.ai"
|
||||
target="_blank" rel="noopener noreferrer"
|
||||
class="px-3 py-1.5 text-sm font-medium rounded-md bg-accent text-black hover:opacity-90">Claim 3 months free →</a>
|
||||
class="px-3 py-1.5 text-sm font-medium rounded-md bg-accent text-black hover:opacity-90">Talk to us →</a>
|
||||
<button type="button" @click="dismissTeamsPerk()" aria-label="Dismiss Headroom for Teams offer"
|
||||
class="text-sm text-gray-500 hover:text-gray-200">✕</button>
|
||||
</div>
|
||||
@@ -2056,12 +2056,12 @@
|
||||
return {
|
||||
stats: {},
|
||||
teamsPerkDismissed: (() => {
|
||||
try { return localStorage.getItem('headroom-teams-perk-dismissed') === '1'; }
|
||||
try { return localStorage.getItem('headroom-teams-contact-dismissed') === '1'; }
|
||||
catch (e) { return false; }
|
||||
})(),
|
||||
dismissTeamsPerk() {
|
||||
this.teamsPerkDismissed = true;
|
||||
try { localStorage.setItem('headroom-teams-perk-dismissed', '1'); } catch (e) {}
|
||||
try { localStorage.setItem('headroom-teams-contact-dismissed', '1'); } catch (e) {}
|
||||
},
|
||||
lifetimeStats: {},
|
||||
historyStats: {},
|
||||
|
||||
@@ -46,7 +46,8 @@ _CCR_MARKER_RE = re.compile(r"Retrieve more: hash=|Retrieve original: hash=|<<cc
|
||||
# A number with its immediate key context ("retry_limit: 3", "port=8787",
|
||||
# JSON's '"latency_ms": 12'). Bare numbers are skipped: without context they
|
||||
# are unverifiable noise.
|
||||
_NUMERIC_RE = re.compile(r"[A-Za-z_][\w.-]{0,24}\"?[ =:]{1,3}\d+(?:\.\d+)?")
|
||||
# Do not extract a decimal prefix from unsupported exponents or numeric suffixes.
|
||||
_NUMERIC_RE = re.compile(r"[A-Za-z_][\w.-]{0,24}\"?[ =:]{1,3}\d+(?:\.\d+)?(?!\w|\.\d)")
|
||||
_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+")
|
||||
_PATH_RE = re.compile(r"(?:~/|\.{1,2}/|/)?(?:[\w.-]+/){2,}[\w.@-]+")
|
||||
# Requires at least one a-f so bare decimal runs (timestamps, row counts) are
|
||||
@@ -233,6 +234,21 @@ def _normalize(text: str) -> str:
|
||||
|
||||
|
||||
def _target_survives(dimension: str, value: str, haystack: str, normalized_haystack: str) -> bool:
|
||||
if dimension == "numerics":
|
||||
# Check numbers before substring shortcuts: 80 is not retained in
|
||||
# 8080, 80.5 or -80. A trailing sentence period is still punctuation.
|
||||
# Format conversions (JSON -> table) may separate key from value.
|
||||
match = _NUMERIC_SPLIT_RE.match(value)
|
||||
if match:
|
||||
key, number = match.groups()
|
||||
normalized_key = _normalize(key)
|
||||
if (
|
||||
normalized_key
|
||||
and normalized_key in normalized_haystack
|
||||
and re.search(rf"(?<![\w.-]){re.escape(number)}(?!\w|\.\d)", normalized_haystack)
|
||||
):
|
||||
return True
|
||||
return False
|
||||
if value in haystack:
|
||||
return True
|
||||
normalized_value = _normalize(value)
|
||||
@@ -244,19 +260,6 @@ def _target_survives(dimension: str, value: str, haystack: str, normalized_hayst
|
||||
_, _, remainder = normalized_value.partition(" ")
|
||||
if len(remainder) >= _MIN_TARGET_LEN and remainder in normalized_haystack:
|
||||
return True
|
||||
if dimension == "numerics":
|
||||
# Format conversions (JSON -> table) separate key from value; count the
|
||||
# probe as retained only when both still appear.
|
||||
match = _NUMERIC_SPLIT_RE.match(value)
|
||||
if match:
|
||||
key, number = match.groups()
|
||||
normalized_key = _normalize(key)
|
||||
if (
|
||||
normalized_key
|
||||
and normalized_key in normalized_haystack
|
||||
and re.search(rf"\b{re.escape(number)}\b", normalized_haystack)
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
|
||||
+25
-6
@@ -230,8 +230,8 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
|
||||
opened world-readable, so a failure to create or narrow the file privately
|
||||
raises :class:`PermissionError` rather than proceeding to open a wide
|
||||
file. The existing-file path is symlink-race resistant — ``O_EXCL`` on the
|
||||
create refuses to reuse a planted file or symlink; an existing file is
|
||||
re-opened with ``O_NOFOLLOW`` and narrowed through that descriptor
|
||||
create refuses to reuse a planted file or symlink; an existing file not
|
||||
already 0600 is opened with ``O_NOFOLLOW`` and narrowed through it
|
||||
(``fstat`` to confirm a regular file, ``fchmod`` to set the mode) rather
|
||||
than by re-resolving the path, which closes the check-then-chmod TOCTOU a
|
||||
``chmod(path)`` would leave open. On Windows POSIX mode bits and
|
||||
@@ -249,15 +249,36 @@ def ensure_private_file(path: str | os.PathLike[str], *, what: str = "file") ->
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
_narrow_existing(path, what=what, missing_ok=False)
|
||||
|
||||
|
||||
def _narrow_existing(path: str | os.PathLike[str], *, what: str, missing_ok: bool) -> None:
|
||||
"""Narrow an existing file to 0600 through an ``O_NOFOLLOW`` descriptor.
|
||||
|
||||
A regular file already 0600 and owned by the caller (any owner as root) is
|
||||
only ``lstat``-ed: closing any descriptor on a sqlite file drops the
|
||||
process's ``fcntl`` locks on it (sqlite.org/howtocorrupt.html, 2.2).
|
||||
Non-regular files are refused; with *missing_ok*, a missing file is skipped.
|
||||
"""
|
||||
if not (hasattr(os, "O_NOFOLLOW") and hasattr(os, "fchmod")):
|
||||
return
|
||||
|
||||
fd = -1
|
||||
try:
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
st = os.lstat(path)
|
||||
if stat.S_ISREG(st.st_mode):
|
||||
euid = os.geteuid() # type: ignore[attr-defined]
|
||||
if stat.S_IMODE(st.st_mode) == OWNER_ONLY_MODE and (st.st_uid == euid or euid == 0):
|
||||
return
|
||||
fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW)
|
||||
except OSError as exc:
|
||||
if missing_ok and isinstance(exc, FileNotFoundError):
|
||||
return
|
||||
raise PermissionError(
|
||||
f"refusing to open {what} at {path}: not a regular file (symlink or open error: {exc})"
|
||||
) from exc
|
||||
if fd < 0:
|
||||
raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
|
||||
try:
|
||||
if not stat.S_ISREG(os.fstat(fd).st_mode):
|
||||
raise PermissionError(f"refusing to open {what} at {path}: not a regular file")
|
||||
@@ -346,9 +367,7 @@ def connect_private_sqlite(
|
||||
# them. Missing ones are left alone: sqlite creates them with the
|
||||
# main file's mode, which is now 0600.
|
||||
for suffix in _SQLITE_SIDECAR_SUFFIXES:
|
||||
sidecar = file_path + suffix
|
||||
if os.path.lexists(sidecar):
|
||||
ensure_private_file(sidecar, what=f"{what} {suffix[1:]} file")
|
||||
_narrow_existing(file_path + suffix, what=f"{what} {suffix[1:]} file", missing_ok=True)
|
||||
# ``**connect_kwargs: Any`` makes mypy type the call as ``Any``; the
|
||||
# annotation pins it back to the real return type.
|
||||
conn: sqlite3.Connection = sqlite3.connect(os.fspath(path), **connect_kwargs)
|
||||
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -11,8 +13,88 @@ import click
|
||||
|
||||
from headroom import paths as _paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PROFILE_RE = re.compile(r"^[A-Za-z0-9._-]+$")
|
||||
|
||||
# A deployment profile directory holds secrets. `headroom install --env
|
||||
# KEY=VALUE` is the supported way to give a supervised proxy a provider API key
|
||||
# — supervisors (launchd, systemd, Task Scheduler, cron) start from a bare
|
||||
# environment, so nothing else reaches the process — and every such value is
|
||||
# persisted verbatim, both into `manifest.json` and into the generated runner
|
||||
# scripts that `export` it before the exec. Those files are therefore written
|
||||
# owner-only rather than at the process umask (which leaves them world-readable
|
||||
# at the common 022). The supervisor always runs them as the installing user
|
||||
# (user scope) or as root (system scope), and neither needs the group/other
|
||||
# bits, so this is not a functional restriction.
|
||||
#
|
||||
# SECURITY.md documents these modes; `tests/test_packaging_extras_and_security_docs.py`
|
||||
# reads the octal values back out of that document and compares them with what
|
||||
# an install actually writes, so the policy and the code cannot drift apart.
|
||||
# Named for what they ARE -- a permission mode -- not for what the files they
|
||||
# protect contain. The previous SECRET_* spelling read as a credential to
|
||||
# CodeQL's sensitive-data heuristic, so `logger.warning("... 0o%o", mode)`
|
||||
# tripped py/clear-text-logging-sensitive-data on a diagnostic that logs a
|
||||
# permission bitmask and no secret at all.
|
||||
OWNER_ONLY_FILE_MODE = 0o600
|
||||
OWNER_ONLY_SCRIPT_MODE = 0o700
|
||||
OWNER_ONLY_DIR_MODE = 0o700
|
||||
|
||||
|
||||
#: Whether this platform actually enforces POSIX permission bits. On Windows
|
||||
#: access is governed by ACLs and ``chmod`` only toggles the read-only flag, so
|
||||
#: it SUCCEEDS without establishing the requested mode — which is why the check
|
||||
#: below reads the mode back rather than trusting the call not to raise.
|
||||
POSIX_MODES_ENFORCED = os.name == "posix"
|
||||
|
||||
|
||||
def chmod_owner_only(path: Path, mode: int) -> bool:
|
||||
"""Narrow ``path`` to an owner-only ``mode``. Returns whether that took.
|
||||
|
||||
This used to swallow every ``OSError`` and return nothing, so a caller had
|
||||
no way to tell a restricted file from an unrestricted one and continued as
|
||||
if the documented mode had been established. That is the wrong default for
|
||||
the two places it matters, because both NARROW A PRE-EXISTING INODE rather
|
||||
than create one: a profile directory made before these modes existed is
|
||||
0755, and a runner script rewritten through ``O_TRUNC`` keeps whatever mode
|
||||
it already had. In both cases this chmod is the only thing standing between
|
||||
a provider API key and every local user, and a silent failure left the
|
||||
caller asserting a guarantee it did not have.
|
||||
|
||||
The result is verified with ``stat`` instead of inferred from ``chmod`` not
|
||||
raising, because on Windows it does not raise and does not work either.
|
||||
|
||||
Returns ``True`` only when the mode is now exactly ``mode``. A caller
|
||||
holding secret-bearing content must act on ``False``; see
|
||||
:func:`headroom.install.supervisors._write_private_text`, which refuses to
|
||||
leave the file behind.
|
||||
"""
|
||||
try:
|
||||
path.chmod(mode)
|
||||
actual = stat.S_IMODE(path.stat().st_mode)
|
||||
except OSError as exc:
|
||||
logger.warning(
|
||||
"Could not restrict %s to mode 0o%o (%s); it may be readable by other local users.",
|
||||
path,
|
||||
mode,
|
||||
exc,
|
||||
)
|
||||
return False
|
||||
if actual != mode:
|
||||
# Expected on Windows, where the bits are advisory -- noise there, a
|
||||
# real finding on POSIX.
|
||||
logger.log(
|
||||
logging.WARNING if POSIX_MODES_ENFORCED else logging.DEBUG,
|
||||
"%s is mode 0o%o after asking for 0o%o; this platform does not "
|
||||
"enforce POSIX permission bits, so the file's protection comes "
|
||||
"from its directory and the system ACLs instead.",
|
||||
path,
|
||||
actual,
|
||||
mode,
|
||||
)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def validate_profile_name(profile: str) -> str:
|
||||
"""Validate and normalize a deployment profile name."""
|
||||
|
||||
@@ -4,6 +4,7 @@ from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shlex
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
@@ -32,7 +33,8 @@ def _merge_marker_block(file_path: Path, block: str, pattern: re.Pattern[str], m
|
||||
if file_path.exists():
|
||||
existing = fsutil.read_text(file_path)
|
||||
if marker in existing:
|
||||
return pattern.sub(block, existing)
|
||||
# A callable replacement: the block is literal text, not a template.
|
||||
return pattern.sub(lambda _match: block, existing)
|
||||
return existing.rstrip() + "\n\n" + block + "\n"
|
||||
return block + "\n"
|
||||
|
||||
@@ -40,7 +42,8 @@ def _merge_marker_block(file_path: Path, block: str, pattern: re.Pattern[str], m
|
||||
def _env_block(values: dict[str, str]) -> str:
|
||||
lines = [_ENV_MARKER_START]
|
||||
for name, value in values.items():
|
||||
lines.append(f'export {name}="{value}"')
|
||||
# shlex.quote so `$`, `"` and backticks reach the shell verbatim.
|
||||
lines.append(f"export {name}={shlex.quote(value)}")
|
||||
lines.append(_ENV_MARKER_END)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -12,7 +12,16 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_now
|
||||
from .paths import deploy_root, manifest_path, profile_root, recovery_manifest_path
|
||||
from .paths import (
|
||||
OWNER_ONLY_DIR_MODE,
|
||||
OWNER_ONLY_FILE_MODE,
|
||||
POSIX_MODES_ENFORCED,
|
||||
chmod_owner_only,
|
||||
deploy_root,
|
||||
manifest_path,
|
||||
profile_root,
|
||||
recovery_manifest_path,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -29,6 +38,15 @@ def _atomic_write_text(path: Path, data: str) -> None:
|
||||
both POSIX and Windows). A crash between truncate and full write therefore
|
||||
leaves either the previous file or the complete new one on disk, never a
|
||||
truncated manifest.
|
||||
|
||||
The file is owner-only. ``manifest.base_env`` holds whatever
|
||||
``headroom install --env KEY=VALUE`` was given, and that is the supported
|
||||
way to hand a provider API key to a supervised proxy (supervisors start
|
||||
from a bare environment), so the manifest must be assumed to contain
|
||||
secrets. :func:`tempfile.mkstemp` already creates the temporary file 0600
|
||||
and :func:`os.replace` carries that mode across, but the chmod is explicit
|
||||
so the guarantee survives a future rewrite of this function and is pinned
|
||||
by a test rather than inherited by accident.
|
||||
"""
|
||||
directory = path.parent
|
||||
fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=f".{path.name}.", suffix=".tmp")
|
||||
@@ -38,6 +56,7 @@ def _atomic_write_text(path: Path, data: str) -> None:
|
||||
handle.write(data)
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
chmod_owner_only(tmp_path, OWNER_ONLY_FILE_MODE)
|
||||
os.replace(tmp_path, path)
|
||||
except BaseException:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
@@ -48,9 +67,39 @@ def _manifest_json(manifest: DeploymentManifest) -> str:
|
||||
return json.dumps(asdict(manifest), indent=2) + "\n"
|
||||
|
||||
|
||||
def _save_manifest_at(path: Path, manifest: DeploymentManifest, *, strict: bool) -> None:
|
||||
def _narrow_profile_root(root: Path) -> None:
|
||||
"""Make a deployment profile directory owner-only.
|
||||
|
||||
``exist_ok=True`` leaves a pre-existing directory's mode alone, and a
|
||||
profile created before owner-only modes is 0755, so narrow it on every save
|
||||
rather than only at creation.
|
||||
|
||||
Not fatal, unlike the runner script: the manifest itself is created through
|
||||
``mkstemp``, which is 0600 from birth, so a directory that could not be
|
||||
narrowed weakens the outer layer without exposing the file. Warn rather
|
||||
than abandon a deployment over it.
|
||||
"""
|
||||
if not chmod_owner_only(root, OWNER_ONLY_DIR_MODE) and POSIX_MODES_ENFORCED:
|
||||
logger.warning(
|
||||
"Deployment profile directory %s is not owner-only; the "
|
||||
"manifest inside it is still 0o%o, but other local users can "
|
||||
"list the directory.",
|
||||
root,
|
||||
OWNER_ONLY_FILE_MODE,
|
||||
)
|
||||
|
||||
|
||||
def _save_manifest_at(
|
||||
path: Path,
|
||||
manifest: DeploymentManifest,
|
||||
*,
|
||||
strict: bool,
|
||||
owner_only_dir: bool = False,
|
||||
) -> None:
|
||||
root = path.parent
|
||||
root.mkdir(parents=True, exist_ok=True)
|
||||
if owner_only_dir:
|
||||
_narrow_profile_root(root)
|
||||
manifest.updated_at = iso_utc_now()
|
||||
try:
|
||||
_atomic_write_text(path, _manifest_json(manifest))
|
||||
@@ -66,9 +115,16 @@ def save_manifest(manifest: DeploymentManifest) -> None:
|
||||
The write is atomic so an interrupted save (SIGKILL, system restart, OOM)
|
||||
cannot leave a truncated ``manifest.json`` behind. Gracefully handles
|
||||
read-only filesystems by logging a warning instead of crashing.
|
||||
|
||||
The profile directory and the manifest are owner-only: ``base_env`` carries
|
||||
whatever ``headroom install --env`` was given, which is the supported way to
|
||||
hand a provider API key to a supervised proxy. See
|
||||
:data:`headroom.install.paths.OWNER_ONLY_DIR_MODE`.
|
||||
"""
|
||||
try:
|
||||
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=False)
|
||||
_save_manifest_at(
|
||||
manifest_path(manifest.profile), manifest, strict=False, owner_only_dir=True
|
||||
)
|
||||
except OSError as e:
|
||||
logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)
|
||||
|
||||
@@ -76,7 +132,7 @@ def save_manifest(manifest: DeploymentManifest) -> None:
|
||||
def save_manifest_strict(manifest: DeploymentManifest) -> None:
|
||||
"""Persist a manifest and propagate errors when recovery depends on it."""
|
||||
|
||||
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=True)
|
||||
_save_manifest_at(manifest_path(manifest.profile), manifest, strict=True, owner_only_dir=True)
|
||||
|
||||
|
||||
def save_recovery_manifest(manifest: DeploymentManifest) -> None:
|
||||
|
||||
@@ -21,6 +21,10 @@ from headroom._subprocess import run
|
||||
|
||||
from .models import ArtifactRecord, DeploymentManifest, SupervisorKind
|
||||
from .paths import (
|
||||
OWNER_ONLY_FILE_MODE,
|
||||
OWNER_ONLY_SCRIPT_MODE,
|
||||
POSIX_MODES_ENFORCED,
|
||||
chmod_owner_only,
|
||||
unix_ensure_script_path,
|
||||
unix_run_script_path,
|
||||
windows_ensure_cmd_path,
|
||||
@@ -47,6 +51,32 @@ def _is_windows() -> bool:
|
||||
return sys.platform.startswith("win")
|
||||
|
||||
|
||||
def _write_private_text(path: Path, data: str, mode: int) -> None:
|
||||
"""Atomically replace a secret-bearing runner with a private new inode.
|
||||
|
||||
Creating the temporary file exclusively avoids following a replacement
|
||||
hard link. Restrict it before publishing; a failed write, restriction or
|
||||
replacement leaves the previous runner and unrelated files untouched.
|
||||
"""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, name = tempfile.mkstemp(prefix=f".{path.name}.", suffix=".tmp", dir=path.parent)
|
||||
temporary = Path(name)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
if hasattr(os, "fchmod"):
|
||||
os.fchmod(handle.fileno(), mode)
|
||||
handle.write(data)
|
||||
if not chmod_owner_only(temporary, mode) and POSIX_MODES_ENFORCED:
|
||||
raise click.ClickException(
|
||||
f"Refusing to write {path}: permissions could not be restricted "
|
||||
f"to {oct(mode)}, and the file carries any API key passed via "
|
||||
f"`headroom install --env`. Fix directory permissions and re-run."
|
||||
)
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def _validated_env_items(env: dict[str, str] | None) -> list[tuple[str, str]]:
|
||||
items = list((env or {}).items())
|
||||
for name, _value in items:
|
||||
@@ -101,14 +131,19 @@ def _render_unix_runner(
|
||||
export_lines = "".join(
|
||||
f"export {name}={shlex.quote(value)}\n" for name, value in _validated_env_items(env)
|
||||
)
|
||||
path.write_text(
|
||||
# 0700, not 0755: `export_lines` can contain a provider API key in
|
||||
# cleartext (see `_write_private_text`). The supervisor runs this script as
|
||||
# the installing user for a user-scope install and as root for a
|
||||
# system-scope one, so dropping group/other loses nothing.
|
||||
_write_private_text(
|
||||
path,
|
||||
"#!/usr/bin/env bash\nset -euo pipefail\n"
|
||||
+ export_lines
|
||||
+ "exec "
|
||||
+ " ".join(shlex.quote(x) for x in command)
|
||||
+ "\n"
|
||||
+ "\n",
|
||||
OWNER_ONLY_SCRIPT_MODE,
|
||||
)
|
||||
path.chmod(0o755)
|
||||
return ArtifactRecord(kind="script", path=str(path))
|
||||
|
||||
|
||||
@@ -124,9 +159,17 @@ def _render_windows_runner(
|
||||
env_lines = "".join(
|
||||
f"$env:{name} = {_powershell_literal(value)}\n" for name, value in _validated_env_items(env)
|
||||
)
|
||||
ps1_path.write_text(
|
||||
f"$ErrorActionPreference = 'Stop'\n{env_lines}& {escaped}\nexit $LASTEXITCODE\n"
|
||||
# The .ps1 carries `$env:` assignments that can include a provider API key,
|
||||
# so it is owner-only like its POSIX counterpart. Windows resolves access
|
||||
# by ACL rather than by these bits, but the mode still matters when the
|
||||
# profile directory is read from WSL, a backup, or a synced home.
|
||||
_write_private_text(
|
||||
ps1_path,
|
||||
f"$ErrorActionPreference = 'Stop'\n{env_lines}& {escaped}\nexit $LASTEXITCODE\n",
|
||||
OWNER_ONLY_FILE_MODE,
|
||||
)
|
||||
# The .cmd shim holds no secrets — it only invokes the .ps1 — so it keeps a
|
||||
# conventional mode.
|
||||
cmd_path.write_text(
|
||||
'@echo off\r\npowershell -NoProfile -ExecutionPolicy Bypass -File "%~dp0'
|
||||
+ ps1_path.name
|
||||
@@ -651,6 +694,11 @@ def start_supervisor(manifest: DeploymentManifest) -> None:
|
||||
subprocess.run(["schtasks", "/Run", "/TN", f"{manifest.service_name}-startup"], check=True)
|
||||
|
||||
|
||||
# `/End` and `/Delete` report a missing task as "cannot find the file specified";
|
||||
# `/Change` reports it as "does not exist in the system".
|
||||
_SCHTASKS_ABSENT_TEXT = ("cannot find the file specified", "does not exist in the system")
|
||||
|
||||
|
||||
def stop_supervisor(manifest: DeploymentManifest) -> None:
|
||||
"""Stop the installed supervisor for a deployment."""
|
||||
|
||||
@@ -687,10 +735,32 @@ def stop_supervisor(manifest: DeploymentManifest) -> None:
|
||||
)
|
||||
return
|
||||
if _is_windows():
|
||||
# Session-scoped fallback installs never create the `-startup` task,
|
||||
# and the health task may already be absent, so tolerate missing
|
||||
# tasks (schtasks reports "cannot find the file specified") while
|
||||
# still surfacing real failures such as access denied.
|
||||
health = f"{manifest.service_name}-health"
|
||||
subprocess.run(["schtasks", "/Change", "/TN", health, "/DISABLE"], check=True)
|
||||
result = run(
|
||||
["schtasks", "/Change", "/TN", health, "/DISABLE"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
_require_removal_success(
|
||||
result,
|
||||
"schtasks health disable",
|
||||
absent_text=_SCHTASKS_ABSENT_TEXT,
|
||||
)
|
||||
subprocess.run(["schtasks", "/End", "/TN", health], check=False)
|
||||
subprocess.run(["schtasks", "/End", "/TN", f"{manifest.service_name}-startup"], check=True)
|
||||
result = run(
|
||||
["schtasks", "/End", "/TN", f"{manifest.service_name}-startup"],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
_require_removal_success(
|
||||
result,
|
||||
"schtasks startup end",
|
||||
absent_text=_SCHTASKS_ABSENT_TEXT,
|
||||
)
|
||||
|
||||
|
||||
def remove_supervisor(manifest: DeploymentManifest) -> None:
|
||||
@@ -785,7 +855,7 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:
|
||||
_require_removal_success(
|
||||
result,
|
||||
f"schtasks {suffix} delete",
|
||||
absent_text="cannot find the file specified",
|
||||
absent_text=_SCHTASKS_ABSENT_TEXT,
|
||||
)
|
||||
|
||||
|
||||
@@ -800,7 +870,7 @@ def _require_removal_success(
|
||||
operation: str,
|
||||
*,
|
||||
absent_codes: set[int] | None = None,
|
||||
absent_text: str | None = None,
|
||||
absent_text: str | tuple[str, ...] | None = None,
|
||||
) -> None:
|
||||
returncode = int(getattr(result, "returncode", 0))
|
||||
detail = _result_text(result)
|
||||
@@ -808,6 +878,7 @@ def _require_removal_success(
|
||||
return
|
||||
if absent_codes and returncode in absent_codes:
|
||||
return
|
||||
if absent_text and absent_text in detail.lower():
|
||||
texts = (absent_text,) if isinstance(absent_text, str) else absent_text or ()
|
||||
if any(text in detail.lower() for text in texts):
|
||||
return
|
||||
raise click.ClickException(f"{operation} failed: {detail}")
|
||||
|
||||
@@ -18,6 +18,7 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from headroom import fsutil
|
||||
from headroom.providers.codex.config_toml import loads_codex_config, split_toml_lines
|
||||
|
||||
from .base import (
|
||||
MCPRegistrar,
|
||||
@@ -84,7 +85,7 @@ def _evict_foreign_tables(content: str, server_name: str, start: str, end: str)
|
||||
move, so a header-like line inside a multi-line string can split a table
|
||||
wrongly; callers must check the final result with ``_only_server_changed``.
|
||||
"""
|
||||
lines = content.splitlines(keepends=True)
|
||||
lines = split_toml_lines(content, keepends=True)
|
||||
try:
|
||||
i = next(n for n, line in enumerate(lines) if line.rstrip("\r\n") == start)
|
||||
j = next(n for n in range(i + 1, len(lines)) if lines[n].rstrip("\r\n") == end)
|
||||
@@ -121,7 +122,7 @@ def _only_server_changed(old: str, new: str, server_name: str) -> bool:
|
||||
makes this False and the rewrite is refused instead of written.
|
||||
"""
|
||||
try:
|
||||
before, after = tomllib.loads(old), tomllib.loads(new)
|
||||
before, after = loads_codex_config(old), loads_codex_config(new)
|
||||
except tomllib.TOMLDecodeError:
|
||||
return False
|
||||
return _without_server(before, server_name) == _without_server(after, server_name)
|
||||
@@ -255,7 +256,7 @@ class _TomlMarkerRegistrar(MCPRegistrar):
|
||||
# Read via fsutil (UTF-8 with locale fallback) so a config that a
|
||||
# tool wrote in the system locale (e.g. GBK) still parses instead
|
||||
# of failing tomllib's UTF-8 requirement. See #733.
|
||||
data = tomllib.loads(fsutil.read_text(self._config_file))
|
||||
data = loads_codex_config(fsutil.read_text(self._config_file))
|
||||
except (tomllib.TOMLDecodeError, OSError):
|
||||
return {}
|
||||
return data if isinstance(data, dict) else {}
|
||||
@@ -276,7 +277,7 @@ class _TomlMarkerRegistrar(MCPRegistrar):
|
||||
if not raw.strip():
|
||||
return None
|
||||
try:
|
||||
data = tomllib.loads(fsutil.read_text(self._config_file))
|
||||
data = loads_codex_config(fsutil.read_text(self._config_file))
|
||||
except (tomllib.TOMLDecodeError, OSError) as exc:
|
||||
return f"{self._config_file} is not valid TOML ({exc})"
|
||||
if not isinstance(data, dict):
|
||||
|
||||
@@ -49,6 +49,20 @@ def _validate_metadata_key(key: str) -> bool:
|
||||
return _SAFE_METADATA_KEY_PATTERN.match(key) is not None
|
||||
|
||||
|
||||
def _utc_timestamp(value: str | None) -> str | None:
|
||||
"""Comparable UTC text without rewriting stored offsets or rounding fractions.
|
||||
|
||||
Naive timestamps follow Memory's UTC defaults. SQLite date functions round
|
||||
fractional seconds, so keep Python's full microsecond precision here.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
timestamp = datetime.fromisoformat(value)
|
||||
if timestamp.tzinfo is not None:
|
||||
timestamp = timestamp.astimezone(timezone.utc).replace(tzinfo=None)
|
||||
return timestamp.isoformat(timespec="microseconds")
|
||||
|
||||
|
||||
class SQLiteMemoryStore:
|
||||
"""SQLite-based memory store implementing the MemoryStore protocol.
|
||||
|
||||
@@ -67,6 +81,12 @@ class SQLiteMemoryStore:
|
||||
Schema:
|
||||
The memories table stores all Memory fields with appropriate indexes
|
||||
for efficient querying by scope, category, importance, and time.
|
||||
|
||||
Metadata filters:
|
||||
Numbers and booleans use native SQLite equality. String filters match only
|
||||
JSON text values. None matches an explicit JSON null,
|
||||
not a missing key. Arrays and objects compare as compact JSON text with
|
||||
matching JSON types; array order and object member order are significant.
|
||||
"""
|
||||
|
||||
def __init__(self, db_path: str | Path = "headroom_memory.db") -> None:
|
||||
@@ -89,6 +109,7 @@ class SQLiteMemoryStore:
|
||||
"""
|
||||
conn = connect_private_sqlite(self.db_path, what="memory store")
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.create_function("headroom_utc_timestamp", 1, _utc_timestamp, deterministic=True)
|
||||
try:
|
||||
with conn:
|
||||
yield conn
|
||||
@@ -501,21 +522,23 @@ class SQLiteMemoryStore:
|
||||
if scope_conditions:
|
||||
conditions.append(f"({' OR '.join(scope_conditions)})")
|
||||
|
||||
# Normalize both sides: persisted rows may retain explicit UTC offsets.
|
||||
# This favors correct instant comparisons over the raw TEXT time index.
|
||||
# Temporal filtering
|
||||
if filter.created_after is not None:
|
||||
conditions.append("created_at >= ?")
|
||||
params.append(filter.created_after.isoformat())
|
||||
conditions.append("headroom_utc_timestamp(created_at) >= ?")
|
||||
params.append(_utc_timestamp(filter.created_after.isoformat()))
|
||||
|
||||
if filter.created_before is not None:
|
||||
conditions.append("created_at <= ?")
|
||||
params.append(filter.created_before.isoformat())
|
||||
conditions.append("headroom_utc_timestamp(created_at) <= ?")
|
||||
params.append(_utc_timestamp(filter.created_before.isoformat()))
|
||||
|
||||
# Point-in-time query
|
||||
if filter.valid_at is not None:
|
||||
valid_at_str = filter.valid_at.isoformat()
|
||||
conditions.append("valid_from <= ?")
|
||||
valid_at_str = _utc_timestamp(filter.valid_at.isoformat())
|
||||
conditions.append("headroom_utc_timestamp(valid_from) <= ?")
|
||||
params.append(valid_at_str)
|
||||
conditions.append("(valid_until IS NULL OR valid_until > ?)")
|
||||
conditions.append("(valid_until IS NULL OR headroom_utc_timestamp(valid_until) > ?)")
|
||||
params.append(valid_at_str)
|
||||
|
||||
# Superseded filtering
|
||||
@@ -571,13 +594,22 @@ class SQLiteMemoryStore:
|
||||
# equates ``5 = '5'`` — so an int/float/bool metadata filter matched
|
||||
# nothing. ``bool`` is a subclass of ``int``, so it is covered here
|
||||
# (a JSON ``true`` extracts to 1, and Python ``True`` binds to 1).
|
||||
# A non-scalar value (dict/list) is not a bindable SQLite type, so it
|
||||
# keeps the JSON-text comparison rather than raising.
|
||||
conditions.append(f"json_extract(metadata, '$.{key}') = ?")
|
||||
if isinstance(value, (str, int, float)):
|
||||
if value is None:
|
||||
# json_extract returns SQL NULL for both null and missing keys.
|
||||
conditions.append(f"json_type(metadata, '$.{key}') = 'null'")
|
||||
elif isinstance(value, (str, int, float)):
|
||||
if isinstance(value, str):
|
||||
# Extracted JSON arrays/objects are also SQL text.
|
||||
conditions.append(f"json_type(metadata, '$.{key}') = 'text'")
|
||||
conditions.append(f"json_extract(metadata, '$.{key}') = ?")
|
||||
params.append(value)
|
||||
else:
|
||||
params.append(json.dumps(value))
|
||||
# Normalize whitespace like json_extract does for arrays/objects,
|
||||
# and do not match a string containing the same JSON text.
|
||||
encoded = json.dumps(value)
|
||||
conditions.append(f"json_type(metadata, '$.{key}') = json_type(?)")
|
||||
conditions.append(f"json_extract(metadata, '$.{key}') = json(?)")
|
||||
params.extend([encoded, encoded])
|
||||
|
||||
return conditions, params
|
||||
|
||||
@@ -605,6 +637,9 @@ class SQLiteMemoryStore:
|
||||
):
|
||||
order_column = "created_at"
|
||||
|
||||
if order_column in ("created_at", "last_accessed"):
|
||||
order_column = f"headroom_utc_timestamp({order_column})"
|
||||
|
||||
order_direction = "DESC" if filter.order_desc else "ASC"
|
||||
|
||||
# Build full query
|
||||
@@ -798,23 +833,28 @@ class SQLiteMemoryStore:
|
||||
return []
|
||||
|
||||
history: list[Memory] = [current]
|
||||
# Imported or reused IDs can form cycles. Visit each stored record
|
||||
# only once across both directions instead of blocking indefinitely.
|
||||
visited = {current.id}
|
||||
|
||||
# Follow chain backwards (supersedes)
|
||||
back_id = current.supersedes
|
||||
while back_id is not None:
|
||||
while back_id is not None and back_id not in visited:
|
||||
prev = await self.get(back_id)
|
||||
if prev is None:
|
||||
break
|
||||
visited.add(prev.id)
|
||||
history.insert(0, prev) # Add to beginning
|
||||
back_id = prev.supersedes
|
||||
|
||||
# Follow chain forwards (superseded_by) if requested
|
||||
if include_future:
|
||||
forward_id = current.superseded_by
|
||||
while forward_id is not None:
|
||||
while forward_id is not None and forward_id not in visited:
|
||||
next_mem = await self.get(forward_id)
|
||||
if next_mem is None:
|
||||
break
|
||||
visited.add(next_mem.id)
|
||||
history.append(next_mem)
|
||||
forward_id = next_mem.superseded_by
|
||||
|
||||
|
||||
@@ -248,7 +248,9 @@ class SQLiteGraphStore:
|
||||
async def add_entity(self, entity: Entity) -> None:
|
||||
"""Add an entity to the graph store.
|
||||
|
||||
If an entity with the same ID already exists, it will be replaced.
|
||||
If an entity with the same ID and owner already exists, its fields are
|
||||
updated without deleting its relationships. Reassigning the owner
|
||||
replaces the entity and deletes its old relationships.
|
||||
|
||||
Args:
|
||||
entity: The entity to add.
|
||||
@@ -257,6 +259,14 @@ class SQLiteGraphStore:
|
||||
|
||||
with self._lock:
|
||||
with self._get_conn() as conn:
|
||||
# Keep owner reassignment's replacement/cascade semantics. Both
|
||||
# statements share a transaction, so a failed save restores the old graph.
|
||||
conn.execute(
|
||||
"DELETE FROM entities WHERE id = :id AND user_id != :user_id",
|
||||
row,
|
||||
)
|
||||
# The ID upsert overrides REPLACE's delete. Keep REPLACE's existing
|
||||
# NOT NULL/default handling for the incoming row.
|
||||
conn.execute(
|
||||
"""
|
||||
INSERT OR REPLACE INTO entities (
|
||||
@@ -266,6 +276,16 @@ class SQLiteGraphStore:
|
||||
:id, :user_id, :name, :name_lower, :entity_type, :description,
|
||||
:properties, :created_at, :updated_at, :metadata
|
||||
)
|
||||
ON CONFLICT(id) DO UPDATE SET
|
||||
user_id = excluded.user_id,
|
||||
name = excluded.name,
|
||||
name_lower = excluded.name_lower,
|
||||
entity_type = excluded.entity_type,
|
||||
description = excluded.description,
|
||||
properties = excluded.properties,
|
||||
created_at = excluded.created_at,
|
||||
updated_at = excluded.updated_at,
|
||||
metadata = excluded.metadata
|
||||
""",
|
||||
row,
|
||||
)
|
||||
|
||||
@@ -91,7 +91,7 @@ class ResolvedScope:
|
||||
"""
|
||||
|
||||
mode: MemoryStorageMode
|
||||
db_path: Path
|
||||
db_path: Path | None
|
||||
display_name: str # human-readable label, e.g. project basename
|
||||
project_key: str | None # stable hash, None for USER/GLOBAL
|
||||
|
||||
@@ -304,13 +304,20 @@ class BackendRouter:
|
||||
self._backends: OrderedDict[Path, LocalBackend] = OrderedDict()
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def backend_for(self, ctx: RequestContext) -> tuple[LocalBackend, ResolvedScope]:
|
||||
def backend_for(self, ctx: RequestContext) -> tuple[LocalBackend | None, ResolvedScope]:
|
||||
"""Return the backend + scope metadata to use for this request."""
|
||||
|
||||
scope = self._resolve_scope(ctx)
|
||||
if scope.db_path is None:
|
||||
return None, scope
|
||||
backend = self._get_or_create_backend(scope.db_path)
|
||||
return backend, scope
|
||||
|
||||
def scope_for(self, ctx: RequestContext) -> ResolvedScope:
|
||||
"""Resolve request scope without acquiring or creating a backend."""
|
||||
|
||||
return self._resolve_scope(ctx)
|
||||
|
||||
def _resolve_scope(self, ctx: RequestContext) -> ResolvedScope:
|
||||
mode = self._config.mode
|
||||
|
||||
@@ -363,7 +370,7 @@ class BackendRouter:
|
||||
)
|
||||
return ResolvedScope(
|
||||
mode=MemoryStorageMode.PROJECT,
|
||||
db_path=self._config.global_db_path, # Unused — caller checks project_key.
|
||||
db_path=None,
|
||||
display_name="unresolved (no memory)",
|
||||
project_key=None,
|
||||
)
|
||||
|
||||
@@ -320,6 +320,10 @@ def _paths_related_as_typo(failed: str, success: str) -> bool:
|
||||
return False
|
||||
if a == b:
|
||||
return True
|
||||
# Basenames come from tool traffic; no real file name comes near this cap,
|
||||
# so skip the quadratic edit distance rather than run it on an oversized one.
|
||||
if max(len(a), len(b)) > _RETRY_EDIT_DISTANCE_MAX_CHARS:
|
||||
return False
|
||||
threshold = max(2, max(len(a), len(b)) // 3)
|
||||
return _levenshtein(a, b) <= threshold
|
||||
|
||||
@@ -375,13 +379,21 @@ def _bash_binaries_match(a: str, b: str) -> bool:
|
||||
b_base = b.rsplit("/", 1)[-1]
|
||||
if a_base == b_base:
|
||||
return True
|
||||
if (a_base.startswith(b_base) or b_base.startswith(a_base)) and _levenshtein(
|
||||
a_base, b_base
|
||||
# One name is a prefix of the other, so their edit distance is exactly the
|
||||
# length difference: compare lengths instead of running _levenshtein (O(n*m))
|
||||
# on an unbounded, traffic-derived token.
|
||||
if (a_base.startswith(b_base) or b_base.startswith(a_base)) and abs(
|
||||
len(a_base) - len(b_base)
|
||||
) <= 2:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# _levenshtein is O(n*m) pure Python and runs on the request path; longer
|
||||
# commands skip it and are judged by the linear token-overlap check below.
|
||||
_RETRY_EDIT_DISTANCE_MAX_CHARS = 512
|
||||
|
||||
|
||||
def _commands_related_as_retry(failed: str, success: str) -> bool:
|
||||
"""Heuristic: is `success` plausibly a corrected retry of `failed`?
|
||||
|
||||
@@ -405,7 +417,10 @@ def _commands_related_as_retry(failed: str, success: str) -> bool:
|
||||
return False
|
||||
|
||||
max_len = max(len(failed), len(success))
|
||||
if max_len > 0 and _levenshtein(failed, success) / max_len <= 0.40:
|
||||
if (
|
||||
0 < max_len <= _RETRY_EDIT_DISTANCE_MAX_CHARS
|
||||
and _levenshtein(failed, success) / max_len <= 0.40
|
||||
):
|
||||
return True
|
||||
|
||||
def _substantive(cmd: str, binary: str) -> set[str]:
|
||||
|
||||
@@ -224,6 +224,7 @@ class PerfReport:
|
||||
oldest_kept_ts: str | None = None
|
||||
newest_kept_ts: str | None = None
|
||||
records_filtered_out: int = 0
|
||||
records_malformed: int = 0
|
||||
# True when no time cutoff was applied (--hours 0, or a value so large it
|
||||
# overflows datetime arithmetic). The header says "all data" instead of a
|
||||
# misleading "last 0h".
|
||||
@@ -386,9 +387,8 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport:
|
||||
if not _within_window(ts):
|
||||
report.records_filtered_out += 1
|
||||
continue
|
||||
_track_window(ts)
|
||||
report.perf_records.append(
|
||||
PerfRecord(
|
||||
try:
|
||||
record = PerfRecord(
|
||||
timestamp=ts,
|
||||
request_id=m.group("rid"),
|
||||
model=kv.get("model", ""),
|
||||
@@ -411,7 +411,13 @@ def parse_log_files(last_n_hours: float = 168.0) -> PerfReport:
|
||||
from_response_cache=kv.get("cached", "0") == "1",
|
||||
stages=stages_by_rid.get(m.group("rid"), {}),
|
||||
)
|
||||
)
|
||||
except ValueError:
|
||||
# A non-numeric field is not a PERF line Headroom wrote
|
||||
# (e.g. text a client put into a logged model id).
|
||||
report.records_malformed += 1
|
||||
continue
|
||||
_track_window(ts)
|
||||
report.perf_records.append(record)
|
||||
continue
|
||||
|
||||
# content_router summary lines
|
||||
@@ -544,6 +550,11 @@ def format_report(report: PerfReport) -> str:
|
||||
f"Records outside window: {report.records_filtered_out:,} "
|
||||
"(filtered out — increase --hours to include them)"
|
||||
)
|
||||
if report.records_malformed > 0:
|
||||
lines.append(
|
||||
f"Malformed PERF lines: {report.records_malformed:,} "
|
||||
"(skipped — a numeric field did not parse)"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
records = report.perf_records
|
||||
@@ -1400,6 +1411,7 @@ def build_perf_summary(report: PerfReport) -> dict:
|
||||
"newest": report.newest_kept_ts,
|
||||
},
|
||||
"records_filtered_out": report.records_filtered_out,
|
||||
"records_malformed": report.records_malformed,
|
||||
"total_requests": len(records),
|
||||
"total_tokens_before": total_before,
|
||||
"total_tokens_after": total_after,
|
||||
|
||||
@@ -56,6 +56,7 @@ from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import lru_cache
|
||||
@@ -68,11 +69,16 @@ from headroom.pricing.cache_ttl import CACHE_WRITE_MULTIPLIERS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
#: Context size at which the major catalogs publish a second, higher price tier
|
||||
#: (LiteLLM spells it ``*_above_200k_tokens``). A request's billed prompt is
|
||||
#: compared against this to pick which rate applies.
|
||||
#: Prompt size at which most catalog rows publish a second, higher price tier
|
||||
#: (LiteLLM spells it ``*_above_200k_tokens``). The default for
|
||||
#: :func:`long_context_threshold`, which reads each model's own threshold from
|
||||
#: its row: Claude Haiku 5.5's tier starts at 100K, GPT-5.x's at 272K.
|
||||
LONG_CONTEXT_THRESHOLD_TOKENS = 200_000
|
||||
|
||||
#: A catalog row's long-context input rate; the number is the threshold in
|
||||
#: thousands of tokens. Each row publishes at most one.
|
||||
_LONG_CONTEXT_INPUT_FIELD = re.compile(r"^input_cost_per_token_above_(\d+)k_tokens$")
|
||||
|
||||
#: Provider-level cache discount ratios, as a fraction of the base input price.
|
||||
#: FALLBACK ONLY — the per-model LiteLLM catalog is always preferred, because
|
||||
#: these go stale per model and per context tier. The lightweight shared table
|
||||
@@ -237,15 +243,18 @@ class CacheMix:
|
||||
"""
|
||||
return self.billed > 0
|
||||
|
||||
def is_long_context(self, *, local_tokens: int = 0) -> bool:
|
||||
"""True when this request billed above the catalogs' 200k price tier.
|
||||
def is_long_context(
|
||||
self, *, local_tokens: int = 0, threshold: int = LONG_CONTEXT_THRESHOLD_TOKENS
|
||||
) -> bool:
|
||||
"""True when this request billed above ``threshold``, the model's long-context tier.
|
||||
|
||||
``local_tokens`` lets a caller contribute its own forwarded-token count
|
||||
for requests where the provider reported no breakdown; the larger of the
|
||||
two decides, so a long request is never priced at the cheap tier merely
|
||||
because usage was missing.
|
||||
because usage was missing. Pass :func:`long_context_threshold` for the
|
||||
model so a tier that starts below 200K (Haiku 5.5: 100K) is honoured.
|
||||
"""
|
||||
return max(self.billed, _coerce_int(local_tokens)) > LONG_CONTEXT_THRESHOLD_TOKENS
|
||||
return max(self.billed, _coerce_int(local_tokens)) > threshold
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -358,6 +367,34 @@ def _litellm() -> Any | None:
|
||||
_RATE_CACHE_MAXSIZE = 256
|
||||
|
||||
|
||||
def long_context_tier(info: dict[str, Any]) -> tuple[int, str] | None:
|
||||
"""The row's long-context tier as ``(threshold_tokens, field_suffix)``, if any."""
|
||||
for key in info:
|
||||
match = _LONG_CONTEXT_INPUT_FIELD.match(key)
|
||||
if match:
|
||||
thousands = int(match.group(1))
|
||||
return thousands * 1000, f"_above_{thousands}k_tokens"
|
||||
return None
|
||||
|
||||
|
||||
@lru_cache(maxsize=_RATE_CACHE_MAXSIZE)
|
||||
def long_context_threshold(model: str) -> int:
|
||||
"""Billed-prompt size above which ``model`` pays its long-context rates.
|
||||
|
||||
Read from the catalog row's ``*_above_<N>k_tokens`` fields: 100K for Claude
|
||||
Haiku 5.5, 200K for Sonnet 4 / 4.5, 272K for GPT-5.x. A model with no tier,
|
||||
or one the catalog does not list, gets :data:`LONG_CONTEXT_THRESHOLD_TOKENS`;
|
||||
that is harmless, because :func:`resolve_rates` then has no tier to switch
|
||||
to and returns the base rates either way.
|
||||
|
||||
Memoized on the same bounded LRU as :func:`resolve_rates`, for the same
|
||||
reason; tests that swap the catalog clear both.
|
||||
"""
|
||||
row = _catalog_row(model)
|
||||
tier = long_context_tier(row[2]) if row is not None else None
|
||||
return tier[0] if tier else LONG_CONTEXT_THRESHOLD_TOKENS
|
||||
|
||||
|
||||
def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context: bool) -> float:
|
||||
"""Resolve an absent catalog slice through the model's billing calculator.
|
||||
|
||||
@@ -365,7 +402,7 @@ def _canonical_cache_rate(litellm: Any, model: str, field: str, *, long_context:
|
||||
zero, list price, or a model-specific discount. Never invent our own ratio.
|
||||
The result is cached with the other resolved rates, not probed per request.
|
||||
"""
|
||||
tokens = LONG_CONTEXT_THRESHOLD_TOKENS + 1 if long_context else 1
|
||||
tokens = long_context_threshold(model) + 1 if long_context else 1
|
||||
try:
|
||||
input_cost, _ = litellm.cost_per_token(
|
||||
model=model, prompt_tokens=tokens, completion_tokens=0, **{field: tokens}
|
||||
@@ -413,11 +450,12 @@ def resolve_rates(
|
||||
Preference order, strongest first:
|
||||
|
||||
1. **Catalog** — LiteLLM's per-model published rates, including the
|
||||
long-context tier and, where the row has it,
|
||||
long-context tier (the row's own ``*_above_<N>k_tokens`` fields, see
|
||||
:func:`long_context_threshold`) and, where the row has it,
|
||||
``cache_creation_input_token_cost_above_1hr``. Anthropic publishes a real
|
||||
1h write rate (Sonnet: 2.00x base), and reading it beats deriving it.
|
||||
2. **Catalog + TTL ratio** — a catalog row that prices 5m writes but not 1h
|
||||
ones, which is every row at the above-200k tier. The 1h rate is derived
|
||||
ones, which is every row at its long-context tier. The 1h rate is derived
|
||||
from the tier's base input price and the structural multiplier in
|
||||
:mod:`headroom.pricing.cache_ttl`. Only applied when the row shows a real
|
||||
write premium: a provider that does not bill for cache writes at all
|
||||
@@ -446,14 +484,17 @@ def resolve_rates(
|
||||
# savings on a model that costs nothing.
|
||||
if base is None:
|
||||
return None
|
||||
if long_context:
|
||||
base = info.get("input_cost_per_token_above_200k_tokens") or base
|
||||
tier = long_context_tier(info) if long_context else None
|
||||
if tier:
|
||||
long_base = info.get(f"input_cost_per_token{tier[1]}")
|
||||
if long_base is not None:
|
||||
base = long_base
|
||||
base = float(base)
|
||||
|
||||
def _tier(field: str, default: float) -> float:
|
||||
"""Read ``field``, preferring its above-200k variant on long requests."""
|
||||
if long_context:
|
||||
hi = info.get(f"{field}_above_200k_tokens")
|
||||
"""Read ``field``, preferring its long-context variant on long requests."""
|
||||
if tier:
|
||||
hi = info.get(f"{field}{tier[1]}")
|
||||
if hi is not None:
|
||||
return float(hi)
|
||||
value = info.get(field)
|
||||
@@ -468,10 +509,10 @@ def resolve_rates(
|
||||
# billed provider-reported writes require their own catalog rate.
|
||||
write_5m = _tier("cache_creation_input_token_cost", missing_cache_rate)
|
||||
read_is_catalog = info.get("cache_read_input_token_cost") is not None or (
|
||||
long_context and info.get("cache_read_input_token_cost_above_200k_tokens") is not None
|
||||
tier is not None and info.get(f"cache_read_input_token_cost{tier[1]}") is not None
|
||||
)
|
||||
write_is_catalog = info.get("cache_creation_input_token_cost") is not None or (
|
||||
long_context and info.get("cache_creation_input_token_cost_above_200k_tokens") is not None
|
||||
tier is not None and info.get(f"cache_creation_input_token_cost{tier[1]}") is not None
|
||||
)
|
||||
if for_billing and not read_is_catalog:
|
||||
read = _canonical_cache_rate(
|
||||
@@ -484,10 +525,12 @@ def resolve_rates(
|
||||
|
||||
basis = BASIS_CATALOG
|
||||
write_1h_raw = info.get("cache_creation_input_token_cost_above_1hr")
|
||||
if long_context:
|
||||
# No catalog publishes a combined 1h + above-200k rate, so the long
|
||||
# tier always derives. Ratio basis, and labelled as such.
|
||||
write_1h_raw = None
|
||||
if tier:
|
||||
# The long tier's 1h rate is its own field
|
||||
# (``cache_creation_input_token_cost_above_1hr_above_100k_tokens``);
|
||||
# most tiered rows omit it, and the rate then derives below. Ratio
|
||||
# basis, and labelled as such.
|
||||
write_1h_raw = info.get(f"cache_creation_input_token_cost_above_1hr{tier[1]}")
|
||||
if write_1h_raw is not None:
|
||||
write_1h = float(write_1h_raw)
|
||||
elif write_5m > base:
|
||||
@@ -540,6 +583,63 @@ def resolve_rates(
|
||||
)
|
||||
|
||||
|
||||
def long_context_premium_avoided_usd(
|
||||
model: str,
|
||||
mix: CacheMix,
|
||||
*,
|
||||
local_tokens: int = 0,
|
||||
output_tokens: int = 0,
|
||||
provider: str | None = None,
|
||||
) -> float:
|
||||
"""Long-context premium a request's forwarded input and output avoided, in USD.
|
||||
|
||||
For a request whose uncompressed prompt would have passed ``model``'s
|
||||
long-context threshold but whose forwarded prompt did not: the provider
|
||||
re-prices the WHOLE request above the threshold, so compression saved the
|
||||
long card on every forwarded input token, on top of the removed tokens
|
||||
themselves. This is that difference, forwarded input at the long card minus
|
||||
the same input at the base card. When ``output_tokens`` is supplied, include
|
||||
the premium avoided on that generated output too. These tokens were still
|
||||
generated; this is compression savings, not output-shaping savings.
|
||||
|
||||
``local_tokens`` stands in as uncached input when the provider reported no
|
||||
breakdown. Returns 0.0 for a model with no tier or no catalog row.
|
||||
"""
|
||||
try:
|
||||
long_rates = resolve_rates(model, long_context=True, provider=provider)
|
||||
base_rates = resolve_rates(model, long_context=False, provider=provider)
|
||||
except Exception: # pragma: no cover - defensive; pricing must never raise
|
||||
return 0.0
|
||||
if long_rates is None or base_rates is None:
|
||||
return 0.0
|
||||
if mix.has_signal():
|
||||
n = mix.normalized()
|
||||
split = TokenSplit(
|
||||
read=float(n.read),
|
||||
write_5m=float(n.write_5m),
|
||||
write_1h=float(n.write_1h),
|
||||
uncached=float(n.uncached),
|
||||
)
|
||||
else:
|
||||
split = TokenSplit(uncached=float(max(_coerce_int(local_tokens), 0)))
|
||||
input_premium = max(0.0, long_rates.price(split) - base_rates.price(split))
|
||||
output_premium = 0.0
|
||||
if max(_coerce_int(output_tokens), 0):
|
||||
try:
|
||||
row = _catalog_row(model)
|
||||
info = row[2] if row is not None else {}
|
||||
tier = long_context_tier(info)
|
||||
base_output = info.get("output_cost_per_token")
|
||||
long_output = info.get(f"output_cost_per_token{tier[1]}") if tier else None
|
||||
if base_output is not None and long_output is not None:
|
||||
output_premium = max(0.0, float(long_output) - float(base_output)) * max(
|
||||
_coerce_int(output_tokens), 0
|
||||
)
|
||||
except Exception: # Pricing metadata must not break telemetry.
|
||||
pass
|
||||
return input_premium + output_premium
|
||||
|
||||
|
||||
def catalog_min_cacheable_tokens(model: str) -> int | None:
|
||||
"""The catalog's smallest prompt the provider will cache for ``model``.
|
||||
|
||||
@@ -613,7 +713,9 @@ def price_savings(
|
||||
|
||||
mix = (mix or CacheMix()).normalized()
|
||||
if long_context is None:
|
||||
long_context = mix.is_long_context(local_tokens=local_tokens)
|
||||
long_context = mix.is_long_context(
|
||||
local_tokens=local_tokens, threshold=long_context_threshold(model)
|
||||
)
|
||||
|
||||
try:
|
||||
rates = resolve_rates(model, long_context=long_context, provider=provider)
|
||||
@@ -664,6 +766,9 @@ __all__ = [
|
||||
"PricedSavings",
|
||||
"Region",
|
||||
"TokenSplit",
|
||||
"long_context_premium_avoided_usd",
|
||||
"long_context_threshold",
|
||||
"long_context_tier",
|
||||
"catalog_min_cacheable_tokens",
|
||||
"price_savings",
|
||||
"resolve_rates",
|
||||
|
||||
+106
-26
@@ -132,6 +132,8 @@ ANTHROPIC_CONTEXT_LIMITS: dict[str, int] = {
|
||||
"claude-sonnet-4-6": 1000000,
|
||||
# Claude Sonnet 4.5
|
||||
"claude-sonnet-4-5": 200000,
|
||||
# Claude Haiku 5.5 - 1M context
|
||||
"claude-haiku-5-5": 1000000,
|
||||
# Claude 4 (Sonnet 4, Haiku 4)
|
||||
"claude-sonnet-4-20250514": 200000,
|
||||
"claude-haiku-4-5-20251001": 200000,
|
||||
@@ -155,9 +157,9 @@ ANTHROPIC_CONTEXT_LIMITS: dict[str, int] = {
|
||||
# NOTE: These are ESTIMATES. Always verify against actual Anthropic billing.
|
||||
# Last updated: 2026-09-29 (platform.claude.com/docs/en/about-claude/pricing)
|
||||
#
|
||||
# Newer ids come before their prefixes: `_get_pricing` falls back to substring
|
||||
# matching in insertion order, so a dated or suffixed "claude-sonnet-5-5-..."
|
||||
# must hit the Sonnet 5.5 row before it can hit "claude-sonnet-5".
|
||||
# Newer ids still come before their prefixes, but order no longer decides a
|
||||
# match: `_is_release_of` refuses to read "claude-sonnet-5-5-..." as a release
|
||||
# of "claude-sonnet-5".
|
||||
ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
|
||||
# Claude Fable 5.1: $10 in / $50 out, cache read $0.25 (0.025x input).
|
||||
"claude-fable-5-1": {"input": 10.00, "output": 50.00, "cached_input": 0.25},
|
||||
@@ -183,6 +185,9 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
|
||||
# Claude Sonnet 4.6 / 4.5: $3 in / $15 out, cache read $0.30
|
||||
"claude-sonnet-4-6": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
|
||||
"claude-sonnet-4-5": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
|
||||
# Claude Haiku 5.5: $0.10 in / $0.50 out, cache read $0.01, for prompts up
|
||||
# to 100K tokens; longer prompts use a 5x rate card (`_LONG_CONTEXT_TIERS`).
|
||||
"claude-haiku-5-5": {"input": 0.10, "output": 0.50, "cached_input": 0.01},
|
||||
# Claude 4 (Sonnet/Haiku tier pricing)
|
||||
"claude-sonnet-4-20250514": {"input": 3.00, "output": 15.00, "cached_input": 0.30},
|
||||
"claude-haiku-4-5-20251001": {"input": 1.00, "output": 5.00, "cached_input": 0.10},
|
||||
@@ -204,10 +209,10 @@ ANTHROPIC_PRICING: dict[str, dict[str, float]] = {
|
||||
# derived from LiteLLM's `*_above_200k_tokens` fields ($3->$6 in, $15->$22.50
|
||||
# out, $0.30->$0.60 cache read).
|
||||
#
|
||||
# Only the Sonnet 4 / 4.5 family is tiered: Opus, and Sonnet 4.6 onward, are
|
||||
# flat-rated across their whole window. This is the same population that needs
|
||||
# the `[1m]` suffix to reach 1M at all, so a session that fills the window this
|
||||
# unlocks is billed at these rates.
|
||||
# Among the 200K-threshold models only the Sonnet 4 / 4.5 family is tiered:
|
||||
# Opus, and Sonnet 4.6 onward, are flat-rated across their whole window. This
|
||||
# is the same population that needs the `[1m]` suffix to reach 1M at all, so a
|
||||
# session that fills the window this unlocks is billed at these rates.
|
||||
_LONG_CONTEXT_THRESHOLD = 200_000
|
||||
_LONG_CONTEXT_PREMIUM: dict[str, float] = {"input": 2.0, "output": 1.5, "cached_input": 2.0}
|
||||
_LONG_CONTEXT_TIERED_MODELS = (
|
||||
@@ -216,6 +221,55 @@ _LONG_CONTEXT_TIERED_MODELS = (
|
||||
"claude-4-sonnet-20250514",
|
||||
)
|
||||
|
||||
# Claude Haiku 5.5 has its own rate card for prompts above 100K tokens: every
|
||||
# rate is 5x (LiteLLM's `*_above_100k_tokens` fields: $0.10->$0.50 in,
|
||||
# $0.50->$2.50 out, $0.01->$0.05 cache read).
|
||||
_HAIKU_5_5_LONG_PROMPT_THRESHOLD = 100_000
|
||||
_HAIKU_5_5_LONG_PROMPT_PREMIUM: dict[str, float] = {
|
||||
"input": 5.0,
|
||||
"output": 5.0,
|
||||
"cached_input": 5.0,
|
||||
}
|
||||
|
||||
# (model ids, prompt-token threshold, per-rate multipliers above it). Ids match
|
||||
# releases and provider-wrapped forms too (`_long_context_tier`).
|
||||
_LONG_CONTEXT_TIERS: tuple[tuple[tuple[str, ...], int, dict[str, float]], ...] = (
|
||||
(_LONG_CONTEXT_TIERED_MODELS, _LONG_CONTEXT_THRESHOLD, _LONG_CONTEXT_PREMIUM),
|
||||
(("claude-haiku-5-5",), _HAIKU_5_5_LONG_PROMPT_THRESHOLD, _HAIKU_5_5_LONG_PROMPT_PREMIUM),
|
||||
)
|
||||
|
||||
|
||||
# A dated release (`-20250929`) or `-latest` alias of a model id.
|
||||
_RELEASE_SUFFIX_RE = re.compile(r"-(?:\d{8}|latest)")
|
||||
# A further version segment (the `-5` in `claude-haiku-5-5`): another model.
|
||||
_VERSION_SEGMENT_RE = re.compile(r"-\d{1,2}(?!\d)")
|
||||
|
||||
|
||||
def _is_release_of(model: str, known_model: str) -> bool:
|
||||
"""True when ``model`` names ``known_model``, or a release or alias of it.
|
||||
|
||||
The fallback tables are keyed by a mix of undated and dated ids, so a
|
||||
lookup accepts:
|
||||
|
||||
- a longer id containing ``known_model``: a dated release or a provider
|
||||
wrapped id (``claude-sonnet-4-5-20250929``,
|
||||
``anthropic.claude-haiku-5-5-v1:0``);
|
||||
- an undated alias of a dated key (``claude-3-5-sonnet`` for
|
||||
``claude-3-5-sonnet-20241022``), or its ``-latest`` form.
|
||||
|
||||
An id one version segment away is a different model and does not match:
|
||||
``claude-haiku-5`` is not ``claude-haiku-5-5``, and ``claude-sonnet-5-5``
|
||||
is not ``claude-sonnet-5``.
|
||||
"""
|
||||
if model == known_model:
|
||||
return True
|
||||
if known_model.startswith(model):
|
||||
return bool(_RELEASE_SUFFIX_RE.fullmatch(known_model[len(model) :]))
|
||||
at = model.find(known_model)
|
||||
if at < 0:
|
||||
return False
|
||||
return not _VERSION_SEGMENT_RE.match(model, at + len(known_model))
|
||||
|
||||
|
||||
def _apply_long_context_premium(
|
||||
model: str, pricing: dict[str, float], input_tokens: int
|
||||
@@ -226,11 +280,24 @@ def _apply_long_context_premium(
|
||||
LiteLLM entry has no above-threshold rate (see
|
||||
``_litellm_lacks_long_context_rate``).
|
||||
"""
|
||||
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
|
||||
tier = _long_context_tier(model)
|
||||
if tier is None or input_tokens <= tier[0]:
|
||||
return pricing
|
||||
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
|
||||
return pricing
|
||||
return {key: rate * _LONG_CONTEXT_PREMIUM.get(key, 1.0) for key, rate in pricing.items()}
|
||||
_threshold, premium = tier
|
||||
return {key: rate * premium.get(key, 1.0) for key, rate in pricing.items()}
|
||||
|
||||
|
||||
def _long_context_tier(model: str) -> tuple[int, dict[str, float]] | None:
|
||||
"""Return the ``(threshold, premium)`` of the long-context tier ``model`` is in.
|
||||
|
||||
Matches with ``_is_release_of``, as the manual rate lookup does, so a
|
||||
Bedrock id such as ``us.anthropic.claude-haiku-5-5-v1:0`` that the manual
|
||||
table prices as Haiku 5.5 also gets Haiku 5.5's long-prompt card.
|
||||
"""
|
||||
for known_models, threshold, premium in _LONG_CONTEXT_TIERS:
|
||||
if any(_is_release_of(model, known) for known in known_models):
|
||||
return threshold, premium
|
||||
return None
|
||||
|
||||
|
||||
def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
|
||||
@@ -240,17 +307,29 @@ def _litellm_lacks_long_context_rate(model: str, input_tokens: int) -> bool:
|
||||
from the bare Anthropic ids before, and pricing such a request through it then
|
||||
bills the base tier: about half the real cost of a long session. The manual
|
||||
table still carries the premium, so the caller uses it instead.
|
||||
|
||||
Each tier is checked against its own field, named after its threshold
|
||||
(``*_above_200k_tokens`` for Sonnet 4 / 4.5, ``*_above_100k_tokens`` for
|
||||
Haiku 5.5).
|
||||
|
||||
The row is resolved as ``estimate_cost_from_tokens`` resolves it, so a
|
||||
gateway id such as ``anthropic/claude-haiku-5-5`` is checked against the
|
||||
bare row it is billed from. Only Anthropic's own rows count: a reseller row
|
||||
without the field (``perplexity/anthropic/claude-haiku-5-5``) is
|
||||
deliberately flat-rated, not a dropped rate.
|
||||
"""
|
||||
if input_tokens <= _LONG_CONTEXT_THRESHOLD:
|
||||
return False
|
||||
if not any(model.startswith(tiered) for tiered in _LONG_CONTEXT_TIERED_MODELS):
|
||||
return False
|
||||
cost_data = get_litellm_model_cost()
|
||||
for candidate in pricing_lookup_candidates(model):
|
||||
info = cost_data.get(candidate)
|
||||
if isinstance(info, dict):
|
||||
return "input_cost_per_token_above_200k_tokens" not in info
|
||||
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
|
||||
row = next((c for c in pricing_lookup_candidates(model) if c in cost_data), None)
|
||||
if row is None:
|
||||
return False # LiteLLM doesn't know the model; it returns None and we fall back anyway
|
||||
info = cost_data[row]
|
||||
if not isinstance(info, dict) or info.get("litellm_provider") != "anthropic":
|
||||
return False
|
||||
tier = _long_context_tier(row)
|
||||
if tier is None or input_tokens <= tier[0]:
|
||||
return False
|
||||
threshold, _premium = tier
|
||||
return f"input_cost_per_token_above_{threshold // 1000}k_tokens" not in info
|
||||
|
||||
|
||||
# Default limits for pattern-based inference
|
||||
@@ -778,9 +857,10 @@ class AnthropicProvider(Provider):
|
||||
if model in self._context_limits:
|
||||
return self._context_limits[model]
|
||||
|
||||
# Check for partial matches (e.g., "claude-3-5-sonnet" matches "claude-3-5-sonnet-20241022")
|
||||
# Releases and aliases of a known id (e.g. "claude-3-5-sonnet" matches
|
||||
# "claude-3-5-sonnet-20241022"); see `_is_release_of`.
|
||||
for known_model, limit in self._context_limits.items():
|
||||
if model in known_model or known_model in model:
|
||||
if _is_release_of(model, known_model):
|
||||
return limit
|
||||
|
||||
# Try LiteLLM for context limit
|
||||
@@ -853,8 +933,8 @@ class AnthropicProvider(Provider):
|
||||
|
||||
Tries LiteLLM first for up-to-date pricing, falls back to manual pricing.
|
||||
Both paths apply Anthropic's long-context premium: on the Sonnet 4 / 4.5
|
||||
family a prompt over 200K re-prices the whole request (see
|
||||
``_LONG_CONTEXT_PREMIUM``).
|
||||
family a prompt over 200K, and on Haiku 5.5 a prompt over 100K,
|
||||
re-prices the whole request (see ``_LONG_CONTEXT_TIERS``).
|
||||
|
||||
``now`` selects a DeepSeek peak/off-peak tier (see
|
||||
:mod:`headroom.pricing.deepseek_tiers`); ``None`` reads the wall clock.
|
||||
@@ -898,9 +978,9 @@ class AnthropicProvider(Provider):
|
||||
if model in self._pricing:
|
||||
return self._pricing[model]
|
||||
|
||||
# Partial match
|
||||
# Releases and aliases of a known id; see `_is_release_of`.
|
||||
for known_model, prices in self._pricing.items():
|
||||
if model in known_model or known_model in model:
|
||||
if _is_release_of(model, known_model):
|
||||
return prices
|
||||
|
||||
# Pattern-based inference
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Parse Codex ``config.toml`` with the syntax Codex itself accepts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import Any, cast
|
||||
|
||||
import tomlkit
|
||||
from tomlkit.exceptions import TOMLKitError
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
import tomllib
|
||||
else: # pragma: no cover - exercised only on Python 3.10
|
||||
import tomli as tomllib # type: ignore[no-redef]
|
||||
|
||||
|
||||
def loads_codex_config(content: str) -> dict[str, Any]:
|
||||
"""Parse Codex config text, falling back to ``tomlkit`` for TOML 1.1 syntax.
|
||||
|
||||
Codex accepts TOML 1.1, where an inline table may span several lines and
|
||||
end with a trailing comma (#4378). ``tomllib`` before Python 3.15 only
|
||||
implements TOML 1.0 and rejects that, so retry with ``tomlkit`` (0.15+),
|
||||
which parses it. Text neither parser accepts raises the original
|
||||
``tomllib.TOMLDecodeError``, so callers keep their existing error handling.
|
||||
"""
|
||||
try:
|
||||
return cast("dict[str, Any]", tomllib.loads(content))
|
||||
except tomllib.TOMLDecodeError as exc:
|
||||
try:
|
||||
return cast("dict[str, Any]", tomlkit.parse(content).unwrap())
|
||||
except TOMLKitError:
|
||||
raise exc from None
|
||||
|
||||
|
||||
def split_toml_lines(content: str, keepends: bool = False) -> list[str]:
|
||||
"""Split ``content`` like ``str.splitlines``, but only at TOML newlines.
|
||||
|
||||
TOML ends a line at LF or CRLF only. ``str.splitlines`` also breaks at
|
||||
characters such as U+2028 and U+0085, which TOML allows inside a
|
||||
single-line string or comment, so its pieces are not always TOML lines.
|
||||
"""
|
||||
lines = [line + "\n" for line in content.split("\n")]
|
||||
lines[-1] = lines[-1][:-1]
|
||||
if not lines[-1]:
|
||||
lines.pop()
|
||||
if keepends:
|
||||
return lines
|
||||
return [line.removesuffix("\n").removesuffix("\r") for line in lines]
|
||||
|
||||
|
||||
def root_assignment_lines(content: str) -> set[int]:
|
||||
"""Return indexes of the lines that start a key/value pair at the document root.
|
||||
|
||||
Indexes refer to ``split_toml_lines(content)``. Only lines before
|
||||
the first table header count, and a line inside a multi-line inline table,
|
||||
array, or string is skipped, so a key nested in a multi-line inline table
|
||||
(TOML 1.1, e.g. ``profiles = {\\n work = { model_provider = ... } }``) is
|
||||
never mistaken for a root key.
|
||||
"""
|
||||
roots: set[int] = set()
|
||||
depth = 0
|
||||
open_string: str | None = None # delimiter of an open multi-line string
|
||||
for index, line in enumerate(split_toml_lines(content, keepends=True)):
|
||||
if depth == 0 and open_string is None:
|
||||
stripped = line.lstrip(" \t")
|
||||
if stripped.startswith("["):
|
||||
break # first table header ends the root
|
||||
if stripped and stripped[0] not in "#\r\n":
|
||||
roots.add(index)
|
||||
pos = 0
|
||||
while pos < len(line):
|
||||
if open_string is not None:
|
||||
if line.startswith(open_string, pos):
|
||||
pos += 3
|
||||
# Up to two quotes may directly precede the closing delimiter.
|
||||
for _ in range(2):
|
||||
if pos < len(line) and line[pos] == open_string[0]:
|
||||
pos += 1
|
||||
open_string = None
|
||||
elif open_string == '"""' and line[pos] == "\\":
|
||||
pos += 2
|
||||
else:
|
||||
pos += 1
|
||||
continue
|
||||
char = line[pos]
|
||||
if char == "#":
|
||||
break
|
||||
if line.startswith('"""', pos) or line.startswith("'''", pos):
|
||||
open_string = line[pos : pos + 3]
|
||||
pos += 3
|
||||
elif char in "\"'":
|
||||
pos += 1
|
||||
while pos < len(line) and line[pos] not in (char, "\n"):
|
||||
pos += 2 if char == '"' and line[pos] == "\\" else 1
|
||||
pos += 1
|
||||
else:
|
||||
if char in "[{":
|
||||
depth += 1
|
||||
elif char in "]}" and depth:
|
||||
depth -= 1
|
||||
pos += 1
|
||||
return roots
|
||||
@@ -21,6 +21,7 @@ from headroom._subprocess import run
|
||||
from headroom.install.models import ConfigScope, DeploymentManifest, ManagedMutation, ToolTarget
|
||||
from headroom.install.paths import codex_config_path
|
||||
|
||||
from .config_toml import loads_codex_config, root_assignment_lines, split_toml_lines
|
||||
from .runtime import proxy_base_url
|
||||
from .threads import retag_to_headroom, retag_to_native
|
||||
|
||||
@@ -65,7 +66,7 @@ print(key, end="")
|
||||
|
||||
def _codex_credential_store(config_dir: Path) -> str | None:
|
||||
try:
|
||||
config = tomllib.loads((config_dir / "config.toml").read_text(encoding="utf-8"))
|
||||
config = loads_codex_config((config_dir / "config.toml").read_text(encoding="utf-8"))
|
||||
except (OSError, tomllib.TOMLDecodeError):
|
||||
return None
|
||||
store = config.get("cli_auth_credentials_store")
|
||||
@@ -261,7 +262,7 @@ def codex_auth_helper_is_referenced(content: str, helper_path: str) -> bool | No
|
||||
the helper instead of risking deletion of a pre-existing user file.
|
||||
"""
|
||||
try:
|
||||
document = tomllib.loads(content)
|
||||
document = loads_codex_config(content)
|
||||
except (tomllib.TOMLDecodeError, TypeError):
|
||||
return None
|
||||
|
||||
@@ -358,7 +359,7 @@ def _insert_block_at_root(content: str, block: str) -> str:
|
||||
root instead.
|
||||
"""
|
||||
block = block.strip()
|
||||
lines = content.splitlines()
|
||||
lines = split_toml_lines(content)
|
||||
for index, line in enumerate(lines):
|
||||
if _TOML_TABLE_HEADER_RE.search(line):
|
||||
head = "\n".join(lines[:index]).rstrip()
|
||||
@@ -369,19 +370,18 @@ def _insert_block_at_root(content: str, block: str) -> str:
|
||||
|
||||
|
||||
def _strip_root_provider_assignments(content: str) -> str:
|
||||
"""Remove root provider assignments without touching table-scoped settings."""
|
||||
lines = content.splitlines(keepends=True)
|
||||
kept: list[str] = []
|
||||
in_root = True
|
||||
for line in lines:
|
||||
if in_root and _TOML_TABLE_HEADER_RE.search(line):
|
||||
in_root = False
|
||||
if in_root and (
|
||||
_ROOT_MODEL_PROVIDER_RE.match(line) or _ROOT_OPENAI_BASE_URL_RE.match(line)
|
||||
):
|
||||
continue
|
||||
kept.append(line)
|
||||
return "".join(kept)
|
||||
"""Remove root provider assignments without touching table-scoped settings.
|
||||
|
||||
Keys nested in a multi-line inline table (e.g. a ``profiles = {`` override)
|
||||
are not root assignments and are kept.
|
||||
"""
|
||||
root_lines = root_assignment_lines(content)
|
||||
return "".join(
|
||||
line
|
||||
for index, line in enumerate(split_toml_lines(content, keepends=True))
|
||||
if index not in root_lines
|
||||
or not (_ROOT_MODEL_PROVIDER_RE.match(line) or _ROOT_OPENAI_BASE_URL_RE.match(line))
|
||||
)
|
||||
|
||||
|
||||
def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None:
|
||||
@@ -411,7 +411,7 @@ def apply_provider_scope(manifest: DeploymentManifest) -> ManagedMutation | None
|
||||
existing = _CODEX_PATTERN.sub("", existing)
|
||||
existing = _strip_root_provider_assignments(existing)
|
||||
merged = _insert_block_at_root(existing, section)
|
||||
tomllib.loads(merged)
|
||||
loads_codex_config(merged)
|
||||
path.write_text(merged, encoding="utf-8")
|
||||
# Pull existing native threads into the headroom-provider menu so Codex's
|
||||
# history list stays whole once it routes through Headroom. Best-effort.
|
||||
|
||||
@@ -14,12 +14,10 @@ from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, cast
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - Python 3.10
|
||||
import tomli as tomllib
|
||||
import tomllib
|
||||
|
||||
from headroom.memory.storage_router import ProjectResolver, RequestContext
|
||||
from headroom.providers.codex.config_toml import loads_codex_config
|
||||
from headroom.providers.codex.threads import _codex_state_db_paths
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -298,7 +296,7 @@ class CodexProjectContextResolver:
|
||||
return self._configured_sqlite_home
|
||||
codex_home = Path(os.environ.get("CODEX_HOME", "").strip() or Path.home() / ".codex")
|
||||
try:
|
||||
config = tomllib.loads((codex_home / "config.toml").read_text(encoding="utf-8"))
|
||||
config = loads_codex_config((codex_home / "config.toml").read_text(encoding="utf-8"))
|
||||
configured_sqlite_home = config.get("sqlite_home")
|
||||
if isinstance(configured_sqlite_home, str) and configured_sqlite_home.strip():
|
||||
return Path(configured_sqlite_home).expanduser()
|
||||
|
||||
@@ -646,7 +646,7 @@ class OpenAIProvider(Provider):
|
||||
if model not in _UNKNOWN_MODEL_WARNINGS:
|
||||
_UNKNOWN_MODEL_WARNINGS.add(model)
|
||||
logger.warning(
|
||||
f"Unknown OpenAI model '{model}': {reason} ({limit:,} tokens). "
|
||||
f"Unknown OpenAI model {safe_id(model)}: {reason} ({limit:,} tokens). "
|
||||
f"To configure explicitly, set HEADROOM_MODEL_LIMITS env var or "
|
||||
f"add to ~/.headroom/models.json"
|
||||
)
|
||||
|
||||
@@ -373,6 +373,34 @@ def _call_openai_transport(
|
||||
return response
|
||||
|
||||
|
||||
def _call_anthropic_create_transport(
|
||||
client: Any,
|
||||
*,
|
||||
model: str,
|
||||
messages: list[dict[str, Any]],
|
||||
stream: bool,
|
||||
metrics: Any,
|
||||
**kwargs: Any,
|
||||
) -> Any:
|
||||
if stream:
|
||||
response = client._original.messages.create(
|
||||
model=model,
|
||||
messages=messages,
|
||||
stream=True,
|
||||
**kwargs,
|
||||
)
|
||||
return client._wrap_stream(response, metrics)
|
||||
|
||||
return _call_anthropic_transport(
|
||||
client,
|
||||
model=model,
|
||||
messages=messages,
|
||||
stream=False,
|
||||
metrics=metrics,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _call_anthropic_transport(
|
||||
client: Any,
|
||||
*,
|
||||
@@ -409,6 +437,7 @@ def _call_anthropic_transport(
|
||||
_ClientTransport = Callable[..., Any]
|
||||
_CLIENT_TRANSPORTS: dict[str, _ClientTransport] = {
|
||||
"anthropic": _call_anthropic_transport,
|
||||
"anthropic_create": _call_anthropic_create_transport,
|
||||
"openai": _call_openai_transport,
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -98,6 +99,23 @@ class CCSwitchReconciler:
|
||||
self._task: asyncio.Task | None = None
|
||||
self._last_mtime_ns: int | None = None
|
||||
|
||||
def _is_self(self, url: str) -> bool:
|
||||
"""True if ``url`` names this proxy (``localhost`` or its bind address, any path).
|
||||
|
||||
Loopback families are distinct sockets: ``[::1]`` is not the proxy when it is
|
||||
bound to ``127.0.0.1``, so another local provider on that address is captured.
|
||||
"""
|
||||
try:
|
||||
want = urlsplit(self.proxy_url)
|
||||
got = urlsplit(url)
|
||||
return (
|
||||
(got.hostname or "").lower() in {"localhost", (want.hostname or "").lower()}
|
||||
and got.scheme == want.scheme
|
||||
and got.port == want.port
|
||||
)
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
async def start(self) -> None:
|
||||
if self._task is not None:
|
||||
return
|
||||
@@ -178,7 +196,7 @@ class CCSwitchReconciler:
|
||||
return False
|
||||
|
||||
# Already pointing at us: nothing to do (loop guard).
|
||||
if url.rstrip("/") == self.proxy_url:
|
||||
if self._is_self(url):
|
||||
return False
|
||||
|
||||
# Third-party / custom endpoint: capture it as upstream, point Claude at us.
|
||||
|
||||
+70
-12
@@ -133,10 +133,21 @@ class CostEntry(NamedTuple):
|
||||
# (Anthropic's >200k rates differ), so they are never preferred over the catalog.
|
||||
|
||||
|
||||
#: Context size at which the major catalogs publish a second, higher price
|
||||
#: tier (LiteLLM spells it ``*_above_200k_tokens``). A request's billed prompt
|
||||
#: is compared against this to pick which rate applies.
|
||||
_LONG_CONTEXT_THRESHOLD_TOKENS = 200_000
|
||||
def _long_context_threshold(model: str) -> int:
|
||||
"""Billed-prompt size above which ``model`` pays its long-context rates.
|
||||
|
||||
Delegates to :func:`headroom.pricing.counterfactual.long_context_threshold`,
|
||||
which reads the model's own tier from the catalog (Haiku 5.5: 100K, Sonnet
|
||||
4.5: 200K). Imported at call time, like ``_get_cache_prices``: importing
|
||||
``headroom.pricing`` pulls in litellm (~4s) and this module is on the
|
||||
proxy's startup path. Falls back to 200K if the lookup fails.
|
||||
"""
|
||||
try:
|
||||
from headroom.pricing.counterfactual import long_context_threshold
|
||||
|
||||
return long_context_threshold(model)
|
||||
except Exception:
|
||||
return 200_000
|
||||
|
||||
|
||||
def _bucket_by_cache_mix(
|
||||
@@ -906,6 +917,13 @@ class CostTracker:
|
||||
self._saved_write_5m_by_tier: dict[tuple[str, bool], float] = {}
|
||||
self._saved_write_1h_by_tier: dict[tuple[str, bool], float] = {}
|
||||
self._saved_list_by_tier: dict[tuple[str, bool], float] = {}
|
||||
# Forwarded tokens of requests that compression kept under the model's
|
||||
# long-context threshold (their uncompressed prompt would have crossed
|
||||
# it), as [read, write 5m, write 1h, uncached, output]. The provider
|
||||
# re-prices the whole request above the threshold, so these tokens
|
||||
# avoided the long card's premium; stats() prices that premium into
|
||||
# the cache-aware savings.
|
||||
self._tier_crossing_by_model: dict[str, list[float]] = {}
|
||||
# Tool-schema deferral per model, DISJOINT from _tokens_saved_by_model
|
||||
# (deferred schemas are never in the message counts). Tracked separately
|
||||
# so the compression-only figure stays available; `stats()` reports both
|
||||
@@ -964,6 +982,7 @@ class CostTracker:
|
||||
self._saved_write_5m_by_tier.clear()
|
||||
self._saved_write_1h_by_tier.clear()
|
||||
self._saved_list_by_tier.clear()
|
||||
self._tier_crossing_by_model.clear()
|
||||
self._tool_saved_by_model.clear()
|
||||
self._tool_saved_read_by_model.clear()
|
||||
self._tool_saved_write_5m_by_model.clear()
|
||||
@@ -1129,7 +1148,24 @@ class CostTracker:
|
||||
write_5m_eff = 0 if cache_inferred else max(0, cache_write_5m_tokens)
|
||||
write_1h_eff = 0 if cache_inferred else max(0, cache_write_1h_tokens)
|
||||
billed_prompt = max(0, cache_read_tokens) + write_eff + max(0, uncached_tokens)
|
||||
long_context = max(billed_prompt, tokens_sent) > _LONG_CONTEXT_THRESHOLD_TOKENS
|
||||
threshold = _long_context_threshold(model)
|
||||
forwarded_prompt = max(billed_prompt, tokens_sent)
|
||||
long_context = forwarded_prompt > threshold
|
||||
# Compression that kept the prompt under the threshold: the removed
|
||||
# tokens would have billed at the long card, and so would everything
|
||||
# forwarded (see `_tier_crossing_by_model`).
|
||||
crossed = not long_context and forwarded_prompt + tokens_saved > threshold
|
||||
if crossed:
|
||||
w1h = min(write_1h_eff, write_eff)
|
||||
row = self._tier_crossing_by_model.setdefault(key, [0.0] * 5)
|
||||
if billed_prompt > 0:
|
||||
row[0] += max(0, cache_read_tokens)
|
||||
row[1] += write_eff - w1h
|
||||
row[2] += w1h
|
||||
row[3] += max(0, uncached_tokens)
|
||||
else:
|
||||
row[3] += max(0, tokens_sent)
|
||||
row[4] += max(0, output_tokens)
|
||||
if tokens_saved > 0:
|
||||
# Message compression works the LIVE ZONE only: handlers freeze the
|
||||
# cached prefix (system + prior turns) byte-for-byte for prefix-cache
|
||||
@@ -1151,7 +1187,7 @@ class CostTracker:
|
||||
uncached_tokens=uncached_tokens,
|
||||
region="live_zone",
|
||||
)
|
||||
wkey = (key, long_context)
|
||||
wkey = (key, long_context or crossed)
|
||||
self._saved_write_5m_by_tier[wkey] = self._saved_write_5m_by_tier.get(wkey, 0.0) + c_w5m
|
||||
self._saved_write_1h_by_tier[wkey] = self._saved_write_1h_by_tier.get(wkey, 0.0) + c_w1h
|
||||
self._saved_list_by_tier[wkey] = self._saved_list_by_tier.get(wkey, 0.0) + c_list
|
||||
@@ -1419,8 +1455,9 @@ class CostTracker:
|
||||
def _get_output_price(self, model: str, *, long_context: bool = False) -> float | None:
|
||||
"""Get the per-token completion price for a model, or None if unpriced.
|
||||
|
||||
``long_context`` selects the catalog's above-200k completion rate where
|
||||
the model publishes one (Anthropic charges 1.5x there).
|
||||
``long_context`` selects the completion rate of the model's
|
||||
long-context tier where it publishes one (Sonnet 4.5 above 200K: 1.5x;
|
||||
Haiku 5.5 above 100K: 5x).
|
||||
"""
|
||||
litellm = _get_litellm_module()
|
||||
if litellm is None:
|
||||
@@ -1432,7 +1469,11 @@ class CostTracker:
|
||||
info = litellm.model_cost.get(resolved, {})
|
||||
base = info.get("output_cost_per_token")
|
||||
if long_context:
|
||||
return info.get("output_cost_per_token_above_200k_tokens") or base or None
|
||||
from headroom.pricing.counterfactual import long_context_tier
|
||||
|
||||
tier = long_context_tier(info)
|
||||
if tier:
|
||||
return info.get(f"output_cost_per_token{tier[1]}") or base or None
|
||||
return base or None
|
||||
except Exception as e:
|
||||
_warn_pricing_once("output_price", "Failed to get output price", model, e)
|
||||
@@ -1540,9 +1581,9 @@ class CostTracker:
|
||||
``headroom.pricing.cache_ttl`` derives it. Previously every 1h write was
|
||||
priced at the 5m rate.
|
||||
|
||||
``long_context`` picks the above-200k tier for each rate, falling back
|
||||
per rate to the base one for a model that publishes no long-context
|
||||
price.
|
||||
``long_context`` picks the model's long-context tier for each rate,
|
||||
falling back per rate to the base one for a model that publishes no
|
||||
long-context price.
|
||||
|
||||
Imported at call time: ``headroom.pricing`` eagerly imports litellm
|
||||
(~4s) and this module is on the proxy's startup path.
|
||||
@@ -1721,6 +1762,23 @@ class CostTracker:
|
||||
+ self._saved_list_by_tier.get(key, 0.0) * uncached_price
|
||||
)
|
||||
|
||||
# The long card's premium that tier-crossing requests' forwarded
|
||||
# tokens avoided (see `_tier_crossing_by_model`).
|
||||
for model, crossing in self._tier_crossing_by_model.items():
|
||||
long_prices = self._get_cache_prices(model, long_context=True)
|
||||
base_prices = self._get_cache_prices(model)
|
||||
if not long_prices or not base_prices:
|
||||
continue
|
||||
premium = sum(
|
||||
tokens * max(0.0, hi - lo)
|
||||
for tokens, hi, lo in zip(crossing[:4], long_prices, base_prices, strict=True)
|
||||
)
|
||||
long_out = self._get_output_price(model, long_context=True)
|
||||
base_out = self._get_output_price(model)
|
||||
if long_out and base_out:
|
||||
premium += crossing[4] * max(0.0, long_out - base_out)
|
||||
cache_aware_savings_usd += premium
|
||||
|
||||
# Tool-schema deferral, priced CACHE-AWARE rather than flat at list:
|
||||
# had the schemas shipped they would have been part of the same prefix
|
||||
# as the rest of the request — cold-written once at the provider's
|
||||
|
||||
@@ -1544,7 +1544,9 @@ class AnthropicHandlerMixin:
|
||||
# it was looked up by — the cache would never hit and would fill with
|
||||
# unreachable entries. Reuse this raw snapshot verbatim at cache.set
|
||||
# (the same reason cache_key_fields is snapshotted here, #327).
|
||||
cache_lookup_messages = messages
|
||||
cache_lookup_messages = (
|
||||
copy.deepcopy(messages) if self.cache and not stream else messages
|
||||
)
|
||||
# Response-cache partition: a cached response is only ever replayed to a
|
||||
# caller presenting the same provider credentials and principal (01-F15).
|
||||
# Snapshotted with the key fields so lookup and store agree. None means
|
||||
@@ -1751,11 +1753,31 @@ class AnthropicHandlerMixin:
|
||||
# parallel sub-calls commonly share model+system+history while
|
||||
# carrying different tool sets. Sharing one tracker across those
|
||||
# requests pins cache reads at the early tools segment (#2671).
|
||||
from headroom.cache.prefix_tracker import segment_fingerprint
|
||||
# Deferred tools are not in that key, so a ToolSearch load that
|
||||
# appends one keeps the conversation's lineage, but only where they
|
||||
# reach the provider still deferred. A third-party upstream gets
|
||||
# them eagerly (the tool-search downgrade below uses this same
|
||||
# predicate) and so does a translating backend; there a load does
|
||||
# change the provider's key, and the key here counts them.
|
||||
from headroom.cache.prefix_tracker import cache_key_tools, segment_fingerprint
|
||||
from headroom.proxy.helpers import anthropic_first_party_tool_search_supported
|
||||
|
||||
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
|
||||
_third_party_anthropic_upstream = provider_name == "anthropic" and (
|
||||
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
|
||||
)
|
||||
affinity_tools = self._tools_for_forwarding(
|
||||
body.get("tools"), preserve_order=preserve_tool_order
|
||||
)
|
||||
from headroom.proxy import runtime_env
|
||||
|
||||
if (
|
||||
not _third_party_anthropic_upstream
|
||||
and getattr(self, "anthropic_backend", None) is None
|
||||
and runtime_env.getenv("HEADROOM_DEFERRED_TOOL_AFFINITY", "1").strip().lower()
|
||||
not in ("0", "false", "no", "off")
|
||||
):
|
||||
affinity_tools = cache_key_tools(affinity_tools)
|
||||
cache_affinity = segment_fingerprint(
|
||||
{
|
||||
"model": model,
|
||||
@@ -2623,9 +2645,14 @@ class AnthropicHandlerMixin:
|
||||
prefix_tracker.restored_maturation_state = None
|
||||
maturation_mgr.restore_state(state)
|
||||
_maturation_mgr = maturation_mgr
|
||||
# Never below the floor finalize_turn just replayed: the
|
||||
# pipeline's freeze is clamped to what the local compression
|
||||
# cache can byte-replay, which stops at the first tool_result
|
||||
# it never stored, so on its own it lets a cached Read mature
|
||||
# and re-write the provider cache from that Read onward.
|
||||
maturation = maturation_mgr.apply(
|
||||
optimized_messages,
|
||||
frozen_message_count=frozen_message_count,
|
||||
frozen_message_count=max(frozen_message_count, _confirmed_frozen),
|
||||
)
|
||||
if maturation.replacements_applied or maturation.holding_msg_indices:
|
||||
optimized_messages = maturation.messages
|
||||
@@ -2910,7 +2937,10 @@ class AnthropicHandlerMixin:
|
||||
)
|
||||
|
||||
# Traffic Learner: Extract patterns from inbound tool results
|
||||
if self.traffic_learner:
|
||||
if self.traffic_learner and not (
|
||||
self.memory_handler
|
||||
and self.memory_handler.is_project_unresolved(memory_request_ctx)
|
||||
):
|
||||
try:
|
||||
# Wire backend on first use (lazy init after memory handler is ready)
|
||||
if (
|
||||
@@ -3324,14 +3354,10 @@ class AnthropicHandlerMixin:
|
||||
tokens_saved = max(0, original_tokens - optimized_tokens)
|
||||
|
||||
from headroom.proxy.helpers import (
|
||||
anthropic_first_party_tool_search_supported,
|
||||
strip_first_party_tool_search_tools_for_third_party_upstream,
|
||||
)
|
||||
|
||||
_anthropic_target_base_url = upstream_base_url or self.ANTHROPIC_API_URL
|
||||
_third_party_anthropic_upstream = provider_name == "anthropic" and (
|
||||
not anthropic_first_party_tool_search_supported(_anthropic_target_base_url)
|
||||
)
|
||||
# _third_party_anthropic_upstream is decided before the lineage key.
|
||||
if _third_party_anthropic_upstream:
|
||||
_tools_before_strip = body.get("tools")
|
||||
_tools_after_strip = strip_first_party_tool_search_tools_for_third_party_upstream(
|
||||
|
||||
@@ -1551,28 +1551,29 @@ class GeminiHandlerMixin:
|
||||
max(0, original_tokens - compressed_tokens) if compressed_tokens > 0 else 0
|
||||
)
|
||||
|
||||
# Fallback denominator (see comment on the main gemini
|
||||
# record_request site) — pre-comp request size.
|
||||
# countTokens is a sizing helper; it never generates output
|
||||
# tokens and never touches cache. The funnel handles the
|
||||
# "nothing to report" shape with all-zero cache defaults.
|
||||
await self._record_request_outcome(
|
||||
RequestOutcome(
|
||||
request_id=request_id,
|
||||
provider=provider_name,
|
||||
model=model,
|
||||
status_code=response.status_code,
|
||||
original_tokens=original_tokens,
|
||||
optimized_tokens=compressed_tokens,
|
||||
output_tokens=0,
|
||||
tokens_saved=tokens_saved,
|
||||
attempted_input_tokens=compressed_tokens + tokens_saved,
|
||||
total_latency_ms=total_latency,
|
||||
transforms_applied=tuple(transforms_applied),
|
||||
tags=tags,
|
||||
client=client,
|
||||
# countTokens is a sizing helper: the provider generates and bills
|
||||
# nothing for it, so a successful call is logged below but never
|
||||
# booked as a request or as savings. An upstream rejection (429,
|
||||
# 5xx) still goes through the funnel, which records it under the
|
||||
# rate-limit / failure counters and stops before any success stats.
|
||||
if response.status_code >= 400:
|
||||
await self._record_request_outcome(
|
||||
RequestOutcome(
|
||||
request_id=request_id,
|
||||
provider=provider_name,
|
||||
model=model,
|
||||
status_code=response.status_code,
|
||||
original_tokens=original_tokens,
|
||||
optimized_tokens=compressed_tokens,
|
||||
output_tokens=0,
|
||||
tokens_saved=tokens_saved,
|
||||
attempted_input_tokens=compressed_tokens + tokens_saved,
|
||||
total_latency_ms=total_latency,
|
||||
transforms_applied=tuple(transforms_applied),
|
||||
tags=tags,
|
||||
client=client,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
if tokens_saved > 0:
|
||||
logger.info(
|
||||
|
||||
@@ -1965,6 +1965,7 @@ class OpenAIHandlerMixin:
|
||||
body: dict[str, Any],
|
||||
*,
|
||||
request_id: str,
|
||||
request_context: Any = None,
|
||||
) -> None:
|
||||
"""Feed one Responses HTTP request into the live traffic learner."""
|
||||
traffic_learner = getattr(self, "traffic_learner", None)
|
||||
@@ -1972,6 +1973,8 @@ class OpenAIHandlerMixin:
|
||||
return
|
||||
try:
|
||||
memory_handler = getattr(self, "memory_handler", None)
|
||||
if memory_handler and memory_handler.is_project_unresolved(request_context):
|
||||
return
|
||||
if (
|
||||
traffic_learner._backend is None
|
||||
and memory_handler
|
||||
@@ -2001,6 +2004,7 @@ class OpenAIHandlerMixin:
|
||||
messages: list[dict[str, Any]],
|
||||
*,
|
||||
request_id: str,
|
||||
request_context: Any = None,
|
||||
) -> None:
|
||||
"""Feed one chat/completions request into the live traffic learner.
|
||||
|
||||
@@ -2017,6 +2021,8 @@ class OpenAIHandlerMixin:
|
||||
return
|
||||
try:
|
||||
memory_handler = getattr(self, "memory_handler", None)
|
||||
if memory_handler and memory_handler.is_project_unresolved(request_context):
|
||||
return
|
||||
if (
|
||||
traffic_learner._backend is None
|
||||
and memory_handler
|
||||
@@ -2044,6 +2050,7 @@ class OpenAIHandlerMixin:
|
||||
seen_call_ids: set[str],
|
||||
baseline: bool,
|
||||
request_id: str,
|
||||
request_context: Any = None,
|
||||
) -> None:
|
||||
"""Feed one Codex WS ``response.create`` turn into the traffic learner.
|
||||
|
||||
@@ -2067,6 +2074,8 @@ class OpenAIHandlerMixin:
|
||||
return
|
||||
try:
|
||||
memory_handler = getattr(self, "memory_handler", None)
|
||||
if memory_handler and memory_handler.is_project_unresolved(request_context):
|
||||
return
|
||||
if (
|
||||
traffic_learner._backend is None
|
||||
and memory_handler
|
||||
@@ -3454,8 +3463,6 @@ class OpenAIHandlerMixin:
|
||||
# compression mutates it, mirroring the Responses and Anthropic
|
||||
# ingestion paths. Without this, chat/completions traffic (Copilot CLI,
|
||||
# opencode, OpenAI SDKs) fed nothing to the learner (part of #2060).
|
||||
await self._observe_openai_chat_traffic(original_client_messages, request_id=request_id)
|
||||
|
||||
# Bypass: skip ALL compression for explicit opt-out
|
||||
_bypass = self._headroom_bypass_enabled(request.headers)
|
||||
if _bypass:
|
||||
@@ -3605,6 +3612,12 @@ class OpenAIHandlerMixin:
|
||||
),
|
||||
)
|
||||
|
||||
await self._observe_openai_chat_traffic(
|
||||
original_client_messages,
|
||||
request_id=request_id,
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
|
||||
# Canonical memory-injection gate (parallels Anthropic). Pre-
|
||||
# PR-this the inline conjunction at the memory site silently
|
||||
# ignored `x-headroom-bypass: true`, mutating request bytes
|
||||
@@ -3688,7 +3701,7 @@ class OpenAIHandlerMixin:
|
||||
# `messages`, but it runs before this snapshot, so its output is already
|
||||
# captured — keep this snapshot after image compression, or a reorder
|
||||
# silently reintroduces the drift.
|
||||
cache_lookup_messages = messages
|
||||
cache_lookup_messages = copy.deepcopy(messages) if self.cache and not stream else messages
|
||||
# Response-cache partition: a cached response is only ever replayed to a
|
||||
# caller presenting the same provider credentials and principal (01-F15).
|
||||
# Snapshotted with the key fields so lookup and store agree. None means
|
||||
@@ -4321,6 +4334,24 @@ class OpenAIHandlerMixin:
|
||||
optimized_messages, original_client_messages
|
||||
)
|
||||
|
||||
from headroom.proxy.helpers import enforce_cache_breakpoint_budget
|
||||
|
||||
# Bypass (x-headroom-bypass: true, _bypass computed at ~L3458) opts out
|
||||
# of ALL compression: never mutate those requests here.
|
||||
# client_messages=None: passing the client list runs
|
||||
# normalize_message_cache_control, whose block-level legacy fallback can
|
||||
# add a marker to a list-content user message on chat requests and then
|
||||
# cost a real system marker. Without it only the oldest-first drop runs
|
||||
# — stale replayed markers are oldest, so they go first.
|
||||
if not _bypass:
|
||||
_, optimized_messages, tools, _ = enforce_cache_breakpoint_budget(
|
||||
None,
|
||||
optimized_messages,
|
||||
tools,
|
||||
client_messages=None,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
# Memory: inject context and tools for OpenAI requests.
|
||||
#
|
||||
# PR-A3 follow-up to A2: memory context now routes exclusively to
|
||||
@@ -6010,12 +6041,6 @@ class OpenAIHandlerMixin:
|
||||
resolved_project_root_override = codex_project_root_override or (
|
||||
str(codex_project.cwd) if codex_project and codex_project.cwd else None
|
||||
)
|
||||
# The shared learner cannot isolate project state. Scoped Codex turns
|
||||
# skip learning until a project-scoped learner is available.
|
||||
if not codex_project_scope_required and (
|
||||
codex_project is None or codex_project.reason == "metadata_missing"
|
||||
):
|
||||
await self._observe_openai_responses_traffic(body, request_id=request_id)
|
||||
memory_client = (
|
||||
"codex"
|
||||
if is_chatgpt_auth
|
||||
@@ -6090,6 +6115,17 @@ class OpenAIHandlerMixin:
|
||||
project_root_override=resolved_project_root_override,
|
||||
)
|
||||
|
||||
# The shared learner cannot isolate project state. Scoped Codex turns
|
||||
# skip learning until a project-scoped learner is available.
|
||||
if not codex_project_scope_required and (
|
||||
codex_project is None or codex_project.reason == "metadata_missing"
|
||||
):
|
||||
await self._observe_openai_responses_traffic(
|
||||
body,
|
||||
request_id=request_id,
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
|
||||
# Rate limiting
|
||||
if self.rate_limiter:
|
||||
rate_key = rate_limit_identity(request, headers)
|
||||
@@ -6998,13 +7034,13 @@ class OpenAIHandlerMixin:
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = {}
|
||||
|
||||
await self.memory_handler._ensure_initialized()
|
||||
if self.memory_handler._backend:
|
||||
result = await self.memory_handler._execute_memory_tool(
|
||||
name, args, memory_user_id, "openai"
|
||||
)
|
||||
else:
|
||||
result = json.dumps({"error": "Memory backend not initialized"})
|
||||
result = await self.memory_handler._execute_memory_tool(
|
||||
name,
|
||||
args,
|
||||
memory_user_id,
|
||||
"openai",
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
|
||||
tool_outputs.append(
|
||||
{
|
||||
@@ -8131,21 +8167,37 @@ class OpenAIHandlerMixin:
|
||||
resolved_project.project_key if ws_turn_project_features_allowed else None
|
||||
)
|
||||
|
||||
if first_project_resolution and ws_turn_learning_allowed:
|
||||
inner_payload = frame_body.get("response", frame_body)
|
||||
if isinstance(inner_payload, dict):
|
||||
await self._observe_openai_ws_response_create(
|
||||
inner_payload,
|
||||
seen_call_ids=ws_learner_seen_call_ids,
|
||||
baseline=True,
|
||||
request_id=request_id,
|
||||
)
|
||||
|
||||
memory_user_id_candidate = (
|
||||
resolve_memory_identity(websocket)
|
||||
if self.memory_handler and ws_turn_project_features_allowed
|
||||
else None
|
||||
)
|
||||
ws_response_body = frame_body.get("response", frame_body)
|
||||
if memory_user_id_candidate is not None and isinstance(ws_response_body, dict):
|
||||
from headroom.memory.storage_router import (
|
||||
RequestContext as _MemRequestContext,
|
||||
)
|
||||
|
||||
memory_request_ctx = _MemRequestContext(
|
||||
headers=dict(ws_headers),
|
||||
system_prompt=str(ws_response_body.get("instructions") or ""),
|
||||
base_user_id=memory_user_id_candidate,
|
||||
project_root_override=resolved_project_root_override,
|
||||
)
|
||||
|
||||
if (
|
||||
first_project_resolution
|
||||
and ws_turn_learning_allowed
|
||||
and isinstance(ws_response_body, dict)
|
||||
):
|
||||
await self._observe_openai_ws_response_create(
|
||||
ws_response_body,
|
||||
seen_call_ids=ws_learner_seen_call_ids,
|
||||
baseline=True,
|
||||
request_id=request_id,
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
|
||||
memory_decision = MemoryDecision.decide(
|
||||
headers=ws_headers,
|
||||
memory_handler=(
|
||||
@@ -8162,24 +8214,8 @@ class OpenAIHandlerMixin:
|
||||
|
||||
memory_user_id = memory_user_id_candidate
|
||||
try:
|
||||
# Unwrap response.create envelope to access the response body
|
||||
ws_response_body = frame_body.get("response", frame_body)
|
||||
if not isinstance(ws_response_body, dict):
|
||||
return frame_raw
|
||||
# Per-project memory routing (GH #462). For WS,
|
||||
# ``ws_response_body`` carries ``instructions`` —
|
||||
# that's the system-prompt-equivalent we feed to the
|
||||
# resolver.
|
||||
from headroom.memory.storage_router import (
|
||||
RequestContext as _MemRequestContext,
|
||||
)
|
||||
|
||||
memory_request_ctx = _MemRequestContext(
|
||||
headers=dict(ws_headers),
|
||||
system_prompt=str(ws_response_body.get("instructions") or ""),
|
||||
base_user_id=memory_user_id,
|
||||
project_root_override=resolved_project_root_override,
|
||||
)
|
||||
|
||||
# Debug: log what Codex sends so we can see the full tool list
|
||||
existing_tool_names = [
|
||||
@@ -8761,6 +8797,7 @@ class OpenAIHandlerMixin:
|
||||
seen_call_ids=ws_learner_seen_call_ids,
|
||||
baseline=False,
|
||||
request_id=request_id,
|
||||
request_context=memory_request_ctx,
|
||||
)
|
||||
store_forced = _ensure_chatgpt_responses_store_false(
|
||||
inner_payload,
|
||||
@@ -9627,19 +9664,19 @@ class OpenAIHandlerMixin:
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
fc_args = {}
|
||||
|
||||
await self.memory_handler._ensure_initialized()
|
||||
if not self.memory_handler.is_project_unresolved(
|
||||
response_memory_request_ctx
|
||||
):
|
||||
await self.memory_handler._ensure_initialized()
|
||||
if response_memory_generation != ws_memory_generation:
|
||||
break
|
||||
if self.memory_handler._backend:
|
||||
result = await self.memory_handler._execute_memory_tool(
|
||||
fc_name,
|
||||
fc_args,
|
||||
response_memory_user_id,
|
||||
"openai",
|
||||
request_context=response_memory_request_ctx,
|
||||
)
|
||||
else:
|
||||
result = json.dumps({"error": "backend not ready"})
|
||||
result = await self.memory_handler._execute_memory_tool(
|
||||
fc_name,
|
||||
fc_args,
|
||||
response_memory_user_id,
|
||||
"openai",
|
||||
request_context=response_memory_request_ctx,
|
||||
)
|
||||
|
||||
tool_outputs.append(
|
||||
{
|
||||
|
||||
@@ -458,6 +458,12 @@ def count_cache_breakpoints(
|
||||
if not isinstance(msg, dict):
|
||||
continue
|
||||
found = 1 if "cache_control" in msg else 0
|
||||
# OpenAI chat shape: markers on an assistant's tool_calls entries.
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if isinstance(tool_calls, list):
|
||||
found += sum(
|
||||
1 for tc in tool_calls if isinstance(tc, dict) and "cache_control" in tc
|
||||
)
|
||||
content = msg.get("content")
|
||||
if isinstance(content, list):
|
||||
for block in content:
|
||||
@@ -847,6 +853,16 @@ def enforce_cache_breakpoint_budget(
|
||||
budget["left"] -= 1
|
||||
new_blocks.append(block)
|
||||
msg = {**msg, "content": new_blocks}
|
||||
tool_calls = msg.get("tool_calls")
|
||||
if isinstance(tool_calls, list) and budget["left"] > 0:
|
||||
new_calls = []
|
||||
for tc in tool_calls:
|
||||
if budget["left"] > 0 and isinstance(tc, dict) and "cache_control" in tc:
|
||||
tc = {k: v for k, v in tc.items() if k != "cache_control"}
|
||||
budget["left"] -= 1
|
||||
new_calls.append(tc)
|
||||
if new_calls != tool_calls:
|
||||
msg = {**msg, "tool_calls": new_calls}
|
||||
if budget["left"] > 0 and "cache_control" in msg:
|
||||
msg = {k: v for k, v in msg.items() if k != "cache_control"}
|
||||
budget["left"] -= 1
|
||||
|
||||
@@ -9,6 +9,7 @@ from dataclasses import dataclass
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from headroom.cache.compression_cache import (
|
||||
_can_swap_tool_result_content,
|
||||
_extract_tool_result_content,
|
||||
_is_tool_result_message,
|
||||
_swap_tool_result_content,
|
||||
@@ -253,6 +254,10 @@ def apply_to_messages(
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
if not _can_swap_tool_result_content(msg):
|
||||
new_messages.append(msg)
|
||||
continue
|
||||
|
||||
original = _extract_tool_result_content(msg)
|
||||
if not isinstance(original, str) or not original:
|
||||
new_messages.append(msg)
|
||||
|
||||
@@ -185,12 +185,18 @@ class MemoryHandler:
|
||||
self.config = config
|
||||
self.agent_type = agent_type
|
||||
self._backend: LocalBackend | Any = None
|
||||
# Per-project routing for the local backend. Built in
|
||||
# ``_init_backend_locked`` so a single, shared resolver / LRU is
|
||||
# kept on the handler. Qdrant deployments use composite user-id
|
||||
# partitioning instead (see ``_compose_effective_user_id``) — the
|
||||
# router stays None in that case.
|
||||
self._router: BackendRouter | None = None
|
||||
storage_root = (
|
||||
Path(config.storage_root)
|
||||
if config.storage_root
|
||||
else (Path(config.db_path).resolve().parent / "memories")
|
||||
)
|
||||
self._router = BackendRouter(
|
||||
BackendRouterConfig(
|
||||
mode=config.storage_mode,
|
||||
root_dir=storage_root,
|
||||
global_db_path=Path(config.db_path).resolve(),
|
||||
)
|
||||
)
|
||||
self._initialized = False
|
||||
# Async singleflight guard for backend init. Ensures concurrent first
|
||||
# callers land on one init (double-checked pattern inside
|
||||
@@ -410,19 +416,8 @@ class MemoryHandler:
|
||||
# remains the GLOBAL-mode fallback / legacy compatibility
|
||||
# backend; callers that pass a ``RequestContext`` route
|
||||
# through ``self._router`` instead.
|
||||
storage_root = (
|
||||
Path(self.config.storage_root)
|
||||
if self.config.storage_root
|
||||
else (Path(self.config.db_path).resolve().parent / "memories")
|
||||
)
|
||||
global_db_path = Path(self.config.db_path).resolve()
|
||||
router_cfg = BackendRouterConfig(
|
||||
mode=self.config.storage_mode,
|
||||
root_dir=storage_root,
|
||||
global_db_path=global_db_path,
|
||||
backend_config_template=backend_config,
|
||||
)
|
||||
self._router = BackendRouter(router_cfg)
|
||||
self._router._config.backend_config_template = backend_config
|
||||
# Seed the router's LRU with the already-initialized
|
||||
# legacy backend so GLOBAL-mode requests reuse it instead
|
||||
# of opening a second handle to the same file.
|
||||
@@ -431,7 +426,7 @@ class MemoryHandler:
|
||||
logger.info(
|
||||
"event=memory_router_initialized mode=%s root=%s global_db=%s",
|
||||
self.config.storage_mode.value,
|
||||
storage_root,
|
||||
self._router._config.root_dir,
|
||||
global_db_path,
|
||||
)
|
||||
|
||||
@@ -670,7 +665,9 @@ class MemoryHandler:
|
||||
# Non-local backends: derive scope but keep one shared backend
|
||||
# and compose the user_id so the partition lives in the user_id
|
||||
# column instead of in a separate file.
|
||||
scope = self._router._resolve_scope(request_context)
|
||||
scope = self._router.scope_for(request_context)
|
||||
if scope.db_path is None:
|
||||
return None, scope, base_user_id
|
||||
composed = (
|
||||
base_user_id
|
||||
if scope.project_key is None or scope.mode is MemoryStorageMode.GLOBAL
|
||||
@@ -678,6 +675,18 @@ class MemoryHandler:
|
||||
)
|
||||
return self._backend, scope, composed
|
||||
|
||||
def is_project_unresolved(self, request_context: RequestContext | None) -> bool:
|
||||
"""Return whether fail-closed project routing has no usable target."""
|
||||
|
||||
if request_context is None or self._router is None:
|
||||
return False
|
||||
scope = self._router.scope_for(request_context)
|
||||
return (
|
||||
scope.mode is MemoryStorageMode.PROJECT
|
||||
and scope.project_key is None
|
||||
and scope.db_path is None
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _unresolved_project_error(scope: ResolvedScope | None) -> str | None:
|
||||
if (
|
||||
@@ -771,6 +780,8 @@ class MemoryHandler:
|
||||
)
|
||||
return None
|
||||
|
||||
if self.is_project_unresolved(request_context):
|
||||
return None
|
||||
await self._ensure_initialized()
|
||||
if not self._backend:
|
||||
return None
|
||||
@@ -1105,6 +1116,8 @@ your responses, not to drive new actions."""
|
||||
"""
|
||||
tool_calls = self._extract_tool_calls(response, provider)
|
||||
results: list[dict[str, Any]] = []
|
||||
project_unresolved = self.is_project_unresolved(request_context)
|
||||
unresolved_result = json.dumps({"status": "skipped", "reason": "project_unresolved"})
|
||||
|
||||
for tc in tool_calls:
|
||||
# `tc.get("function", {})` returns None for an explicit
|
||||
@@ -1129,8 +1142,14 @@ your responses, not to drive new actions."""
|
||||
input_data = {}
|
||||
|
||||
# Handle native memory tool
|
||||
if tool_name == NATIVE_MEMORY_TOOL_NAME:
|
||||
result_content = await self._execute_native_memory_tool(input_data, user_id)
|
||||
if tool_name in ({NATIVE_MEMORY_TOOL_NAME} | MEMORY_TOOL_NAMES) and project_unresolved:
|
||||
result_content = unresolved_result
|
||||
elif tool_name == NATIVE_MEMORY_TOOL_NAME:
|
||||
result_content = await self._execute_native_memory_tool(
|
||||
input_data,
|
||||
user_id,
|
||||
request_context=request_context,
|
||||
)
|
||||
elif tool_name in MEMORY_TOOL_NAMES:
|
||||
# Custom memory tools need backend
|
||||
await self._ensure_initialized()
|
||||
@@ -1178,7 +1197,12 @@ your responses, not to drive new actions."""
|
||||
request_context: RequestContext | None = None,
|
||||
) -> str:
|
||||
"""Execute a memory tool and return result string."""
|
||||
if self.is_project_unresolved(request_context):
|
||||
return json.dumps({"status": "skipped", "reason": "project_unresolved"})
|
||||
try:
|
||||
await self._ensure_initialized()
|
||||
if self._backend is None:
|
||||
return json.dumps({"status": "error", "error": "Memory backend not initialized"})
|
||||
if tool_name == "memory_save":
|
||||
return await self._execute_save(input_data, user_id, provider, request_context)
|
||||
elif tool_name == "memory_search":
|
||||
@@ -1537,7 +1561,13 @@ your responses, not to drive new actions."""
|
||||
# str_replace → Update memory content
|
||||
# =========================================================================
|
||||
|
||||
async def _execute_native_memory_tool(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _execute_native_memory_tool(
|
||||
self,
|
||||
input_data: dict[str, Any],
|
||||
user_id: str,
|
||||
*,
|
||||
request_context: RequestContext | None = None,
|
||||
) -> str:
|
||||
"""Execute Anthropic's native memory tool with semantic backend.
|
||||
|
||||
This is a TRANSLATION LAYER: Claude thinks it's doing file operations,
|
||||
@@ -1551,24 +1581,40 @@ your responses, not to drive new actions."""
|
||||
- delete: Remove from vector store
|
||||
- rename: Update memory tags/path
|
||||
"""
|
||||
if self.is_project_unresolved(request_context):
|
||||
return json.dumps({"status": "skipped", "reason": "project_unresolved"})
|
||||
|
||||
# Ensure our semantic backend is initialized
|
||||
await self._ensure_initialized()
|
||||
|
||||
command = input_data.get("command", "")
|
||||
|
||||
try:
|
||||
backend, _scope, effective_user_id = self._resolve_for_request(user_id, request_context)
|
||||
if command == "view":
|
||||
return await self._native_view_semantic(input_data, user_id)
|
||||
return await self._native_view_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
elif command == "create":
|
||||
return await self._native_create_semantic(input_data, user_id)
|
||||
return await self._native_create_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
elif command == "str_replace":
|
||||
return await self._native_update_semantic(input_data, user_id)
|
||||
return await self._native_update_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
elif command == "insert":
|
||||
return await self._native_append_semantic(input_data, user_id)
|
||||
return await self._native_append_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
elif command == "delete":
|
||||
return await self._native_delete_semantic(input_data, user_id)
|
||||
return await self._native_delete_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
elif command == "rename":
|
||||
return await self._native_rename_semantic(input_data, user_id)
|
||||
return await self._native_rename_semantic(
|
||||
input_data, effective_user_id, backend=backend
|
||||
)
|
||||
else:
|
||||
return f"Error: Unknown command '{command}'"
|
||||
except Exception as e:
|
||||
@@ -1579,7 +1625,9 @@ your responses, not to drive new actions."""
|
||||
# Semantic Translation Methods (Native Tool → Vector Store)
|
||||
# =========================================================================
|
||||
|
||||
async def _native_view_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_view_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle VIEW command with semantic search capabilities.
|
||||
|
||||
Path patterns:
|
||||
@@ -1602,31 +1650,35 @@ your responses, not to drive new actions."""
|
||||
query = subpath[len("search/") :]
|
||||
if not query:
|
||||
return "Error: Please provide a search query. Example: view /memories/search/food preferences"
|
||||
return await self._semantic_search(query, user_id)
|
||||
return await self._semantic_search(query, user_id, backend=backend)
|
||||
|
||||
# CASE 2: /memories/recent → Recent memories
|
||||
if subpath == "recent":
|
||||
return await self._get_recent_memories(user_id, limit=10)
|
||||
return await self._get_recent_memories(user_id, limit=10, backend=backend)
|
||||
|
||||
# CASE 3: /memories/all → List all (paginated)
|
||||
if subpath == "all":
|
||||
return await self._list_all_memories(user_id, limit=20)
|
||||
return await self._list_all_memories(user_id, limit=20, backend=backend)
|
||||
|
||||
# CASE 4: /memories (root) → Overview with instructions
|
||||
if not subpath or subpath == "":
|
||||
return await self._get_memory_overview(user_id)
|
||||
return await self._get_memory_overview(user_id, backend=backend)
|
||||
|
||||
# CASE 5: /memories/<something> → Search by topic
|
||||
# Treat the path as a search query
|
||||
return await self._semantic_search(subpath.replace("/", " ").replace("_", " "), user_id)
|
||||
return await self._semantic_search(
|
||||
subpath.replace("/", " ").replace("_", " "), user_id, backend=backend
|
||||
)
|
||||
|
||||
async def _semantic_search(self, query: str, user_id: str, top_k: int = 5) -> str:
|
||||
async def _semantic_search(
|
||||
self, query: str, user_id: str, top_k: int = 5, *, backend: Any
|
||||
) -> str:
|
||||
"""Perform semantic search and format results."""
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query=query,
|
||||
user_id=user_id,
|
||||
top_k=top_k,
|
||||
@@ -1657,15 +1709,15 @@ your responses, not to drive new actions."""
|
||||
logger.error(f"Memory: Semantic search failed: {e}")
|
||||
return f"Error searching memories: {e}"
|
||||
|
||||
async def _get_recent_memories(self, user_id: str, limit: int = 10) -> str:
|
||||
async def _get_recent_memories(self, user_id: str, limit: int = 10, *, backend: Any) -> str:
|
||||
"""Get most recent memories."""
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
# Use a generic query to get recent items
|
||||
# Most backends will return by recency when query is broad
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query="recent memories",
|
||||
user_id=user_id,
|
||||
top_k=limit,
|
||||
@@ -1692,14 +1744,14 @@ your responses, not to drive new actions."""
|
||||
logger.error(f"Memory: Get recent failed: {e}")
|
||||
return f"Error getting recent memories: {e}"
|
||||
|
||||
async def _list_all_memories(self, user_id: str, limit: int = 20) -> str:
|
||||
async def _list_all_memories(self, user_id: str, limit: int = 20, *, backend: Any) -> str:
|
||||
"""List all memories (paginated)."""
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
# Get all memories with a broad search
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query="*", # Broad query
|
||||
user_id=user_id,
|
||||
top_k=limit,
|
||||
@@ -1724,14 +1776,14 @@ your responses, not to drive new actions."""
|
||||
logger.error(f"Memory: List all failed: {e}")
|
||||
return f"Error listing memories: {e}"
|
||||
|
||||
async def _get_memory_overview(self, user_id: str) -> str:
|
||||
async def _get_memory_overview(self, user_id: str, *, backend: Any) -> str:
|
||||
"""Get memory directory overview with search instructions."""
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
# Get count of memories
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query="*",
|
||||
user_id=user_id,
|
||||
top_k=100, # Just to get a count
|
||||
@@ -1783,7 +1835,9 @@ To see RECENT: view /memories/recent
|
||||
To SAVE: create /memories/<topic>.txt "content"
|
||||
"""
|
||||
|
||||
async def _native_create_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_create_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle CREATE command - save to semantic vector store."""
|
||||
path = input_data.get("path", "")
|
||||
file_text = input_data.get("file_text", "")
|
||||
@@ -1793,7 +1847,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
if not file_text:
|
||||
return "Error: file_text is required (the memory content)"
|
||||
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
@@ -1806,7 +1860,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
)
|
||||
|
||||
# Save to our semantic backend
|
||||
memory = await self._backend.save_memory(
|
||||
memory = await backend.save_memory(
|
||||
content=file_text,
|
||||
user_id=user_id,
|
||||
importance=0.5,
|
||||
@@ -1820,7 +1874,9 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
logger.error(f"Memory: Semantic create failed: {e}")
|
||||
return f"Error: {e}"
|
||||
|
||||
async def _native_update_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_update_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle STR_REPLACE command - update memory content."""
|
||||
path = input_data.get("path", "")
|
||||
old_str = input_data.get("old_str", "")
|
||||
@@ -1831,12 +1887,12 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
if not old_str:
|
||||
return "Error: old_str is required"
|
||||
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
# Search for memory containing old_str
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query=old_str,
|
||||
user_id=user_id,
|
||||
top_k=5,
|
||||
@@ -1860,15 +1916,15 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
new_content = matching_memory.content.replace(old_str, new_str, 1)
|
||||
|
||||
# Update via delete + create (or update if backend supports it)
|
||||
if hasattr(self._backend, "update_memory"):
|
||||
await self._backend.update_memory(
|
||||
if hasattr(backend, "update_memory"):
|
||||
await backend.update_memory(
|
||||
memory_id=matching_memory.id,
|
||||
new_content=new_content,
|
||||
user_id=user_id,
|
||||
)
|
||||
else:
|
||||
await self._backend.delete_memory(matching_memory.id)
|
||||
await self._backend.save_memory(
|
||||
await backend.delete_memory(matching_memory.id)
|
||||
await backend.save_memory(
|
||||
content=new_content,
|
||||
user_id=user_id,
|
||||
importance=0.5,
|
||||
@@ -1885,7 +1941,9 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
logger.error(f"Memory: Semantic update failed: {e}")
|
||||
return f"Error: {e}"
|
||||
|
||||
async def _native_append_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_append_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle INSERT command - append to memory or create new."""
|
||||
path = input_data.get("path", "")
|
||||
insert_text = input_data.get("insert_text", "")
|
||||
@@ -1896,7 +1954,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
if not insert_text:
|
||||
return "Error: insert_text is required"
|
||||
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
@@ -1904,7 +1962,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
# with the additional context
|
||||
topic = path.replace("/memories/", "").replace("/", "_").replace(".txt", "")
|
||||
|
||||
await self._backend.save_memory(
|
||||
await backend.save_memory(
|
||||
content=insert_text,
|
||||
user_id=user_id,
|
||||
importance=0.5,
|
||||
@@ -1918,14 +1976,16 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
logger.error(f"Memory: Semantic append failed: {e}")
|
||||
return f"Error: {e}"
|
||||
|
||||
async def _native_delete_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_delete_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle DELETE command - remove from vector store."""
|
||||
path = input_data.get("path", "")
|
||||
|
||||
if not path:
|
||||
return "Error: path is required"
|
||||
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
@@ -1937,7 +1997,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
.replace(".txt", "")
|
||||
)
|
||||
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query=topic,
|
||||
user_id=user_id,
|
||||
top_k=10,
|
||||
@@ -1952,7 +2012,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
# Check if metadata matches path
|
||||
metadata = getattr(r.memory, "metadata", {}) or {}
|
||||
if metadata.get("virtual_path") == path or r.score > 0.8:
|
||||
await self._backend.delete_memory(r.memory.id)
|
||||
await backend.delete_memory(r.memory.id)
|
||||
deleted_count += 1
|
||||
|
||||
if deleted_count == 0:
|
||||
@@ -1967,7 +2027,9 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
logger.error(f"Memory: Semantic delete failed: {e}")
|
||||
return f"Error: {e}"
|
||||
|
||||
async def _native_rename_semantic(self, input_data: dict[str, Any], user_id: str) -> str:
|
||||
async def _native_rename_semantic(
|
||||
self, input_data: dict[str, Any], user_id: str, *, backend: Any
|
||||
) -> str:
|
||||
"""Handle RENAME command - update memory path/topic."""
|
||||
old_path = input_data.get("old_path", "")
|
||||
new_path = input_data.get("new_path", "")
|
||||
@@ -1977,7 +2039,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
if not new_path:
|
||||
return "Error: new_path is required"
|
||||
|
||||
if not self._backend:
|
||||
if not backend:
|
||||
return "Error: Memory backend not initialized"
|
||||
|
||||
try:
|
||||
@@ -1989,7 +2051,7 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
.replace(".txt", "")
|
||||
)
|
||||
|
||||
results = await self._backend.search_memories(
|
||||
results = await backend.search_memories(
|
||||
query=old_topic,
|
||||
user_id=user_id,
|
||||
top_k=10,
|
||||
@@ -2006,8 +2068,8 @@ To SAVE: create /memories/<topic>.txt "content"
|
||||
metadata = getattr(r.memory, "metadata", {}) or {}
|
||||
if metadata.get("virtual_path") == old_path or r.score > 0.8:
|
||||
# Delete old and create with new path
|
||||
await self._backend.delete_memory(r.memory.id)
|
||||
await self._backend.save_memory(
|
||||
await backend.delete_memory(r.memory.id)
|
||||
await backend.save_memory(
|
||||
content=r.memory.content,
|
||||
user_id=user_id,
|
||||
importance=getattr(r.memory, "importance", 0.5),
|
||||
|
||||
@@ -932,6 +932,7 @@ class PrometheusMetrics:
|
||||
compression_tokens_saved=tokens_saved,
|
||||
tool_schema_tokens_saved=tool_search_saved,
|
||||
output_tokens_saved=output_tokens_saved,
|
||||
output_tokens=output_tokens,
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
cache_write_5m_tokens=cache_write_5m_tokens,
|
||||
|
||||
@@ -84,6 +84,12 @@ RUNTIME_ENV_KNOBS: tuple[Knob, ...] = (
|
||||
"int",
|
||||
"Max on-disk file size (bytes) the disk-verify fallback will read.",
|
||||
),
|
||||
Knob(
|
||||
"HEADROOM_DEFERRED_TOOL_AFFINITY",
|
||||
"bool",
|
||||
"Keep defer_loading tools out of the Anthropic prefix-tracker lineage key "
|
||||
"(default on; 0 counts them again).",
|
||||
),
|
||||
Knob(
|
||||
"HEADROOM_WEB_SEARCH_HISTORY",
|
||||
"bool",
|
||||
|
||||
@@ -543,6 +543,7 @@ def estimate_request_savings_usd(
|
||||
compression_tokens_saved: int = 0,
|
||||
tool_schema_tokens_saved: int = 0,
|
||||
output_tokens_saved: int = 0,
|
||||
output_tokens: int = 0,
|
||||
cache_read_tokens: int = 0,
|
||||
cache_write_tokens: int = 0,
|
||||
cache_write_5m_tokens: int = 0,
|
||||
@@ -588,6 +589,8 @@ def estimate_request_savings_usd(
|
||||
CacheMix,
|
||||
PricedSavings,
|
||||
Region,
|
||||
long_context_premium_avoided_usd,
|
||||
long_context_threshold,
|
||||
price_savings,
|
||||
weakest_basis,
|
||||
)
|
||||
@@ -600,26 +603,45 @@ def estimate_request_savings_usd(
|
||||
uncached_input_tokens=uncached_input_tokens,
|
||||
cache_inferred=cache_inferred,
|
||||
)
|
||||
long_context = mix.is_long_context(local_tokens=local_input_tokens)
|
||||
threshold = long_context_threshold(model)
|
||||
long_context = mix.is_long_context(local_tokens=local_input_tokens, threshold=threshold)
|
||||
# Compression that kept the prompt under the threshold avoided the long
|
||||
# card on the whole request: the removed tokens are priced at that card,
|
||||
# and the forwarded input and generated output premiums are added on top.
|
||||
forwarded = max(mix.billed, max(_coerce_int(local_input_tokens), 0))
|
||||
crossed = (
|
||||
not long_context and forwarded + max(_coerce_int(compression_tokens_saved), 0) > threshold
|
||||
)
|
||||
|
||||
def _price(tokens: Any, region: Region) -> PricedSavings:
|
||||
def _price(tokens: Any, region: Region, *, long_tier: bool) -> PricedSavings:
|
||||
return price_savings(
|
||||
max(_coerce_int(tokens), 0),
|
||||
model=model,
|
||||
mix=mix,
|
||||
region=region,
|
||||
long_context=long_context,
|
||||
long_context=long_tier,
|
||||
provider=provider,
|
||||
# Matches the blended rate the flat estimators fall back to, so an
|
||||
# unpriceable model reports the same dollars it always has.
|
||||
fallback_rate_per_token=DEFAULT_FALLBACK_INPUT_COST_PER_TOKEN,
|
||||
)
|
||||
|
||||
compression = _price(compression_tokens_saved, Region.LIVE_ZONE)
|
||||
tool_schema = _price(tool_schema_tokens_saved, Region.PREFIX)
|
||||
compression = _price(
|
||||
compression_tokens_saved, Region.LIVE_ZONE, long_tier=long_context or crossed
|
||||
)
|
||||
tool_schema = _price(tool_schema_tokens_saved, Region.PREFIX, long_tier=long_context)
|
||||
compression_usd = compression.usd
|
||||
if crossed:
|
||||
compression_usd += long_context_premium_avoided_usd(
|
||||
model,
|
||||
mix,
|
||||
local_tokens=local_input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
provider=provider,
|
||||
)
|
||||
|
||||
return {
|
||||
"compression": compression.usd,
|
||||
"compression": compression_usd,
|
||||
"compression_list": compression.usd_list,
|
||||
"tool_schema": tool_schema.usd,
|
||||
"tool_schema_list": tool_schema.usd_list,
|
||||
@@ -1461,6 +1483,7 @@ class SavingsTracker:
|
||||
priced = estimate_request_savings_usd(
|
||||
model,
|
||||
compression_tokens_saved=_coerce_int(metrics.get("tokens_saved")),
|
||||
output_tokens=_coerce_int(metrics.get("output_tokens")),
|
||||
cache_read_tokens=cache_read_tokens,
|
||||
cache_write_tokens=cache_write_tokens,
|
||||
cache_write_5m_tokens=_coerce_int(metrics.get("cache_write_5m_tokens")),
|
||||
|
||||
@@ -2004,7 +2004,12 @@ class HeadroomProxy(
|
||||
if raw not in ("1", "true", "yes", "on"):
|
||||
import platform
|
||||
|
||||
libc, version = platform.libc_ver()
|
||||
try:
|
||||
libc, version = platform.libc_ver()
|
||||
except OSError:
|
||||
# libc_ver() reads sys.executable; Microsoft Store Python's
|
||||
# python.exe is an app-execution alias that can't be opened.
|
||||
libc, version = "", ""
|
||||
if libc == "glibc":
|
||||
try:
|
||||
major, minor = (int(part) for part in version.split(".")[:2])
|
||||
|
||||
@@ -230,10 +230,7 @@ def compute_release_version(
|
||||
def get_canonical_version(root: Path) -> str:
|
||||
"""Read the canonical project version from pyproject.toml."""
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError: # pragma: no cover - Python 3.10 compatibility
|
||||
import tomli as tomllib
|
||||
import tomllib
|
||||
|
||||
with open(root / "pyproject.toml", "rb") as file:
|
||||
project = tomllib.load(file)["project"]
|
||||
|
||||
@@ -41,6 +41,12 @@ from headroom.onnx_runtime import _resolve_revision, hf_hub_download_local_first
|
||||
|
||||
from .base import RelevanceScore, RelevanceScorer
|
||||
|
||||
# Texts per ONNX Runtime run. fastembed pads each batch to its longest text, and
|
||||
# its default batch of 256 grows ORT's CPU arena (which never shrinks) to fit
|
||||
# the biggest batch a proxy ever scores: 2.5 GB retained in a long-lived macOS
|
||||
# proxy. 16 holds ~10x less and runs faster, with less padding per batch.
|
||||
EMBED_BATCH_SIZE = 16
|
||||
|
||||
# numpy is an optional dependency - import lazily
|
||||
_numpy = None
|
||||
|
||||
@@ -335,7 +341,7 @@ class EmbeddingScorer(RelevanceScorer):
|
||||
"""
|
||||
np = _get_numpy()
|
||||
model = self._get_model()
|
||||
embeddings = list(model.embed(texts))
|
||||
embeddings = list(model.embed(texts, batch_size=EMBED_BATCH_SIZE))
|
||||
return np.array(embeddings)
|
||||
|
||||
def score(self, item: str, context: str) -> RelevanceScore:
|
||||
|
||||
@@ -999,13 +999,32 @@ def save(values: dict[str, Any]) -> None:
|
||||
_atomic_write_text(paths.settings_path(), payload)
|
||||
|
||||
|
||||
# The env vars Headroom itself inserted at startup, mapped to the value it
|
||||
# inserted. This is provenance, not value equality: an operator export that
|
||||
# happens to carry the same text as the settings file is still an operator
|
||||
# export, and Headroom's own seed stays Headroom's own even after the file
|
||||
# changes underneath it.
|
||||
_SEEDED_ENV: dict[str, str] = {}
|
||||
|
||||
|
||||
def seeded_environment() -> dict[str, str]:
|
||||
"""Return the env vars Headroom inserted at startup, with their values."""
|
||||
return dict(_SEEDED_ENV)
|
||||
|
||||
|
||||
def apply_to_environ(values: dict[str, Any]) -> None:
|
||||
"""``setdefault`` each stored value into ``os.environ`` (explicit export wins)."""
|
||||
for key, value in values.items():
|
||||
field = _BY_KEY.get(key)
|
||||
if field is None or value is None:
|
||||
continue
|
||||
os.environ.setdefault(field.env, _serialize(field, value))
|
||||
if field.env in os.environ:
|
||||
# An explicit export is already there and keeps precedence, so
|
||||
# Headroom did not insert this one.
|
||||
continue
|
||||
serialized = _serialize(field, value)
|
||||
os.environ[field.env] = serialized
|
||||
_SEEDED_ENV[field.env] = serialized
|
||||
|
||||
|
||||
def _coerce_runtime_values(runtime_values: dict[str, Any] | None) -> dict[str, Any]:
|
||||
@@ -1079,6 +1098,13 @@ def to_schema(
|
||||
effective = {**environment_effective, **runtime}
|
||||
fields: list[dict[str, Any]] = []
|
||||
for field in SETTINGS:
|
||||
# An env var is an operator export only when Headroom did not put it
|
||||
# there. The startup seed is recorded when it happens, so the answer
|
||||
# does not depend on how the value compares with the settings file:
|
||||
# a value saved after the seed is still Headroom's own seed, and an
|
||||
# operator export that matches the stored text is still an export.
|
||||
env_raw = os.environ.get(field.env)
|
||||
seeded_env = _SEEDED_ENV.get(field.env)
|
||||
fields.append(
|
||||
{
|
||||
"key": field.key,
|
||||
@@ -1094,7 +1120,8 @@ def to_schema(
|
||||
"minimum": field.minimum,
|
||||
"maximum": field.maximum,
|
||||
"tier": field.tier,
|
||||
"env_override": bool(os.environ.get(field.env))
|
||||
"env_override": bool(env_raw)
|
||||
and env_raw != seeded_env
|
||||
and field.env not in env_override_exclusions,
|
||||
"runtime_override": field.key in runtime
|
||||
and runtime[field.key] != environment_effective[field.key],
|
||||
|
||||
+91
-27
@@ -68,7 +68,7 @@ import threading
|
||||
import time
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import Any, Final
|
||||
|
||||
from .models import FieldSemantics, ToolSignature
|
||||
@@ -409,9 +409,16 @@ class ToolPattern:
|
||||
"last_updated",
|
||||
}
|
||||
filtered = {k: v for k, v in data.items() if k in valid_fields}
|
||||
# A value of the wrong kind (a string where a list belongs) would be
|
||||
# accepted as-is and then iterated or added to when patterns merge.
|
||||
for name, value in filtered.items():
|
||||
if not isinstance(value, _PATTERN_FIELD_KINDS.get(name, object)):
|
||||
raise TypeError(f"{name} is a {type(value).__name__}")
|
||||
|
||||
# Handle seen_instance_hashes (serialized without underscore prefix)
|
||||
seen_hashes = data.get("seen_instance_hashes", [])
|
||||
if not isinstance(seen_hashes, list) or not all(isinstance(h, str) for h in seen_hashes):
|
||||
raise TypeError("seen_instance_hashes is not a list of strings")
|
||||
|
||||
pattern = cls(**filtered)
|
||||
pattern._seen_instance_hashes = seen_hashes[:100] # Limit on load
|
||||
@@ -438,6 +445,38 @@ class ToolPattern:
|
||||
return pattern
|
||||
|
||||
|
||||
# What each stored ToolPattern field may hold, by the base of its annotation.
|
||||
# Merges store some counts as floats, so numeric fields accept either.
|
||||
_KIND_BY_ANNOTATION: Final[dict[str, type | tuple[type, ...]]] = {
|
||||
"str": str,
|
||||
"int": (int, float),
|
||||
"float": (int, float),
|
||||
"bool": bool,
|
||||
"list": list,
|
||||
"dict": dict,
|
||||
}
|
||||
_PATTERN_FIELD_KINDS: Final[dict[str, type | tuple[type, ...]]] = {
|
||||
f.name: _KIND_BY_ANNOTATION.get(str(f.type).split("[")[0], object) for f in fields(ToolPattern)
|
||||
}
|
||||
_PATTERN_CONTAINER_FIELDS: Final = tuple(
|
||||
f.name for f in fields(ToolPattern) if f.default_factory in (list, dict, set)
|
||||
)
|
||||
|
||||
|
||||
def _detached_copy(pattern: ToolPattern) -> ToolPattern:
|
||||
"""Copy ``pattern`` so that merging into the copy leaves the original intact.
|
||||
|
||||
``_merge_patterns`` reassigns scalars and adds to the pattern's lists, dicts
|
||||
and sets, so those containers are the parts that need their own copies.
|
||||
"""
|
||||
import copy
|
||||
|
||||
clone = copy.copy(pattern)
|
||||
for name in _PATTERN_CONTAINER_FIELDS:
|
||||
setattr(clone, name, getattr(pattern, name).copy())
|
||||
return clone
|
||||
|
||||
|
||||
@dataclass
|
||||
class TOINConfig:
|
||||
"""Configuration for the Tool Output Intelligence Network."""
|
||||
@@ -1397,40 +1436,65 @@ class ToolIntelligenceNetwork:
|
||||
patterns_data = data.get("patterns", {})
|
||||
source_instance = data.get("instance_id", "unknown")
|
||||
|
||||
skipped = 0
|
||||
first_error: Exception | None = None
|
||||
with self._lock:
|
||||
for serialized_key, pattern_dict in patterns_data.items():
|
||||
key = _deserialize_pattern_key(serialized_key)
|
||||
imported = ToolPattern.from_dict(pattern_dict)
|
||||
self._sanitize_loaded_pattern(imported)
|
||||
# Make sure dataclass fields agree with the dict key — pre-F3
|
||||
# dumps don't carry tenant_key on the pattern, and pre-B5
|
||||
# dumps don't carry auth_mode/model_family either; promote
|
||||
# from the (possibly default) key.
|
||||
imported.tenant_key = key[0]
|
||||
imported.auth_mode = key[1]
|
||||
imported.model_family = key[2]
|
||||
# Each entry is parsed, tracked and merged into a pattern that is
|
||||
# only stored once all of that succeeds. One unreadable entry must
|
||||
# not abort the rest (the store that did load would be saved over
|
||||
# the whole file) or leave a half-merged pattern behind.
|
||||
try:
|
||||
key = _deserialize_pattern_key(serialized_key)
|
||||
imported = ToolPattern.from_dict(pattern_dict)
|
||||
self._sanitize_loaded_pattern(imported)
|
||||
# Make sure dataclass fields agree with the dict key — pre-F3
|
||||
# dumps don't carry tenant_key on the pattern, and pre-B5
|
||||
# dumps don't carry auth_mode/model_family either; promote
|
||||
# from the (possibly default) key.
|
||||
imported.tenant_key = key[0]
|
||||
imported.auth_mode = key[1]
|
||||
imported.model_family = key[2]
|
||||
|
||||
if key in self._patterns:
|
||||
# Merge with existing
|
||||
self._merge_patterns(self._patterns[key], imported)
|
||||
else:
|
||||
# Add new pattern - need to track source instance
|
||||
self._patterns[key] = imported
|
||||
existing = self._patterns.get(key)
|
||||
if existing is not None:
|
||||
# Merge with existing
|
||||
pattern = _detached_copy(existing)
|
||||
self._merge_patterns(pattern, imported)
|
||||
else:
|
||||
# Add new pattern - need to track source instance
|
||||
pattern = imported
|
||||
|
||||
# For NEW patterns from another instance, track the source in
|
||||
# _seen_instance_hashes so user_count reflects cross-user data
|
||||
if source_instance != self._instance_id:
|
||||
pattern = self._patterns[key]
|
||||
if source_instance not in pattern._seen_instance_hashes:
|
||||
# Limit storage to 100 unique instances to bound memory
|
||||
if len(pattern._seen_instance_hashes) < 100:
|
||||
pattern._seen_instance_hashes.append(source_instance)
|
||||
# CRITICAL: Always increment user_count (even after cap)
|
||||
pattern.user_count += 1
|
||||
# For NEW patterns from another instance, track the source in
|
||||
# _seen_instance_hashes so user_count reflects cross-user data
|
||||
if source_instance != self._instance_id:
|
||||
if source_instance not in pattern._seen_instance_hashes:
|
||||
# Limit storage to 100 unique instances to bound memory
|
||||
if len(pattern._seen_instance_hashes) < 100:
|
||||
pattern._seen_instance_hashes.append(source_instance)
|
||||
# CRITICAL: Always increment user_count (even after cap)
|
||||
pattern.user_count += 1
|
||||
except (AttributeError, TypeError, ValueError) as e:
|
||||
skipped += 1
|
||||
first_error = first_error or e
|
||||
continue
|
||||
self._patterns[key] = pattern
|
||||
|
||||
self._prune_patterns_locked()
|
||||
self._dirty = True
|
||||
|
||||
if first_error is not None:
|
||||
logger.warning(
|
||||
"TOIN skipped malformed patterns on import",
|
||||
extra={
|
||||
"event": "toin_import_skipped",
|
||||
"skipped": skipped,
|
||||
"imported": len(patterns_data) - skipped,
|
||||
"error_type": type(first_error).__name__,
|
||||
"error": str(first_error),
|
||||
},
|
||||
)
|
||||
|
||||
def _merge_patterns(self, existing: ToolPattern, imported: ToolPattern) -> None:
|
||||
"""Merge imported pattern into existing."""
|
||||
total = existing.sample_size + imported.sample_size
|
||||
|
||||
+386
-10
@@ -6,7 +6,11 @@ tokenizer backends must implement.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
import json
|
||||
import re
|
||||
import zlib
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
@@ -25,6 +29,323 @@ _COUNT_CACHE_MIN_CHARS = 256
|
||||
_COUNT_CACHE_MAX_ENTRIES = 4096
|
||||
_COUNT_CACHE_MAX_CHARS = 8_000_000
|
||||
|
||||
#: Document pricing (see ``BaseTokenizer._estimate_document_tokens``). Anthropic
|
||||
#: documents ~1,500-3,000 text tokens per PDF page; the low end is used. Pages
|
||||
#: come from the PDF's active page tree, read the way a PDF reader reads it: the
|
||||
#: cross-reference data the last ``startxref`` chains to (tables and xref streams,
|
||||
#: through ``/Prev``) gives ``/Root`` and where each object lives, and the catalog
|
||||
#: and its ``/Pages`` node are read there (an ``N G obj`` at that offset, or entry
|
||||
#: i of an object stream), each verified, never wherever their text happens to
|
||||
#: appear in the file. Comments and string literals are cut from an object's
|
||||
#: syntax, and each key comes from its own dictionary's top level, never from a
|
||||
#: dict, array or hex string nested in it. A missing, damaged or unverifiable xref,
|
||||
#: an absurd ``/Count``, or a spent budget gives ~3KB of file per page instead.
|
||||
_DOCUMENT_TOKENS_PER_PAGE = 1500
|
||||
_DOCUMENT_BYTES_PER_PAGE = 3000
|
||||
_PDF_MAX_PAGES = 10_000 # a larger /Count is treated as malformed, not priced
|
||||
# One budget per file of bytes scanned or inflated, and one of xref rows read
|
||||
# (repeats included), so no chain of sections or streams can multiply the work.
|
||||
_PDF_OBJECT_STREAM_MAX_BYTES = 4_000_000
|
||||
_PDF_MAX_OBJECTS = 100_000
|
||||
_PDF_MAX_XREF_SECTIONS = 1_000 # a longer /Prev chain is treated as malformed
|
||||
# Each pattern opens with a lookahead on its first byte so the scan skips binary fast.
|
||||
# A stream's data starts after its keyword, any spaces and an EOL if present (a
|
||||
# lone CR too): the spec requires CRLF or LF, but readers accept the others.
|
||||
_PDF_SYNTAX_STOP = re.compile(
|
||||
rb"(?=[%(se])(?:%[^\r\n]*|\(|(?<![A-Za-z/])(?:stream *(?:\r\n?|\n)?|endobj|startxref))"
|
||||
)
|
||||
_PDF_STRING_TOKEN = re.compile(rb"\\.|[()]", re.S)
|
||||
_PDF_STREAM_LENGTH = re.compile(rb"/Length\s+(\d{1,10})(?!\d|\s+\d+\s+R)")
|
||||
_PDF_STREAM_END = re.compile(rb"\s*endstream")
|
||||
_PDF_ROOT = re.compile(rb"/Root\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
_PDF_PAGES = re.compile(rb"/Pages\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
# A longer digit run never reaches int(): it is malformed, and the size estimate stands in.
|
||||
_PDF_COUNT = re.compile(rb"/Count\s+(\d{1,5})(?!\d)")
|
||||
_PDF_FIRST = re.compile(rb"/First\s+(\d{1,10})")
|
||||
_PDF_NESTING = re.compile(rb"<<|>>|<[^<>]*>|(\[[\d\s]*\])|\[|\]")
|
||||
_PDF_LENGTH_REF = re.compile(rb"/Length\s+(\d{1,10})\s+(\d{1,5})\s+R")
|
||||
_PDF_LENGTH_OBJECT = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\s*(\d{1,10})\s*endobj")
|
||||
_PDF_OBJECT_HEADER = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+obj\b")
|
||||
_PDF_STARTXREF = re.compile(rb"startxref\s+(\d{1,10})")
|
||||
_PDF_XREF_TABLE = re.compile(rb"\s*xref\b")
|
||||
_PDF_XREF_SUBSECTION = re.compile(rb"\s*(\d{1,10})\s+(\d{1,5})\s+")
|
||||
_PDF_XREF_ROW = re.compile(rb"(\d{10}) (\d{5}) ([nf])")
|
||||
_PDF_XREF_TRAILER = re.compile(rb"\s*trailer\b")
|
||||
_PDF_XREF_WIDTHS = re.compile(rb"/W\s*\[\s*(\d)\s+(\d)\s+(\d)\s*\]")
|
||||
|
||||
|
||||
def _pdf_syntax(data: bytes, pos: int = 0, end: int | None = None) -> tuple[bytes, int, bool]:
|
||||
"""PDF syntax from ``pos`` to the next ``stream``/``endobj``/``startxref`` (or ``end``),
|
||||
minus comments and string literals; also where it stopped and whether a stream
|
||||
starts there."""
|
||||
end = len(data) if end is None else min(end, len(data))
|
||||
parts = []
|
||||
while match := _PDF_SYNTAX_STOP.search(data, pos, end):
|
||||
parts.append(data[pos : match.start()])
|
||||
pos = match.end()
|
||||
if match.group() == b"(":
|
||||
depth = 0 # a string literal may nest balanced parentheses
|
||||
pos = end
|
||||
for token in _PDF_STRING_TOKEN.finditer(data, match.start(), end):
|
||||
depth += {b"(": 1, b")": -1}.get(token.group(), 0)
|
||||
if not depth:
|
||||
pos = token.end()
|
||||
break
|
||||
elif not match.group().startswith(b"%"):
|
||||
return b"".join(parts), pos, match.group().startswith(b"stream")
|
||||
return b"".join(parts) + data[pos:end], end, False
|
||||
|
||||
|
||||
def _pdf_own_keys(syntax: bytes, nested: list[bytes] | None = None) -> bytes:
|
||||
"""The top level of the dictionary ``syntax`` opens with, each nested dict, array
|
||||
and hex string cut to ``_<i>``: a key read from it is that dictionary's own. An
|
||||
array of plain integers holds no key, so it stays (an xref stream's ``/W`` and
|
||||
``/Index``). ``nested`` collects what was cut, ``_<i>`` being entry i."""
|
||||
body = syntax.lstrip(b"\0\t\n\x0c\r ")
|
||||
if not body.startswith(b"<<"):
|
||||
return b""
|
||||
cut: list[bytes] = [] if nested is None else nested
|
||||
parts: list[bytes] = []
|
||||
depth = pos = opened = 0
|
||||
for token in _PDF_NESTING.finditer(body):
|
||||
step = {b"<<": 1, b"[": 1, b">>": -1, b"]": -1}.get(token.group(), 0)
|
||||
if depth == 1:
|
||||
parts.append(body[pos : token.start()])
|
||||
if step < 0:
|
||||
break # the dictionary's own end
|
||||
if token.group(1):
|
||||
parts.append(token.group(1))
|
||||
else:
|
||||
parts.append(b" _%d " % len(cut))
|
||||
opened = token.start()
|
||||
if not step:
|
||||
cut.append(token.group()) # a hex string
|
||||
depth += step
|
||||
if step < 0 and depth == 1:
|
||||
cut.append(body[opened : token.end()])
|
||||
if not depth:
|
||||
break
|
||||
pos = token.end()
|
||||
return b"".join(parts)
|
||||
|
||||
|
||||
def _pdf_png_rows(data: bytes, columns: int) -> bytes:
|
||||
"""``data`` decoded from PNG predictor rows: each a filter byte, then ``columns``
|
||||
bytes, one per sample (an xref stream's /Colors 1 /BitsPerComponent 8)."""
|
||||
out, prior = bytearray(), bytes(columns)
|
||||
for at in range(0, len(data), columns + 1):
|
||||
kind, row = data[at], bytearray(data[at + 1 : at + 1 + columns])
|
||||
if kind == 2:
|
||||
row = bytearray((a + b) & 255 for a, b in zip(row, prior))
|
||||
elif kind:
|
||||
for i in range(len(row)):
|
||||
left, up, corner = row[i - 1] if i else 0, prior[i], prior[i - 1] if i else 0
|
||||
if kind == 1:
|
||||
row[i] = (row[i] + left) & 255
|
||||
elif kind == 3:
|
||||
row[i] = (row[i] + (left + up) // 2) & 255
|
||||
elif kind == 4:
|
||||
guess = left + up - corner # Paeth: the neighbour nearest the guess
|
||||
pa, pb, pc = abs(guess - left), abs(guess - up), abs(guess - corner)
|
||||
row[i] = (
|
||||
row[i] + (left if pa <= pb and pa <= pc else up if pb <= pc else corner)
|
||||
) & 255
|
||||
else:
|
||||
raise _PdfUnreadable
|
||||
out += row
|
||||
prior = bytes(row)
|
||||
return bytes(out)
|
||||
|
||||
|
||||
class _PdfUnreadable(Exception):
|
||||
"""The PDF fails verification or a budget runs out: its size estimate stands in."""
|
||||
|
||||
|
||||
class _PdfReader:
|
||||
"""Reads objects at the locations the PDF's cross-reference data gives for them."""
|
||||
|
||||
def __init__(self, doc: bytes) -> None:
|
||||
self.doc = doc
|
||||
self.budget = _PDF_OBJECT_STREAM_MAX_BYTES
|
||||
self.rows = _PDF_MAX_OBJECTS
|
||||
self.entries: dict[int, tuple[int, int, int]] = {} # number: (type, field 2, field 3)
|
||||
self.packed: dict[int, tuple[list[bytes], int, bytes]] = {} # header, /First, data
|
||||
self.root: re.Match[bytes] | None = None
|
||||
eof = doc.rfind(b"%%EOF") # bytes after the last revision hold no objects
|
||||
last = doc.rfind(b"startxref", 0, len(doc) if eof < 0 else eof)
|
||||
start = _PDF_STARTXREF.match(doc, last) if last >= 0 else None
|
||||
todo = [int(start.group(1))] if start else []
|
||||
seen: set[int] = set()
|
||||
while todo:
|
||||
at = todo.pop(0)
|
||||
if at in seen:
|
||||
continue
|
||||
seen.add(at)
|
||||
if len(seen) > _PDF_MAX_XREF_SECTIONS:
|
||||
raise _PdfUnreadable
|
||||
self.spend(rows=1)
|
||||
table = _PDF_XREF_TABLE.match(doc, at)
|
||||
keys = self.read_table(table.end()) if table else self.read_xref_stream(at)
|
||||
self.root = self.root or _PDF_ROOT.search(keys) # the newest trailer's
|
||||
for key in (rb"/XRefStm", rb"/Prev"): # a table's stream entries, then older sections
|
||||
if found := re.search(key + rb"\s+(\d{1,10})(?!\d)", keys):
|
||||
todo.append(int(found.group(1)))
|
||||
if not self.root:
|
||||
raise _PdfUnreadable
|
||||
|
||||
def spend(self, scanned: int = 0, rows: int = 0) -> None:
|
||||
self.budget -= scanned
|
||||
self.rows -= rows
|
||||
if self.budget < 0 or self.rows < 0:
|
||||
raise _PdfUnreadable
|
||||
|
||||
def keys_at(self, pos: int, nested: list[bytes] | None = None) -> tuple[bytes, int, bool]:
|
||||
"""Own keys of the dictionary at ``pos``, where its syntax stops, and whether a
|
||||
stream starts there; the scan is charged to the budget."""
|
||||
syntax, stop, is_stream = _pdf_syntax(self.doc, pos, pos + self.budget + 1)
|
||||
self.spend(stop - pos) # a scan past the budget stops one byte beyond it
|
||||
return _pdf_own_keys(syntax, nested), stop, is_stream
|
||||
|
||||
def read_table(self, pos: int) -> bytes:
|
||||
while sub := _PDF_XREF_SUBSECTION.match(self.doc, pos):
|
||||
self.spend(rows=int(sub.group(2)) + 1) # also caps the distinct counts compiled
|
||||
rows = re.compile(rb"(?:\d{10} \d{5} [nf]\s{1,2}){%s}" % sub.group(2))
|
||||
if not (block := rows.match(self.doc, sub.end())):
|
||||
raise _PdfUnreadable
|
||||
for number, (offset, gen, kind) in enumerate(
|
||||
_PDF_XREF_ROW.findall(block.group()), int(sub.group(1))
|
||||
):
|
||||
if kind == b"n": # the newest section is read first, so its entry wins
|
||||
self.entries.setdefault(number, (1, int(offset), int(gen)))
|
||||
pos = block.end()
|
||||
if not (trailer := _PDF_XREF_TRAILER.match(self.doc, pos)):
|
||||
raise _PdfUnreadable
|
||||
return self.keys_at(trailer.end())[0]
|
||||
|
||||
def read_xref_stream(self, at: int) -> bytes:
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, at)
|
||||
nested: list[bytes] = []
|
||||
keys, data = self.stream(header.end(), nested) if header else (b"", b"")
|
||||
widths = _PDF_XREF_WIDTHS.search(keys)
|
||||
size = re.search(rb"/Size\s+(\d{1,10})", keys)
|
||||
if not (widths and size and re.search(rb"/Type\s*/XRef\b", keys)):
|
||||
raise _PdfUnreadable
|
||||
kind, field, gen = map(int, widths.groups())
|
||||
width = kind + field + gen
|
||||
predicted = 0
|
||||
if parms := re.search(rb"/DecodeParms\s+_(\d{1,4})(?!\d)", keys):
|
||||
raw = nested[int(parms.group(1))] if int(parms.group(1)) < len(nested) else b""
|
||||
if not raw.startswith(b"<<"):
|
||||
raise _PdfUnreadable # an array, or "_<i>" typed into the dict itself
|
||||
values = _pdf_own_keys(raw)
|
||||
predictor = re.search(rb"/Predictor\s+(\d{1,2})(?!\d)", values)
|
||||
columns = re.search(rb"/Columns\s+(\d{1,2})(?!\d)", values)
|
||||
if re.search(rb"/Colors\s+(?!1(?!\d))|/BitsPerComponent\s+(?!8(?!\d))", values):
|
||||
raise _PdfUnreadable
|
||||
if predictor and int(predictor.group(1)) >= 10:
|
||||
predicted = 1
|
||||
if int(columns.group(1) if columns else 1) != width:
|
||||
raise _PdfUnreadable
|
||||
elif predictor and int(predictor.group(1)) != 1:
|
||||
raise _PdfUnreadable # TIFF prediction: not written for xref streams
|
||||
if not width:
|
||||
raise _PdfUnreadable
|
||||
index = re.search(rb"/Index\s*\[([\d\s]*)\]", keys)
|
||||
bounds = (index.group(1) if index else b"0 " + size.group(1)).split()
|
||||
if any(len(n) > 10 for n in bounds):
|
||||
raise _PdfUnreadable
|
||||
starts, lengths = [int(n) for n in bounds[::2]], [int(n) for n in bounds[1::2]]
|
||||
count = min(sum(lengths), len(data) // (width + predicted))
|
||||
self.spend(rows=count)
|
||||
data = data[: count * (width + predicted)]
|
||||
if predicted:
|
||||
data = _pdf_png_rows(data, width)
|
||||
numbers = (n for a, length in zip(starts, lengths) for n in range(a, a + length))
|
||||
for number, at_row in zip(numbers, range(0, count * width, width)):
|
||||
entry = data[at_row : at_row + width]
|
||||
entry_type = int.from_bytes(entry[:kind] or b"\x01", "big")
|
||||
if entry_type in (1, 2):
|
||||
second = int.from_bytes(entry[kind : kind + field], "big")
|
||||
self.entries.setdefault(
|
||||
number, (entry_type, second, int.from_bytes(entry[kind + field :], "big"))
|
||||
)
|
||||
return keys
|
||||
|
||||
def stream(self, pos: int, nested: list[bytes]) -> tuple[bytes, bytes]:
|
||||
"""Own keys and inflated data of the stream object whose dict starts at ``pos``.
|
||||
Its end must be verified: ``endstream`` right after ``/Length`` bytes."""
|
||||
keys, start, is_stream = self.keys_at(pos, nested)
|
||||
length = _PDF_STREAM_LENGTH.search(keys)
|
||||
size = int(length.group(1)) if length else self.integer(_PDF_LENGTH_REF.search(keys))
|
||||
if not (is_stream and _PDF_STREAM_END.match(self.doc, start + size)):
|
||||
raise _PdfUnreadable
|
||||
data = self.doc[start : start + size]
|
||||
if b"/Filter" not in keys:
|
||||
if b"/DecodeParms" in keys:
|
||||
raise _PdfUnreadable
|
||||
return keys, data
|
||||
if not re.search(rb"/Filter\s*/FlateDecode\b", keys) or self.budget <= 0:
|
||||
raise _PdfUnreadable # (max_length 0 would mean no limit)
|
||||
inflate = zlib.decompressobj()
|
||||
try:
|
||||
data = inflate.decompress(data, self.budget)
|
||||
except zlib.error:
|
||||
raise _PdfUnreadable from None
|
||||
if inflate.unconsumed_tail:
|
||||
raise _PdfUnreadable
|
||||
self.spend(len(data))
|
||||
return keys, data
|
||||
|
||||
def integer(self, ref: re.Match[bytes] | None) -> int:
|
||||
"""The integer an indirect ``/Length`` names, at its xref offset."""
|
||||
entry = self.entries.get(int(ref.group(1))) if ref else None
|
||||
value = _PDF_LENGTH_OBJECT.match(self.doc, entry[1]) if entry and entry[0] == 1 else None
|
||||
if not (ref and entry and value and value.group(1, 2) == ref.group(1, 2)):
|
||||
raise _PdfUnreadable
|
||||
if int(ref.group(2)) != entry[2]: # the generation the xref holds
|
||||
raise _PdfUnreadable
|
||||
return int(value.group(3))
|
||||
|
||||
def get(self, ref: re.Match[bytes] | None) -> bytes:
|
||||
"""Own keys of the object ``ref`` names, read where the xref says it lives."""
|
||||
number, gen = (int(ref.group(1)), int(ref.group(2))) if ref else (-1, -1)
|
||||
kind, where, extra = self.entries.get(number, (0, 0, 0))
|
||||
if kind == 1 and extra == gen:
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, where)
|
||||
if header and (int(header.group(1)), int(header.group(2))) == (number, gen):
|
||||
return self.keys_at(header.end())[0]
|
||||
elif kind == 2 and gen == 0:
|
||||
head, first, data = self.packed.get(where) or self.unpack(where)
|
||||
i = 2 * extra # entry i of the stream's header of (number, offset) pairs
|
||||
if i + 1 < len(head) and int(head[i]) == number:
|
||||
stop = first + int(head[i + 3]) if i + 3 < len(head) else len(data)
|
||||
return _pdf_own_keys(_pdf_syntax(data[first + int(head[i + 1]) : stop])[0])
|
||||
raise _PdfUnreadable
|
||||
|
||||
def unpack(self, number: int) -> tuple[list[bytes], int, bytes]:
|
||||
"""An object stream's header numbers, its /First and its inflated data."""
|
||||
kind, where, gen = self.entries.get(number, (0, 0, 0))
|
||||
header = _PDF_OBJECT_HEADER.match(self.doc, where) if kind == 1 else None
|
||||
if not (header and (int(header.group(1)), int(header.group(2))) == (number, gen)):
|
||||
raise _PdfUnreadable
|
||||
keys, data = self.stream(header.end(), [])
|
||||
first = _PDF_FIRST.search(keys)
|
||||
if not (first and re.search(rb"/Type\s*/ObjStm\b", keys)) or b"/DecodeParms" in keys:
|
||||
raise _PdfUnreadable
|
||||
head = re.findall(rb"\d{1,10}", data[: int(first.group(1))])
|
||||
self.packed[number] = head, int(first.group(1)), data
|
||||
return self.packed[number]
|
||||
|
||||
|
||||
def _pdf_page_count(doc: bytes) -> int:
|
||||
"""Pages in the PDF's active page tree, else 0."""
|
||||
try:
|
||||
reader = _PdfReader(doc)
|
||||
count = _PDF_COUNT.search(reader.get(_PDF_PAGES.search(reader.get(reader.root))))
|
||||
except _PdfUnreadable:
|
||||
return 0
|
||||
return int(count.group(1)) if count and 0 < int(count.group(1)) <= _PDF_MAX_PAGES else 0
|
||||
|
||||
|
||||
class TokenCountCache:
|
||||
"""Exact memo for ``count_text``.
|
||||
@@ -78,12 +399,15 @@ class TokenCountCache:
|
||||
must not run the eviction check (a dict hit would clear the whole
|
||||
cache and force a full re-encode of every cached message on the
|
||||
next miss — the storm this exists to prevent).
|
||||
Oversized texts are never admitted, so warm entries survive.
|
||||
"""
|
||||
if text in self._counts:
|
||||
self._counts[text] = count
|
||||
return
|
||||
if len(text) < self._min_chars:
|
||||
return
|
||||
if len(text) > self._max_chars:
|
||||
return
|
||||
if len(self._counts) >= self._max_entries or self._chars >= self._max_chars:
|
||||
self._counts.clear()
|
||||
self._chars = 0
|
||||
@@ -211,16 +535,22 @@ class BaseTokenizer(ABC):
|
||||
|
||||
# Count role
|
||||
role = message.get("role", "")
|
||||
total += self.count_text(role)
|
||||
total += self.count_text(coerce_countable_text(role))
|
||||
|
||||
# Count content
|
||||
content = message.get("content")
|
||||
if isinstance(content, dict):
|
||||
# A single block sent without its list wrapper is priced like
|
||||
# the wrapped block, so images keep their fixed estimate (#4273).
|
||||
content = [content]
|
||||
if content is not None:
|
||||
if isinstance(content, str):
|
||||
total += self.count_text(content)
|
||||
elif isinstance(content, list):
|
||||
# Multi-part content (images, tool results, etc.)
|
||||
total += self._count_content_parts(content)
|
||||
else:
|
||||
total += self.count_text(coerce_countable_text(content))
|
||||
|
||||
# Count tool calls
|
||||
tool_calls = message.get("tool_calls")
|
||||
@@ -235,7 +565,7 @@ class BaseTokenizer(ABC):
|
||||
# Count name field
|
||||
name = message.get("name")
|
||||
if name:
|
||||
total += self.count_text(name)
|
||||
total += self.count_text(coerce_countable_text(name))
|
||||
total += 1 # Name field overhead
|
||||
|
||||
# Reply start overhead
|
||||
@@ -255,7 +585,7 @@ class BaseTokenizer(ABC):
|
||||
part_type = part.get("type", "")
|
||||
|
||||
if part_type == "text":
|
||||
total += self.count_text(part.get("text", ""))
|
||||
total += self.count_text(coerce_countable_text(part.get("text", "")))
|
||||
elif part_type in ("image_url", "image", "input_image"):
|
||||
# Images are NOT tokenized as text — they have a pixel-based cost.
|
||||
# Anthropic: tokens = (width * height) / 750, max ~1600 after resize.
|
||||
@@ -264,6 +594,12 @@ class BaseTokenizer(ABC):
|
||||
# This prevents the base64 blob from being json.dumps'd and counted
|
||||
# as text tokens (1MB image = ~330K fake tokens without this).
|
||||
total += 1600
|
||||
elif part_type == "document":
|
||||
# Anthropic document block. Its base64 PDF used to reach the
|
||||
# JSON catch-all below and be priced as prose: a 1 MB PDF
|
||||
# counted ~907K tokens, against the ~1,600 an image of the
|
||||
# same bytes gets.
|
||||
total += self._count_document_block(part)
|
||||
elif part_type in ("input_audio", "audio"):
|
||||
# Audio has fixed token cost, not tokenized as text
|
||||
total += 200
|
||||
@@ -281,15 +617,15 @@ class BaseTokenizer(ABC):
|
||||
else:
|
||||
total += self._count_serialized(content)
|
||||
elif part_type == "tool_use":
|
||||
total += self.count_text(part.get("name", ""))
|
||||
total += self.count_text(coerce_countable_text(part.get("name", "")))
|
||||
total += self._count_serialized(part.get("input", {}))
|
||||
elif not part_type and "text" in part:
|
||||
# Strands SDK format: {"text": "..."} without "type" field
|
||||
total += self.count_text(part["text"])
|
||||
total += self.count_text(coerce_countable_text(part["text"]))
|
||||
elif not part_type and "toolUse" in part:
|
||||
# Strands SDK tool_use: {"toolUse": {"name": ..., "input": ...}}
|
||||
tool_use = part["toolUse"]
|
||||
total += self.count_text(tool_use.get("name", ""))
|
||||
total += self.count_text(coerce_countable_text(tool_use.get("name", "")))
|
||||
total += self._count_serialized(tool_use.get("input", {}))
|
||||
elif not part_type and "toolResult" in part:
|
||||
# Strands SDK tool_result: {"toolResult": {"content": [...]}}
|
||||
@@ -308,7 +644,9 @@ class BaseTokenizer(ABC):
|
||||
reasoning = part["reasoningContent"]
|
||||
reasoning_text = reasoning.get("reasoningText", {})
|
||||
if isinstance(reasoning_text, dict):
|
||||
total += self.count_text(reasoning_text.get("text", ""))
|
||||
total += self.count_text(
|
||||
coerce_countable_text(reasoning_text.get("text", ""))
|
||||
)
|
||||
elif isinstance(reasoning_text, str):
|
||||
total += self.count_text(reasoning_text)
|
||||
elif not part_type and "document" in part:
|
||||
@@ -321,8 +659,7 @@ class BaseTokenizer(ABC):
|
||||
source = doc.get("source", {})
|
||||
doc_bytes = source.get("bytes", b"")
|
||||
if isinstance(doc_bytes, bytes | bytearray):
|
||||
estimated_pages = max(1, len(doc_bytes) // 3000)
|
||||
total += estimated_pages * 1500
|
||||
total += self._estimate_document_tokens(bytes(doc_bytes))
|
||||
else:
|
||||
total += self.count_text(str(doc_bytes))
|
||||
elif not part_type and "image" in part:
|
||||
@@ -358,7 +695,7 @@ class BaseTokenizer(ABC):
|
||||
# ~4 bytes/token (len * 3/4 / 4), which lands that session
|
||||
# near the provider figure and never prices an omitted
|
||||
# block at zero.
|
||||
total += self.count_text(part.get("thinking", "") or "")
|
||||
total += self.count_text(coerce_countable_text(part.get("thinking", "") or ""))
|
||||
total += len(part.get("signature") or "") * 3 // 16
|
||||
else:
|
||||
# Unknown type - estimate from JSON
|
||||
@@ -393,6 +730,45 @@ class BaseTokenizer(ABC):
|
||||
)
|
||||
return int(self.count_text(sample) * len(s) / len(sample))
|
||||
|
||||
def _count_document_block(self, part: dict[str, Any]) -> int:
|
||||
"""Price an Anthropic ``document`` block by its source type."""
|
||||
source = part.get("source")
|
||||
source = source if isinstance(source, dict) else {}
|
||||
source_type = source.get("type")
|
||||
if source_type == "base64":
|
||||
try:
|
||||
doc_bytes = base64.b64decode(source.get("data") or b"")
|
||||
except (binascii.Error, TypeError, ValueError):
|
||||
doc_bytes = b""
|
||||
total = self._estimate_document_tokens(doc_bytes)
|
||||
elif source_type == "content":
|
||||
content = source.get("content")
|
||||
if isinstance(content, list):
|
||||
total = self._count_content_parts(content)
|
||||
else:
|
||||
total = self.count_text(str(content or ""))
|
||||
elif source_type in ("url", "file"):
|
||||
# The bytes live with the provider; one page as a nominal estimate.
|
||||
total = _DOCUMENT_TOKENS_PER_PAGE
|
||||
else:
|
||||
# A plain-text source, or an untyped one carrying text.
|
||||
total = self.count_text(str(source.get("data") or ""))
|
||||
for key in ("title", "context"):
|
||||
value = part.get(key)
|
||||
if isinstance(value, str):
|
||||
total += self.count_text(value)
|
||||
return total
|
||||
|
||||
@staticmethod
|
||||
def _estimate_document_tokens(doc_bytes: bytes) -> int:
|
||||
"""Estimate a document file's tokens from its page count (see the constants)."""
|
||||
if b"%PDF-" not in doc_bytes[:1024]:
|
||||
# Only a PDF has a page tree: other bytes (a Strands txt/csv/docx
|
||||
# document) keep the size estimate even when their text quotes PDF syntax.
|
||||
return max(1, len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE) * _DOCUMENT_TOKENS_PER_PAGE
|
||||
pages = _pdf_page_count(doc_bytes) or len(doc_bytes) // _DOCUMENT_BYTES_PER_PAGE
|
||||
return max(1, pages) * _DOCUMENT_TOKENS_PER_PAGE
|
||||
|
||||
@staticmethod
|
||||
def _estimate_image_tokens(image_data: dict[str, Any]) -> int:
|
||||
"""Estimate tokens for an image using Anthropic's formula: (w*h)/750.
|
||||
|
||||
@@ -297,6 +297,9 @@ class TiktokenCounter(BaseTokenizer):
|
||||
continue
|
||||
|
||||
if key == "content":
|
||||
if isinstance(value, dict):
|
||||
# Unwrapped single block: price it like the list form (#4273).
|
||||
value = [value]
|
||||
if isinstance(value, str):
|
||||
total += self.count_text(value)
|
||||
elif isinstance(value, list):
|
||||
@@ -304,7 +307,9 @@ class TiktokenCounter(BaseTokenizer):
|
||||
for part in value:
|
||||
if isinstance(part, dict):
|
||||
if part.get("type") == "text":
|
||||
total += self.count_text(part.get("text", ""))
|
||||
total += self.count_text(
|
||||
coerce_countable_text(part.get("text", ""))
|
||||
)
|
||||
elif part.get("type") == "image_url":
|
||||
# Image tokens vary by detail level
|
||||
detail = part.get("image_url", {}).get("detail", "auto")
|
||||
@@ -324,6 +329,8 @@ class TiktokenCounter(BaseTokenizer):
|
||||
total += self._count_content_parts([part])
|
||||
elif isinstance(part, str):
|
||||
total += self.count_text(part)
|
||||
else:
|
||||
total += self.count_text(coerce_countable_text(value))
|
||||
elif key == "role":
|
||||
total += self.count_text(coerce_countable_text(value))
|
||||
elif key == "name":
|
||||
|
||||
@@ -383,9 +383,12 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
|
||||
if not eq or not name.startswith("@"):
|
||||
return None
|
||||
try:
|
||||
legends[name[1:]] = json.loads(payload)
|
||||
result = json.loads(payload)
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
if not isinstance(result, list):
|
||||
return None
|
||||
legends[name[1:]] = result
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for cells in _split_data_rows(rest, declared):
|
||||
@@ -404,7 +407,10 @@ def expand_compacted(text: str) -> list[dict[str, Any]] | None:
|
||||
_assign(record, col.path, None)
|
||||
continue
|
||||
try:
|
||||
_assign(record, col.path, legends[col.name][int(value)])
|
||||
idx_val = int(value)
|
||||
if idx_val < 0:
|
||||
return None
|
||||
_assign(record, col.path, legends[col.name][idx_val])
|
||||
except (ValueError, IndexError):
|
||||
return None
|
||||
continue
|
||||
|
||||
@@ -16,7 +16,9 @@ from .compression_units import (
|
||||
RoutedCompressionUnit,
|
||||
TokenCounterLike,
|
||||
UnitCompressionResult,
|
||||
_categorize_reason,
|
||||
_is_structured_shell_output,
|
||||
check_unit_protection,
|
||||
)
|
||||
from .content_router import RouterCompressionResult
|
||||
from .tag_protector import protect_tags, restore_tags
|
||||
@@ -104,7 +106,12 @@ def build_compression_batches(
|
||||
for entry in entries:
|
||||
entry_bytes = _text_bytes(entry.routed.unit.text)
|
||||
entry_key = _compatibility_key(entry)
|
||||
if entry_bytes >= min_batch_bytes or entry_bytes > max_batch_bytes:
|
||||
protection_reason = check_unit_protection(entry.routed.unit)
|
||||
if (
|
||||
protection_reason is not None
|
||||
or entry_bytes >= min_batch_bytes
|
||||
or entry_bytes > max_batch_bytes
|
||||
):
|
||||
flush()
|
||||
skipped.append(entry)
|
||||
continue
|
||||
@@ -197,7 +204,7 @@ def _passthrough_batch_results(
|
||||
batch: CompressionBatch,
|
||||
*,
|
||||
tokenizer: TokenCounterLike,
|
||||
reason: str,
|
||||
reason: str | None = None,
|
||||
router_result: RouterCompressionResult | None = None,
|
||||
) -> list[tuple[object, UnitCompressionResult]]:
|
||||
strategy = (
|
||||
@@ -217,11 +224,13 @@ def _passthrough_batch_results(
|
||||
tokens_saved=0,
|
||||
transforms_applied=[],
|
||||
strategy=strategy,
|
||||
reason=reason,
|
||||
reason=reason or check_unit_protection(entry.routed.unit) or "batch_invalid",
|
||||
router_result=router_result,
|
||||
text_bytes=_text_bytes(entry.routed.unit.text),
|
||||
min_bytes=entry.routed.unit.min_bytes,
|
||||
reason_category=reason,
|
||||
reason_category=_categorize_reason(
|
||||
reason or check_unit_protection(entry.routed.unit) or "batch_invalid"
|
||||
),
|
||||
),
|
||||
)
|
||||
for entry in batch.entries
|
||||
@@ -241,6 +250,9 @@ def compress_batch_with_router(
|
||||
``deadline_started_at`` works as in ``compress_unit_with_router``.
|
||||
"""
|
||||
|
||||
if any(check_unit_protection(entry.routed.unit) is not None for entry in batch.entries):
|
||||
return _passthrough_batch_results(batch, tokenizer=tokenizer)
|
||||
|
||||
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
|
||||
return _passthrough_batch_results(batch, tokenizer=tokenizer, reason="deadline_exceeded")
|
||||
|
||||
|
||||
@@ -218,6 +218,21 @@ def _compress_marker_free_text(
|
||||
)
|
||||
|
||||
|
||||
def check_unit_protection(unit: CompressionUnit) -> str | None:
|
||||
"""Return the early refusal reason if a unit is protected, else None."""
|
||||
if not unit.mutable:
|
||||
return "immutable"
|
||||
if unit.role == "user" and unit.metadata.get("compress_user") != "true":
|
||||
return "protected_user_message"
|
||||
if unit.role in {"system", "developer"}:
|
||||
return "protected_system_message"
|
||||
if unit.role == "assistant" and unit.metadata.get("compress_assistant") != "true":
|
||||
return "protected_assistant_message"
|
||||
if unit.cache_zone != "live":
|
||||
return f"cache_zone_{unit.cache_zone}"
|
||||
return None
|
||||
|
||||
|
||||
def compress_unit_with_router(
|
||||
unit: CompressionUnit,
|
||||
*,
|
||||
@@ -263,16 +278,9 @@ def compress_unit_with_router(
|
||||
kw["reason_category"] = _categorize_reason(reason_val)
|
||||
return replace(base, **kw) # type: ignore[arg-type]
|
||||
|
||||
if not unit.mutable:
|
||||
return _with_reason(reason="immutable")
|
||||
if unit.role == "user" and unit.metadata.get("compress_user") != "true":
|
||||
return _with_reason(reason="protected_user_message")
|
||||
if unit.role in {"system", "developer"}:
|
||||
return _with_reason(reason="protected_system_message")
|
||||
if unit.role == "assistant" and unit.metadata.get("compress_assistant") != "true":
|
||||
return _with_reason(reason="protected_assistant_message")
|
||||
if unit.cache_zone != "live":
|
||||
return _with_reason(reason=f"cache_zone_{unit.cache_zone}")
|
||||
protection_reason = check_unit_protection(unit)
|
||||
if protection_reason is not None:
|
||||
return _with_reason(reason=protection_reason)
|
||||
if text_bytes < unit.min_bytes:
|
||||
return _with_reason(reason="below_unit_floor")
|
||||
if deadline_started_at is not None and not router.share_request_deadline(deadline_started_at):
|
||||
|
||||
@@ -53,7 +53,12 @@ _COMMENT_RES = {
|
||||
# Content where whole-line elision is unsafe: a '#' line inside a YAML block
|
||||
# scalar or a TOML multi-line string is data, not a comment. Detection is
|
||||
# deliberately over-broad — when in doubt, Tier 2 stays off.
|
||||
_YAML_BLOCK_SCALAR_RE = re.compile(r":\s*[|>][+-]?\d*\s*$", re.MULTILINE)
|
||||
# Block scalars can be sequence items as well as mapping values; indentation
|
||||
# and chomping indicators may appear in either order, and a header can carry
|
||||
# a trailing comment. Conservatively skip elision for every such header.
|
||||
_YAML_BLOCK_SCALAR_RE = re.compile(
|
||||
r"(?:^|[ \t:])[|>][1-9+-]*[ \t]*(?:#[^\r\n]*)?\r?$", re.MULTILINE
|
||||
)
|
||||
_TOML_MULTILINE_RE = re.compile(r'"""|\'\'\'')
|
||||
|
||||
|
||||
|
||||
@@ -770,11 +770,52 @@ def _try_detect_tabular(content: str) -> DetectionResult | None:
|
||||
|
||||
md_result = _try_detect_markdown_table(lines)
|
||||
if md_result:
|
||||
return md_result
|
||||
# The tabular compressor replaces the whole payload with the table's
|
||||
# rows, so a table with a heading, prose or other output around it is
|
||||
# not tabular content: every line outside the table would be dropped.
|
||||
if _is_whole_markdown_table(content):
|
||||
return md_result
|
||||
# It can still be CSV holding markdown in a quoted field. Its pipes
|
||||
# belong to that markdown, so they are never the delimiter here.
|
||||
delimited = _try_detect_delimited(lines)
|
||||
if delimited and delimited.metadata.get("delimiter") != "|":
|
||||
return delimited
|
||||
return None
|
||||
|
||||
return _try_detect_delimited(lines)
|
||||
|
||||
|
||||
def _is_whole_markdown_table(content: str) -> bool:
|
||||
"""True when ``content`` is one markdown table and nothing else.
|
||||
|
||||
A blank line ends a markdown table, so every line after the header and
|
||||
separator has to be a row: written in the header's leading-pipe style,
|
||||
and neither another separator nor a list item. A note or a second table
|
||||
straight after the rows means the content is not just the table.
|
||||
|
||||
Every row must also split into exactly the header's cells, the way the
|
||||
tabular parser splits it, or the records built from it would truncate or
|
||||
pad the row. The parser does not unescape ``\\|``, so a table that uses it
|
||||
is left alone too.
|
||||
"""
|
||||
if "\\|" in content:
|
||||
return False
|
||||
rows = content.strip().split("\n")
|
||||
if len(rows) < 3 or not _is_md_separator(rows[1]):
|
||||
return False
|
||||
cols = _md_cell_count(rows[0])
|
||||
if cols < 2:
|
||||
return False
|
||||
leading_pipe = rows[0].lstrip().startswith("|")
|
||||
return all(
|
||||
_md_cell_count(row) == cols
|
||||
and row.lstrip().startswith("|") == leading_pipe
|
||||
and not _is_md_separator(row)
|
||||
and not _FIXED_WIDTH_LIST_RE.match(row)
|
||||
for row in rows[2:]
|
||||
)
|
||||
|
||||
|
||||
# Fixed-width (space-aligned) command output: `ls -l`, `ps aux`, `df -h`,
|
||||
# `docker ps`, `kubectl get`. These rows have no delimiter, so without this
|
||||
# check they fall through to PLAIN_TEXT and the prose compressor drops fields
|
||||
|
||||
@@ -360,9 +360,18 @@ def _invoke_smart_crusher(router: ContentRouter, inp: CompressInput) -> str | No
|
||||
crusher = router._get_smart_crusher()
|
||||
if crusher is None:
|
||||
return None
|
||||
# F2.2: the router never calls ``crusher.apply()``, so the crusher's own
|
||||
# policy field stays None. Forward the request's TOIN read-only gate per
|
||||
# call instead (the crusher is shared across concurrent requests). The
|
||||
# kwarg is only sent when gating, so crushers exposing the plain
|
||||
# ``crush(content, query, bias)`` signature keep working.
|
||||
policy = router._runtime_compression_policy
|
||||
gate: dict[str, bool] = {}
|
||||
if policy is not None and policy.toin_read_only:
|
||||
gate["record_toin"] = False
|
||||
# ``_get_*`` getters are typed ``Any``; pin the result to the contract type.
|
||||
compressed: str = crusher.crush(
|
||||
inp.content, query=inp.query, bias=_adapter_bias(inp)
|
||||
inp.content, query=inp.query, bias=_adapter_bias(inp), **gate
|
||||
).compressed
|
||||
return compressed
|
||||
|
||||
@@ -596,6 +605,24 @@ def _external_compressor_matches(descriptor: CompressorDescriptor, content_mime:
|
||||
return any(d in ("*", "*/*") or d == type_wildcard for d in declared)
|
||||
|
||||
|
||||
def _malformed_tool_call_ids(messages: list[dict[str, Any]]) -> set[str]:
|
||||
"""Ids of OpenAI tool calls whose ``function`` is not an object.
|
||||
|
||||
Such a call has no readable tool name, so its result cannot be classified;
|
||||
it is passed through like a verbatim-excluded tool rather than compressed.
|
||||
"""
|
||||
ids: set[str] = set()
|
||||
for msg in messages:
|
||||
if msg.get("role") != "assistant":
|
||||
continue
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
if isinstance(tc, dict) and not isinstance(tc.get("function"), dict):
|
||||
tc_id = tc.get("id")
|
||||
if isinstance(tc_id, str) and tc_id:
|
||||
ids.add(tc_id)
|
||||
return ids
|
||||
|
||||
|
||||
def _tool_call_args_text(raw: Any) -> str:
|
||||
"""Compact, query-usable text from a tool call's args.
|
||||
|
||||
@@ -2113,6 +2140,30 @@ def _is_small_mcp_result(tool_name: Any, text: Any) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def _shell_result_min_chars() -> int:
|
||||
"""Below this many characters a shell tool result is never lossily compressed."""
|
||||
try:
|
||||
return max(0, int(os.environ.get("HEADROOM_SHELL_RESULT_MIN_CHARS", "2000")))
|
||||
except ValueError:
|
||||
return 2000
|
||||
|
||||
|
||||
def _is_small_shell_result(tool_name: Any, text: Any, bash_tool_names: frozenset[str]) -> bool:
|
||||
"""True when ``text`` is a short shell result that must not be lossily compressed.
|
||||
|
||||
A 200-token ``ls -la``, ``git status``, or JSON dump is cheap to keep and
|
||||
expensive to get wrong: lossy compression can leave a numbers-only blob
|
||||
that still looks structured, so the model treats it as data instead of
|
||||
retrying the command (issue #3590). Large build/test logs still take the
|
||||
lossy path. ``HEADROOM_SHELL_RESULT_MIN_CHARS=0`` restores the old
|
||||
behaviour.
|
||||
"""
|
||||
if not isinstance(text, str) or not text:
|
||||
return False
|
||||
name = str(tool_name or "").lower()
|
||||
return name in bash_tool_names and len(text) < _shell_result_min_chars()
|
||||
|
||||
|
||||
@dataclass
|
||||
class ContentRouterConfig:
|
||||
"""Configuration for intelligent content routing.
|
||||
@@ -2317,8 +2368,22 @@ class ContentRouterConfig:
|
||||
# BUT a read-only *search* run through them (grep/rg/git grep) yields byte-
|
||||
# losslessly foldable output — folded instead of lossy-compressed. See
|
||||
# ``_bash_search_fold``. Config so new harness tool names / search programs
|
||||
# can be added without code changes.
|
||||
bash_tool_names: frozenset[str] = frozenset({"bash", "shell", "local_shell"})
|
||||
# can be added without code changes. The aliases mirror
|
||||
# ``headroom.learn._shared._TOOL_NAME_MAP`` so the same harnesses are
|
||||
# covered here.
|
||||
bash_tool_names: frozenset[str] = frozenset(
|
||||
{
|
||||
"bash",
|
||||
"shell",
|
||||
"local_shell",
|
||||
"run_shell_command",
|
||||
"execute_command",
|
||||
"exec_command",
|
||||
"terminal",
|
||||
"run_command",
|
||||
"run_terminal_command",
|
||||
}
|
||||
)
|
||||
bash_search_commands: frozenset[str] = frozenset(
|
||||
{"grep", "egrep", "fgrep", "rg", "ripgrep", "ag", "ack"}
|
||||
)
|
||||
@@ -2349,6 +2414,12 @@ class ContentRouterConfig:
|
||||
# Default False; the proxy enables it in token mode.
|
||||
search_group_by_file: bool = False
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
# Normalize once so the per-message shell check stays a set lookup.
|
||||
self.bash_tool_names = frozenset(
|
||||
name.lower() for name in self.bash_tool_names if isinstance(name, str)
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class _PerRequestRuntimeState:
|
||||
@@ -3716,6 +3787,31 @@ class ContentRouter(Transform):
|
||||
|
||||
return cand, f"lossless_{kind}"
|
||||
|
||||
@staticmethod
|
||||
def _validate_provider_output(supplied: Any) -> tuple[str, str] | None:
|
||||
"""Contract check shared by the general and excluded-tool provider paths.
|
||||
|
||||
Returns ``(candidate, safe_kind)`` or ``None`` ("ignore"). Never raises.
|
||||
"""
|
||||
if supplied is None:
|
||||
return None
|
||||
if not isinstance(supplied, tuple | list) or len(supplied) != 2:
|
||||
logger.debug("lossless provider returned a malformed result; ignoring")
|
||||
return None
|
||||
if not isinstance(supplied[0], str) or not isinstance(supplied[1], str):
|
||||
logger.debug("lossless provider returned non-str members; ignoring")
|
||||
return None
|
||||
# An empty / whitespace-only "fold" wins every length comparison and
|
||||
# would silently delete the block's content. That is not lossless.
|
||||
if not supplied[0].strip():
|
||||
logger.debug("lossless provider returned an empty result; ignoring")
|
||||
return None
|
||||
# fullmatch, not match: Python's `$` also matches just BEFORE a
|
||||
# trailing newline, so `re.match` would let "log\n" through and
|
||||
# put a newline into a metric label.
|
||||
kind = supplied[1] if _PROVIDER_KIND_RE.fullmatch(supplied[1]) else _PROVIDER_KIND_FALLBACK
|
||||
return (supplied[0], kind)
|
||||
|
||||
def _lossless_provider_result(self, content: str) -> tuple[str, str] | None:
|
||||
"""Validated + memoized ``(candidate, safe_kind)`` from the provider.
|
||||
|
||||
@@ -3746,29 +3842,7 @@ class ContentRouter(Transform):
|
||||
|
||||
result: tuple[str, str] | None = None
|
||||
try:
|
||||
supplied = provider(content)
|
||||
if supplied is None:
|
||||
result = None
|
||||
# Validate defensively *inside* the try: unpacking a 3-tuple / a bare
|
||||
# string / a non-sequence raises, and that exception used to escape
|
||||
# all the way out of the request.
|
||||
elif not isinstance(supplied, tuple | list) or len(supplied) != 2:
|
||||
logger.debug("lossless provider returned a malformed result; ignoring")
|
||||
elif not isinstance(supplied[0], str) or not isinstance(supplied[1], str):
|
||||
logger.debug("lossless provider returned non-str members; ignoring")
|
||||
# An empty / whitespace-only "fold" wins every length comparison and
|
||||
# would silently delete the block's content. That is not lossless.
|
||||
elif not supplied[0].strip():
|
||||
logger.debug("lossless provider returned an empty result; ignoring")
|
||||
else:
|
||||
raw_kind = supplied[1]
|
||||
# fullmatch, not match: Python's `$` also matches just BEFORE a
|
||||
# trailing newline, so `re.match` would let "log\n" through and
|
||||
# put a newline into a metric label.
|
||||
kind = (
|
||||
raw_kind if _PROVIDER_KIND_RE.fullmatch(raw_kind) else _PROVIDER_KIND_FALLBACK
|
||||
)
|
||||
result = (supplied[0], kind)
|
||||
result = self._validate_provider_output(provider(content))
|
||||
except Exception: # noqa: BLE001 - a broken provider must not break routing
|
||||
logger.debug("lossless provider failed in _lossless_first", exc_info=True)
|
||||
result = None
|
||||
@@ -5788,7 +5862,9 @@ class ContentRouter(Transform):
|
||||
for tc in msg.get("tool_calls") or []:
|
||||
if isinstance(tc, dict):
|
||||
tc_id = tc.get("id", "")
|
||||
fn = tc.get("function", {})
|
||||
fn = tc.get("function")
|
||||
if not isinstance(fn, dict): # "function": null / non-object
|
||||
continue
|
||||
name = fn.get("name", "")
|
||||
call_args = fn.get("arguments")
|
||||
if name:
|
||||
@@ -6140,6 +6216,9 @@ class ContentRouter(Transform):
|
||||
for tool_id, name in tool_name_map.items()
|
||||
if is_tool_excluded(name, exclude_tools)
|
||||
}
|
||||
# A call with no readable name keeps its result untouched, as it was
|
||||
# before such calls stopped raising out of apply().
|
||||
excluded_tool_ids |= _malformed_tool_call_ids(messages)
|
||||
|
||||
# CCR-retrieve tool IDs, precomputed once (mirrors excluded_tool_ids
|
||||
# above, rather than calling is_tool_excluded() per message/block). A
|
||||
@@ -6311,6 +6390,7 @@ class ContentRouter(Transform):
|
||||
"ccr_retrieve": 0,
|
||||
"user_msg": 0,
|
||||
"small": 0,
|
||||
"shell_small_result": 0,
|
||||
"recent_code": 0,
|
||||
"analysis_ctx": 0,
|
||||
"ratio_too_high": 0,
|
||||
@@ -6534,7 +6614,7 @@ class ContentRouter(Transform):
|
||||
if role == "tool":
|
||||
if tool_call_id in excluded_tool_ids:
|
||||
tool_name = tool_name_map.get(tool_call_id, "")
|
||||
if tool_name and is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
|
||||
if not tool_name or is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
|
||||
result_slots[i] = message
|
||||
transforms_applied.append("router:excluded:tool")
|
||||
route_counts["excluded_tool"] += 1
|
||||
@@ -6600,6 +6680,36 @@ class ContentRouter(Transform):
|
||||
_diag[i] = "compressed:bash_lossless_search"
|
||||
continue
|
||||
|
||||
# Small shell results stay verbatim (#3590). A short `ls`,
|
||||
# `git status`, or JSON dump is cheap to keep; lossy
|
||||
# compression can leave a numbers-only blob that still looks
|
||||
# structured, so the model treats it as data instead of
|
||||
# retrying. Large build/test logs still take the lossy path.
|
||||
if _is_small_shell_result(tool_name, content, self.config.bash_tool_names):
|
||||
result_slots[i] = message
|
||||
transforms_applied.append("router:shell_small_result_verbatim")
|
||||
route_counts["shell_small_result"] = (
|
||||
route_counts.get("shell_small_result", 0) + 1
|
||||
)
|
||||
if collect_diagnostics:
|
||||
_diag[i] = "protected:shell_small_result"
|
||||
continue
|
||||
|
||||
# Legacy OpenAI `role:"function"` carries the tool name on the
|
||||
# message itself, not in a tool_call_id. Resolve it directly so the
|
||||
# small-shell floor also covers this wire shape (#3590 review).
|
||||
if role == "function":
|
||||
function_name = message.get("name", "")
|
||||
if _is_small_shell_result(function_name, content, self.config.bash_tool_names):
|
||||
result_slots[i] = message
|
||||
transforms_applied.append("router:shell_small_result_verbatim")
|
||||
route_counts["shell_small_result"] = (
|
||||
route_counts.get("shell_small_result", 0) + 1
|
||||
)
|
||||
if collect_diagnostics:
|
||||
_diag[i] = "protected:shell_small_result"
|
||||
continue
|
||||
|
||||
# Read protection (ROLE / SHAPE-AGNOSTIC). An observation produced by
|
||||
# a file read command (cat/sed/head/…) is passed VERBATIM so the agent
|
||||
# keeps exact bytes to patch — regardless of how THIS harness labels
|
||||
@@ -7239,7 +7349,7 @@ class ContentRouter(Transform):
|
||||
try:
|
||||
# A registered provider is authoritative for excluded-tool
|
||||
# compaction; fall back to the built-in folds only if it raises.
|
||||
return provider(content)
|
||||
return self._validate_provider_output(provider(content))
|
||||
except Exception: # noqa: BLE001
|
||||
logger.debug(
|
||||
"lossless provider failed; using built-in compaction",
|
||||
@@ -7569,6 +7679,21 @@ class ContentRouter(Transform):
|
||||
any_compressed = False
|
||||
role = message.get("role", "")
|
||||
|
||||
# Top-level text blocks are fragments of one tool result. Measure the
|
||||
# whole result so splitting a large log cannot activate the small-result
|
||||
# floor for every fragment. Sum lengths without copying the payload.
|
||||
shell_result_chars = (
|
||||
sum(
|
||||
len(block["text"])
|
||||
for block in content_blocks
|
||||
if isinstance(block, dict)
|
||||
and block.get("type") == "text"
|
||||
and isinstance(block.get("text"), str)
|
||||
)
|
||||
if role in ("tool", "function")
|
||||
else 0
|
||||
)
|
||||
|
||||
# Role-based gate for `text` blocks. Tool/function roles are tool
|
||||
# outputs and compress freely; assistant defaults to skip (cache
|
||||
# safety) with explicit opt-in; unknown roles default to skip.
|
||||
@@ -7663,7 +7788,7 @@ class ContentRouter(Transform):
|
||||
continue
|
||||
if tool_use_id in excluded_tool_ids:
|
||||
tool_name = tool_name_map.get(tool_use_id, "") if tool_name_map else ""
|
||||
if tool_name and is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
|
||||
if not tool_name or is_tool_excluded(tool_name, DEFAULT_VERBATIM_EXCLUDE_TOOLS):
|
||||
new_blocks.append(block)
|
||||
transforms_applied.append("router:excluded:tool")
|
||||
if route_counts is not None:
|
||||
@@ -7767,6 +7892,18 @@ class ContentRouter(Transform):
|
||||
any_compressed = True
|
||||
continue
|
||||
|
||||
# Small shell results stay verbatim (#3590), same contract as
|
||||
# the string-form path above. Build/test logs above the floor
|
||||
# still go through lossy compression.
|
||||
if _is_small_shell_result(tool_name, tool_text, self.config.bash_tool_names):
|
||||
new_blocks.append(block)
|
||||
transforms_applied.append("router:shell_small_result_verbatim")
|
||||
if route_counts is not None:
|
||||
route_counts["shell_small_result"] = (
|
||||
route_counts.get("shell_small_result", 0) + 1
|
||||
)
|
||||
continue
|
||||
|
||||
# Protection: failed tool calls / error outputs stay verbatim
|
||||
# (issue #847). `is_error` is Anthropic's explicit failure
|
||||
# flag and suffices alone; the indicator scan catches error
|
||||
@@ -7864,6 +8001,29 @@ class ContentRouter(Transform):
|
||||
if route_counts is not None:
|
||||
route_counts["ccr_retrieve"] = route_counts.get("ccr_retrieve", 0) + 1
|
||||
continue
|
||||
|
||||
# A normalized tool/function message can carry its output
|
||||
# as top-level text blocks rather than a tool_result
|
||||
# wrapper. Apply the same small-shell floor here (#3590
|
||||
# review).
|
||||
_shell_name = (
|
||||
(tool_name_map or {}).get(_msg_tool_id, "")
|
||||
if role == "tool"
|
||||
else message.get("name", "")
|
||||
)
|
||||
if (
|
||||
_is_small_shell_result(
|
||||
_shell_name, block.get("text", ""), self.config.bash_tool_names
|
||||
)
|
||||
and shell_result_chars < _shell_result_min_chars()
|
||||
):
|
||||
new_blocks.append(block)
|
||||
transforms_applied.append("router:shell_small_result_verbatim")
|
||||
if route_counts is not None:
|
||||
route_counts["shell_small_result"] = (
|
||||
route_counts.get("shell_small_result", 0) + 1
|
||||
)
|
||||
continue
|
||||
text_content = block.get("text", "")
|
||||
if isinstance(text_content, str) and (
|
||||
len(text_content) > min_chars or self._has_lossless_fold(text_content)
|
||||
|
||||
@@ -39,9 +39,9 @@ from ..log_safety import describe_exception, safe_id
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# A run must be at least this many lines AND this many chars to be worth a
|
||||
# pointer. Small dups are left alone (fragmenting context is not worth it) —
|
||||
# and a larger floor keeps the pointer comfortably shorter than the span it
|
||||
# replaces, so a fold is always a net byte win.
|
||||
# pointer. Small dups are left alone (fragmenting context is not worth it).
|
||||
# The floor alone does not make a fold a net byte win (the pointer can still be
|
||||
# as long as a short span); dedup_blocks checks the UTF-8 sizes before folding.
|
||||
DEFAULT_MIN_LINES = 3
|
||||
DEFAULT_MIN_CHARS = 40
|
||||
# Cap anchor candidates examined per line so a hot line (e.g. `` return``)
|
||||
@@ -54,7 +54,7 @@ MAX_ANCHOR_CANDIDATES = 16
|
||||
# line must start with the digit, so ``number + delta`` re-numbers byte-exactly
|
||||
# without touching alignment padding, keeping renumbered folds strictly lossless.
|
||||
#
|
||||
# ``[1-9]\d*`` (not ``\d+``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
|
||||
# The leading ``[1-9]`` (not ``\d``) is load-bearing: a LEADING-ZERO run (``08:00:01`` in
|
||||
# a timestamped log, ``007:...``) is not an unpadded grep/sed line number, and
|
||||
# recovery is ``str(int(number) + delta)``, which drops the zero pad — ``int("08")
|
||||
# + 1`` renders ``"9"``, not ``"09"``. Matching those would fold them under a
|
||||
@@ -63,7 +63,11 @@ MAX_ANCHOR_CANDIDATES = 16
|
||||
# fold only on an EXACT match (delta 0) — the "prefer false negatives" posture.
|
||||
# Real grep -n / sed -n / rg -n numbers never carry a leading zero, so the
|
||||
# intended renumber-fold feature is unaffected.
|
||||
_LINENO_RE = re.compile(r"^([1-9]\d*)(:|\t)(.*)$", re.DOTALL)
|
||||
#
|
||||
# ``\d{0,17}`` bounds the run at 18 digits: Python refuses ``int()`` on a run of
|
||||
# 4,301+ digits (ValueError), which would abort dedup for the whole request. A
|
||||
# longer run is not a real line number, so the line stays non-numbered.
|
||||
_LINENO_RE = re.compile(r"^([1-9]\d{0,17})(:|\t)(.*)$", re.DOTALL)
|
||||
|
||||
|
||||
def _num_and_key(line: str) -> tuple[int | None, str, str]:
|
||||
@@ -264,9 +268,12 @@ def dedup_blocks(
|
||||
if m is not None and m[0] >= min_lines:
|
||||
span = lines[i : i + m[0]]
|
||||
span_text = "\n".join(span)
|
||||
if len(span_text) >= min_chars:
|
||||
ref_turn = blocks[m[1]].turn
|
||||
ptr = _pointer(span, ref_turn, m[3])
|
||||
ptr = _pointer(span, blocks[m[1]].turn, m[3])
|
||||
# Never fold when the pointer is not shorter than the span, in
|
||||
# UTF-8 bytes ("↑" is 3 bytes, so character counts mislead).
|
||||
if len(span_text) >= min_chars and len(ptr.encode("utf-8")) < len(
|
||||
span_text.encode("utf-8")
|
||||
):
|
||||
out.append(ptr)
|
||||
# Folded span is NOT verbatim in this block's output:
|
||||
# mark None so it can't seed a later contiguous match,
|
||||
|
||||
@@ -7,6 +7,7 @@ import re
|
||||
from dataclasses import dataclass
|
||||
|
||||
from .content_detector import ContentType, _is_search_result_line
|
||||
from .lossless_compaction import _TIMESTAMP_ROW_RE
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -45,7 +46,13 @@ def _is_search_line(line: str) -> bool:
|
||||
Sharing it keeps the splitter carving the same lines the detector claims
|
||||
(#3580); otherwise code in context lines would strand in PLAIN_TEXT
|
||||
sections and reach the prose compressor.
|
||||
|
||||
Timestamped log rows (``2026-09-23T10:05:07Z ...``) match the legacy
|
||||
pattern but are never grep output; the detector rejects them (#3736), so
|
||||
the splitter must too or it hands them to the lossy search compressor.
|
||||
"""
|
||||
if _TIMESTAMP_ROW_RE.match(line):
|
||||
return False
|
||||
return bool(_SEARCH_RESULT_PATTERN.match(line)) or _is_search_result_line(line)
|
||||
|
||||
|
||||
|
||||
@@ -96,6 +96,16 @@ class ReadLifecycleResult:
|
||||
ccr_hashes: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _rejected_before_running(content: Any) -> bool:
|
||||
"""True when a tool_result's text is a ``<tool_use_error>`` validation error."""
|
||||
if isinstance(content, list):
|
||||
content = next(
|
||||
(b.get("text") for b in content if isinstance(b, dict) and b.get("type") == "text"),
|
||||
None,
|
||||
)
|
||||
return isinstance(content, str) and content.lstrip().startswith("<tool_use_error>")
|
||||
|
||||
|
||||
class ReadLifecycleManager:
|
||||
"""Event-driven Read lifecycle management.
|
||||
|
||||
@@ -181,7 +191,9 @@ class ReadLifecycleManager:
|
||||
if not isinstance(tc, dict):
|
||||
continue
|
||||
tc_id = tc.get("id", "")
|
||||
func = tc.get("function", {})
|
||||
func = tc.get("function")
|
||||
if not isinstance(func, dict): # "function": null / non-object
|
||||
continue
|
||||
name = func.get("name", "")
|
||||
if not tc_id or not name:
|
||||
continue
|
||||
@@ -191,11 +203,15 @@ class ReadLifecycleManager:
|
||||
limit = None
|
||||
try:
|
||||
args = json.loads(func.get("arguments", "{}"))
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
args = None
|
||||
if isinstance(args, dict): # arguments may decode to a list/str/number
|
||||
file_path = args.get("file_path") or args.get("path")
|
||||
offset = args.get("offset")
|
||||
limit = args.get("limit")
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
pass
|
||||
# file_path is a dict key downstream; an unhashable value must not reach it.
|
||||
if not isinstance(file_path, str):
|
||||
file_path = None
|
||||
metadata[tc_id] = (name, file_path, offset, limit)
|
||||
|
||||
# Anthropic format: content blocks with type=tool_use
|
||||
@@ -218,6 +234,8 @@ class ReadLifecycleManager:
|
||||
file_path = inp.get("file_path") or inp.get("path")
|
||||
offset = inp.get("offset")
|
||||
limit = inp.get("limit")
|
||||
if not isinstance(file_path, str):
|
||||
file_path = None
|
||||
metadata[tc_id] = (name, file_path, offset, limit)
|
||||
|
||||
return metadata
|
||||
@@ -232,6 +250,7 @@ class ReadLifecycleManager:
|
||||
Groups all Read/Edit/Write operations by file_path for lifecycle analysis.
|
||||
"""
|
||||
file_ops: dict[str, list[FileOperation]] = defaultdict(list)
|
||||
failed_ids = self._failed_tool_use_ids(messages)
|
||||
|
||||
for tc_id, (name, file_path, offset, limit) in tool_metadata.items():
|
||||
if not file_path:
|
||||
@@ -239,7 +258,8 @@ class ReadLifecycleManager:
|
||||
|
||||
if name in _READ_TOOL_NAMES:
|
||||
operation = "read"
|
||||
elif name in _MUTATING_TOOL_NAMES:
|
||||
elif name in _MUTATING_TOOL_NAMES and tc_id not in failed_ids:
|
||||
# A failed Edit/Write left the file unchanged, so earlier Reads stay valid.
|
||||
operation = "edit"
|
||||
else:
|
||||
continue
|
||||
@@ -263,6 +283,33 @@ class ReadLifecycleManager:
|
||||
|
||||
return dict(file_ops)
|
||||
|
||||
@staticmethod
|
||||
def _failed_tool_use_ids(messages: list[dict[str, Any]]) -> set[str]:
|
||||
"""Ids of Anthropic tool_use blocks that were rejected before they ran.
|
||||
|
||||
Only an is_error tool_result whose text is a ``<tool_use_error>`` (the
|
||||
harness's pre-execution validation, e.g. "String to replace not found")
|
||||
proves the file was not touched. Any other error, such as a call that was
|
||||
interrupted and "may have partially or fully completed", still counts as
|
||||
an edit.
|
||||
"""
|
||||
failed: set[str] = set()
|
||||
for msg in messages:
|
||||
content = msg.get("content")
|
||||
if not isinstance(content, list):
|
||||
continue
|
||||
for block in content:
|
||||
if not (
|
||||
isinstance(block, dict)
|
||||
and block.get("type") == "tool_result"
|
||||
and block.get("is_error")
|
||||
):
|
||||
continue
|
||||
tool_use_id = block.get("tool_use_id")
|
||||
if isinstance(tool_use_id, str) and _rejected_before_running(block.get("content")):
|
||||
failed.add(tool_use_id)
|
||||
return failed
|
||||
|
||||
def _find_tool_call_msg_index(
|
||||
self, messages: list[dict[str, Any]], tool_call_id: str
|
||||
) -> int | None:
|
||||
|
||||
@@ -284,15 +284,21 @@ class ReadMaturationManager:
|
||||
for tc in msg.get("tool_calls", []) or []:
|
||||
if not isinstance(tc, dict):
|
||||
continue
|
||||
func = tc.get("function", {})
|
||||
func = tc.get("function")
|
||||
if not isinstance(func, dict): # "function": null / non-object
|
||||
continue
|
||||
name = func.get("name", "")
|
||||
if name not in _TOUCH_TOOLS:
|
||||
continue
|
||||
try:
|
||||
args = json.loads(func.get("arguments", "{}"))
|
||||
except (ValueError, TypeError):
|
||||
args = {}
|
||||
except (ValueError, TypeError): # not JSON: unknown file
|
||||
continue
|
||||
if not isinstance(args, dict): # list/str/number: unknown file
|
||||
continue
|
||||
fp = args.get("file_path") or args.get("path") or ""
|
||||
if not isinstance(fp, str): # unknown file: never mature this call
|
||||
continue
|
||||
if fp:
|
||||
act.file_last_touch[fp] = turn
|
||||
if name in _READ_TOOLS:
|
||||
@@ -306,8 +312,12 @@ class ReadMaturationManager:
|
||||
name = b.get("name", "")
|
||||
if name not in _TOUCH_TOOLS:
|
||||
continue
|
||||
inp = b.get("input") or {}
|
||||
inp = b.get("input")
|
||||
if not isinstance(inp, dict): # unknown file: never mature this call
|
||||
continue
|
||||
fp = inp.get("file_path") or inp.get("path") or ""
|
||||
if not isinstance(fp, str): # unknown file: never mature this call
|
||||
continue
|
||||
if fp:
|
||||
act.file_last_touch[fp] = turn
|
||||
if name in _READ_TOOLS:
|
||||
|
||||
@@ -497,6 +497,7 @@ class SmartCrusher(Transform):
|
||||
query: str = "",
|
||||
bias: float = 1.0,
|
||||
lossless_only: bool | None = None,
|
||||
record_toin: bool = True,
|
||||
) -> CrushResult:
|
||||
"""Crush a single JSON content string.
|
||||
|
||||
@@ -510,6 +511,12 @@ class SmartCrusher(Transform):
|
||||
but any path that would need a CCR marker (row-drop or
|
||||
opaque-blob offload) leaves the content uncompacted instead.
|
||||
`None` (default) uses the instance's configured value.
|
||||
|
||||
`record_toin=False` skips the TOIN write for this call only. The
|
||||
ContentRouter passes it when the request's ``CompressionPolicy``
|
||||
has ``toin_read_only=True`` (Subscription, F2.2): the crusher is
|
||||
shared across concurrent requests, so the gate has to travel with
|
||||
the call rather than through ``_runtime_compression_policy``.
|
||||
"""
|
||||
# Web search tools often return space-separated JSON objects
|
||||
# (``{...} {...} {...}``) rather than a real array. The Rust crusher
|
||||
@@ -548,7 +555,7 @@ class SmartCrusher(Transform):
|
||||
# JSON re-canonicalization (whitespace normalization) without
|
||||
# actually compressing — the strategy stays `"passthrough"` in
|
||||
# that case, and there's no learning value in recording it.
|
||||
if r.was_modified and r.strategy != "passthrough":
|
||||
if record_toin and r.was_modified and r.strategy != "passthrough":
|
||||
self._record_to_toin(
|
||||
original=content,
|
||||
compressed=r.compressed,
|
||||
|
||||
@@ -64,10 +64,10 @@ The canonical, always-current documentation index lives at the docs site below.
|
||||
|
||||
## Headroom for Teams
|
||||
|
||||
For teams and companies that want Headroom across every developer rather than one laptop: Headroom Enterprise adds more savings (tool search for every model, harness tuning, extra compressors), automatic model routing (open source has opt-in, rule-based routing), prompt-injection and risky-tool-call removal, SSO, managed or VPC deployment, an org-wide savings dashboard, and support. Companies that sign a 12-month Enterprise plan get the first 3 months free.
|
||||
Headroom OSS is a framework built to show what's possible on one developer's laptop, so it saves less than Headroom Enterprise. Enterprise adds more cost savings, tokens secured, on premise deployment, deep attribution and many more features. Teams and enterprises: contact hello@headroomlabs.ai or visit https://headroomlabs.ai.
|
||||
|
||||
- [Headroom for Teams](https://docs.headroomlabs.ai/docs/teams): Open source vs Enterprise, and the 3-months-free offer.
|
||||
- [Claim 3 months free](https://headroom-perks.vercel.app/?source=github&utm_source=llms-txt): Perk sign-up form. One perk per company.
|
||||
- [Headroom for Teams](https://docs.headroomlabs.ai/docs/teams): Open source vs Enterprise, and how to reach us.
|
||||
- [headroomlabs.ai](https://headroomlabs.ai): Company site. Email hello@headroomlabs.ai.
|
||||
|
||||
## Licensing
|
||||
|
||||
|
||||
@@ -207,7 +207,8 @@ Compression is lossless via CCR (Compress-Cache-Retrieve): originals are stored
|
||||
| `pythonPath` | auto-detected | Optional Python executable override for Python fallback launcher. |
|
||||
| `autoStart` | `false` | Opt-in auto-start for a local `headroom proxy` if not already running (local URLs only; ignored for remote proxies). Keep `false` when systemd owns the proxy. |
|
||||
| `startupTimeoutMs` | `20000` | Time to wait for auto-started proxy to become healthy |
|
||||
| `requestTimeoutMs` | `30000` | Maximum milliseconds to wait for a single `compress()` call. If the proxy hangs or is slow, the call is cancelled after this deadline and the original uncompressed messages are used as a fallback. |
|
||||
| `requestTimeoutMs` | `2000` | Maximum milliseconds to wait for a single `compress()` call. The SDK aborts the request at this deadline and the original messages are used as a fallback. |
|
||||
| `minContextChars` | `800` | Skip proxy compression below this approximate 200-token context floor. Large single tool results still qualify; set to `0` to disable the gate. |
|
||||
| `circuitBreakerThreshold` | `3` | Number of consecutive `assemble()` errors before the circuit breaker opens and all requests bypass the proxy. Prevents cascading failures when the proxy is unhealthy. |
|
||||
| `circuitBreakerCooldownMs` | `60000` | How long (ms) the circuit breaker stays open after the threshold is reached. After the cool-down the breaker resets automatically and the next request re-probes the proxy via `/health`. |
|
||||
| `routeCodexViaProxy` | `true` | Rewrite OpenClaw's built-in `openai-codex` provider to use the active Headroom proxy in memory so upstream Codex requests pass through Headroom. |
|
||||
|
||||
@@ -26,6 +26,10 @@
|
||||
"label": "Compression Request Timeout",
|
||||
"help": "Maximum milliseconds to wait for one compression request before returning the original messages."
|
||||
},
|
||||
"minContextChars": {
|
||||
"label": "Minimum Context Size",
|
||||
"help": "Skip the proxy for smaller contexts that cannot produce meaningful savings. Set to 0 to disable."
|
||||
},
|
||||
"circuitBreakerThreshold": {
|
||||
"label": "Circuit Breaker Threshold",
|
||||
"help": "Consecutive compression failures before temporarily bypassing Headroom."
|
||||
@@ -88,7 +92,12 @@
|
||||
"requestTimeoutMs": {
|
||||
"type": "integer",
|
||||
"minimum": 1,
|
||||
"default": 30000
|
||||
"default": 2000
|
||||
},
|
||||
"minContextChars": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"default": 800
|
||||
},
|
||||
"circuitBreakerThreshold": {
|
||||
"type": "integer",
|
||||
|
||||
@@ -17,17 +17,6 @@ import {
|
||||
type OpenClawCompactResult,
|
||||
} from "./openclaw-compaction.js";
|
||||
|
||||
/** Race a promise against a timeout and always release the timer. */
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number): Promise<T> {
|
||||
let timerId: ReturnType<typeof setTimeout> | undefined;
|
||||
const timer = new Promise<never>((_, reject) => {
|
||||
timerId = setTimeout(() => reject(new Error(`headroom compress() timed out after ${ms}ms`)), ms);
|
||||
});
|
||||
return Promise.race([promise, timer]).finally(() => {
|
||||
if (timerId !== undefined) clearTimeout(timerId);
|
||||
});
|
||||
}
|
||||
|
||||
// Must stay static — OpenClaw prepends it to the cached system prompt.
|
||||
const HEADROOM_COMPRESSION_NOTICE =
|
||||
"[Headroom is compressing tool outputs in this session. Use headroom_retrieve if you need the original, uncompressed content.]";
|
||||
@@ -35,6 +24,7 @@ const HEADROOM_COMPRESSION_NOTICE =
|
||||
export interface HeadroomEngineConfig extends ProxyManagerConfig {
|
||||
enabled?: boolean;
|
||||
requestTimeoutMs?: number;
|
||||
minContextChars?: number;
|
||||
circuitBreakerThreshold?: number;
|
||||
circuitBreakerCooldownMs?: number;
|
||||
/** Where to durably record committed turn-advancement keys (see
|
||||
@@ -160,17 +150,26 @@ export class HeadroomContextEngine {
|
||||
try {
|
||||
// Convert AgentMessage → OpenAI format
|
||||
const openaiMessages = agentToOpenAIIndexed(params.messages);
|
||||
const minContextChars = this.config.minContextChars ?? 800;
|
||||
if (minContextChars > 0 && countContextChars(openaiMessages) < minContextChars) {
|
||||
this.logger.debug("[headroom] Context below compression threshold — using original messages");
|
||||
return {
|
||||
messages: normalizeAgentMessages(params.messages),
|
||||
estimatedTokens: 0,
|
||||
systemPromptAddition: this.sessionCompressionNotice(params.sessionId),
|
||||
};
|
||||
}
|
||||
|
||||
// Compress via proxy — pass tokenBudget so RollingWindow enforces it
|
||||
const result = await withTimeout(
|
||||
compress(openaiMessages, {
|
||||
model: params.model ?? "claude-sonnet-4-5",
|
||||
baseUrl: this.proxyUrl,
|
||||
fallback: true,
|
||||
tokenBudget: params.tokenBudget,
|
||||
} as any),
|
||||
this.config.requestTimeoutMs ?? 30_000,
|
||||
);
|
||||
const result = await compress(openaiMessages, {
|
||||
model: params.model ?? "claude-sonnet-4-5",
|
||||
baseUrl: this.proxyUrl,
|
||||
fallback: true,
|
||||
tokenBudget: params.tokenBudget,
|
||||
// The SDK applies this to AbortSignal.timeout(), so a timed-out call is
|
||||
// cancelled instead of continuing in the background after fail-open.
|
||||
timeout: this.config.requestTimeoutMs ?? 2_000,
|
||||
} as any);
|
||||
|
||||
const shouldAnnounceCompression =
|
||||
(this.hasAnnouncedCompression(params.sessionId) || result.tokensSaved > 100) &&
|
||||
@@ -400,3 +399,14 @@ export class HeadroomContextEngine {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function countContextChars(messages: ReturnType<typeof agentToOpenAIIndexed>): number {
|
||||
let total = 0;
|
||||
for (const message of messages) {
|
||||
total += message.content?.length ?? 0;
|
||||
for (const toolCall of message.tool_calls ?? []) {
|
||||
total += toolCall.function?.arguments?.length ?? 0;
|
||||
}
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
@@ -277,40 +277,77 @@ describe("HeadroomContextEngine proxy startup helpers", () => {
|
||||
expect(mocked.start).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("clears the request timeout after successful compression", async () => {
|
||||
vi.useFakeTimers();
|
||||
try {
|
||||
vi.mocked(compress).mockResolvedValue({
|
||||
compressed: false,
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
tokensBefore: 5,
|
||||
tokensAfter: 5,
|
||||
tokensSaved: 0,
|
||||
});
|
||||
it("passes the configured request timeout to the aborting SDK client", async () => {
|
||||
vi.mocked(compress).mockResolvedValue({
|
||||
compressed: false,
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
tokensBefore: 5,
|
||||
tokensAfter: 5,
|
||||
tokensSaved: 0,
|
||||
});
|
||||
|
||||
const engine = new HeadroomContextEngine({ requestTimeoutMs: 30_000 });
|
||||
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
const engine = new HeadroomContextEngine({ requestTimeoutMs: 1_234, minContextChars: 0 });
|
||||
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
|
||||
await expect(
|
||||
engine.assemble({
|
||||
sessionId: "session-1",
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
}),
|
||||
).resolves.toEqual({
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
estimatedTokens: 5,
|
||||
});
|
||||
await engine.assemble({
|
||||
sessionId: "session-1",
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
});
|
||||
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
expect(compress).toHaveBeenCalledWith(
|
||||
expect.any(Array),
|
||||
expect.objectContaining({ timeout: 1_234 }),
|
||||
);
|
||||
});
|
||||
|
||||
it("skips proxy compression for short contexts with no meaningful savings", async () => {
|
||||
const messages = [{ role: "user", content: "hello" }];
|
||||
const engine = new HeadroomContextEngine();
|
||||
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
|
||||
await expect(engine.assemble({ sessionId: "session-1", messages })).resolves.toEqual({
|
||||
messages,
|
||||
estimatedTokens: 0,
|
||||
});
|
||||
expect(compress).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("counts a single large tool result as eligible context", async () => {
|
||||
const messages = [{ role: "toolResult", toolCallId: "call-1", content: "x".repeat(800) }];
|
||||
vi.mocked(compress).mockResolvedValue({
|
||||
compressed: false,
|
||||
messages: [{ role: "tool", tool_call_id: "call-1", content: "x".repeat(800) }],
|
||||
tokensBefore: 200,
|
||||
tokensAfter: 200,
|
||||
tokensSaved: 0,
|
||||
});
|
||||
const engine = new HeadroomContextEngine();
|
||||
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
|
||||
await engine.assemble({ sessionId: "session-1", messages });
|
||||
expect(compress).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("allows the short-context gate to be disabled", async () => {
|
||||
vi.mocked(compress).mockResolvedValue({
|
||||
compressed: false,
|
||||
messages: [{ role: "user", content: "hello" }],
|
||||
tokensBefore: 2,
|
||||
tokensAfter: 2,
|
||||
tokensSaved: 0,
|
||||
});
|
||||
const engine = new HeadroomContextEngine({ minContextChars: 0 });
|
||||
(engine as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
|
||||
await engine.assemble({ sessionId: "session-1", messages: [{ role: "user", content: "hello" }] });
|
||||
expect(compress).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("opens the circuit after consecutive compression failures", async () => {
|
||||
vi.mocked(compress).mockRejectedValue(new Error("proxy stalled"));
|
||||
const messages = [{ role: "user", content: "hello" }];
|
||||
const engine = new HeadroomContextEngine({
|
||||
minContextChars: 0,
|
||||
circuitBreakerThreshold: 2,
|
||||
circuitBreakerCooldownMs: 60_000,
|
||||
});
|
||||
@@ -451,7 +488,7 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
|
||||
}
|
||||
|
||||
function readyEngine(config?: HeadroomEngineConfig) {
|
||||
const engine = new HeadroomContextEngine(config);
|
||||
const engine = new HeadroomContextEngine({ minContextChars: 0, ...config });
|
||||
(engine as unknown as { proxyUrl: string | null }).proxyUrl = "http://127.0.0.1:8787";
|
||||
return engine;
|
||||
}
|
||||
@@ -491,6 +528,18 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
|
||||
expect(second.systemPromptAddition).toBe(HEADROOM_COMPRESSION_NOTICE);
|
||||
});
|
||||
|
||||
it("keeps an earned notice on a later context below the compression threshold", async () => {
|
||||
vi.mocked(compress).mockResolvedValueOnce(mockCompressResult({ tokensSaved: 150 }));
|
||||
const engine = readyEngine({ minContextChars: 800 });
|
||||
const first = await engine.assemble({
|
||||
sessionId: "s1", messages: [{ role: "user", content: "x".repeat(1000) }],
|
||||
});
|
||||
const second = await engine.assemble({ sessionId: "s1", messages });
|
||||
expect(first.systemPromptAddition).toBe(HEADROOM_COMPRESSION_NOTICE);
|
||||
expect(second.systemPromptAddition).toBe(first.systemPromptAddition);
|
||||
expect(compress).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("never returns a notice when announceCompression is false", async () => {
|
||||
vi.mocked(compress).mockResolvedValueOnce(mockCompressResult({ tokensSaved: 500 }));
|
||||
|
||||
@@ -571,7 +620,9 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
|
||||
const firstNotice = turn1.systemPromptAddition;
|
||||
expect(firstNotice).toBe(HEADROOM_COMPRESSION_NOTICE);
|
||||
|
||||
vi.mocked(compress).mockImplementationOnce(() => new Promise(() => {}));
|
||||
vi.mocked(compress).mockImplementationOnce(() => new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error("SDK request timed out")), requestTimeoutMs);
|
||||
}));
|
||||
const turn2Promise = engine.assemble({ sessionId: "s1", messages });
|
||||
await vi.advanceTimersByTimeAsync(requestTimeoutMs);
|
||||
const turn2 = await turn2Promise;
|
||||
@@ -619,7 +670,9 @@ describe("HeadroomContextEngine assemble() compression notice", () => {
|
||||
const turn1 = await engine.assemble({ sessionId: "s1", messages });
|
||||
expect(turn1.systemPromptAddition).toBeUndefined();
|
||||
|
||||
vi.mocked(compress).mockImplementationOnce(() => new Promise(() => {}));
|
||||
vi.mocked(compress).mockImplementationOnce(() => new Promise((_, reject) => {
|
||||
setTimeout(() => reject(new Error("SDK request timed out")), requestTimeoutMs);
|
||||
}));
|
||||
const turn2Promise = engine.assemble({ sessionId: "s1", messages });
|
||||
await vi.advanceTimersByTimeAsync(requestTimeoutMs);
|
||||
const turn2 = await turn2Promise;
|
||||
|
||||
@@ -47,8 +47,8 @@
|
||||
},
|
||||
"overrides": {
|
||||
"esbuild": "^0.28.1",
|
||||
"zod": "^4.6.6",
|
||||
"toml": "^5.0.1"
|
||||
"zod": "4.6.7",
|
||||
"toml": "5.0.1"
|
||||
},
|
||||
"license": "Apache-2.0"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import { tool } from "@opencode-ai/plugin";
|
||||
import { Effect, FileSystem, Path } from "effect";
|
||||
import { Primitive } from "effect/unstable/cli";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
function parseConfig(content: string): Promise<unknown> {
|
||||
const fs = FileSystem.makeNoop({
|
||||
exists: () => Effect.succeed(true),
|
||||
stat: () => Effect.succeed({ type: "File" } as FileSystem.File.Info),
|
||||
readFileString: () => Effect.succeed(content),
|
||||
});
|
||||
return Effect.runPromise(
|
||||
Primitive.fileParse({ format: "toml" }).parse("/headroom/config.toml").pipe(
|
||||
Effect.provideService(FileSystem.FileSystem, fs),
|
||||
Effect.provide(Path.layer),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
describe("patched OpenCode dependency compatibility", () => {
|
||||
it("builds and validates a tool with the upstream Zod schema interface", () => {
|
||||
const schema = tool.schema.object({
|
||||
hash: tool.schema.string().regex(/^[a-f0-9]{24}$/i),
|
||||
});
|
||||
const retrieve = tool({
|
||||
description: "Retrieve an original tool result",
|
||||
args: schema.shape,
|
||||
async execute(args) {
|
||||
return args.hash;
|
||||
},
|
||||
});
|
||||
expect(schema.parse({ hash: "a".repeat(24) })).toEqual({ hash: "a".repeat(24) });
|
||||
expect(schema.safeParse({ hash: "invalid" }).success).toBe(false);
|
||||
expect(retrieve.args.hash).toBe(schema.shape.hash);
|
||||
});
|
||||
|
||||
it("parses typed TOML through Effect's actual file parser", async () => {
|
||||
const content = `
|
||||
name = "headroom"
|
||||
enabled = true
|
||||
ports = [8787, 8788]
|
||||
[proxy]
|
||||
host = "127.0.0.1"
|
||||
ratio = 0.5
|
||||
[[models]]
|
||||
name = "coding"
|
||||
[[models]]
|
||||
name = "review"
|
||||
`;
|
||||
await expect(parseConfig(content)).resolves.toEqual({
|
||||
name: "headroom",
|
||||
enabled: true,
|
||||
ports: [8787, 8788],
|
||||
proxy: { host: "127.0.0.1", ratio: 0.5 },
|
||||
models: [{ name: "coding" }, { name: "review" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps malformed TOML on Effect's parse-error channel", async () => {
|
||||
await expect(parseConfig("[unclosed")).rejects.toThrow("Failed to parse '.toml'");
|
||||
});
|
||||
});
|
||||
+32
-19
@@ -8,7 +8,7 @@ version = "0.42.0"
|
||||
description = "The Context Optimization Layer for LLM Applications - Cut costs by 50-90%"
|
||||
readme = "README.md"
|
||||
license = "Apache-2.0"
|
||||
requires-python = ">=3.10"
|
||||
requires-python = ">=3.11"
|
||||
authors = [
|
||||
{ name = "Headroom Contributors" }
|
||||
]
|
||||
@@ -35,7 +35,6 @@ classifiers = [
|
||||
"Intended Audience :: Developers",
|
||||
"Operating System :: OS Independent",
|
||||
"Programming Language :: Python :: 3",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Programming Language :: Python :: 3.13",
|
||||
@@ -50,12 +49,12 @@ dependencies = [
|
||||
"pydantic>=2.0.0", # Config and data models
|
||||
# Model registry, pricing and non-core providers; all lazily imported and
|
||||
# ImportError-guarded. 1.96.2 is the first litellm with prebuilt wheels for
|
||||
# Python 3.10-3.14 on Linux, macOS and Windows: older releases either reject
|
||||
# Python 3.11-3.14 on Linux, macOS and Windows: older releases either reject
|
||||
# Python 3.14 (GH #956) or fall back to a Rust source build.
|
||||
"litellm>=1.96.2,<2.0",
|
||||
"click>=8.3.3", # CLI framework; PYSEC-2026-2132 fix (command injection in click.edit())
|
||||
"rich>=13.0.0", # Rich terminal output
|
||||
"opentelemetry-api>=1.24.0", # Safe no-op OTEL API for instrumentation
|
||||
"opentelemetry-api>=1.45.1", # CVE-2026-114603: bounded, linear-time tracestate parsing
|
||||
# AST-aware code slicing (CodeCompressor); binary wheel. 0.44.1 is excluded:
|
||||
# that PyPI release was a compromised supply-chain build shipping an
|
||||
# info-stealer `sg.exe` (Trojan:Win64/Lazy!MTB) alongside the real binary
|
||||
@@ -63,8 +62,7 @@ dependencies = [
|
||||
# future patched one.
|
||||
"ast-grep-cli>=0.30.0,!=0.44.1",
|
||||
"pyyaml>=6.0", # omp wrap: parse/merge omp's models.yml registry
|
||||
"tomli>=2.0.0; python_version < '3.11'", # tomllib backport for helper scripts
|
||||
"tomlkit>=0.13.0,<1.0", # Loss-minimizing Codex config recovery
|
||||
"tomlkit>=0.15.0,<1.0", # Loss-minimizing Codex config recovery
|
||||
# Verify upstream TLS through the OS trust store (macOS Keychain, Windows
|
||||
# cert store), where IT installs corporate TLS-inspection roots (Zscaler,
|
||||
# Netskope). Pure Python, no deps; the same library pip uses by default.
|
||||
@@ -82,9 +80,8 @@ dependencies = [
|
||||
# CVE-2026-112907/112913/112940 are fixed in 4.2.1, but litellm 1.106 caps
|
||||
# filelock <4, so without this floor pip picks that litellm and 3.32.7;
|
||||
# with it pip takes litellm 1.103.7 (uv.lock has 1.101.0). Declared here so
|
||||
# the floor reaches pip installs. 4.2 needs Python 3.11, so pip on 3.10
|
||||
# still gets an affected filelock.
|
||||
"filelock>=4.2.1; python_version >= '3.11'",
|
||||
# the floor reaches pip installs.
|
||||
"filelock>=4.2.1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
@@ -103,6 +100,7 @@ proxy = [
|
||||
"magika>=0.6.0; python_version<'3.14' or sys_platform != 'darwin' or platform_machine != 'x86_64'", # ML detection needs ORT wheels
|
||||
"zstandard>=0.20.0", # Decompress zstd request bodies (Codex, etc.)
|
||||
"websockets>=13.0", # WebSocket proxy for /v1/responses (Codex gpt-5.4+)
|
||||
"python-socks[asyncio]>=2.7.1", # SOCKS support when websockets discovers a system proxy
|
||||
# ORT <1.24 deadlocks in Rust fastembed instead of returning an init error.
|
||||
# Python 3.10 keeps Python-only ORT and bypasses native detection. Intel
|
||||
# macOS also needs older ORT because newer releases have no x86_64 wheels.
|
||||
@@ -361,8 +359,13 @@ dev = [
|
||||
all = [
|
||||
"headroom-ai[proxy,code,ml,memory,relevance,image,reports,otel,evals,voice,html,mcp,spreadsheet]",
|
||||
]
|
||||
# Sandbox: a lean proxy with ALL torch-free capability — for running Headroom in
|
||||
# a locked-down/low-resource sandbox and offloading heavy ML elsewhere.
|
||||
# Lean: a proxy with ALL torch-free capability — for running Headroom in a
|
||||
# constrained or low-resource environment and offloading heavy ML elsewhere.
|
||||
#
|
||||
# This is a DEPENDENCY PROFILE. It provides no isolation, no confinement and
|
||||
# no privilege reduction of any kind. It was previously named `sandbox`, which
|
||||
# invited exactly that assumption; `sandbox` is retained below as a
|
||||
# deprecated alias for one release so existing installs keep resolving.
|
||||
#
|
||||
# = [all] MINUS:
|
||||
# - image (SigLIP/OCR — excluded by request)
|
||||
@@ -373,13 +376,18 @@ all = [
|
||||
# - memory + evals (both pull sentence-transformers -> torch, i.e. the very
|
||||
# ML weight a sandbox avoids; evals is a dev/test harness, not a
|
||||
# runtime feature). Opt back in explicitly if you accept torch:
|
||||
# pip install headroom-ai[sandbox,memory]
|
||||
# pip install headroom-ai[lean,memory]
|
||||
#
|
||||
# Everything kept here is torch-free: code-aware compression (tree-sitter),
|
||||
# embedding relevance (fastembed), HTML/spreadsheet ingestion, reports, OTel.
|
||||
sandbox = [
|
||||
lean = [
|
||||
"headroom-ai[proxy,code,relevance,reports,otel,html,mcp,spreadsheet]",
|
||||
]
|
||||
# Deprecated alias for `lean`, kept for one release. The name implied an
|
||||
# isolation guarantee the extra never provided. Remove after 0.39.x.
|
||||
sandbox = [
|
||||
"headroom-ai[lean]",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
headroom = "headroom.cli:main"
|
||||
@@ -408,6 +416,8 @@ Changelog = "https://github.com/headroomlabs-ai/headroom/blob/main/CHANGELOG.md"
|
||||
# transitively; the floor pins below ensure uv resolves to patched versions.
|
||||
[tool.uv]
|
||||
constraint-dependencies = [
|
||||
# PYSEC-2026-4320 — remove distrusted root certificates from the CA bundle.
|
||||
"certifi>=2026.10.19",
|
||||
# GHSA-5239-wwwm-4pmq (Low) — transitive via rich; fix at 2.20.0
|
||||
"pygments>=2.20.0",
|
||||
# GHSA-4xgf-cpjx-pc3j (Medium) — transitive via mcp; fix at 2.14.2
|
||||
@@ -423,8 +433,11 @@ constraint-dependencies = [
|
||||
"lxml-html-clean>=0.4.5",
|
||||
# CVE-2026-5241 (High) — direct optional dep for proxy/ml/voice; fix at 5.5.0
|
||||
"transformers>=5.5.0",
|
||||
# PYSEC-2026-3447 — transitive dependency; fix at 83.0.0
|
||||
"setuptools>=83.0.0",
|
||||
# GHSA-xmj3-9gf7-h52w — Windows wheel namespace path traversal.
|
||||
"setuptools>=84.1.0",
|
||||
# CVE-2026-112940/112913/112907. The patched series requires Python 3.11;
|
||||
# Python 3.10 retains the legacy locked version; it is not remediated by this marker.
|
||||
"filelock>=4.2.1; python_version >= '3.11'",
|
||||
# PYSEC-2026-3545/3546/3547 — transitive HTTP/WebSocket parser fixes;
|
||||
# CVE-2026-111063/111064, GHSA-2g87-pp6c-29x5; fix at 3.14.4
|
||||
"aiohttp>=3.14.4",
|
||||
@@ -448,10 +461,8 @@ constraint-dependencies = [
|
||||
# `pip install headroom-ai[all]` still resolves against fastembed's cap; on
|
||||
# that Pillow, headroom/image/decoder_guard.py forwards images undecoded.
|
||||
# Drop this override once fastembed allows Pillow 13.
|
||||
# Pillow 13 needs Python 3.11; on 3.10 no fixed release exists.
|
||||
override-dependencies = [
|
||||
"pillow>=13.0.0; python_version >= '3.11'",
|
||||
"pillow>=12.3.0,<13; python_version < '3.11'",
|
||||
"pillow>=13.0.0",
|
||||
]
|
||||
|
||||
[[tool.uv.index]]
|
||||
@@ -486,6 +497,8 @@ features = ["extension-module"]
|
||||
bindings = "pyo3"
|
||||
|
||||
[tool.ruff]
|
||||
# Kept at py310 until the py311 idiom migration lands (UP017/UP036/UP041/UP042
|
||||
# touch ~200 sites); requires-python is 3.11.
|
||||
target-version = "py310"
|
||||
line-length = 100
|
||||
|
||||
@@ -513,7 +526,7 @@ quote-style = "double"
|
||||
indent-style = "space"
|
||||
|
||||
[tool.mypy]
|
||||
python_version = "3.10"
|
||||
python_version = "3.11"
|
||||
warn_return_any = true
|
||||
warn_unused_configs = true
|
||||
disallow_untyped_defs = true
|
||||
|
||||
+4
-4
@@ -5,7 +5,7 @@
|
||||
# to the Kompress-v2 Modal endpoint. Then confirm everything loaded.
|
||||
#
|
||||
# Plugins : lossless_guard, skill_search, observability, tier_router, tool_search
|
||||
# Extra : headroom-ai[sandbox] (torch-free proxy; ML offloaded to Modal)
|
||||
# Extra : headroom-ai[lean] (torch-free proxy; ML offloaded to Modal)
|
||||
# Profile : coding (HEADROOM_SAVINGS_PROFILE) + cache mode (prefix-cache safe)
|
||||
#
|
||||
# Secrets are SOURCED from ~/env.txt and ~/.headroom/plugins.env — never inlined.
|
||||
@@ -29,7 +29,7 @@ source "$VENV/bin/activate"
|
||||
PY="$VENV/bin/python"
|
||||
|
||||
# ── 2. install (guarded) ──────────────────────────────────────────────────────
|
||||
# headroom-ai[sandbox] pulls proxy,code,relevance,reports,otel,html,mcp,spreadsheet
|
||||
# headroom-ai[lean] pulls proxy,code,relevance,reports,otel,html,mcp,spreadsheet
|
||||
# (all torch-free — heavy ML is offloaded to the Modal Kompress endpoint below).
|
||||
# The 5 plugins install --no-deps so pip won't drag PyPI's headroom-ai over the
|
||||
# local editable one; headroom-license is their shared Ed25519 verifier.
|
||||
@@ -44,14 +44,14 @@ if [ "$need_install" = "1" ]; then
|
||||
if [ -n "$avail" ] && [ "$avail" -lt 2 ]; then
|
||||
echo "!! <2Gi free — aborting before heavy install (free space, then re-run)"; exit 1
|
||||
fi
|
||||
echo "▶ installing pip+maturin, then headroom-ai[sandbox] + license + 5 plugins (editable)…"
|
||||
echo "▶ installing pip+maturin, then headroom-ai[lean] + license + 5 plugins (editable)…"
|
||||
"$PY" -m pip install -U pip maturin
|
||||
# litellm >=1.92 ships an sdist-only Rust bridge whose AWS-SDK crates need rustc>=1.94.1;
|
||||
# the default rustup toolchain here is older (pip builds litellm in a temp dir that misses
|
||||
# the repo's 1.95 pin), so pin to the last pure-Python wheel line (1.91.4). Satisfies
|
||||
# headroom's litellm>=1.86.2,<2.0 and skips the Rust build entirely.
|
||||
"$PY" -m pip install "litellm<1.92"
|
||||
"$PY" -m pip install -e "${HR}[sandbox]" "litellm<1.92"
|
||||
"$PY" -m pip install -e "${HR}[lean]" "litellm<1.92"
|
||||
"$PY" -m pip install -e /Users/tcms/demo/headroom-license
|
||||
for p in lossless-guard skill-search observability tier-router tool-search; do
|
||||
"$PY" -m pip install -e "/Users/tcms/demo/headroom-${p}" --no-deps
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user