mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 15:10:00 +08:00
Two stale identities left over from moves that were only half applied. ## GitHub org `install/state.py` already migrated Docker images to `ghcr.io/headroomlabs-ai/headroom` and carries a `_DEPRECATED_IMAGE_REPO` constant that rewrites the old path on load — so this move was decided long ago. Everything else still said `chopratejas/headroom`. Rewritten across packaging (`pyproject.toml`, `Cargo.toml`), both plugin marketplaces, the agent-hooks manifests, install scripts, CI templates, crate docs and tests. **Two deliberate exclusions:** - **`ghcr.io/chopratejas/headroom` is untouched everywhere.** It is the *input* to `_migrate_deprecated_image()` and to the tests covering it. Rewriting it would silently disable the migration. - **`CHANGELOG.md` keeps its 238 historical references.** GitHub's rename redirect resolves them, and rewriting would bury this diff in noise. **HuggingFace model ids are NOT affected and were never in scope.** `chopratejas/kompress-base`, `kompress-v2-base`, `kompress-finance`, `technique-router`, `technique-router-onnx` and `siglip-image-encoder-onnx` are account-scoped identifiers on a different service. All six were validated against the HF API before this change: | Model | HTTP | |---|---| | `kompress-base` | 200 | | `kompress-v2-base` | 200 | | `technique-router` | 200 | | `technique-router-onnx` | 200 | | `siglip-image-encoder-onnx` | 200 | | `kompress-finance` | **401 (private)** | The 401 is unrelated to this migration, but worth knowing: `kompress_compressor.py:1255` references it, so that path is not reachable by an OSS user without access. **One functional fix, not cosmetic:** `_marketplace_source()` (`headroom/cli/init.py`) returned the old org as its fallback, so plugin installs were pointed at the retired repo. ## Docs domain The site moved to `docs.headroomlabs.ai`, but eleven references still named the vercel.app host — including **`llms.txt` (27 links)**, which is served at `docs.headroomlabs.ai/llms.txt` and read by LLM crawlers, the **dashboard's own Documentation link**, and a `headroom wrap` help message. The docs app's canonical-URL fallbacks were fixed separately in #3388. ## Also folded in `RUST_DEV.md` corrections found while verifying the above: - `make build-wheel` points at `Cargo.toml`, not `pyproject.toml` - the Rust toolchain is pinned, not tracking `stable` - `headroom-simulators` was missing from the workspace listing - the Python `CompressionStore` defaults to **SQLite**, not in-memory, when `HEADROOM_CCR_BACKEND` is unset (verified by execution) — this one matters for multi-worker reasoning And the install scripts pointed at `docs/docker-install.md`, a path that no longer exists; they now point at the live docs page. ## Verification 297 tests pass, including the plugin-manifest, install-state and release-workflow suites that assert on these exact strings. `ruff check` clean. Unrelated lockfile churn (`uv.lock`, both `package-lock.json`) is deliberately excluded. 🤖 Generated with [Claude Code](https://claude.com/claude-code) https://claude.ai/code/session_01HRaEPjQcCSAwD2xbQMNENu --------- Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
109 lines
3.9 KiB
Python
109 lines
3.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Paired report for the code-mode steering A/B (benchmarks/codemode_ab.py)."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import statistics
|
|
import sys
|
|
from collections import defaultdict
|
|
|
|
|
|
def paired(rows, key):
|
|
"""Per-(task,rep) steered-minus-control deltas for a numeric field."""
|
|
by = defaultdict(dict)
|
|
for r in rows:
|
|
if r.get("error") or r.get(key) is None:
|
|
continue
|
|
by[(r["task"], r.get("rep"))][r["arm"]] = r[key]
|
|
return [
|
|
(k, v["steered"] - v["control"], v["control"], v["steered"])
|
|
for k, v in by.items()
|
|
if "steered" in v and "control" in v
|
|
]
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--results", required=True)
|
|
args = ap.parse_args()
|
|
rows = json.loads(open(args.results).read())
|
|
# Re-grade stored answers with the current scorer so a grading fix does not
|
|
# require re-running (and re-paying for) the suite.
|
|
sys.path.insert(0, str(__import__("pathlib").Path(__file__).resolve().parent))
|
|
from codemode_ab import TASKS, TASKS_MULTI, score
|
|
|
|
spec = {t[0]: (t[2], t[3]) for t in list(TASKS) + list(TASKS_MULTI)}
|
|
for r in rows:
|
|
if r.get("task") in spec and "answer" in r:
|
|
truth, kind = spec[r["task"]]
|
|
r["correct"] = score(kind, truth, r["answer"])
|
|
errs = [r for r in rows if r.get("error")]
|
|
rows = [r for r in rows if not r.get("error")]
|
|
print(f"runs: {len(rows)} errors: {len(errs)}")
|
|
|
|
for arm in ("control", "steered"):
|
|
a = [r for r in rows if r["arm"] == arm]
|
|
if not a:
|
|
continue
|
|
ok = sum(1 for r in a if r.get("correct"))
|
|
print(f"\n[{arm}] n={len(a)} correct={ok}/{len(a)} ({100 * ok / len(a):.0f}%)")
|
|
for f, label, unit in (
|
|
("cost_usd", "cost", "$"),
|
|
("num_turns", "turns", ""),
|
|
("tool_calls", "tool calls", ""),
|
|
("bash_calls", "bash calls", ""),
|
|
("compound_bash", "compound bash", ""),
|
|
("fetched_chars", "bytes fetched", ""),
|
|
("wall_s", "wall", "s"),
|
|
):
|
|
vals = [r.get(f) or 0 for r in a]
|
|
print(
|
|
f" {label:16s} mean {unit}{statistics.mean(vals):>10.4f} "
|
|
f"median {unit}{statistics.median(vals):>10.4f} total {unit}{sum(vals):>12.2f}"
|
|
)
|
|
|
|
print("\n=== PAIRED (steered - control), per task ===")
|
|
for f, label in (
|
|
("cost_usd", "cost $"),
|
|
("num_turns", "turns"),
|
|
("tool_calls", "tool calls"),
|
|
("fetched_chars", "bytes fetched"),
|
|
("compound_bash", "compound bash"),
|
|
):
|
|
d = paired(rows, f)
|
|
if not d:
|
|
continue
|
|
deltas = [x[1] for x in d]
|
|
wins = sum(1 for x in deltas if x < 0)
|
|
losses = sum(1 for x in deltas if x > 0)
|
|
mean = statistics.mean(deltas)
|
|
ctrl_tot = sum(x[2] for x in d)
|
|
pct = 100 * sum(deltas) / ctrl_tot if ctrl_tot else 0
|
|
line = (
|
|
f" {label:16s} mean Δ {mean:+12.4f} total Δ {sum(deltas):+12.4f} "
|
|
f"({pct:+.1f}%) steered better/worse/tie: {wins}/{losses}/{len(deltas) - wins - losses}"
|
|
)
|
|
if len(deltas) > 1:
|
|
sd = statistics.stdev(deltas)
|
|
se = sd / (len(deltas) ** 0.5)
|
|
line += f" 95%CI [{mean - 1.96 * se:+.4f}, {mean + 1.96 * se:+.4f}]"
|
|
print(line)
|
|
|
|
print("\n=== per-task cost detail ===")
|
|
d = paired(rows, "cost_usd")
|
|
print(f" {'task':24s} {'control':>10s} {'steered':>10s} {'delta':>10s} {'Δ%':>7s}")
|
|
for (task, _rep), delta, c, s in sorted(d):
|
|
print(f" {task:24s} {c:10.4f} {s:10.4f} {delta:+10.4f} {100 * delta / c:+6.1f}%")
|
|
|
|
d = paired(rows, "correct")
|
|
regress = [k for k, dd, c, s in d if c and not s]
|
|
if regress:
|
|
print(f"\n !! correctness REGRESSED on: {regress}")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|