mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
feat(pipeline): put per-request saved tokens on OutcomeSnapshot (#4231)
## Description Expose per-request message-compression savings and the novel share booked to conversation totals on the immutable OutcomeSnapshot passed to pipeline extensions. Extensions can consume the same readings as the proxy without parsing PERF logs. The previous body described the separate remote-Kompress startup fix in #4234. This description has been corrected by the maintainer to match this PR's diff. ## Type of Change - [x] New feature (non-breaking additive outcome fields) ## Changes Made - Add tokens_saved and novel_tokens_saved to OutcomeSnapshot, with defaults preserving existing construction. - Emit OUTCOME_OBSERVED after the proxy computes the novel share. - Populate both fields from the proxy's accounting result. - Add a regression observing successive Responses outcomes through PipelineExtensionManager. ## Testing - [x] Focused conversation-savings tests pass. - [x] Touched-file Ruff and whitespace checks pass. ### Test Output ```text python -m pytest tests/test_proxy/test_conversation_savings.py tests/test_original_snapshot_in_place_hook.py -q 25 passed, 1 warning in 8.72s ruff check headroom/pipeline.py headroom/proxy/outcome.py tests/test_proxy/test_conversation_savings.py All checks passed! ``` Maintainer verification used the contributor's unchanged outcome implementation. The latest clean main sync incorporates #4241's snapshot-preservation fix; 25 combined conversation/snapshot tests passed. Exact-head hosted CI remains in progress. ## Real Behavior Proof - Environment: maintainer isolated Windows/Python 3.12.13 worktree; actual outcome emitter and PipelineExtensionManager exercised by the focused suite. - Exact command / steps: python -m pytest tests/test_proxy/test_conversation_savings.py tests/test_original_snapshot_in_place_hook.py -q; inspect test_pipeline_extensions_read_both_figures_from_the_snapshot. - Observed result: 25 tests passed, including the regression asserting that extension snapshots carry per-request and novel savings across successive Responses outcomes. - Not tested: maintainer live provider round trip with an installed entry-point extension. The author separately reports replaying recorded Codex outcomes through a real create_app proxy and observing tokens_saved=8575 and novel_tokens_saved=791, matching the recorded PERF figures; this is author-supplied replay evidence, not an independent live inference claim. ## Runtime Rollout Safety - Rollout-managed feature(s): no additional rollout-managed switch introduced. - Minimum rollout channel: existing pipeline-extension policy applies. - Stable/default behavior changed: enabled extensions receive two additive fields; the outcome event is emitted after novelty accounting. - Kill switch / disable path: disable the consuming pipeline extension. - Unsafe override required: no. - Qualification impact: exact-head hosted CI and final human review remain required. - Rollback path: revert this change or retain the previous qualified release. ## Review Readiness - [ ] I performed self-review - [ ] This PR is ready for human review Maintainer qualification above does not invent an author self-review attestation. Final merge readiness remains pending hosted CI and review. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
JerrettDavis
parent
00093ceddc
commit
8cedf7be4d
@@ -77,6 +77,15 @@ class OutcomeSnapshot:
|
||||
cache_write_tokens: int = 0
|
||||
turn_index: int = 0
|
||||
transforms_applied: tuple[str, ...] = ()
|
||||
#: Input tokens Headroom removed from THIS request's payload (the wire
|
||||
#: figure PERF logs as ``tok_saved``), message compression only.
|
||||
tokens_saved: int = 0
|
||||
#: The share of ``tokens_saved`` new to this conversation, which is what
|
||||
#: running totals book (PERF ``tok_novel``). The proxy funnel always sets
|
||||
#: both; they differ only on ``/v1/responses``, where the whole transcript
|
||||
#: is re-sent and recompressed every turn, and ``tokens_saved -
|
||||
#: novel_tokens_saved`` is the re-sent history removed again.
|
||||
novel_tokens_saved: int = 0
|
||||
|
||||
@property
|
||||
def truncated(self) -> bool:
|
||||
|
||||
+43
-38
@@ -571,44 +571,6 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
|
||||
await handler.metrics.record_failed(provider=outcome.provider)
|
||||
return
|
||||
|
||||
# Outcome stage: hand the meter's reading to any pipeline extension that
|
||||
# adapts to it — a ceiling that was hit, an effort setting that did or did
|
||||
# not pay off. The snapshot is frozen, so an extension can learn from the
|
||||
# measurement without being able to rewrite it: extensions declare what
|
||||
# they did, the core records what happened, and attribution is the core's
|
||||
# arithmetic over both.
|
||||
#
|
||||
# Best-effort and last in line: a handler without a pipeline manager, or an
|
||||
# extension that raises, must not affect the response or the stats below.
|
||||
_pipeline = getattr(handler, "pipeline_extensions", None)
|
||||
if _pipeline is not None and getattr(_pipeline, "enabled", False):
|
||||
try:
|
||||
from headroom.pipeline import OutcomeSnapshot, PipelineStage
|
||||
|
||||
_pipeline.emit(
|
||||
PipelineStage.OUTCOME_OBSERVED,
|
||||
operation="proxy.outcome",
|
||||
request_id=outcome.request_id,
|
||||
provider=outcome.provider,
|
||||
model=outcome.model,
|
||||
outcome=OutcomeSnapshot(
|
||||
request_id=outcome.request_id,
|
||||
provider=outcome.provider,
|
||||
model=outcome.model,
|
||||
output_tokens=outcome.output_tokens,
|
||||
thinking_tokens=outcome.thinking_tokens,
|
||||
thinking_inferred=outcome.thinking_inferred,
|
||||
stop_reason=outcome.stop_reason,
|
||||
input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,
|
||||
cache_read_tokens=outcome.cache_read_tokens,
|
||||
cache_write_tokens=outcome.cache_write_tokens,
|
||||
turn_index=outcome.turn_index,
|
||||
transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 - telemetry must never break a response
|
||||
logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)
|
||||
|
||||
# Output-shaping savings ledger (counterfactual estimator). The shaper
|
||||
# tags each request's (arm, stratum) onto ``transforms_applied``; feed the
|
||||
# observed output tokens to the recorder so it can produce an honest
|
||||
@@ -653,6 +615,49 @@ async def emit_request_outcome(handler: Any, outcome: RequestOutcome) -> None:
|
||||
if novel_tokens_saved is None:
|
||||
novel_tokens_saved = outcome.tokens_saved
|
||||
|
||||
# Outcome stage: hand the meter's reading to any pipeline extension that
|
||||
# adapts to it — a ceiling that was hit, an effort setting that did or did
|
||||
# not pay off. The snapshot is frozen, so an extension can learn from the
|
||||
# measurement without being able to rewrite it: extensions declare what
|
||||
# they did, the core records what happened, and attribution is the core's
|
||||
# arithmetic over both.
|
||||
#
|
||||
# Emitted after the novel share is known, so an extension reads the same
|
||||
# per-request savings the running totals book instead of parsing PERF lines.
|
||||
#
|
||||
# Best-effort: a handler without a pipeline manager, or an extension that
|
||||
# raises, must not affect the response or the stats below.
|
||||
_pipeline = getattr(handler, "pipeline_extensions", None)
|
||||
if _pipeline is not None and getattr(_pipeline, "enabled", False):
|
||||
try:
|
||||
from headroom.pipeline import OutcomeSnapshot, PipelineStage
|
||||
|
||||
_pipeline.emit(
|
||||
PipelineStage.OUTCOME_OBSERVED,
|
||||
operation="proxy.outcome",
|
||||
request_id=outcome.request_id,
|
||||
provider=outcome.provider,
|
||||
model=outcome.model,
|
||||
outcome=OutcomeSnapshot(
|
||||
request_id=outcome.request_id,
|
||||
provider=outcome.provider,
|
||||
model=outcome.model,
|
||||
output_tokens=outcome.output_tokens,
|
||||
thinking_tokens=outcome.thinking_tokens,
|
||||
thinking_inferred=outcome.thinking_inferred,
|
||||
stop_reason=outcome.stop_reason,
|
||||
input_tokens=outcome.provider_input_tokens or outcome.optimized_tokens,
|
||||
cache_read_tokens=outcome.cache_read_tokens,
|
||||
cache_write_tokens=outcome.cache_write_tokens,
|
||||
turn_index=outcome.turn_index,
|
||||
transforms_applied=tuple(str(t) for t in (outcome.transforms_applied or ())),
|
||||
tokens_saved=outcome.tokens_saved,
|
||||
novel_tokens_saved=novel_tokens_saved,
|
||||
),
|
||||
)
|
||||
except Exception: # noqa: BLE001 - telemetry must never break a response
|
||||
logger.debug("OUTCOME_OBSERVED emit failed", exc_info=True)
|
||||
|
||||
# Tool-schema savings (deferral + turn-hook tool shrink) live in per-request
|
||||
# tags and never move tok_before/after; aggregate them into Metrics so the
|
||||
# session summary / cost summary / all-layers total can surface the layer.
|
||||
|
||||
@@ -337,3 +337,29 @@ def test_server_side_state_means_incremental_input() -> None:
|
||||
savings_conversation_key(_responses_body("more", conversation={"id": "conv_abc"})) is None
|
||||
)
|
||||
assert savings_conversation_key({"model": "gpt-6-astra", "messages": []}) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pipeline_extensions_read_both_figures_from_the_snapshot() -> None:
|
||||
"""An extension sees the wire figure and the novel share the totals book,
|
||||
so it never has to parse PERF lines to tell re-sent history from new."""
|
||||
from headroom.pipeline import PipelineExtensionManager, PipelineStage
|
||||
|
||||
seen: list[Any] = []
|
||||
|
||||
class _Recorder:
|
||||
def on_pipeline_event(self, event: Any) -> None:
|
||||
if event.stage is PipelineStage.OUTCOME_OBSERVED:
|
||||
seen.append(event.outcome)
|
||||
|
||||
handler = _Handler()
|
||||
handler.pipeline_extensions = PipelineExtensionManager(extensions=[_Recorder()], discover=False)
|
||||
await emit_request_outcome(
|
||||
handler, _responses_outcome(tokens_saved=62_806, conversation_tokens_saved=62_806)
|
||||
)
|
||||
await emit_request_outcome(handler, _responses_outcome())
|
||||
|
||||
assert [(s.tokens_saved, s.novel_tokens_saved) for s in seen] == [
|
||||
(62_806, 62_806),
|
||||
(76_020, 13_214),
|
||||
]
|
||||
|
||||
Reference in New Issue
Block a user