fix(proxy): skip memory tool injection on streamed OpenAI requests

The OpenAI chat and Responses handlers injected memory_* tool definitions
into streamed requests. Only the Anthropic stream path executes memory tool
calls (streaming.py memory_enabled), so a memory_* call from the model
reached the client unhandled. Skip tool injection when stream is true;
memory context injection and buffered requests are unchanged.

Closes #4420
This commit is contained in:
danieldutu
2026-10-23 22:13:28 +03:00
parent e995eb9f1d
commit d440495a47
3 changed files with 94 additions and 4 deletions
+9 -3
View File
@@ -4254,9 +4254,12 @@ class OpenAIHandlerMixin:
apply_session_sticky_memory_tools as _apply_sticky_mem_tools,
)
# Only the Anthropic stream path executes memory tool calls
# (streaming.py `memory_enabled`); skip tools on streamed OpenAI.
inject_mem_tools = bool(self.memory_handler.config.inject_tools) and not stream
memory_tool_defs = (
self.memory_handler.compute_memory_tool_definitions("openai")
if self.memory_handler.config.inject_tools
if inject_mem_tools
else []
)
tools, mem_tools_injected = _apply_sticky_mem_tools(
@@ -4265,7 +4268,7 @@ class OpenAIHandlerMixin:
request_id=request_id,
existing_tools=tools,
memory_tools_to_inject=memory_tool_defs,
inject_this_turn=bool(self.memory_handler.config.inject_tools),
inject_this_turn=inject_mem_tools,
client_declared_tools=bool(_original_tools),
)
if mem_tools_injected:
@@ -5875,7 +5878,10 @@ class OpenAIHandlerMixin:
) or _client_can_receive_memory_tools(memory_client)
if _ensure_chatgpt_responses_store_false(body, is_chatgpt_auth=is_chatgpt_auth):
logger.info(f"[{request_id}] Responses: forced store=false for ChatGPT auth")
responses_memory_tools_allowed = _allow_responses_memory_tools(is_chatgpt_auth)
# Streamed Responses turns do not execute memory tool calls; keep tools off.
responses_memory_tools_allowed = (
_allow_responses_memory_tools(is_chatgpt_auth) and not stream
)
# PR-A6 (P5-50, preps P0-6): session-sticky `OpenAI-Beta` merge
# for /v1/responses. Compute a session_id off the same store the
+61 -1
View File
@@ -801,7 +801,7 @@ class TestHandlerPassesServerMemoryTools:
captured: dict[str, Any] = {}
async def fake_stream_response(*args: Any, **kwargs: Any) -> StreamingResponse:
captured.update(kwargs)
captured.update(kwargs, body=args[2])
async def gen() -> Any:
yield b""
@@ -838,6 +838,66 @@ class TestHandlerPassesServerMemoryTools:
)
assert captured["server_memory_tool_names"] == MEMORY_TOOLS
assert MEMORY_TOOLS <= {t["name"] for t in captured["body"]["tools"]}
def test_streamed_openai_chat_gets_no_memory_tools(self) -> None:
from types import SimpleNamespace
from fastapi.responses import StreamingResponse
from fastapi.testclient import TestClient
from headroom.proxy.server import ProxyConfig, create_app
config = ProxyConfig(
optimize=False,
cache_enabled=False,
rate_limit_enabled=False,
cost_tracking_enabled=False,
log_requests=False,
ccr_inject_tool=False,
ccr_handle_responses=False,
ccr_context_tracking=False,
image_optimize=False,
)
captured: dict[str, Any] = {}
async def fake_stream_response(url: str, headers: Any, body: Any, *a: Any, **kw: Any):
captured["body"] = body
async def gen() -> Any:
yield b""
return StreamingResponse(gen(), media_type="text/event-stream")
memory_tools = [
{
"type": "function",
"function": {"name": name, "description": name, "parameters": {}},
}
for name in sorted(MEMORY_TOOLS)
]
with TestClient(create_app(config)) as client:
proxy = client.app.state.proxy
proxy.memory_handler = SimpleNamespace(
config=SimpleNamespace(inject_context=False, inject_tools=True),
compute_memory_tool_definitions=lambda provider: memory_tools,
get_beta_headers=lambda: {},
has_memory_tool_calls=lambda resp, provider: False,
)
proxy._stream_response = fake_stream_response
client.post(
"/v1/chat/completions",
headers={"authorization": "Bearer sk-test", "x-headroom-user-id": "u1"},
json={
"model": "gpt-4o-mini",
"stream": True,
"tools": [{"type": "function", "function": {"name": "Bash", "parameters": {}}}],
"messages": [{"role": "user", "content": "remember the deploy region"}],
},
)
names = {t["function"]["name"] for t in captured["body"]["tools"]}
assert names == {"Bash"}
class TestStreamingRetainedByteLimit:
+24
View File
@@ -581,6 +581,30 @@ def test_handle_openai_responses_api_auth_store_false_injects_stateless_memory_t
assert memory_handler.compute_calls == 1
def test_handle_openai_responses_stream_skips_memory_tools(monkeypatch):
request = _build_request(
{"model": "gpt-4o-mini", "input": "hello", "store": False, "stream": True},
{
"Authorization": "Bearer sk-test",
"User-Agent": "codex-cli/0.5",
"x-headroom-user-id": "user-1",
},
)
handler = _DummyOpenAIHandler()
handler.memory_handler = _MemoryToolsOnlyHandler()
handler.session_tracker_store = SimpleNamespace(
compute_session_id=lambda *a, **k: "sess-api-memory-tools-stream",
)
monkeypatch.setattr("headroom.tokenizers.get_tokenizer", lambda model: _DummyTokenizer())
anyio.run(handler.handle_openai_responses, request)
assert handler.captured_stream_request is not None
assert "memory_search" not in json.dumps(handler.captured_stream_request[2].get("tools", []))
assert handler.memory_handler.compute_calls == 0
@pytest.mark.parametrize("store", [pytest.param(None, id="omitted"), True, False])
@pytest.mark.parametrize(
"include",