mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 12:05:34 +08:00
fix(proxy): retry a TLS error on a reused upstream connection
A corrupt TLS record on a reused keep-alive connection surfaces as a raw ssl.SSLError: anyio re-raises it and httpcore maps SSL errors only during the handshake. It is not an httpx.TransportError, so the #1639 retry in _stream_response and _retry_request let it through as a failed request. Both loops and the streaming exhaustion handler now catch it too. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,7 @@ import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import logging
|
||||
import ssl
|
||||
import time
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import TYPE_CHECKING, Any
|
||||
@@ -1502,7 +1503,10 @@ class StreamingMixin:
|
||||
# `StreamReset`) from a poisoned shared h2 connection. This runs
|
||||
# before any body byte is forwarded to the client, so re-sending
|
||||
# on a fresh connection is safe and avoids a 502. (#1639)
|
||||
except httpx.TransportError as e:
|
||||
# A corrupt TLS record on a reused keep-alive connection is a
|
||||
# raw ssl.SSLError, not a TransportError: anyio re-raises it and
|
||||
# httpcore maps SSL errors only during the handshake.
|
||||
except (httpx.TransportError, ssl.SSLError) as e:
|
||||
last_connect_error = e
|
||||
if attempt >= retry_attempts - 1:
|
||||
raise
|
||||
@@ -1534,7 +1538,7 @@ class StreamingMixin:
|
||||
# 502 keeps the structured SSE body for clients that read it while
|
||||
# letting SDK retry logic treat a transient upstream transport failure
|
||||
# as what it is.
|
||||
except httpx.TransportError as e:
|
||||
except (httpx.TransportError, ssl.SSLError) as e:
|
||||
error_msg = str(e) or repr(e)
|
||||
logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
|
||||
self.metrics.record_upstream_connection_error(provider)
|
||||
|
||||
@@ -35,6 +35,7 @@ import logging
|
||||
import math
|
||||
import os
|
||||
import signal
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
@@ -2849,7 +2850,9 @@ class HeadroomProxy(
|
||||
# HTTP/2 `StreamReset`) that a poisoned shared h2 connection raises on
|
||||
# every in-flight request. Retrying drops the bad connection and
|
||||
# re-sends on a fresh one instead of collapsing to a 502. (#1639)
|
||||
except (httpx.TransportError, httpx.HTTPStatusError) as e:
|
||||
# ssl.SSLError: a corrupt TLS record on a reused keep-alive
|
||||
# connection escapes httpx's TransportError mapping.
|
||||
except (httpx.TransportError, httpx.HTTPStatusError, ssl.SSLError) as e:
|
||||
last_error = e
|
||||
|
||||
if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
|
||||
|
||||
@@ -6,10 +6,15 @@ every in-flight request. Those are transport errors, so the proxy must retry
|
||||
them (dropping the bad connection and re-sending on a fresh one) instead of
|
||||
collapsing to a 502. These tests drive the real `_retry_request` and
|
||||
`_stream_response` paths.
|
||||
|
||||
A corrupt TLS record on a reused HTTP/1.1 keep-alive connection is the same
|
||||
class of failure, but surfaces as a raw `ssl.SSLError` that is not an
|
||||
`httpx.TransportError`, so it gets the same retry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import ssl
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import httpx
|
||||
@@ -177,3 +182,53 @@ async def test_successful_stream_still_returns_200():
|
||||
|
||||
assert result.status_code == 200
|
||||
proxy.metrics.record_upstream_connection_error.assert_not_called()
|
||||
|
||||
|
||||
def _bad_record() -> ssl.SSLError:
|
||||
# What httpx raises on a reused connection whose next TLS record is corrupt.
|
||||
return ssl.SSLError(1, "[SSL] record layer failure (_ssl.c:2580)")
|
||||
|
||||
|
||||
async def test_retry_request_retries_ssl_error_on_reused_connection():
|
||||
proxy = _mock_proxy()
|
||||
good = MagicMock()
|
||||
good.status_code = 200
|
||||
good.request = MagicMock()
|
||||
proxy.http_client.post = AsyncMock(side_effect=[_bad_record(), good])
|
||||
|
||||
result = await proxy._retry_request(
|
||||
"POST",
|
||||
"https://api.anthropic.com/v1/messages",
|
||||
{"x-api-key": "sk-test"},
|
||||
{"model": "claude-sonnet-4-20250514", "messages": []},
|
||||
)
|
||||
|
||||
assert result is good
|
||||
assert proxy.http_client.post.await_count == 2
|
||||
|
||||
|
||||
async def test_stream_retries_ssl_error_on_reused_connection():
|
||||
proxy = _mock_proxy()
|
||||
good = _good_stream_response([b'event: message_stop\ndata: {"type":"message_stop"}\n\n'])
|
||||
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
|
||||
proxy.http_client.send = AsyncMock(side_effect=[_bad_record(), good])
|
||||
|
||||
result = await _run_stream(proxy)
|
||||
body = b"".join([chunk async for chunk in result.body_iterator])
|
||||
|
||||
assert proxy.http_client.send.await_count == 2
|
||||
assert result.status_code == 200
|
||||
assert b"connection_error" not in body
|
||||
|
||||
|
||||
async def test_stream_ssl_error_exhaustion_is_a_tls_error_502():
|
||||
proxy = _mock_proxy()
|
||||
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
|
||||
proxy.http_client.send = AsyncMock(side_effect=_bad_record())
|
||||
|
||||
result = await _run_stream(proxy)
|
||||
body = b"".join([chunk async for chunk in result.body_iterator])
|
||||
|
||||
assert proxy.http_client.send.await_count == 2
|
||||
assert result.status_code == 502
|
||||
assert b"upstream_tls_error" in body
|
||||
|
||||
Reference in New Issue
Block a user