fix(proxy): retry a TLS error on a reused upstream connection

A corrupt TLS record on a reused keep-alive connection surfaces as a raw
ssl.SSLError: anyio re-raises it and httpcore maps SSL errors only during
the handshake. It is not an httpx.TransportError, so the #1639 retry in
_stream_response and _retry_request let it through as a failed request.
Both loops and the streaming exhaustion handler now catch it too.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Garm
2026-11-11 11:20:03 +01:00
co-authored by Claude Opus 5.5
parent 200c838c73
commit 240680dff0
3 changed files with 65 additions and 3 deletions
+6 -2
View File
@@ -9,6 +9,7 @@ import asyncio
import contextlib
import json
import logging
import ssl
import time
from collections.abc import AsyncIterator
from typing import TYPE_CHECKING, Any
@@ -1502,7 +1503,10 @@ class StreamingMixin:
# `StreamReset`) from a poisoned shared h2 connection. This runs
# before any body byte is forwarded to the client, so re-sending
# on a fresh connection is safe and avoids a 502. (#1639)
except httpx.TransportError as e:
# A corrupt TLS record on a reused keep-alive connection is a
# raw ssl.SSLError, not a TransportError: anyio re-raises it and
# httpcore maps SSL errors only during the handshake.
except (httpx.TransportError, ssl.SSLError) as e:
last_connect_error = e
if attempt >= retry_attempts - 1:
raise
@@ -1534,7 +1538,7 @@ class StreamingMixin:
# 502 keeps the structured SSE body for clients that read it while
# letting SDK retry logic treat a transient upstream transport failure
# as what it is.
except httpx.TransportError as e:
except (httpx.TransportError, ssl.SSLError) as e:
error_msg = str(e) or repr(e)
logger.error(f"[{request_id}] Connection error to upstream API: {error_msg}")
self.metrics.record_upstream_connection_error(provider)
+4 -1
View File
@@ -35,6 +35,7 @@ import logging
import math
import os
import signal
import ssl
import sys
import threading
import time
@@ -2849,7 +2850,9 @@ class HeadroomProxy(
# HTTP/2 `StreamReset`) that a poisoned shared h2 connection raises on
# every in-flight request. Retrying drops the bad connection and
# re-sends on a fresh one instead of collapsing to a 502. (#1639)
except (httpx.TransportError, httpx.HTTPStatusError) as e:
# ssl.SSLError: a corrupt TLS record on a reused keep-alive
# connection escapes httpx's TransportError mapping.
except (httpx.TransportError, httpx.HTTPStatusError, ssl.SSLError) as e:
last_error = e
if not self.config.retry_enabled or attempt >= self.config.retry_max_attempts - 1:
+55
View File
@@ -6,10 +6,15 @@ every in-flight request. Those are transport errors, so the proxy must retry
them (dropping the bad connection and re-sending on a fresh one) instead of
collapsing to a 502. These tests drive the real `_retry_request` and
`_stream_response` paths.
A corrupt TLS record on a reused HTTP/1.1 keep-alive connection is the same
class of failure, but surfaces as a raw `ssl.SSLError` that is not an
`httpx.TransportError`, so it gets the same retry.
"""
from __future__ import annotations
import ssl
from unittest.mock import AsyncMock, MagicMock
import httpx
@@ -177,3 +182,53 @@ async def test_successful_stream_still_returns_200():
assert result.status_code == 200
proxy.metrics.record_upstream_connection_error.assert_not_called()
def _bad_record() -> ssl.SSLError:
# What httpx raises on a reused connection whose next TLS record is corrupt.
return ssl.SSLError(1, "[SSL] record layer failure (_ssl.c:2580)")
async def test_retry_request_retries_ssl_error_on_reused_connection():
proxy = _mock_proxy()
good = MagicMock()
good.status_code = 200
good.request = MagicMock()
proxy.http_client.post = AsyncMock(side_effect=[_bad_record(), good])
result = await proxy._retry_request(
"POST",
"https://api.anthropic.com/v1/messages",
{"x-api-key": "sk-test"},
{"model": "claude-sonnet-4-20250514", "messages": []},
)
assert result is good
assert proxy.http_client.post.await_count == 2
async def test_stream_retries_ssl_error_on_reused_connection():
proxy = _mock_proxy()
good = _good_stream_response([b'event: message_stop\ndata: {"type":"message_stop"}\n\n'])
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
proxy.http_client.send = AsyncMock(side_effect=[_bad_record(), good])
result = await _run_stream(proxy)
body = b"".join([chunk async for chunk in result.body_iterator])
assert proxy.http_client.send.await_count == 2
assert result.status_code == 200
assert b"connection_error" not in body
async def test_stream_ssl_error_exhaustion_is_a_tls_error_502():
proxy = _mock_proxy()
proxy.http_client.build_request = MagicMock(return_value=MagicMock())
proxy.http_client.send = AsyncMock(side_effect=_bad_record())
result = await _run_stream(proxy)
body = b"".join([chunk async for chunk in result.body_iterator])
assert proxy.http_client.send.await_count == 2
assert result.status_code == 502
assert b"upstream_tls_error" in body