Files
W fb26a7ab91 fix(shell): stop dropping cookies whose values SimpleCookie rejects (#541)
* fix(shell): stop dropping cookies whose values SimpleCookie rejects

_CookieParser fed the cookie string to http.cookies.SimpleCookie, which
stops at the first value it considers illegal (a space, a double quote,
...) and silently discards that cookie and everything after it. Cookie
strings copied from DevTools contain such values regularly, so curl
commands parsed by uncurl/curl2fetcher and `extract --cookies` lost
cookies without any warning.

Split the string on ";" instead and keep each value as sent, including
surrounding quotes, which SimpleCookie used to strip. Set-Cookie
attribute names are still skipped, as before.

* fix(shell): keep request cookies named partitioned
2026-11-04 16:08:55 +02:00

60 lines
2.5 KiB
Python

from scrapling.core._types import (
List,
Dict,
Tuple,
Generator,
)
# `Set-Cookie` attributes, skipped so a pasted `Set-Cookie` value still gives only the cookie itself.
# Bare flags (`Secure`, `HttpOnly`, `Partitioned`) have no `=` and are skipped anyway, so `partitioned` isn't listed:
# it's a valid cookie name, and listing it would drop a real `partitioned=...` request cookie.
_COOKIE_ATTRIBUTES = frozenset(
{"expires", "path", "comment", "domain", "max-age", "secure", "httponly", "version", "samesite"}
)
def _CookieParser(cookie_string: str) -> Generator[Tuple[str, str], None, None]:
"""Split a cookie string like `a=1; b=2` into (name, value) pairs, keeping each value as it was sent.
`http.cookies.SimpleCookie` isn't used here because it stops at the first value it considers invalid (a space,
a double quote, etc.) and silently drops that cookie and every one after it, while browsers send such values.
"""
for pair in cookie_string.split(";"):
name, separator, value = pair.partition("=")
name = name.strip()
if not separator or not name or name.lower() in _COOKIE_ATTRIBUTES:
continue
yield name, value.strip()
def _ParseHeaders(header_lines: List[str], parse_cookies: bool = True) -> Tuple[Dict[str, str], Dict[str, str]]:
"""Parses headers into separate header and cookie dictionaries."""
header_dict = dict()
cookie_dict = dict()
for header_line in header_lines:
if ":" not in header_line:
if header_line.endswith(";"):
header_key = header_line[:-1].strip()
header_value = ""
header_dict[header_key] = header_value
else:
raise ValueError(f"Could not parse header without colon: '{header_line}'.")
else:
header_key, header_value = header_line.split(":", 1)
header_key = header_key.strip()
header_value = header_value.strip()
if parse_cookies:
if header_key.lower() == "cookie":
try:
cookie_dict = {key: value for key, value in _CookieParser(header_value)}
except Exception as e: # pragma: no cover
raise ValueError(f"Could not parse cookie string from header '{header_value}': {e}")
else:
header_dict[header_key] = header_value
else:
header_dict[header_key] = header_value
return header_dict, cookie_dict