mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(storage): page JSONL queries after sorting (#3872)
## Description `JSONLStorage.query()` applied `limit` and `offset` while reading the append-only file, then sorted only the truncated page. SQLite sorts the full filtered set newest-first before paging, so the two storage backends returned different records for the same page. Closes #3822 ## Type of Change - [x] Bug fix (non-breaking change that fixes an issue) - [ ] New feature - [ ] Breaking change - [ ] Documentation update - [ ] Performance improvement - [ ] Code refactoring ## Changes Made - Sort the complete filtered JSONL result set newest-first before applying offset and limit. - Remove the early read-time truncation that returned old rows. - Add comprehensive JSONL/SQLite parity coverage for multiple pages and filters, plus focused JSONL tests. ## Testing - [x] Unit tests pass - [x] Linting passes - [x] Type checking passes - [x] New tests added - [ ] Manual testing performed ### Test Output ```text 41 passed: tests/test_storage_backends.py tests/test_storage tests/test_storage/test_jsonl.py ruff check: passed ruff format --check: passed mypy headroom/storage/jsonl.py: passed ``` ## Real Behavior Proof - Environment: macOS arm64, Python 3.12.14 using the headroom project virtual environment; current main (`ffc6edb4`) merged into this branch. The test uses local temporary JSONL and SQLite files, with no network or provider credentials. - Exact command / steps: From the repository root run `PYTHONPATH=. /Users/ankurkothari/Documents/workspace/headroom/.venv/bin/python -m pytest -q tests/test_storage_backends.py tests/test_storage`. The regression fixture appends 250 timestamped requests oldest-first into both storage backends, then queries `limit=5` at offsets 0, 5, and 10 and repeats with model, mode, and combined filters. - Observed result: `41 passed` on the updated branch. Both backends return identical newest-first pages; the first JSONL page starts at `req-249`, and adjacent pages do not overlap. On the unfixed baseline, JSONL's first page contains old records instead. - Not tested: performance and peak memory for multi-gigabyte JSONL files, or Windows path handling. This change necessarily scans the filtered JSONL data before paging. ## Runtime Rollout Safety - Rollout-managed feature(s): None. - Minimum rollout channel: Stable; the existing JSONL backend receives the correction without a feature flag. - Stable/default behavior changed: JSONL pages now match the existing SQLite newest-first contract. - Kill switch / disable path: Not applicable. - Unsafe override required: No. - Qualification impact: Storage backend query parity and existing storage tests; no provider or proxy change. - Rollback path: Revert this PR. ## Review Readiness - [x] I have performed a self-review - [x] This PR is ready for human review ## Checklist - [x] My code follows the project's style guidelines - [x] I have performed a self-review - [x] I have added tests proving the fix - [x] New and existing tests pass locally - [x] I did not edit `CHANGELOG.md` ## Screenshots (if applicable) Not applicable. --------- Co-authored-by: JerrettDavis <mxjerrett@gmail.com>
This commit is contained in:
co-authored by
JerrettDavis
parent
d75eecd814
commit
91237cae70
+39
-22
@@ -2,6 +2,7 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import heapq
|
||||
import json
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
@@ -101,35 +102,51 @@ class JSONLStorage(Storage):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[RequestMetrics]:
|
||||
"""Query metrics with filters."""
|
||||
results: list[RequestMetrics] = []
|
||||
skipped = 0
|
||||
"""Query metrics with filters.
|
||||
|
||||
for metrics in self.iter_all():
|
||||
# Apply filters
|
||||
Matches are ordered by timestamp descending, then request_id
|
||||
descending (the order SQLiteStorage.query() applies), before the
|
||||
``[offset : offset + limit]`` page is returned. Selection is bounded
|
||||
to that window instead of materializing the full match set.
|
||||
|
||||
Args:
|
||||
start_time: Filter by timestamp >= start_time.
|
||||
end_time: Filter by timestamp <= end_time.
|
||||
model: Filter by model name.
|
||||
mode: Filter by mode (audit/optimize).
|
||||
limit: Maximum results to return; must be non-negative.
|
||||
offset: Number of results to skip; must be non-negative.
|
||||
|
||||
Returns:
|
||||
List of matching RequestMetrics for the requested page.
|
||||
|
||||
Raises:
|
||||
ValueError: If limit or offset is negative.
|
||||
"""
|
||||
if limit < 0:
|
||||
raise ValueError(f"limit must be non-negative, got {limit}")
|
||||
if offset < 0:
|
||||
raise ValueError(f"offset must be non-negative, got {offset}")
|
||||
if limit == 0:
|
||||
return []
|
||||
|
||||
def _matches(metrics: RequestMetrics) -> bool:
|
||||
if start_time is not None and metrics.timestamp < start_time:
|
||||
continue
|
||||
return False
|
||||
if end_time is not None and metrics.timestamp > end_time:
|
||||
continue
|
||||
return False
|
||||
if model is not None and metrics.model != model:
|
||||
continue
|
||||
return False
|
||||
if mode is not None and metrics.mode != mode:
|
||||
continue
|
||||
return False
|
||||
return True
|
||||
|
||||
# Handle offset
|
||||
if skipped < offset:
|
||||
skipped += 1
|
||||
continue
|
||||
filtered = (metrics for metrics in self.iter_all() if _matches(metrics))
|
||||
|
||||
results.append(metrics)
|
||||
|
||||
# Handle limit
|
||||
if len(results) >= limit:
|
||||
break
|
||||
|
||||
# Sort by timestamp descending
|
||||
results.sort(key=lambda m: m.timestamp, reverse=True)
|
||||
return results
|
||||
# Keep only the top offset + limit matches under the shared
|
||||
# timestamp DESC, request_id DESC order, then drop the offset prefix.
|
||||
top = heapq.nlargest(offset + limit, filtered, key=lambda m: (m.timestamp, m.request_id))
|
||||
return top[offset : offset + limit]
|
||||
|
||||
def count(
|
||||
self,
|
||||
|
||||
@@ -142,7 +142,31 @@ class SQLiteStorage(Storage):
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[RequestMetrics]:
|
||||
"""Query metrics with filters."""
|
||||
"""Query metrics with filters.
|
||||
|
||||
Matches are ordered by timestamp descending, then request_id
|
||||
descending (the order JSONLStorage.query() applies), before the
|
||||
``LIMIT/OFFSET`` page is returned.
|
||||
|
||||
Args:
|
||||
start_time: Filter by timestamp >= start_time.
|
||||
end_time: Filter by timestamp <= end_time.
|
||||
model: Filter by model name.
|
||||
mode: Filter by mode (audit/optimize).
|
||||
limit: Maximum results to return; must be non-negative.
|
||||
offset: Number of results to skip; must be non-negative.
|
||||
|
||||
Returns:
|
||||
List of matching RequestMetrics for the requested page.
|
||||
|
||||
Raises:
|
||||
ValueError: If limit or offset is negative.
|
||||
"""
|
||||
if limit < 0:
|
||||
raise ValueError(f"limit must be non-negative, got {limit}")
|
||||
if offset < 0:
|
||||
raise ValueError(f"offset must be non-negative, got {offset}")
|
||||
|
||||
conn = self._get_conn()
|
||||
cursor = conn.cursor()
|
||||
|
||||
@@ -162,7 +186,9 @@ class SQLiteStorage(Storage):
|
||||
query += " AND mode = ?"
|
||||
params.append(mode)
|
||||
|
||||
query += " ORDER BY timestamp DESC LIMIT ? OFFSET ?"
|
||||
# Timestamp DESC, id DESC: the same fully-defined order
|
||||
# JSONLStorage.query() applies, so tied timestamps page identically.
|
||||
query += " ORDER BY timestamp DESC, id DESC LIMIT ? OFFSET ?"
|
||||
params.extend([limit, offset])
|
||||
|
||||
cursor.execute(query, params)
|
||||
|
||||
@@ -0,0 +1,246 @@
|
||||
"""Tests for JSONL storage query paging (shared newest-first contract)."""
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.config import RequestMetrics
|
||||
from headroom.storage.jsonl import JSONLStorage
|
||||
from headroom.storage.sqlite import SQLiteStorage
|
||||
|
||||
|
||||
def _metrics(
|
||||
request_id: str,
|
||||
timestamp: datetime,
|
||||
model: str = "gpt-4o",
|
||||
mode: str = "audit",
|
||||
) -> RequestMetrics:
|
||||
return RequestMetrics(
|
||||
request_id=request_id,
|
||||
timestamp=timestamp,
|
||||
model=model,
|
||||
stream=False,
|
||||
mode=mode,
|
||||
tokens_input_before=1000,
|
||||
tokens_input_after=800,
|
||||
tokens_output=200,
|
||||
block_breakdown={"system": 100},
|
||||
waste_signals={"json_bloat": 5},
|
||||
stable_prefix_hash=f"prefix-{request_id}",
|
||||
cache_alignment_score=80.0,
|
||||
cached_tokens=50,
|
||||
transforms_applied=["SmartCrusher"],
|
||||
tool_units_dropped=1,
|
||||
turns_dropped=0,
|
||||
messages_hash=f"messages-{request_id}",
|
||||
error=None,
|
||||
)
|
||||
|
||||
|
||||
def _expected_page(
|
||||
rows: list[RequestMetrics],
|
||||
*,
|
||||
model: str | None = None,
|
||||
mode: str | None = None,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[str]:
|
||||
"""Reference contract: newest-first ordering, then the [offset:offset+limit] page."""
|
||||
filtered = [
|
||||
row
|
||||
for row in rows
|
||||
if (start_time is None or row.timestamp >= start_time)
|
||||
and (end_time is None or row.timestamp <= end_time)
|
||||
and (model is None or row.model == model)
|
||||
and (mode is None or row.mode == mode)
|
||||
]
|
||||
ordered = sorted(filtered, key=lambda row: row.timestamp, reverse=True)
|
||||
return [row.request_id for row in ordered[offset : offset + limit]]
|
||||
|
||||
|
||||
def _ids(metrics: list[RequestMetrics]) -> list[str]:
|
||||
return [m.request_id for m in metrics]
|
||||
|
||||
|
||||
def _assert_timestamp_desc(metrics: list[RequestMetrics]) -> None:
|
||||
stamps = [m.timestamp for m in metrics]
|
||||
assert stamps == sorted(stamps, reverse=True)
|
||||
|
||||
|
||||
class TestJSONLQueryPagingContract:
|
||||
"""JSONLStorage.query() must page like SQLiteStorage.query() (issue #3822).
|
||||
|
||||
Rows are appended in timestamp order, so paging before ordering returns the
|
||||
oldest remaining rows per page. The contract requires the filtered set to be
|
||||
ordered newest-first first and only then sliced to [offset:offset+limit].
|
||||
"""
|
||||
|
||||
@pytest.fixture
|
||||
def backends(self, tmp_path):
|
||||
jsonl = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
sqlite = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
base = datetime(2025, 1, 6, 12, 0, 0)
|
||||
rows = [
|
||||
_metrics(
|
||||
f"req-{i:03d}",
|
||||
base + timedelta(seconds=i),
|
||||
model="claude" if i % 2 == 0 else "gpt-4o",
|
||||
mode="audit" if i % 3 == 0 else "optimize",
|
||||
)
|
||||
for i in range(250)
|
||||
]
|
||||
for row in rows:
|
||||
jsonl.save(row)
|
||||
sqlite.save(row)
|
||||
yield jsonl, sqlite, rows
|
||||
jsonl.close()
|
||||
sqlite.close()
|
||||
|
||||
def test_multi_page_agrees_with_sqlite_and_starts_at_newest(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
for offset in (0, 5, 10):
|
||||
kw = {"limit": 5, "offset": offset}
|
||||
jsonl_page = jsonl.query(**kw)
|
||||
sqlite_page = sqlite.query(**kw)
|
||||
|
||||
assert _ids(jsonl_page) == _ids(sqlite_page)
|
||||
assert _ids(jsonl_page) == _expected_page(rows, **kw)
|
||||
_assert_timestamp_desc(jsonl_page)
|
||||
|
||||
first_page = jsonl.query(limit=5, offset=0)
|
||||
assert first_page[0].request_id == "req-249"
|
||||
assert _ids(first_page) == [
|
||||
"req-249",
|
||||
"req-248",
|
||||
"req-247",
|
||||
"req-246",
|
||||
"req-245",
|
||||
]
|
||||
|
||||
def test_pages_are_disjoint_and_tile_the_newest_first_order(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
paged: list[str] = []
|
||||
jsonl_pages: list[list[str]] = []
|
||||
offset = 0
|
||||
while True:
|
||||
ids = _ids(jsonl.query(limit=5, offset=offset))
|
||||
if not ids:
|
||||
break
|
||||
jsonl_pages.append(ids)
|
||||
assert ids == _ids(sqlite.query(limit=5, offset=offset))
|
||||
paged.extend(ids)
|
||||
offset += 5
|
||||
|
||||
assert sum(len(page) for page in jsonl_pages) == 250
|
||||
for i, page in enumerate(jsonl_pages):
|
||||
for other in jsonl_pages[i + 1 :]:
|
||||
assert set(page).isdisjoint(other)
|
||||
|
||||
assert paged == _expected_page(rows, limit=250, offset=0)
|
||||
|
||||
def test_filtered_pages_agree_with_sqlite(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
cases = [
|
||||
{"model": "claude", "limit": 5, "offset": 0},
|
||||
{"model": "claude", "limit": 5, "offset": 5},
|
||||
{"mode": "audit", "limit": 5, "offset": 0},
|
||||
{"mode": "audit", "limit": 4, "offset": 2},
|
||||
{"model": "claude", "mode": "audit", "limit": 3, "offset": 1},
|
||||
{
|
||||
"model": "gpt-4o",
|
||||
"mode": "optimize",
|
||||
"start_time": datetime(2025, 1, 6, 12, 0, 30),
|
||||
"limit": 6,
|
||||
"offset": 3,
|
||||
},
|
||||
]
|
||||
for kw in cases:
|
||||
jsonl_page = jsonl.query(**kw)
|
||||
sqlite_page = sqlite.query(**kw)
|
||||
|
||||
assert _ids(jsonl_page) == _ids(sqlite_page), kw
|
||||
assert _ids(jsonl_page) == _expected_page(rows, **kw), kw
|
||||
_assert_timestamp_desc(jsonl_page)
|
||||
|
||||
def test_filtered_pages_tile_the_filtered_set(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
kw = {"mode": "audit"}
|
||||
total = jsonl.count(**kw)
|
||||
assert total == sqlite.count(**kw) == len(_expected_page(rows, **kw, limit=250))
|
||||
|
||||
tiled: list[str] = []
|
||||
offset = 0
|
||||
while True:
|
||||
page_kw = {**kw, "limit": 7, "offset": offset}
|
||||
ids = _ids(jsonl.query(**page_kw))
|
||||
if not ids:
|
||||
break
|
||||
assert ids == _ids(sqlite.query(**page_kw)), page_kw
|
||||
tiled.extend(ids)
|
||||
offset += 7
|
||||
|
||||
assert tiled == _expected_page(rows, **kw, limit=250, offset=0)
|
||||
|
||||
def test_page_one_is_newest_matching_row_not_newest_overall(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
# req-249 is newest overall and an audit row (249 % 3 == 0), but the
|
||||
# newest optimize row is req-248; a filtered page must start there.
|
||||
page = jsonl.query(mode="optimize", limit=5, offset=0)
|
||||
assert page[0].request_id == "req-248"
|
||||
assert _ids(page) == _ids(sqlite.query(mode="optimize", limit=5, offset=0))
|
||||
|
||||
def test_count_summary_and_iter_all_are_unchanged(self, backends):
|
||||
jsonl, sqlite, rows = backends
|
||||
|
||||
assert jsonl.count() == 250
|
||||
assert jsonl.count(mode="audit") == sqlite.count(mode="audit")
|
||||
assert jsonl.count(model="claude") == sqlite.count(model="claude")
|
||||
assert (
|
||||
jsonl.count(
|
||||
start_time=datetime(2025, 1, 6, 12, 0, 30),
|
||||
end_time=datetime(2025, 1, 6, 12, 1, 0),
|
||||
)
|
||||
== 31
|
||||
)
|
||||
|
||||
summary = jsonl.get_summary_stats()
|
||||
assert summary["total_requests"] == 250
|
||||
assert summary["total_tokens_before"] == 250 * 1000
|
||||
assert summary["total_tokens_after"] == 250 * 800
|
||||
assert summary["audit_count"] == jsonl.count(mode="audit")
|
||||
assert summary["optimize_count"] == jsonl.count(mode="optimize")
|
||||
|
||||
# iter_all() keeps file (append) order: paging fixes must not sort it.
|
||||
assert _ids(list(jsonl.iter_all())) == [row.request_id for row in rows]
|
||||
|
||||
|
||||
class TestJSONLQuerySingleRowRegression:
|
||||
"""The 3-row scenario from tests/test_storage_backends.py:162 (issue #3822)."""
|
||||
|
||||
def test_offset_window_over_filtered_set_is_newest_first(self, tmp_path):
|
||||
jsonl = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
sqlite = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
now = datetime(2026, 4, 23, 12, 0, 0)
|
||||
first = _metrics("one", now - timedelta(hours=2), mode="audit")
|
||||
second = _metrics("two", now - timedelta(hours=1), model="claude", mode="optimize")
|
||||
third = _metrics("three", now, mode="audit")
|
||||
for row in (first, second, third):
|
||||
jsonl.save(row)
|
||||
sqlite.save(row)
|
||||
|
||||
kw = {
|
||||
"start_time": now - timedelta(hours=1, minutes=30),
|
||||
"offset": 1,
|
||||
"limit": 1,
|
||||
}
|
||||
assert _ids(jsonl.query(**kw)) == _ids(sqlite.query(**kw)) == ["two"]
|
||||
|
||||
jsonl.close()
|
||||
sqlite.close()
|
||||
@@ -1,9 +1,14 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import weakref
|
||||
from collections.abc import Callable, Iterator
|
||||
from dataclasses import dataclass
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from headroom.config import RequestMetrics
|
||||
from headroom.storage import JSONLStorage, SQLiteStorage, Storage, create_storage
|
||||
@@ -49,16 +54,34 @@ class DummyStorage(Storage):
|
||||
def get(self, request_id: str) -> RequestMetrics | None:
|
||||
return None
|
||||
|
||||
def query(self, **kwargs) -> list[RequestMetrics]:
|
||||
def query(
|
||||
self,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
model: str | None = None,
|
||||
mode: str | None = None,
|
||||
limit: int = 100,
|
||||
offset: int = 0,
|
||||
) -> list[RequestMetrics]:
|
||||
return []
|
||||
|
||||
def count(self, **kwargs) -> int:
|
||||
def count(
|
||||
self,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
model: str | None = None,
|
||||
mode: str | None = None,
|
||||
) -> int:
|
||||
return 0
|
||||
|
||||
def iter_all(self):
|
||||
def iter_all(self) -> Iterator[RequestMetrics]:
|
||||
return iter(())
|
||||
|
||||
def get_summary_stats(self, **kwargs) -> dict[str, int]:
|
||||
def get_summary_stats(
|
||||
self,
|
||||
start_time: datetime | None = None,
|
||||
end_time: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
def close(self) -> None:
|
||||
@@ -81,7 +104,9 @@ def test_storage_base_context_manager_calls_close() -> None:
|
||||
assert Storage.close(storage) is None
|
||||
|
||||
|
||||
def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: Path) -> None:
|
||||
def test_create_storage_builtin_entrypoint_and_fallback(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
sqlite_storage = create_storage(f"sqlite://{tmp_path}\\metrics.db")
|
||||
jsonl_storage = create_storage(f"jsonl://{tmp_path}\\metrics.jsonl")
|
||||
assert isinstance(sqlite_storage, SQLiteStorage)
|
||||
@@ -101,7 +126,7 @@ def test_create_storage_builtin_entrypoint_and_fallback(monkeypatch, tmp_path: P
|
||||
class FakeEntryPoint:
|
||||
name = "custom"
|
||||
|
||||
def load(self):
|
||||
def load(self) -> Callable[[str], DummyStorage]:
|
||||
return lambda store_url: created
|
||||
|
||||
monkeypatch.setattr(
|
||||
@@ -160,7 +185,7 @@ def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> Non
|
||||
assert storage.get("missing") is None
|
||||
|
||||
results = storage.query(start_time=now - timedelta(hours=1, minutes=30), offset=1, limit=1)
|
||||
assert [item.request_id for item in results] == ["three"]
|
||||
assert [item.request_id for item in results] == ["two"]
|
||||
assert storage.query(model="claude")[0].request_id == "two"
|
||||
assert storage.query(mode="optimize")[0].request_id == "two"
|
||||
assert storage.query(end_time=now - timedelta(hours=1, minutes=30))[0].request_id == "one"
|
||||
@@ -183,6 +208,59 @@ def test_jsonl_storage_round_trip_query_count_and_summary(tmp_path: Path) -> Non
|
||||
storage.close()
|
||||
|
||||
|
||||
def test_jsonl_query_paging_matches_sqlite_newest_first(tmp_path: Path) -> None:
|
||||
"""JSONL query pages must be the newest-first window SQLite returns (issue #3822)."""
|
||||
jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
base = datetime(2026, 4, 23, 12, 0, 0)
|
||||
rows = [
|
||||
_metrics(
|
||||
f"req-{i:03d}",
|
||||
base + timedelta(seconds=i),
|
||||
model="claude" if i % 2 == 0 else "gpt-4o",
|
||||
mode="audit" if i % 3 == 0 else "optimize",
|
||||
)
|
||||
for i in range(250)
|
||||
]
|
||||
for row in rows:
|
||||
jsonl_storage.save(row)
|
||||
sqlite_storage.save(row)
|
||||
|
||||
def ids(metrics: list[RequestMetrics]) -> list[str]:
|
||||
return [item.request_id for item in metrics]
|
||||
|
||||
newest_first = [
|
||||
row.request_id for row in sorted(rows, key=lambda item: item.timestamp, reverse=True)
|
||||
]
|
||||
|
||||
for kw in (
|
||||
{"limit": 5, "offset": 0},
|
||||
{"limit": 5, "offset": 5},
|
||||
{"limit": 5, "offset": 10},
|
||||
{"model": "claude", "limit": 5, "offset": 0},
|
||||
{"model": "claude", "limit": 5, "offset": 5},
|
||||
{"mode": "audit", "limit": 4, "offset": 2},
|
||||
{"model": "claude", "mode": "audit", "limit": 3, "offset": 1},
|
||||
):
|
||||
jsonl_page = jsonl_storage.query(**kw)
|
||||
assert ids(jsonl_page) == ids(sqlite_storage.query(**kw)), kw
|
||||
stamps = [item.timestamp for item in jsonl_page]
|
||||
assert stamps == sorted(stamps, reverse=True), kw
|
||||
|
||||
assert ids(jsonl_storage.query(limit=5)) == newest_first[:5]
|
||||
assert ids(jsonl_storage.query(limit=5))[0] == "req-249"
|
||||
|
||||
page1 = ids(jsonl_storage.query(limit=5, offset=0))
|
||||
page2 = ids(jsonl_storage.query(limit=5, offset=5))
|
||||
page3 = ids(jsonl_storage.query(limit=5, offset=10))
|
||||
assert set(page1).isdisjoint(page2)
|
||||
assert set(page2).isdisjoint(page3)
|
||||
assert set(page1).isdisjoint(page3)
|
||||
|
||||
jsonl_storage.close()
|
||||
sqlite_storage.close()
|
||||
|
||||
|
||||
def test_jsonl_storage_handles_missing_file_malformed_lines_and_defaults(tmp_path: Path) -> None:
|
||||
path = tmp_path / "events.jsonl"
|
||||
storage = JSONLStorage(str(path))
|
||||
@@ -278,7 +356,7 @@ def test_sqlite_storage_round_trip_filters_summary_and_defaults(tmp_path: Path)
|
||||
|
||||
|
||||
def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypoint(
|
||||
monkeypatch, tmp_path: Path
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
storage = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
first = storage._get_conn()
|
||||
@@ -292,3 +370,140 @@ def test_sqlite_storage_get_conn_reuses_connection_and_create_storage_entrypoint
|
||||
lambda group: [SimpleNamespace(name="custom", load=lambda: lambda url: created)],
|
||||
)
|
||||
assert create_storage("custom://db") is created
|
||||
|
||||
|
||||
def test_query_pagination_contract_validates_and_returns_empty_pages(tmp_path: Path) -> None:
|
||||
"""Negative limit/offset raise ValueError; zero limit and past-end offsets page empty."""
|
||||
jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
now = datetime(2026, 4, 23, 12, 0, 0)
|
||||
rows = [
|
||||
_metrics("one", now - timedelta(hours=2)),
|
||||
_metrics("two", now - timedelta(hours=1)),
|
||||
_metrics("three", now),
|
||||
]
|
||||
for row in rows:
|
||||
jsonl_storage.save(row)
|
||||
sqlite_storage.save(row)
|
||||
|
||||
for storage in (jsonl_storage, sqlite_storage):
|
||||
with pytest.raises(ValueError):
|
||||
storage.query(limit=-1)
|
||||
with pytest.raises(ValueError):
|
||||
storage.query(offset=-1)
|
||||
with pytest.raises(ValueError):
|
||||
storage.query(limit=-5, offset=-5)
|
||||
|
||||
assert storage.query(limit=0) == []
|
||||
assert storage.query(limit=0, offset=0) == []
|
||||
assert storage.query(limit=0, offset=9) == []
|
||||
assert storage.query(limit=5, offset=3) == []
|
||||
assert storage.query(limit=5, offset=10) == []
|
||||
assert [m.request_id for m in storage.query(limit=2, offset=0)] == ["three", "two"]
|
||||
assert [m.request_id for m in storage.query(limit=2, offset=2)] == ["one"]
|
||||
|
||||
jsonl_storage.close()
|
||||
sqlite_storage.close()
|
||||
|
||||
|
||||
def test_query_equal_timestamp_pages_match_across_backends(tmp_path: Path) -> None:
|
||||
"""Timestamp ties break on request_id DESC and pages agree page-for-page."""
|
||||
jsonl_storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
sqlite_storage = SQLiteStorage(str(tmp_path / "metrics.db"))
|
||||
base = datetime(2026, 4, 23, 12, 0, 0)
|
||||
|
||||
rows: list[RequestMetrics] = []
|
||||
# Rows are appended in a shuffled order on purpose: the contract orders by
|
||||
# timestamp DESC then request_id DESC, never by append order.
|
||||
for group in (1, 2, 3):
|
||||
for suffix in (3, 1, 5, 2, 4):
|
||||
rows.append(
|
||||
_metrics(
|
||||
f"g{group}-{suffix}",
|
||||
base + timedelta(minutes=group),
|
||||
model="claude" if suffix % 2 == 0 else "gpt-4o",
|
||||
)
|
||||
)
|
||||
for row in rows:
|
||||
jsonl_storage.save(row)
|
||||
sqlite_storage.save(row)
|
||||
|
||||
expected = [f"g{group}-{suffix}" for group in (3, 2, 1) for suffix in (5, 4, 3, 2, 1)]
|
||||
|
||||
for offset in (0, 4, 8, 12):
|
||||
page_expected = expected[offset : offset + 4]
|
||||
jsonl_page = jsonl_storage.query(limit=4, offset=offset)
|
||||
sqlite_page = sqlite_storage.query(limit=4, offset=offset)
|
||||
assert [m.request_id for m in jsonl_page] == page_expected, offset
|
||||
assert [m.request_id for m in sqlite_page] == page_expected, offset
|
||||
stamps = [m.timestamp for m in jsonl_page]
|
||||
assert stamps == sorted(stamps, reverse=True)
|
||||
# The same window must be stable across repeated queries.
|
||||
assert [m.request_id for m in jsonl_storage.query(limit=4, offset=offset)] == page_expected
|
||||
assert [m.request_id for m in sqlite_storage.query(limit=4, offset=offset)] == page_expected
|
||||
|
||||
# Filtered ties must stay consistent too: a page size of 3 splits the g2
|
||||
# pair across pages so the tie-breaker decides both sides of the boundary.
|
||||
filtered_expected = [f"g{group}-{suffix}" for group in (3, 2, 1) for suffix in (4, 2)]
|
||||
for offset in (0, 3):
|
||||
page_expected = filtered_expected[offset : offset + 3]
|
||||
assert [
|
||||
m.request_id for m in jsonl_storage.query(model="claude", limit=3, offset=offset)
|
||||
] == page_expected
|
||||
assert [
|
||||
m.request_id for m in sqlite_storage.query(model="claude", limit=3, offset=offset)
|
||||
] == page_expected
|
||||
|
||||
jsonl_storage.close()
|
||||
sqlite_storage.close()
|
||||
|
||||
|
||||
def test_jsonl_query_retains_at_most_requested_window(
|
||||
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
|
||||
) -> None:
|
||||
"""query() must select a bounded window instead of materializing every match."""
|
||||
storage = JSONLStorage(str(tmp_path / "metrics.jsonl"))
|
||||
base = datetime(2026, 4, 23, 12, 0, 0)
|
||||
rows = [
|
||||
_metrics(
|
||||
f"req-{i:03d}",
|
||||
base + timedelta(seconds=i),
|
||||
model="claude" if i % 2 == 0 else "gpt-4o",
|
||||
)
|
||||
for i in range(60)
|
||||
]
|
||||
for row in rows:
|
||||
storage.save(row)
|
||||
|
||||
limit, offset = 5, 10
|
||||
window = offset + limit
|
||||
matches = [row for row in rows if row.model == "claude"]
|
||||
assert len(matches) > window
|
||||
|
||||
original_iter_all = storage.iter_all
|
||||
refs: list[weakref.ReferenceType[RequestMetrics]] = []
|
||||
kept: list[bool] = []
|
||||
retained: list[int] = []
|
||||
|
||||
def tracked_iter_all() -> Iterator[RequestMetrics]:
|
||||
for metrics in original_iter_all():
|
||||
matched = metrics.model == "claude"
|
||||
# Count matching records yielded earlier that the query still
|
||||
# holds; the record being handed over is not in refs yet.
|
||||
retained.append(sum(1 for keep, ref in zip(kept, refs) if keep and ref() is not None))
|
||||
kept.append(matched)
|
||||
refs.append(weakref.ref(metrics))
|
||||
yield metrics
|
||||
|
||||
monkeypatch.setattr(storage, "iter_all", tracked_iter_all)
|
||||
|
||||
page = storage.query(model="claude", limit=limit, offset=offset)
|
||||
|
||||
expected = sorted(matches, key=lambda m: (m.timestamp, m.request_id), reverse=True)
|
||||
assert [m.request_id for m in page] == [m.request_id for m in expected[offset : offset + limit]]
|
||||
assert len(refs) == len(rows)
|
||||
# Selection may hold the requested window plus at most one matching record
|
||||
# still in flight while the next record is pulled; it must never retain the
|
||||
# whole match set.
|
||||
assert max(retained) <= window + 1
|
||||
assert max(retained) < len(matches)
|
||||
|
||||
Reference in New Issue
Block a user