fix: [BUG] Volatile SQLite metrics stores lose their schema before the first save (#4633)

<!-- oss-harness:v2 -->
Fixes #4262.

Focused Python fix for: [BUG] Volatile SQLite metrics stores lose their
schema before the first save

Changed implementation: headroom/storage/sqlite.py.
Regression tests: tests/test_storage/test_sqlite.py.

Verification (working directory `.`): `.venv/bin/python -m pytest -q -x
-p no:cacheprovider -- /workspace/tests/test_storage/test_sqlite.py`
- Patched code: 35 tests, zero failures/errors.
- Original code with the same regression tests: 1 failures, zero
collection/setup errors.
- Base commit: `12dca26aa11d99b3af6c6d68cdf96ff90ef415aa`.

AI-assisted contribution: OpenCode using Autoloops wrote this patch. The
automated harness independently ran the tests listed above. No human
review is claimed.

<!-- devin-review-badge-begin -->

---

<a href="https://app.devin.ai/review/headroomlabs-ai/headroom/pull/4633"
target="_blank"><picture><source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-devin-review-dark.svg?v=4"><img
src="https://static.devin.ai/assets/gh-devin-review-light.svg?v=4"
alt="Devin Review"></picture></a>
<!-- devin-review-badge-end -->

### Maintainer real behavior proof

Reviewed `55a6c8e3807c3d59da81a457514c82d4a3b48032` against main
`12dca26aa11d99b3af6c6d68cdf96ff90ef415aa` in isolated Windows
worktrees, Python 3.12.13. Current main is an ancestor; no branch update
was needed.

Executed `.venv/Scripts/python.exe -m pytest
tests/test_storage/test_sqlite.py -q -k 'not test_creates_db_file'
--disable-warnings`: **34 passed**, including disk migration/concurrency
and volatile-store regressions. Ruff and `git diff --check` passed. The
excluded test fails before exercising storage because its fixture holds
the temporary file open on Windows; the identical WinError 32 was
reproduced independently on current main.

Also ran a direct API smoke script, separately loading `git show
upstream/main:headroom/storage/sqlite.py` and `git show
HEAD:headroom/storage/sqlite.py`. For each of
`SQLiteStorage(':memory:')` and `SQLiteStorage('')`, it saved one
`RequestMetrics` row, checked equality with `get`, `count == 1`, one
result from `query`, `get_summary_stats()['total_requests'] == 1`, then
closed and verified a recreated store has zero rows. **Main: first save
raises `OperationalError('no such table: requests')` for both paths. PR:
all operations pass for both paths.**

Limits: this is a direct local storage API exercise, not a running proxy
or production-load qualification. Hosted CI's four test shards and
applicable checks passed on the exact reviewed head. No release or merge
performed.
This commit is contained in:
Nandana Dileep
2026-11-04 12:28:05 -06:00
committed by GitHub
parent 12dca26aa1
commit d8ff0fe9d2
2 changed files with 131 additions and 59 deletions
+76 -59
View File
@@ -37,9 +37,78 @@ class SQLiteStorage(Storage):
db_path: Path to SQLite database file.
"""
self.db_path = db_path
self._ensure_db_exists()
# Volatile databases (":memory:" and the empty string) belong to the
# connection that opens them: a schema created on a throwaway
# connection is gone once that connection closes, so the serving
# connection must create its own. Disk-backed files still get their
# eager initialization here.
self._volatile = db_path == "" or db_path == ":memory:"
if not self._volatile:
self._ensure_db_exists()
self._conn: sqlite3.Connection | None = None
@staticmethod
def _create_schema(cursor: sqlite3.Cursor) -> None:
"""Create the requests table, indices, and migrate legacy columns."""
cursor.execute("""
CREATE TABLE IF NOT EXISTS requests (
id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
model TEXT NOT NULL,
stream INTEGER NOT NULL,
mode TEXT NOT NULL,
tokens_input_before INTEGER NOT NULL,
tokens_input_after INTEGER NOT NULL,
tokens_output INTEGER,
block_breakdown TEXT NOT NULL,
waste_signals TEXT NOT NULL,
stable_prefix_hash TEXT,
cache_alignment_score REAL,
cached_tokens INTEGER,
cache_optimizer_used TEXT,
cache_optimizer_strategy TEXT,
cacheable_tokens INTEGER DEFAULT 0,
breakpoints_inserted INTEGER DEFAULT 0,
estimated_cache_hit INTEGER DEFAULT 0,
estimated_savings_percent REAL DEFAULT 0.0,
semantic_cache_hit INTEGER DEFAULT 0,
transforms_applied TEXT NOT NULL,
tool_units_dropped INTEGER DEFAULT 0,
turns_dropped INTEGER DEFAULT 0,
messages_hash TEXT,
error TEXT
)
""")
# Upgrade databases created before the cache optimizer columns
# existed, preserving existing rows. On disk the reserved lock in
# _ensure_db_exists serializes this check-and-alter against concurrent
# constructors.
cursor.execute("PRAGMA table_info(requests)")
existing_columns = {row[1] for row in cursor.fetchall()}
for column, ddl in _CACHE_OPTIMIZER_COLUMNS:
if column not in existing_columns:
try:
cursor.execute( # nosec B608
f"ALTER TABLE requests ADD COLUMN {column} {ddl}"
)
except sqlite3.OperationalError as exc:
# Another connection may have added the column
# between our schema read and this ALTER.
if "duplicate column name" not in str(exc).lower():
raise
# Create indices
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_timestamp ON requests(timestamp)
""")
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_model ON requests(model)
""")
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_mode ON requests(mode)
""")
def _ensure_db_exists(self) -> None:
"""Create database and tables if they don't exist."""
path = Path(self.db_path)
@@ -53,64 +122,7 @@ class SQLiteStorage(Storage):
cursor = conn.cursor()
cursor.execute("BEGIN IMMEDIATE")
try:
cursor.execute("""
CREATE TABLE IF NOT EXISTS requests (
id TEXT PRIMARY KEY,
timestamp TEXT NOT NULL,
model TEXT NOT NULL,
stream INTEGER NOT NULL,
mode TEXT NOT NULL,
tokens_input_before INTEGER NOT NULL,
tokens_input_after INTEGER NOT NULL,
tokens_output INTEGER,
block_breakdown TEXT NOT NULL,
waste_signals TEXT NOT NULL,
stable_prefix_hash TEXT,
cache_alignment_score REAL,
cached_tokens INTEGER,
cache_optimizer_used TEXT,
cache_optimizer_strategy TEXT,
cacheable_tokens INTEGER DEFAULT 0,
breakpoints_inserted INTEGER DEFAULT 0,
estimated_cache_hit INTEGER DEFAULT 0,
estimated_savings_percent REAL DEFAULT 0.0,
semantic_cache_hit INTEGER DEFAULT 0,
transforms_applied TEXT NOT NULL,
tool_units_dropped INTEGER DEFAULT 0,
turns_dropped INTEGER DEFAULT 0,
messages_hash TEXT,
error TEXT
)
""")
# Upgrade databases created before the cache optimizer columns
# existed, preserving existing rows. The reserved lock above
# serializes this check-and-alter against concurrent constructors.
cursor.execute("PRAGMA table_info(requests)")
existing_columns = {row[1] for row in cursor.fetchall()}
for column, ddl in _CACHE_OPTIMIZER_COLUMNS:
if column not in existing_columns:
try:
cursor.execute( # nosec B608
f"ALTER TABLE requests ADD COLUMN {column} {ddl}"
)
except sqlite3.OperationalError as exc:
# Another connection may have added the column
# between our schema read and this ALTER.
if "duplicate column name" not in str(exc).lower():
raise
# Create indices
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_timestamp ON requests(timestamp)
""")
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_model ON requests(model)
""")
cursor.execute("""
CREATE INDEX IF NOT EXISTS idx_mode ON requests(mode)
""")
self._create_schema(cursor)
conn.commit()
except Exception:
conn.rollback()
@@ -123,6 +135,11 @@ class SQLiteStorage(Storage):
if self._conn is None:
self._conn = sqlite3.connect(self.db_path)
self._conn.row_factory = sqlite3.Row
if self._volatile:
# The throwaway connection in __init__ cannot initialize a
# volatile database, so build the schema on this connection.
self._create_schema(self._conn.cursor())
self._conn.commit()
return self._conn
def save(self, metrics: RequestMetrics) -> None:
+55
View File
@@ -848,6 +848,61 @@ class TestConcurrency:
storage.close()
class TestVolatileDatabases:
"""Volatile stores must initialize their schema on the serving connection (#4262)."""
@pytest.mark.parametrize("db_path", [":memory:", ""])
def test_save_and_read_back_on_volatile_store(self, db_path):
"""The first save must not fail with 'no such table: requests'."""
storage = SQLiteStorage(db_path)
try:
metrics = RequestMetrics(
request_id="volatile-1",
timestamp=datetime(2025, 1, 6, 12, 0, 0),
model="local",
stream=False,
mode="audit",
tokens_input_before=123,
tokens_input_after=45,
block_breakdown={"system": 10},
waste_signals={},
transforms_applied=[],
)
storage.save(metrics)
assert storage.get("volatile-1") == metrics
assert storage.count() == 1
assert [m.request_id for m in storage.query()] == ["volatile-1"]
assert storage.get_summary_stats()["total_requests"] == 1
finally:
storage.close()
@pytest.mark.parametrize("db_path", [":memory:", ""])
def test_close_discards_volatile_rows(self, db_path):
"""Closing a volatile store drops its rows; the connection is recreated empty."""
storage = SQLiteStorage(db_path)
metrics = RequestMetrics(
request_id="volatile-2",
timestamp=datetime(2025, 1, 6, 12, 0, 0),
model="local",
stream=False,
mode="audit",
tokens_input_before=1000,
tokens_input_after=800,
block_breakdown={},
waste_signals={},
)
storage.save(metrics)
storage.close()
# A fresh connection to a volatile database has no rows, but the
# schema must be rebuilt so the store keeps working.
assert storage.get("volatile-2") is None
assert storage.count() == 0
storage.close()
class TestCacheOptimizerMetrics:
"""Cache-optimizer fields must survive a durable SQLite round trip (#4261)."""