mirror of
https://github.com/headroomlabs-ai/headroom.git
synced 2026-11-22 13:29:19 +08:00
fix(toin): validate a stored pattern and merge it as one step
A malformed entry could still get past the per-entry guard. A seen_instance_hashes string passes from_dict (slicing and set() both work on a str), and the foreign-instance tracking then called .append() on it outside the guard, aborting the load again. Merging an entry into an existing pattern also ran outside the guard and changed the stored pattern in place, so a value that failed part-way left it half merged. from_dict now rejects a field whose value is not the kind its annotation names (numbers accept int or float, since merges store some counts as floats) and a tracking list that is not a list of strings. Tracking and merging now run inside the per-entry guard. A merge goes into a copy of the stored pattern, with its own lists, dicts and sets, which replaces the stored one only when the merge succeeds.
This commit is contained in:
+71
-28
@@ -68,7 +68,7 @@ import threading
|
||||
import time
|
||||
import warnings
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, fields
|
||||
from typing import Any, Final
|
||||
|
||||
from .models import FieldSemantics, ToolSignature
|
||||
@@ -409,9 +409,16 @@ class ToolPattern:
|
||||
"last_updated",
|
||||
}
|
||||
filtered = {k: v for k, v in data.items() if k in valid_fields}
|
||||
# A value of the wrong kind (a string where a list belongs) would be
|
||||
# accepted as-is and then iterated or added to when patterns merge.
|
||||
for name, value in filtered.items():
|
||||
if not isinstance(value, _PATTERN_FIELD_KINDS.get(name, object)):
|
||||
raise TypeError(f"{name} is a {type(value).__name__}")
|
||||
|
||||
# Handle seen_instance_hashes (serialized without underscore prefix)
|
||||
seen_hashes = data.get("seen_instance_hashes", [])
|
||||
if not isinstance(seen_hashes, list) or not all(isinstance(h, str) for h in seen_hashes):
|
||||
raise TypeError("seen_instance_hashes is not a list of strings")
|
||||
|
||||
pattern = cls(**filtered)
|
||||
pattern._seen_instance_hashes = seen_hashes[:100] # Limit on load
|
||||
@@ -438,6 +445,38 @@ class ToolPattern:
|
||||
return pattern
|
||||
|
||||
|
||||
# What each stored ToolPattern field may hold, by the base of its annotation.
|
||||
# Merges store some counts as floats, so numeric fields accept either.
|
||||
_KIND_BY_ANNOTATION: Final[dict[str, type | tuple[type, ...]]] = {
|
||||
"str": str,
|
||||
"int": (int, float),
|
||||
"float": (int, float),
|
||||
"bool": bool,
|
||||
"list": list,
|
||||
"dict": dict,
|
||||
}
|
||||
_PATTERN_FIELD_KINDS: Final[dict[str, type | tuple[type, ...]]] = {
|
||||
f.name: _KIND_BY_ANNOTATION.get(str(f.type).split("[")[0], object) for f in fields(ToolPattern)
|
||||
}
|
||||
_PATTERN_CONTAINER_FIELDS: Final = tuple(
|
||||
f.name for f in fields(ToolPattern) if f.default_factory in (list, dict, set)
|
||||
)
|
||||
|
||||
|
||||
def _detached_copy(pattern: ToolPattern) -> ToolPattern:
|
||||
"""Copy ``pattern`` so that merging into the copy leaves the original intact.
|
||||
|
||||
``_merge_patterns`` reassigns scalars and adds to the pattern's lists, dicts
|
||||
and sets, so those containers are the parts that need their own copies.
|
||||
"""
|
||||
import copy
|
||||
|
||||
clone = copy.copy(pattern)
|
||||
for name in _PATTERN_CONTAINER_FIELDS:
|
||||
setattr(clone, name, getattr(pattern, name).copy())
|
||||
return clone
|
||||
|
||||
|
||||
@dataclass
|
||||
class TOINConfig:
|
||||
"""Configuration for the Tool Output Intelligence Network."""
|
||||
@@ -1401,41 +1440,45 @@ class ToolIntelligenceNetwork:
|
||||
first_error: Exception | None = None
|
||||
with self._lock:
|
||||
for serialized_key, pattern_dict in patterns_data.items():
|
||||
# Each entry is parsed, tracked and merged into a pattern that is
|
||||
# only stored once all of that succeeds. One unreadable entry must
|
||||
# not abort the rest (the store that did load would be saved over
|
||||
# the whole file) or leave a half-merged pattern behind.
|
||||
try:
|
||||
key = _deserialize_pattern_key(serialized_key)
|
||||
imported = ToolPattern.from_dict(pattern_dict)
|
||||
self._sanitize_loaded_pattern(imported)
|
||||
# Make sure dataclass fields agree with the dict key — pre-F3
|
||||
# dumps don't carry tenant_key on the pattern, and pre-B5
|
||||
# dumps don't carry auth_mode/model_family either; promote
|
||||
# from the (possibly default) key.
|
||||
imported.tenant_key = key[0]
|
||||
imported.auth_mode = key[1]
|
||||
imported.model_family = key[2]
|
||||
|
||||
existing = self._patterns.get(key)
|
||||
if existing is not None:
|
||||
# Merge with existing
|
||||
pattern = _detached_copy(existing)
|
||||
self._merge_patterns(pattern, imported)
|
||||
else:
|
||||
# Add new pattern - need to track source instance
|
||||
pattern = imported
|
||||
|
||||
# For NEW patterns from another instance, track the source in
|
||||
# _seen_instance_hashes so user_count reflects cross-user data
|
||||
if source_instance != self._instance_id:
|
||||
if source_instance not in pattern._seen_instance_hashes:
|
||||
# Limit storage to 100 unique instances to bound memory
|
||||
if len(pattern._seen_instance_hashes) < 100:
|
||||
pattern._seen_instance_hashes.append(source_instance)
|
||||
# CRITICAL: Always increment user_count (even after cap)
|
||||
pattern.user_count += 1
|
||||
except (AttributeError, TypeError, ValueError) as e:
|
||||
# One unreadable entry must not abort the rest: the store
|
||||
# that did load would be saved over the whole file.
|
||||
skipped += 1
|
||||
first_error = first_error or e
|
||||
continue
|
||||
# Make sure dataclass fields agree with the dict key — pre-F3
|
||||
# dumps don't carry tenant_key on the pattern, and pre-B5
|
||||
# dumps don't carry auth_mode/model_family either; promote
|
||||
# from the (possibly default) key.
|
||||
imported.tenant_key = key[0]
|
||||
imported.auth_mode = key[1]
|
||||
imported.model_family = key[2]
|
||||
|
||||
if key in self._patterns:
|
||||
# Merge with existing
|
||||
self._merge_patterns(self._patterns[key], imported)
|
||||
else:
|
||||
# Add new pattern - need to track source instance
|
||||
self._patterns[key] = imported
|
||||
|
||||
# For NEW patterns from another instance, track the source in
|
||||
# _seen_instance_hashes so user_count reflects cross-user data
|
||||
if source_instance != self._instance_id:
|
||||
pattern = self._patterns[key]
|
||||
if source_instance not in pattern._seen_instance_hashes:
|
||||
# Limit storage to 100 unique instances to bound memory
|
||||
if len(pattern._seen_instance_hashes) < 100:
|
||||
pattern._seen_instance_hashes.append(source_instance)
|
||||
# CRITICAL: Always increment user_count (even after cap)
|
||||
pattern.user_count += 1
|
||||
self._patterns[key] = pattern
|
||||
|
||||
self._prune_patterns_locked()
|
||||
self._dirty = True
|
||||
|
||||
@@ -116,6 +116,9 @@ def _skip_warnings(caplog) -> list[logging.LogRecord]:
|
||||
lambda entry: {**entry, "field_semantics": {"field_1": "not a dict"}},
|
||||
lambda entry: {**entry, "query_pattern_frequency": "not a dict"},
|
||||
lambda entry: {**entry, "query_pattern_frequency": {"status:*": "many"}},
|
||||
lambda entry: {**entry, "seen_instance_hashes": "bad"},
|
||||
lambda entry: {**entry, "commonly_retrieved_fields": "abc"},
|
||||
lambda entry: {**entry, "sample_size": "x"},
|
||||
lambda entry: "not a pattern",
|
||||
],
|
||||
ids=[
|
||||
@@ -123,6 +126,9 @@ def _skip_warnings(caplog) -> list[logging.LogRecord]:
|
||||
"bad_field_semantics",
|
||||
"bad_query_frequencies",
|
||||
"bad_query_count",
|
||||
"tracking_list_is_a_string",
|
||||
"list_field_is_a_string",
|
||||
"count_is_a_string",
|
||||
"not_a_dict",
|
||||
],
|
||||
)
|
||||
@@ -149,6 +155,70 @@ def test_malformed_pattern_does_not_drop_the_rest_of_the_store(tmp_path, caplog,
|
||||
assert len(saved) == 5
|
||||
|
||||
|
||||
def test_foreign_store_keeps_the_healthy_entries_after_a_malformed_one(tmp_path, caplog) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
data = _saved_store(path, 3)
|
||||
keys = list(data["patterns"])
|
||||
# A string tracking list used to pass from_dict and then fail on .append()
|
||||
# while tracking the foreign source, outside the per-entry guard.
|
||||
data["patterns"][keys[0]]["seen_instance_hashes"] = "bad"
|
||||
data["instance_id"] = "foreign-instance"
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.telemetry.toin"):
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=0))
|
||||
|
||||
assert len(toin._patterns) == 2
|
||||
[warning] = _skip_warnings(caplog)
|
||||
assert warning.skipped == 1
|
||||
assert all("foreign-instance" in p._seen_instance_hashes for p in toin._patterns.values())
|
||||
|
||||
toin.save()
|
||||
saved = json.loads(path.read_text(encoding="utf-8"))["patterns"]
|
||||
assert set(saved) == set(keys[1:])
|
||||
|
||||
|
||||
def test_failed_merge_leaves_the_existing_pattern_unchanged(caplog) -> None:
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=""))
|
||||
for index in range(2):
|
||||
_record(toin, index)
|
||||
exported = toin.export_patterns()
|
||||
keys = list(exported["patterns"])
|
||||
# to_dict() shares the pattern's containers, so snapshot through JSON.
|
||||
before = {key: json.loads(json.dumps(p.to_dict())) for key, p in toin._patterns.items()}
|
||||
|
||||
bad = dict(exported["patterns"][keys[0]])
|
||||
# Passes from_dict, then fails part-way through _merge_patterns: the counts
|
||||
# and the "field_new" key are already added when the "x" value raises.
|
||||
bad["field_retrieval_frequency"] = {"field_new": 1, "field_0": "x"}
|
||||
healthy = dict(exported["patterns"][keys[1]])
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.telemetry.toin"):
|
||||
toin.import_patterns(
|
||||
{"instance_id": "foreign-instance", "patterns": {keys[0]: bad, keys[1]: healthy}}
|
||||
)
|
||||
|
||||
[warning] = _skip_warnings(caplog)
|
||||
assert warning.skipped == 1
|
||||
first, second = (toin._patterns[key] for key in before)
|
||||
assert json.loads(json.dumps(first.to_dict())) == before[next(iter(before))]
|
||||
assert second.total_compressions == 2
|
||||
|
||||
|
||||
def test_float_counts_written_by_merges_still_load(tmp_path, caplog) -> None:
|
||||
# _merge_patterns stores optimal_max_items as a weighted float.
|
||||
path = tmp_path / "toin.json"
|
||||
data = _saved_store(path, 2)
|
||||
for entry in data["patterns"].values():
|
||||
entry["optimal_max_items"] = 37.5
|
||||
path.write_text(json.dumps(data), encoding="utf-8")
|
||||
|
||||
with caplog.at_level(logging.WARNING, logger="headroom.telemetry.toin"):
|
||||
toin = ToolIntelligenceNetwork(TOINConfig(storage_path=str(path), auto_save_interval=0))
|
||||
|
||||
assert len(toin._patterns) == 2
|
||||
assert _skip_warnings(caplog) == []
|
||||
|
||||
|
||||
def test_valid_store_loads_without_a_skip_warning(tmp_path, caplog) -> None:
|
||||
path = tmp_path / "toin.json"
|
||||
_saved_store(path, 5)
|
||||
|
||||
Reference in New Issue
Block a user