fix(overflow): redact diagnostic map identifiers

This commit is contained in:
shehjad-dev
2026-09-25 23:14:15 +06:00
parent efcce8a292
commit cd2a25a110
3 changed files with 33 additions and 3 deletions
+3 -2
View File
@@ -423,8 +423,9 @@ pays the aggregator's real price, 0% markup, disclosed in-band when it ships (st
- **`verify.py`** + `treg-worker overflow verify` — the weekly re-verify: one cheap call per
route through the aggregator (and, when we hold the vendor key, directly), compare the shape
fingerprint (keys and list/leaf markers, values ignored), stamp `last_verified_at` or disable
with the reason. A mismatch prints a bounded key-only structural diff, never response values, so
an operator can distinguish omitted metadata from an incompatible body without exposing PII.
with the reason. A mismatch prints a bounded key-only structural diff, never response values;
identifier-shaped map keys are replaced before logging or persistence, so an operator can
distinguish omitted metadata from an incompatible body without exposing PII.
Two per-route price caps and one run budget: a route that is enabled or was
stamped before is a **renewal**, held to `--renew-max-usd` (default $1); a never-verified pair is
**discovery**, visited only under `--all` and held to `--max-usd` (default 2¢). Renewals go first,
+21 -1
View File
@@ -9,7 +9,9 @@ this module reads no settings itself.
from __future__ import annotations
import ipaddress
import json
import re
from dataclasses import dataclass
from datetime import datetime
@@ -21,6 +23,24 @@ from ...infra.upstream.aggregators import (AGGREGATOR_SIDE, VENDOR_DRY, VENDOR_R
from . import signatures
_SAFE_SHAPE_KEY = re.compile(r"[A-Za-z_][A-Za-z0-9_-]{0,63}")
_UUID_SHAPE_KEY = re.compile(
r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[1-5][0-9a-fA-F]{3}-[89abAB][0-9a-fA-F]{3}-[0-9a-fA-F]{12}"
)
def _safe_shape_key(key) -> str:
"""Keep ordinary schema field names; never persist identifier-shaped map keys."""
text = str(key)
if not _SAFE_SHAPE_KEY.fullmatch(text) or _UUID_SHAPE_KEY.fullmatch(text):
return "<identifier>"
try:
ipaddress.ip_address(text)
except ValueError:
return text
return "<identifier>"
def shape(obj, depth: int = 0):
if depth > 6:
return "…"
@@ -43,7 +63,7 @@ def _shape_paths(value, path: str = "$") -> set[str]:
if isinstance(value, dict):
paths = {f"{path}:object"}
for key, child in value.items():
paths |= _shape_paths(child, f"{path}.{key}")
paths |= _shape_paths(child, f"{path}.{_safe_shape_key(key)}")
return paths
if isinstance(value, list):
return {f"{path}:list"} | (_shape_paths(value[0], f"{path}[]") if value else set())
+9
View File
@@ -468,6 +468,15 @@ def test_shape_fingerprint_ignores_values_but_not_structure():
diff = V.shape_difference(a, c)
assert "$.data.score:leaf" in diff and "$.data.sources:list" in diff
assert "a@x.io" not in diff
keyed = V.shape_difference(
b'{"results":{"ada@acme.com":{"score":1,"title":"x"}}}',
b'{"results":{"ada@acme.com":{"score":1}}}',
)
assert "ada@acme.com" not in keyed
assert "$.results.<identifier>.title:leaf" in keyed
assert V._safe_shape_key("acme.com") == "<identifier>"
assert V._safe_shape_key("550e8400-e29b-41d4-a716-446655440000") == "<identifier>"
assert V._safe_shape_key("title") == "title"
def test_shape_empty_vs_nonempty_list_differs_but_both_empty_match():