mirror of
https://github.com/superdesigndev/treg.git
synced 2026-10-02 03:24:35 +08:00
fix(catalog): verify Firecrawl routing and bounded search price
This commit is contained in:
@@ -2771,7 +2771,7 @@ adapters:
|
||||
out: {results: urls, count: "len(urls)"}
|
||||
miss: "coalesce(urls, []) == []"
|
||||
firecrawl.web.map:
|
||||
accepts: [[url], [url, q]]
|
||||
accepts: [[url, q], [url]]
|
||||
in: {url: body.url, q: body.search}
|
||||
in_expr: {body.limit: limit}
|
||||
out: {results: links, count: "len(links)"}
|
||||
|
||||
@@ -72,8 +72,7 @@ endpoints:
|
||||
test_request: {body: {query: "Firecrawl API documentation", limit: 3}}
|
||||
cost:
|
||||
type: per_success
|
||||
table:
|
||||
- {when: {body.limit: 10}, value: 2}
|
||||
value: 2
|
||||
fallback: {value: 2, note: "The curated request allows at most ten search results and no scrape add-on."}
|
||||
currency: credit
|
||||
unit: call
|
||||
@@ -140,6 +139,7 @@ endpoints:
|
||||
test_request: {body: {url: "https://example.com", limit: 1, maxDiscoveryDepth: 0, scrapeOptions: {parsers: []}}}
|
||||
cost:
|
||||
type: per_success
|
||||
# Table rows require `when`; enumerate limits to hold requested pages rather than all ten.
|
||||
table:
|
||||
- {when: {body.limit: 1}, value: 1, times: body.limit}
|
||||
- {when: {body.limit: 2}, value: 1, times: body.limit}
|
||||
|
||||
+57
-1
@@ -27,7 +27,7 @@ from treg.domain.catalog.routing.plan import Candidate, cost_at, rank
|
||||
from treg.infra.catalog_observations import CachedEndpointObservationReader
|
||||
from treg.models import CallRecord, Hold, LedgerEntry, OverflowRoute
|
||||
|
||||
from test_marketplace_call import _balance, platform_on # noqa: F401
|
||||
from test_marketplace_call import _balance, firecrawl_platform_on, platform_on # noqa: F401
|
||||
|
||||
ROUTED = "treg.people.email.find"
|
||||
|
||||
@@ -119,6 +119,62 @@ def test_every_shipped_adapter_round_trips_its_fixture():
|
||||
assert ad.is_miss({"email": None}) and not ad.is_miss({"email": "x"})
|
||||
|
||||
|
||||
def test_firecrawl_web_adapters_are_verified_routed_children():
|
||||
cat = catalog_store.load()
|
||||
for parent, child in (
|
||||
("treg.web.search", "firecrawl.web.search"),
|
||||
("treg.web.extract", "firecrawl.web.scrape"),
|
||||
("treg.web.map", "firecrawl.web.map"),
|
||||
):
|
||||
adapter = cat.adapters[child]
|
||||
assert adapter.verified, (child, adapter.verify_note)
|
||||
assert not adapter.verify_note, child
|
||||
assert child in cat.by_id[parent]["routed_children"], (parent, child)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("parent", "child", "routed_input", "upstream_body", "expected_body", "expected_cost"), [
|
||||
(
|
||||
"treg.web.search", "firecrawl.web.search",
|
||||
{"q": "example", "limit": 3},
|
||||
{"success": True, "data": {"web": [{"title": "Example", "url": "https://example.com"}]}, "creditsUsed": 2},
|
||||
{"query": "example", "limit": 3}, 10_000,
|
||||
),
|
||||
(
|
||||
"treg.web.extract", "firecrawl.web.scrape",
|
||||
{"url": "https://example.com"},
|
||||
{"success": True, "data": {"markdown": "# Example", "metadata": {"sourceURL": "https://example.com"}}},
|
||||
{"url": "https://example.com", "formats": ["markdown"], "parsers": []}, 5_000,
|
||||
),
|
||||
(
|
||||
"treg.web.map", "firecrawl.web.map",
|
||||
{"url": "https://example.com", "q": "docs", "limit": 3},
|
||||
{"success": True, "links": [{"url": "https://example.com/docs"}]},
|
||||
{"url": "https://example.com", "search": "docs", "limit": 3}, 5_000,
|
||||
),
|
||||
])
|
||||
async def test_firecrawl_web_routed_calls_serve_and_settle(
|
||||
clients, monkeypatch, firecrawl_platform_on,
|
||||
parent, child, routed_input, upstream_body, expected_body, expected_cost,
|
||||
):
|
||||
seen = []
|
||||
monkeypatch.setattr(call_service, "relay", _relay_by_provider({
|
||||
"firecrawl": [(200, upstream_body)],
|
||||
}, seen))
|
||||
|
||||
before = await _balance(clients)
|
||||
response = await clients.post(
|
||||
f"/call/{parent}", json=routed_input,
|
||||
headers={"X-Treg-Route-Prefer": "firecrawl", "X-Treg-Route-Waterfall": "0"},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
doc = response.json()
|
||||
assert doc["_treg"]["served_by"] == child
|
||||
assert doc["_treg"]["charged_micro"] == expected_cost
|
||||
assert response.headers["X-Treg-Route-Outcome"] == "hit"
|
||||
assert before - await _balance(clients) == expected_cost
|
||||
assert len(seen) == 1 and seen[0][3] == expected_body
|
||||
|
||||
|
||||
async def test_tavily_routed_empty_search_is_a_paid_miss_then_falls_through(
|
||||
clients, monkeypatch,
|
||||
):
|
||||
|
||||
Reference in New Issue
Block a user