mirror of
https://github.com/superdesigndev/treg.git
synced 2026-10-02 03:24:35 +08:00
fix(billing): icypeas reads scoped to the team; quickenrich email miss free; icypeas searches billed per row (#705)
* fix(billing): quickenrich email miss free, icypeas bulk billed per row, icypeas reads scoped to the team - quickenrich.people.email.find: QuickEnrich bills a phone-only answer (credits_used=1), but the adapter calls it an email miss; settle it at zero so a routed miss is free and no longer eats the route's max-cost budget before the next provider. - icypeas.bulk.search: the start answer has no rows, so the reserve is the bill; it defaulted to a 20-row page. Reserve data rows x the task's rate (0.1 credit per verification). - icypeas.search.results.read / search.files.read listed every search on treg's shared Icypeas account. Searches and bulk files now record ownership; reads require an owned id and reject the listing modes. Bulk rows move to icypeas.bulk.results.read. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * fix(icypeas): a single email search reserves one credit, not a 20-row page Its answer is {item: {_id}} with no rows, so the reserve is the bill; the 20-row default charged $0.38 for one lookup. Live: $0.38 before, $0.019 after. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Jason Zhou
co-authored by
Claude Opus 5.5
Claude Opus 5.5
parent
2de39612db
commit
8c725204bf
@@ -797,6 +797,15 @@ def _marketplace_pricing(
|
||||
else:
|
||||
unit = (_usd_to_micro(cost["usd"])
|
||||
if cost.get("type") in ("per_result", "quota_rows") and cost.get("usd") else 0)
|
||||
if endpoint_id in ("icypeas.bulk.search", "icypeas.people.email.find") and credit_rate:
|
||||
# The answer is an id ({file} / {item: {_id}}) with no rows, so this reserve IS the bill: one
|
||||
# row per bulk `data` entry at its task's rate, one for a single search. Without it the
|
||||
# 20-row default billed a 3-row job, and a single email lookup, $0.38.
|
||||
# ponytail: a search row is billed as if found; per-found billing needs a settle on job end.
|
||||
doc = _json_object(body)
|
||||
rows = doc.get("data") if isinstance(doc.get("data"), list) else []
|
||||
credits = 0.1 if doc.get("task") == "email-verification" else 1
|
||||
return _usd_to_micro(max(1, min(len(rows), 5000)) * credits * credit_rate), unit
|
||||
if provider == "quickenrich":
|
||||
credit = _usd_to_micro(float(cost.get("usd") or 0))
|
||||
if endpoint_id == "quickenrich.people.search.domain":
|
||||
|
||||
@@ -291,10 +291,20 @@ def _tavily_cost_micro(mk: MarketplaceCall, doc: object) -> int | None:
|
||||
return min(count, _tavily_requested_result_limit(mk)) * mk.unit_micro
|
||||
|
||||
|
||||
def _quickenrich_present(value) -> bool:
|
||||
return isinstance(value, str) and value.strip().lower() not in ("", "n/a", "null", "none")
|
||||
|
||||
|
||||
def _quickenrich_cost_micro(mk: MarketplaceCall, doc: dict) -> int | None:
|
||||
"""Subscription credits at the frozen list rate, independent of the upstream plan fee."""
|
||||
if mk.cost_type == "free":
|
||||
return 0
|
||||
data = doc.get("data")
|
||||
if (mk.endpoint_id == "quickenrich.people.email.find" and isinstance(data, dict)
|
||||
and not _quickenrich_present(data.get("email"))):
|
||||
# QuickEnrich takes its credit for a phone-only answer too (meta.credits_used=1), but this
|
||||
# is the pay-per-success EMAIL finder and the adapter calls that a miss: treg absorbs it.
|
||||
return 0
|
||||
meta = doc.get("meta")
|
||||
if isinstance(meta, dict) and "credits_used" in meta:
|
||||
credits = meta["credits_used"]
|
||||
@@ -306,25 +316,21 @@ def _quickenrich_cost_micro(mk: MarketplaceCall, doc: dict) -> int | None:
|
||||
if data is None or data == [] or data == {}:
|
||||
return 0
|
||||
|
||||
def present(value):
|
||||
return isinstance(value, str) and value.strip().lower() not in ("", "n/a", "null", "none")
|
||||
|
||||
if mk.endpoint_id == "quickenrich.people.search.domain" and isinstance(data, list):
|
||||
if not all(isinstance(row, dict) for row in data):
|
||||
return None
|
||||
title = (mk.request_data.get("queryParams") or {}).get("title", "")
|
||||
credits = (sum(present(row.get("email")) or present(row.get("employee_phone")) for row in data)
|
||||
credits = (sum(_quickenrich_present(row.get("email")) or _quickenrich_present(row.get("employee_phone"))
|
||||
for row in data)
|
||||
if title else 1)
|
||||
return credits * mk.unit_micro
|
||||
if mk.endpoint_id == "quickenrich.companies.search" and isinstance(data, list):
|
||||
return len(data) * mk.unit_micro if all(isinstance(row, dict) for row in data) else None
|
||||
if isinstance(data, dict):
|
||||
if mk.endpoint_id == "quickenrich.people.email.find":
|
||||
if "email" not in data and "employee_phone" not in data:
|
||||
return None
|
||||
return int(present(data.get("email")) or present(data.get("employee_phone"))) * mk.unit_micro
|
||||
return mk.unit_micro # an email is present: the miss returned 0 above
|
||||
if mk.endpoint_id == "quickenrich.people.phone.find" and "employee_phone" in data:
|
||||
return int(present(data["employee_phone"])) * mk.unit_micro
|
||||
return int(_quickenrich_present(data["employee_phone"])) * mk.unit_micro
|
||||
if mk.endpoint_id == "quickenrich.people.enrich":
|
||||
return mk.unit_micro
|
||||
return None
|
||||
|
||||
@@ -54,6 +54,8 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /email-search
|
||||
resource_ownership:
|
||||
produces: [{kind: search, path: item._id}]
|
||||
name: "Find a work email from a name and domain (async)"
|
||||
summary: "Find a single email address from a firstname, a lastname and a company domain name"
|
||||
input:
|
||||
@@ -87,6 +89,8 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /email-verification
|
||||
resource_ownership:
|
||||
produces: [{kind: search, path: item._id}]
|
||||
name: "Verify an email address is deliverable (async)"
|
||||
summary: "Check whether a specific email address exists and is deliverable, including catch-all detection"
|
||||
input:
|
||||
@@ -118,6 +122,8 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /domain-search
|
||||
resource_ownership:
|
||||
produces: [{kind: search, path: item._id}]
|
||||
name: "Find role-based emails at a domain — contact@, support@"
|
||||
summary: "Discover a company's role-based addresses — contact@, support@, info@, admin@ and the rest"
|
||||
input:
|
||||
@@ -149,21 +155,47 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /bulk-single-searchs/read
|
||||
name: "Read any search's results by id (the poll route)"
|
||||
summary: "Collect the result of any single search, or the rows of a bulk search, by id"
|
||||
name: "Read a single search's result by id (the poll route)"
|
||||
summary: "Collect the result of one email find, verification or domain scan by the id it returned"
|
||||
# The listing modes (mode=single|bulk without a file) enumerate every search on the account,
|
||||
# and on treg's shared key that is every team's searches. Only an id this team produced is read.
|
||||
body_allowlist: true
|
||||
resource_ownership:
|
||||
requires: {kind: search, param: id, in: body}
|
||||
input:
|
||||
body:
|
||||
id: {type: string, required: false, note: "the _id returned by a single search — the usual way to collect one result", example: "mP6hHKABeMoKaEB1K1HF"}
|
||||
mode: {type: string, required: false, note: "'single' or 'bulk' — list your searches instead of naming one id"}
|
||||
file: {type: string, required: false, note: "with mode=bulk, the file id returned by icypeas.bulk.search — returns that bulk's rows in input order"}
|
||||
type: {type: string, required: false, note: "with mode=single, filter by task: email-search | email-verification | domain-search. Cannot be combined with mode=bulk"}
|
||||
limit: {type: integer, required: false, note: "rows per page, default 10, MAX 100", example: 10}
|
||||
id: {type: string, required: true, note: "the _id returned by icypeas.people.email.find, icypeas.people.email.verify or icypeas.companies.emails.role", example: "mP6hHKABeMoKaEB1K1HF"}
|
||||
note: "THE POLL ROUTE, and free. Each row's `status` says where the search got to; FOUND / DEBITED mean the payload in `results` is final. Rate limited to 30 calls/min, so poll a couple of seconds apart or register a webhook instead. Bulk rows are read with icypeas.bulk.results.read"
|
||||
test_request:
|
||||
body: {id: "mP6hHKABeMoKaEB1K1HF"}
|
||||
cost: {type: free, value: 0, currency: USD, unit: call, source: docs, source_url: "https://api-doc.icypeas.com/how-works/credit-cost", checked: '2026-08-20', confidence: documented, note: "the rate card prices the search routes; every other route including this one is free"}
|
||||
verified: '2026-08-20'
|
||||
example_response: examples/icypeas.search.results.read.json
|
||||
docs_url: https://api-doc.icypeas.com/fetch-results/search-item
|
||||
|
||||
- id: icypeas.bulk.results.read
|
||||
domain: contacts
|
||||
capability: people.job.results
|
||||
platform: people
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /bulk-single-searchs/read
|
||||
name: "Read a bulk job's rows"
|
||||
summary: "Collect the rows of a bulk search by the file id icypeas.bulk.search returned"
|
||||
body_allowlist: true
|
||||
resource_ownership:
|
||||
requires: {kind: bulk_file, param: file, in: body}
|
||||
input:
|
||||
body:
|
||||
mode: {type: string, required: true, enum: [bulk], note: "always 'bulk'", example: "bulk"}
|
||||
file: {type: string, required: true, note: "the file id returned by icypeas.bulk.search — returns that job's rows in input order", example: "L3mhHKAB9iupLhv96W-F"}
|
||||
limit: {type: integer, required: false, min: 1, max: 100, note: "rows per page, default 10, MAX 100", example: 10}
|
||||
next: {type: boolean, required: false, note: "true = next page, false = previous page, used with `sorts`"}
|
||||
sorts: {type: "array", required: false, note: "the pagination cursor echoed by the previous response — pass it back to page further"}
|
||||
note: "THE POLL ROUTE, and free. Each row's `status` says where the search got to; FOUND / DEBITED mean the payload in `results` is final. Rate limited to 30 calls/min, so poll a couple of seconds apart or register a webhook instead"
|
||||
note: "Free. Watch icypeas.search.files.read for `finished`, then page through the rows here. Rate limited to 30 calls/min"
|
||||
test_request:
|
||||
body: {mode: "single", type: "email-search", limit: 1}
|
||||
cost: {type: free, value: 0, currency: USD, unit: call, source: docs, source_url: "https://api-doc.icypeas.com/how-works/credit-cost", checked: '2026-08-20', confidence: documented, note: "the rate card prices the search routes; every other route including this one is free"}
|
||||
body: {mode: "bulk", file: "L3mhHKAB9iupLhv96W-F", limit: 1}
|
||||
cost: {type: free, value: 0, currency: USD, unit: call, source: docs, source_url: "https://api-doc.icypeas.com/how-works/credit-cost", checked: '2026-08-20', confidence: documented, note: "the rows were billed when the job ran; reading them is free"}
|
||||
verified: '2026-08-20'
|
||||
example_response: examples/icypeas.search.results.read.json
|
||||
docs_url: https://api-doc.icypeas.com/fetch-results/search-item
|
||||
@@ -175,6 +207,8 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /bulk-search
|
||||
resource_ownership:
|
||||
produces: [{kind: bulk_file, path: file}]
|
||||
name: "Start a bulk email job (up to 5,000 rows)"
|
||||
summary: "Queue up to 5,000 rows of email discovery, email verification or domain scanning in one call"
|
||||
input:
|
||||
@@ -183,7 +217,7 @@ endpoints:
|
||||
task: {type: string, required: true, note: "one of email-search | email-verification | domain-search", example: "email-search"}
|
||||
data: {type: "array[array[string]]", required: true, note: "one array per row. email-search rows are [firstname, lastname, domainOrCompany] (one of the two names may be empty); email-verification and domain-search rows hold a single value. MAX 5,000 rows", example: [["Patrick", "Collison", "stripe.com"]]}
|
||||
custom: {type: object, required: false, note: "{webhookUrlItem, webhookUrlBulkDone, externalIds, includeResultsInWebhook} — per-row and end-of-job callbacks, your own row ids, and whether results ride in the final webhook"}
|
||||
note: "Returns {file, status: 'in_progress'} immediately. Track it with icypeas.search.files.read and collect rows with icypeas.search.results.read using mode=bulk and that file id. Rate limited to 1 call/sec"
|
||||
note: "Returns {file, status: 'in_progress'} immediately. Track it with icypeas.search.files.read and collect rows with icypeas.bulk.results.read using that file id. Rate limited to 1 call/sec"
|
||||
test_request:
|
||||
body: {name: "treg catalog verification", task: "email-search", data: [["Patrick", "Collison", "stripe.com"]]}
|
||||
cost:
|
||||
@@ -210,16 +244,16 @@ endpoints:
|
||||
path: /search-files/read
|
||||
name: "Check bulk job progress"
|
||||
summary: "Progress and stats for your bulk searches — how many rows are done, found, aborted or short of credits"
|
||||
# Without `file` this lists every bulk job on the account (every team's, on treg's shared key).
|
||||
body_allowlist: true
|
||||
resource_ownership:
|
||||
requires: {kind: bulk_file, param: file, in: body}
|
||||
input:
|
||||
body:
|
||||
limit: {type: integer, required: false, note: "files per request, default 10, MAX 50", example: 2}
|
||||
file: {type: string, required: false, note: "a single file id, to get just that job's stats"}
|
||||
status: {type: string, required: false, note: "'in_progress' or 'done'"}
|
||||
next: {type: boolean, required: false, note: "true = next page, false = previous page"}
|
||||
sorts: {type: "array", required: false, note: "the cursor echoed by the previous response — [{creationDate}] tuples"}
|
||||
note: "An empty body {} lists every job. Watch `finished` (or status='done'), then collect the rows with icypeas.search.results.read. Rate limited to 15 calls/min"
|
||||
file: {type: string, required: true, note: "the file id returned by icypeas.bulk.search", example: "L3mhHKAB9iupLhv96W-F"}
|
||||
note: "Watch `finished` (or status='done'), then collect the rows with icypeas.bulk.results.read. Rate limited to 15 calls/min"
|
||||
test_request:
|
||||
body: {limit: 2}
|
||||
body: {file: "L3mhHKAB9iupLhv96W-F"}
|
||||
cost: {type: free, value: 0, currency: USD, unit: call, source: docs, source_url: "https://api-doc.icypeas.com/how-works/credit-cost", checked: '2026-08-20', confidence: documented}
|
||||
verified: '2026-08-20'
|
||||
example_response: examples/icypeas.search.files.read.json
|
||||
|
||||
@@ -57,8 +57,8 @@ endpoints:
|
||||
source_url: https://app.quickenrich.io/docs
|
||||
checked: '2026-09-08'
|
||||
confidence: documented
|
||||
note: One credit for a contact with email and/or phone; observed email hit 1 and miss 0. Prefer meta.credits_used;
|
||||
verification date is not a fresh SMTP verification.
|
||||
note: One credit when an email is returned. QuickEnrich also bills a phone-only answer; treg absorbs that
|
||||
credit, so a no-email answer is free here. Verification date is not a fresh SMTP verification.
|
||||
docs_url: https://app.quickenrich.io/docs
|
||||
verified: '2026-09-08'
|
||||
example_response: examples/quickenrich.people.email.find.json
|
||||
|
||||
@@ -767,6 +767,42 @@ async def test_legacy_platform_async_utilities_deny_unknown_ids_before_relay(
|
||||
assert response.json()["detail"]["error"] == "async_resource_not_owned"
|
||||
|
||||
|
||||
async def test_icypeas_shared_key_reads_only_this_teams_searches(clients: AsyncClient, monkeypatch):
|
||||
"""The listing modes enumerate every search on treg's one Icypeas account: every team's."""
|
||||
monkeypatch.setenv("TREG_PLATFORM_KEY_ICYPEAS", "test-platform-token")
|
||||
monkeypatch.setenv("TREG_PLATFORM_PROVIDERS", "icypeas")
|
||||
get_settings.cache_clear()
|
||||
responses = [{"success": True, "item": {"_id": "search-owned", "status": "NONE"}},
|
||||
{"success": True, "status": "in_progress", "file": "file-owned"}]
|
||||
|
||||
async def fake_relay(*args, **kwargs):
|
||||
return _response(200, responses.pop(0) if responses else {"success": True, "items": []})
|
||||
|
||||
monkeypatch.setattr(call_service, "relay", fake_relay)
|
||||
assert (await clients.post("/call/icypeas.people.email.find", json={
|
||||
"firstname": "A", "lastname": "B", "domainOrCompany": "example.com"})).status_code == 200
|
||||
assert (await clients.post("/call/icypeas.bulk.search", json={
|
||||
"name": "x", "task": "email-verification", "data": [["a@example.com"]]})).status_code == 200
|
||||
owned = [("icypeas.search.results.read", {"id": "search-owned"}),
|
||||
("icypeas.bulk.results.read", {"mode": "bulk", "file": "file-owned"}),
|
||||
("icypeas.search.files.read", {"file": "file-owned"})]
|
||||
for endpoint, body in owned:
|
||||
assert (await clients.post(f"/call/{endpoint}", json=body)).status_code == 200, endpoint
|
||||
|
||||
for endpoint, body in [("icypeas.search.results.read", {"mode": "single", "type": "email-search"}),
|
||||
("icypeas.search.results.read", {"id": "search-owned", "mode": "single"}),
|
||||
("icypeas.bulk.results.read", {"mode": "bulk"}),
|
||||
("icypeas.search.files.read", {}),
|
||||
("icypeas.search.results.read", {"id": "someone-elses"})]:
|
||||
response = await clients.post(f"/call/{endpoint}", json=body)
|
||||
assert response.status_code in (400, 403), (endpoint, body, response.status_code)
|
||||
other = await clients.post("/users", json={"email": "icypeas-stranger@example.com"})
|
||||
stranger = {"X-Treg-Token": other.json()["token"]}
|
||||
for endpoint, body in owned:
|
||||
assert (await clients.post(f"/call/{endpoint}", json=body, headers=stranger)).status_code == 403
|
||||
get_settings.cache_clear()
|
||||
|
||||
|
||||
async def test_apify_actor_start_needs_own_key(
|
||||
clients: AsyncClient, monkeypatch, legacy_async_platform,
|
||||
):
|
||||
|
||||
@@ -1864,6 +1864,8 @@ def test_tomba_settles_at_the_estimate_without_countable_evidence(fields, body):
|
||||
*[(endpoint, {'success': True, 'data': data, 'meta': {'credits_used': credits}}, expected)
|
||||
for endpoint, data, credits, expected in [
|
||||
('people.email.find', {'email': 'person@example.com'}, 1, 4834),
|
||||
# QuickEnrich bills a phone-only answer, but an email finder without an email is a free miss
|
||||
('people.email.find', {'email': 'N/A', 'employee_phone': '+15550101000'}, 1, 0),
|
||||
('people.phone.find', {'employee_phone': '+15550101000'}, 1, 4834),
|
||||
('people.enrich', {'email': 'person@example.com'}, 1, 4834),
|
||||
('people.email.find', [], 0, 0),
|
||||
@@ -1882,6 +1884,7 @@ def test_quickenrich_settles_reported_credits_at_frozen_rate(endpoint, doc, expe
|
||||
@pytest.mark.parametrize('endpoint,data,title,credits', [
|
||||
('people.email.find', {'email': 'a@example.com'}, '', 1),
|
||||
('people.email.find', {'email': None, 'employee_phone': 'N/A'}, '', 0),
|
||||
('people.email.find', {'email': None, 'employee_phone': '+15550101000'}, '', 0),
|
||||
('people.phone.find', {'employee_phone': '+15550101000'}, '', 1),
|
||||
('people.phone.find', {'employee_phone': 'N/A'}, '', 0),
|
||||
('people.enrich', {'first_name': 'Example'}, '', 1),
|
||||
@@ -2417,6 +2420,25 @@ def test_icypeas_profile_url_miss_settles_at_zero():
|
||||
mk, b'{"success":true,"result":"https://www.linkedin.com/in/x","status":"FOUND"}') is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task,rows,credits", [
|
||||
("email-verification", 3, 0.3), ("email-search", 3, 3), ("domain-search", 1, 1)])
|
||||
def test_icypeas_bulk_search_bills_its_rows_at_the_task_rate(task, rows, credits):
|
||||
"""The start answer has no rows, so the reserve is the bill: never the 20-row default."""
|
||||
body = {"name": "x", "task": task, "data": [["a@example.com"]] * rows}
|
||||
mk, estimate, credit = _priced("icypeas.bulk.search", None, body, request_data={"body": body})
|
||||
assert estimate == round(credits * credit)
|
||||
assert call_settle._observed_cost_micro(
|
||||
mk, b'{"success":true,"status":"in_progress","file":"f"}') in (None, estimate)
|
||||
|
||||
|
||||
def test_icypeas_single_email_search_bills_one_credit_not_a_page():
|
||||
body = {"firstname": "A", "lastname": "B", "domainOrCompany": "example.com"}
|
||||
mk, estimate, credit = _priced("icypeas.people.email.find", None, body, request_data={"body": body})
|
||||
assert estimate == credit
|
||||
assert call_settle._observed_cost_micro(
|
||||
mk, b'{"success":true,"item":{"_id":"x","status":"NONE"}}') in (None, estimate)
|
||||
|
||||
|
||||
def test_icypeas_company_scrape_bills_the_company_rate():
|
||||
body = {"type": "company", "data": ["https://www.linkedin.com/company/a", "https://www.linkedin.com/company/b"]}
|
||||
mk, estimate, per_row = _priced("icypeas.scrape.bulk", None, body, request_data={"body": body})
|
||||
|
||||
Reference in New Issue
Block a user