mirror of
https://github.com/superdesigndev/treg.git
synced 2026-10-02 03:24:35 +08:00
fix(catalog): let Bright Data dataset rows own the snapshots they start
Every extended Bright Data row runs the same /datasets/v3/scrape runner as the core rows, which falls back to HTTP 202 + snapshot_id past one minute. The core rows declare resource_ownership.produces for that snapshot; the extended rows did not, so a caller who started a slow job (Amazon reviews, Trustpilot, app store reviews) was refused with async_resource_not_owned when collecting it from brightdata.web.scrape.job.status/results on the shared key. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
cae7709870
commit
e5867b2850
@@ -41,6 +41,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Instagram post details by URL"
|
||||
summary: Instagram - Posts
|
||||
input:
|
||||
@@ -91,6 +94,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Instagram post comments by post URL"
|
||||
summary: Instagram - Comments
|
||||
input:
|
||||
@@ -132,6 +138,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Instagram reel details by URL"
|
||||
summary: Instagram - Reels
|
||||
input:
|
||||
@@ -173,6 +182,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Instagram hashtag details by hashtag page URL"
|
||||
summary: Instagram hashtag information
|
||||
input:
|
||||
@@ -215,6 +227,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "TikTok profile by URL"
|
||||
summary: TikTok - Profiles
|
||||
input:
|
||||
@@ -265,6 +280,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "TikTok video details by URL"
|
||||
summary: TikTok - Posts
|
||||
input:
|
||||
@@ -307,6 +325,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "TikTok video comments by video URL"
|
||||
summary: TikTok - Comments
|
||||
input:
|
||||
@@ -348,6 +369,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "TikTok Shop product details by URL"
|
||||
summary: TikTok Shop
|
||||
input:
|
||||
@@ -390,6 +414,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "LinkedIn company profile by company page URL"
|
||||
summary: LinkedIn company information
|
||||
input:
|
||||
@@ -432,6 +459,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "LinkedIn post details by URL"
|
||||
summary: LinkedIn posts
|
||||
input:
|
||||
@@ -474,6 +504,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "LinkedIn job posting details by listing URL"
|
||||
summary: Linkedin job listings information
|
||||
input:
|
||||
@@ -515,6 +548,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Facebook profile or Page by URL"
|
||||
summary: Facebook - Profiles
|
||||
input:
|
||||
@@ -566,6 +602,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Facebook Page posts by Page URL"
|
||||
summary: Facebook - Pages Posts by Profile URL
|
||||
input:
|
||||
@@ -607,6 +646,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Facebook post details by post URL"
|
||||
summary: Facebook - Posts by post URL
|
||||
input:
|
||||
@@ -650,6 +692,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Facebook post comments by post URL"
|
||||
summary: Facebook - Comments
|
||||
input:
|
||||
@@ -691,6 +736,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Facebook Marketplace listing by URL"
|
||||
summary: Facebook Marketplace
|
||||
input:
|
||||
@@ -734,6 +782,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Amazon product details by product URL"
|
||||
summary: Amazon products
|
||||
input:
|
||||
@@ -783,6 +834,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Amazon product reviews by product URL"
|
||||
summary: Amazon Reviews
|
||||
input:
|
||||
@@ -825,6 +879,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Amazon best-seller products by category URL"
|
||||
summary: Amazon best seller products
|
||||
input:
|
||||
@@ -866,6 +923,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Amazon seller profile by storefront URL"
|
||||
summary: Amazon sellers info
|
||||
input:
|
||||
@@ -908,6 +968,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "X (Twitter) profile by URL"
|
||||
summary: X (formerly Twitter) - Profiles
|
||||
input:
|
||||
@@ -958,6 +1021,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "X (Twitter) post details by URL"
|
||||
summary: X (formerly Twitter) - Posts
|
||||
input:
|
||||
@@ -1000,6 +1066,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "YouTube channel profile by URL"
|
||||
summary: YouTube - Channels
|
||||
input:
|
||||
@@ -1050,6 +1119,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "YouTube video details by URL"
|
||||
summary: Youtube - Videos posts
|
||||
input:
|
||||
@@ -1092,6 +1164,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "YouTube video comments by video URL"
|
||||
summary: Youtube - Comments
|
||||
input:
|
||||
@@ -1134,6 +1209,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Reddit post details by thread URL"
|
||||
summary: Reddit - Posts
|
||||
input:
|
||||
@@ -1176,6 +1254,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Reddit post comments by post URL"
|
||||
summary: Reddit - Comments
|
||||
input:
|
||||
@@ -1217,6 +1298,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Walmart product details by product URL"
|
||||
summary: Walmart - products
|
||||
input:
|
||||
@@ -1259,6 +1343,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Walmart product reviews by product URL"
|
||||
summary: Walmart Reviews
|
||||
input:
|
||||
@@ -1301,6 +1388,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "eBay listing details by URL"
|
||||
summary: eBay
|
||||
input:
|
||||
@@ -1343,6 +1433,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Yelp business profile by page URL"
|
||||
summary: Yelp businesses overview
|
||||
input:
|
||||
@@ -1385,6 +1478,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Yelp business reviews by page URL"
|
||||
summary: Yelp businesses reviews
|
||||
input:
|
||||
@@ -1427,6 +1523,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Tripadvisor hotel listing by URL"
|
||||
summary: Tripadvisor Hotel Listings
|
||||
input:
|
||||
@@ -1469,6 +1568,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Trustpilot business reviews by profile URL"
|
||||
summary: Trustpilot business reviews
|
||||
input:
|
||||
@@ -1511,6 +1613,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Google Play app listing by URL"
|
||||
summary: Google Play Store
|
||||
input:
|
||||
@@ -1560,6 +1665,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Google Play app reviews by listing URL"
|
||||
summary: Google Play Store reviews
|
||||
input:
|
||||
@@ -1601,6 +1709,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Apple App Store app listing by URL"
|
||||
summary: Apple App Store
|
||||
input:
|
||||
@@ -1642,6 +1753,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Apple App Store app reviews by listing URL"
|
||||
summary: Apple App Store reviews
|
||||
input:
|
||||
@@ -1684,6 +1798,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "IMDb title details by URL"
|
||||
summary: IMDB media
|
||||
input:
|
||||
@@ -1726,6 +1843,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Xiaohongshu post details by URL"
|
||||
summary: Xiaohongshu - posts
|
||||
input:
|
||||
@@ -1768,6 +1888,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Threads profile by URL"
|
||||
summary: Threads - Profiles
|
||||
input:
|
||||
@@ -1810,6 +1933,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Telegram channel details by URL"
|
||||
summary: Telegram
|
||||
input:
|
||||
@@ -1853,6 +1979,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Crunchbase company profile by page URL"
|
||||
summary: Crunchbase companies information
|
||||
input:
|
||||
@@ -1895,6 +2024,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Glassdoor company profile by page URL"
|
||||
summary: Glassdoor companies information (Public web data)
|
||||
input:
|
||||
@@ -1938,6 +2070,9 @@ endpoints:
|
||||
scope: any_account
|
||||
method: POST
|
||||
path: /datasets/v3/scrape
|
||||
resource_ownership:
|
||||
produces:
|
||||
- {kind: snapshot, path: snapshot_id}
|
||||
name: "Glassdoor company reviews by page URL"
|
||||
summary: Glassdoor companies reviews
|
||||
input:
|
||||
|
||||
Reference in New Issue
Block a user