mirror of
https://github.com/superdesigndev/treg.git
synced 2026-10-02 03:24:35 +08:00
feat(seo): robots, sitemap, a crawlable catalog, and social cards
The catalog had no URLs. ~2,630 endpoints across 80 platform shelves — the whole
substance of the product — existed only as hash routes (/app#platform/<slug>)
behind a login, so a crawler could reach six thin marketing pages and nothing
else. On top of that: no robots.txt, no sitemap.xml, HEAD answering 405 on every
page, no og/twitter tags or image, no structured data, and /docs serving
FastAPI's stock Swagger shell — a kilobyte of JavaScript to anything that does
not run scripts.
Crawler plumbing:
- /robots.txt (bundled, {BASE}-templated) and /sitemap.xml (generated — 80 of
its 88 URLs come from the catalog, lastmod from file and catalog mtimes).
- HEAD widened onto every GET route. FastAPI's APIRoute pins methods to {"GET"}
and never adds HEAD, unlike Starlette's plain Route.
- Canonicals on /support (collapsing /contact and /help), /terms, /privacy,
/tutorial; noindex on the dashboard and on /vendor-listing.md's duplicate URL.
- robots.txt and sitemap.xml 301 from the legacy host, so its name resolves to
one crawlable site rather than a duplicate of it.
Crawlable surfaces:
- /catalog and /catalog/<slug>: server-rendered, no JavaScript, every capability
and price as real text. Registered after the JSON routes, with the reserved
names refused explicitly.
- /docs: a real API reference built from app.openapi(). Swagger UI moves to
/docs/api and is disallowed; the public catalog routes join the schema.
Metadata:
- og/twitter cards everywhere, backed by a new 1200x630 card rendered from
assets/brand/og-card.html. Every brand on it is a real provider.
- SoftwareApplication + Offer + Organization on the landing, ItemList +
BreadcrumbList on the catalog pages, FAQPage over the five questions already
written on /support.
Two things this had to be careful about. Widening HEAD put 58 duplicate
operations into the public openapi.json, so _openapi_without_head() narrows the
widened routes for the duration of schema generation. And landing/legal/tutorial
pages now read-and-substitute {BASE} instead of being served as bare files: a
hardcoded treg.to canonical tells a self-hosted registry's crawler that the real
page lives on someone else's domain.
Counts reconciled to the live catalog (2,630/47); the landing said 2,617/42 and
llms.txt said ~2,600/~48.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jason Zhou
co-authored by
Claude Opus 5
Claude Opus 5
parent
29902e0904
commit
d09f6b8d40
@@ -13,6 +13,7 @@ Regenerate via `scripts/build-map.py`.
|
||||
| `.claude-plugin/marketplace.json` | interface/skill.md |
|
||||
| `.claude-plugin/plugin.json` | interface/skill.md |
|
||||
| `README.md` | foundation/charter.md |
|
||||
| `assets/brand/og-card.html` | interface/seo.md |
|
||||
| `dsh/cordis.patch.yml` | interface/skill.md |
|
||||
| `dsh/index.js` | interface/skill.md |
|
||||
| `examples/proxy-demo/server.js` | architecture/local-proxy.md |
|
||||
@@ -25,7 +26,7 @@ Regenerate via `scripts/build-map.py`.
|
||||
| `src/treg/__main__.py` | ops/deploy.md |
|
||||
| `src/treg/agents.py` | interface/cli.md |
|
||||
| `src/treg/analytics.py` | architecture/data-model.md |
|
||||
| `src/treg/api.py` | architecture/money.md, architecture/multi-tenancy.md, architecture/proxy-model.md, architecture/super-admin.md, guides/expanding-a-category.md, interface/api.md, interface/dashboard.md, interface/landing-sandbox.md |
|
||||
| `src/treg/api.py` | architecture/money.md, architecture/multi-tenancy.md, architecture/proxy-model.md, architecture/super-admin.md, guides/expanding-a-category.md, interface/api.md, interface/dashboard.md, interface/landing-sandbox.md, interface/seo.md |
|
||||
| `src/treg/audit.py` | architecture/data-model.md, ops/deploy.md |
|
||||
| `src/treg/billing.py` | architecture/money.md |
|
||||
| `src/treg/catalog_store.py` | architecture/catalog.md, interface/api.md |
|
||||
@@ -60,11 +61,15 @@ Regenerate via `scripts/build-map.py`.
|
||||
| `src/treg/session.py` | interface/dashboard.md |
|
||||
| `src/treg/shell.py` | interface/shell.md |
|
||||
| `src/treg/skills.py` | interface/env-import.md |
|
||||
| `src/treg/web/catalog.css` | interface/seo.md |
|
||||
| `src/treg/web/connect-demo.html` | architecture/mcp-oauth.md |
|
||||
| `src/treg/web/index.html` | interface/dashboard.md, interface/landing-sandbox.md, interface/onboarding.md |
|
||||
| `src/treg/web/install.sh` | interface/landing-sandbox.md |
|
||||
| `src/treg/web/landing.html` | interface/seo.md |
|
||||
| `src/treg/web/robots.txt` | interface/seo.md |
|
||||
| `src/treg/web/selfhost.sh` | ops/deploy.md |
|
||||
| `src/treg/web/skill.md` | interface/skill.md |
|
||||
| `src/treg/web/support.html` | interface/seo.md |
|
||||
| `src/treg/web/tour/index.html` | interface/dashboard.md |
|
||||
| `src/treg/web/tour/tour.js` | interface/dashboard.md |
|
||||
| `src/treg/web/tutorial.html` | interface/dashboard.md |
|
||||
@@ -93,6 +98,7 @@ Regenerate via `scripts/build-map.py`.
|
||||
| `interface/env-import.md` | `providers.py`, `skills.py` |
|
||||
| `interface/landing-sandbox.md` | `sandbox.py`, `pubfeed.py`, `api.py`, `index.html`, `install.sh` |
|
||||
| `interface/onboarding.md` | `demo.py`, `cli.py`, `index.html` |
|
||||
| `interface/seo.md` | `api.py`, `robots.txt`, `catalog.css`, `landing.html`, `support.html`, `og-card.html` |
|
||||
| `interface/shell.md` | `shell.py`, `cli.py` |
|
||||
| `interface/skill.md` | `skill.md`, `mcp_install.py`, `build_plugin.py`, `plugin.json`, `marketplace.json`, `plugin.json`, `plugin.json`, `package.json`, `cordis.patch.yml`, `index.js` |
|
||||
| `ops/deploy.md` | `__main__.py`, `selfhost.sh`, `config.py`, `db.py`, `email.py`, `audit.py`, `render.yaml` |
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<!--
|
||||
Source for src/treg/web/media/og.png — the 1200x630 social card.
|
||||
|
||||
Regenerate after a repositioning or a big catalog change:
|
||||
open this file in a headless browser at exactly 1200x630 and screenshot #card.
|
||||
It is a BUILD source, not a served page: the favicons below are fetched once at render time and
|
||||
baked into the PNG, so the shipped card has no runtime network dependency.
|
||||
|
||||
Skin is landing.html's token system, verbatim — Geist Pixel on the display line only, DM Mono for
|
||||
identifiers and counts, ink for emphasis, no brand accent. Layout follows docs/assets/treg-hero.png.
|
||||
-->
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Geist+Pixel&family=Inter:wght@400;450;500;600;650;700&family=DM+Mono:ital,wght@0,400;0,500&display=swap" rel="stylesheet">
|
||||
<style>
|
||||
:root{
|
||||
--bg:#f4f4f1; --surface:#fff; --ink:#1a1a1a; --muted:#7c7c7c; --teal:#1a7da6;
|
||||
--shadow-sm:0 1px 2px #00000014;
|
||||
--shadow-md:0 1px 2px -1px #0000000a,0 4px 6px -1px #0000000f;
|
||||
--display:"Geist Pixel",Georgia,serif;
|
||||
--sans:"Inter","Segoe UI",system-ui,sans-serif;
|
||||
--mono:"DM Mono",ui-monospace,"SF Mono",Menlo,monospace;
|
||||
}
|
||||
*{box-sizing:border-box;margin:0;padding:0}
|
||||
body{background:#fff}
|
||||
#card{
|
||||
width:1200px;height:630px;background:var(--bg);color:var(--ink);
|
||||
font-family:var(--sans);display:flex;flex-direction:column;align-items:center;
|
||||
justify-content:center;gap:0;padding:44px 56px;overflow:hidden;
|
||||
}
|
||||
|
||||
/* ---------- wordmark ---------- */
|
||||
.mark{display:flex;align-items:center;gap:12px;background:#1a1a1a;color:#f8f8f7;
|
||||
border-radius:14px;padding:11px 22px 11px 14px;box-shadow:var(--shadow-md)}
|
||||
.mark .glyph{width:30px;height:30px;border-radius:8px;background:#f8f8f7;color:#1a1a1a;
|
||||
display:grid;place-items:center;font-family:var(--mono);font-weight:500;font-size:19px;line-height:1}
|
||||
.mark b{font-family:var(--mono);font-weight:500;font-size:29px;letter-spacing:.01em}
|
||||
|
||||
/* ---------- headline ---------- */
|
||||
h1{font-family:var(--display);font-weight:400;font-size:76px;line-height:1;
|
||||
letter-spacing:-.005em;margin-top:26px;white-space:nowrap}
|
||||
.sub{font-family:var(--mono);font-size:20px;color:var(--muted);margin-top:18px;letter-spacing:-.01em}
|
||||
.sub b{color:var(--teal);font-weight:500}
|
||||
.sub i{font-style:normal;color:#c3c3bf;padding:0 9px}
|
||||
|
||||
/* ---------- the key ---------- */
|
||||
.key{display:flex;align-items:center;gap:13px;background:var(--surface);border-radius:13px;
|
||||
padding:13px 26px;margin-top:26px;box-shadow:var(--shadow-md)}
|
||||
.key .glyph{width:27px;height:27px;border-radius:7px;background:#1a1a1a;color:#f8f8f7;
|
||||
display:grid;place-items:center;font-family:var(--mono);font-weight:500;font-size:17px;line-height:1}
|
||||
.key code{font-family:var(--mono);font-size:22px;letter-spacing:.02em}
|
||||
.key code s{text-decoration:none;color:#b9b9b4;letter-spacing:.14em}
|
||||
|
||||
/* ---------- provider chips ---------- */
|
||||
.rows{display:flex;flex-direction:column;align-items:center;gap:11px;margin-top:32px}
|
||||
.row{display:flex;gap:11px}
|
||||
.chip{display:flex;align-items:center;gap:9px;background:var(--surface);border-radius:11px;
|
||||
padding:9px 16px;box-shadow:var(--shadow-sm);font-weight:600;font-size:17px;white-space:nowrap}
|
||||
.chip img,.chip .ic{width:22px;height:22px;border-radius:5px;display:block}
|
||||
.chip.more{background:none;box-shadow:none;border:1.5px dashed #cfcfc9;color:var(--muted);
|
||||
font-family:var(--mono);font-weight:400;font-size:16px}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div id="card">
|
||||
<div class="mark"><span class="glyph">▚</span><b>treg</b></div>
|
||||
|
||||
<h1>OpenRouter for agent tools</h1>
|
||||
<div class="sub">One unified key for <b>2,600+ tools</b><i>·</i>priced per call<i>·</i>open source</div>
|
||||
|
||||
<div class="key"><span class="glyph">▚</span><code>trg_live_<s>••••••••••••••</s></code></div>
|
||||
|
||||
<div class="rows">
|
||||
<div class="row">
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=semrush.com&sz=64"/>Semrush</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=moz.com&sz=64"/>Moz</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=majestic.com&sz=64"/>Majestic</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=dataforseo.com&sz=64"/>DataForSEO</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=seranking.com&sz=64"/>SE Ranking</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=serpapi.com&sz=64"/>SerpApi</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=tiktok.com&sz=64"/>TikTok</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=instagram.com&sz=64"/>Instagram</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=youtube.com&sz=64"/>YouTube</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=x.com&sz=64"/>X/Twitter</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=reddit.com&sz=64"/>Reddit</div>
|
||||
<!-- LinkedIn's s2 favicon only resolves at 16px and falls back to a generic globe at 64,
|
||||
so the mark is inlined instead. Every other chip fetches cleanly. -->
|
||||
<div class="chip"><svg class="ic" viewBox="0 0 24 24" xmlns="http://www.w3.org/2000/svg"><rect width="24" height="24" rx="4" fill="#0A66C2"/><path fill="#fff" d="M7.1 9.3H4.6V19h2.5V9.3ZM5.85 8.2a1.45 1.45 0 1 0 0-2.9 1.45 1.45 0 0 0 0 2.9ZM19.4 19h-2.5v-4.7c0-1.2-.44-2-1.5-2-.82 0-1.3.55-1.51 1.08-.08.19-.1.45-.1.72V19H11.3s.03-8.8 0-9.7h2.5v1.37c.33-.51.92-1.24 2.24-1.24 1.64 0 2.86 1.07 2.86 3.37V19Z"/></svg>LinkedIn</div>
|
||||
</div>
|
||||
<div class="row">
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=hunter.io&sz=64"/>Hunter</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=apollo.io&sz=64"/>Apollo</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=crunchbase.com&sz=64"/>Crunchbase</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=ads.google.com&sz=64"/>Google Ads</div>
|
||||
<div class="chip"><img src="https://www.google.com/s2/favicons?domain=facebook.com&sz=64"/>Meta Ads</div>
|
||||
<div class="chip more">+2,600 endpoints</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@@ -38,6 +38,7 @@ covers (frontmatter `sources:`). Regenerate this index with
|
||||
| [Import — scan a .env AND/OR a skills dir, auto-register as tools + bundles](interface/env-import.md) | in-progress | providers.py, skills.py |
|
||||
| [Landing sandbox studio — anonymous try-it, hosted skills, CLI installer](interface/landing-sandbox.md) | shipped | sandbox.py, pubfeed.py, api.py, index.html, … |
|
||||
| [Onboarding — the first-run demo team (dashboard + CLI)](interface/onboarding.md) | shipped | demo.py, cli.py, index.html |
|
||||
| [Search surfaces — robots, sitemap, the crawlable catalog, and the social card](interface/seo.md) | shipped | api.py, robots.txt, catalog.css, landing.html, … |
|
||||
| [Shell mode (treg shell) — transparent CLI interception](interface/shell.md) | shipped | shell.py, cli.py |
|
||||
| [The shippable tools-registry skill (3 personas)](interface/skill.md) | shipped | skill.md, mcp_install.py, build_plugin.py, plugin.json, … |
|
||||
|
||||
|
||||
@@ -175,7 +175,8 @@ what they created; `_require_admin_of` gates the org-admin endpoints. See
|
||||
- **Provider catalog:** `providers_catalog` (`GET /providers.json`, open) → `{version, providers}` — the
|
||||
catalog `treg upload` uses to detect env keys → tools; served so the CLI can refresh centrally. See
|
||||
[env-import](env-import.md).
|
||||
- **Endpoint catalog** (open, `include_in_schema=False`, read via `catalog_store`): the operations layer
|
||||
- **Endpoint catalog** (open, and now **in** the OpenAPI schema — these four are the public read API,
|
||||
so they are documented rather than hidden; read via `catalog_store`): the operations layer
|
||||
— what a connected provider can DO. `catalog_platforms` (`GET /catalog/platforms`) → `{platforms:
|
||||
[{slug, label, capabilities, endpoints, verified, providers[]}], generated_from: "catalog"}`, endpoint
|
||||
count desc, platforms nobody implements omitted. `catalog_platform` (`GET /catalog/platforms/{slug}`,
|
||||
@@ -490,6 +491,12 @@ surfaced by `GET /tools` / `/bundles/{id}`).
|
||||
|
||||
Full endpoint list + the running server's OpenAPI: `README.md` and `/docs`. CLI-level usage: `USAGE.md`.
|
||||
|
||||
`/docs` is **ours** — a server-rendered reference built from `app.openapi()`, not Swagger. FastAPI's
|
||||
console moved to `/docs/api` (ReDoc is off), and `/openapi.json` is unchanged. The `/catalog`,
|
||||
`/catalog/<slug>` and `/robots.txt` + `/sitemap.xml` surfaces live alongside it; see
|
||||
[seo](seo.md), which also explains why HEAD is widened onto every GET route after registration and
|
||||
why that widening must be kept out of the schema.
|
||||
|
||||
## OAuth + MCP routes
|
||||
|
||||
treg is an OAuth authorization server for its own MCP endpoint. Detail in
|
||||
|
||||
@@ -356,6 +356,13 @@ be callable (e.g. Google Ads' developer token — surfaced by the `needSecondCre
|
||||
page (Connect looked dead).
|
||||
|
||||
## Endpoint catalog — the platform axis of the marketplace (`view==='platform'`)
|
||||
|
||||
> This is the **signed-in** browse surface, and it is a hash route (`/app#platform/<slug>`), so none
|
||||
> of it is reachable by a crawler. The same catalog is now also served as plain HTML at `/catalog`
|
||||
> and `/catalog/<slug>` from the same `GET /catalog/platforms` data — see [seo](seo.md). The two
|
||||
> render from one shared row builder (`_platform_rows` in `api.py`) so they cannot disagree.
|
||||
> `index.html` itself carries `robots: noindex`: every view here needs a session.
|
||||
|
||||
The marketplace's second browse surface answers "what data can I actually pull?" rather than "whose
|
||||
account can I attach?" — see `architecture/catalog.md` for the data behind it, and it is the marketplace's
|
||||
**default** view. `loadPlatforms` reads **`GET /catalog/platforms`** (once per session; cached on
|
||||
|
||||
@@ -15,7 +15,13 @@ related:
|
||||
|
||||
# Landing sandbox studio
|
||||
|
||||
The logged-out `/` is no longer a login box — it's a **landing page with a live, no-login sandbox
|
||||
> **Where this lives now.** `/` serves `landing.html`, not the SPA — `landing()` in `api.py` only
|
||||
> falls through to `index.html` when the request carries a query string (invite links, OAuth
|
||||
> returns, tour deep-links). The sandbox studio described below is that fall-through branch of
|
||||
> `index.html`, so it is reached from the SPA rather than from the front page. See
|
||||
> `interface/seo.md` for what `/` serves and why it is `{BASE}`-templated.
|
||||
|
||||
The logged-out SPA is not a login box — it's a **landing page with a live, no-login sandbox
|
||||
studio** (the `v-if="!authed"` branch of `index.html`, `.lp` container). A visitor builds a real
|
||||
mini-registry in the browser and keeps using it from their terminal, all without an account. The
|
||||
engine is `src/treg/sandbox.py` + a handful of `api.py` endpoints; the front-end drives it with `sbx*`
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
---
|
||||
title: Search surfaces — robots, sitemap, the crawlable catalog, and the social card
|
||||
status: shipped
|
||||
sources:
|
||||
- src/treg/api.py
|
||||
- src/treg/web/robots.txt
|
||||
- src/treg/web/catalog.css
|
||||
- src/treg/web/landing.html
|
||||
- src/treg/web/support.html
|
||||
- assets/brand/og-card.html
|
||||
related:
|
||||
- interface/api.md
|
||||
- interface/dashboard.md
|
||||
- architecture/catalog.md
|
||||
---
|
||||
|
||||
# Search surfaces
|
||||
|
||||
Everything a crawler, a link unfurler or an AI answer engine sees. It is one subsystem because the
|
||||
pieces only work together: a sitemap is worthless without pages to list, and pages are worthless if
|
||||
`HEAD` 405s before the crawl starts.
|
||||
|
||||
## The problem this fixed
|
||||
|
||||
The catalog — ~2,630 endpoints across 80 platform shelves, the entire substance of the product — had
|
||||
**no URLs**. The dashboard browses platforms through hash routes (`/app#platform/<slug>`) behind a
|
||||
login, and individual endpoints were expandable rows with no address at all. A crawler could reach
|
||||
six thin marketing pages and nothing else. On top of that: no `robots.txt`, no `sitemap.xml`, `HEAD`
|
||||
answering 405 everywhere, no `og:`/`twitter:` tags or image, no structured data, and `/docs` serving
|
||||
FastAPI's stock Swagger shell — a kilobyte of JavaScript to anything that does not run scripts.
|
||||
|
||||
## The pieces
|
||||
|
||||
| Path | What it is |
|
||||
|---|---|
|
||||
| `/robots.txt` | Bundled file, `{BASE}`-templated. Disallows `/app`, `/login`, auth and OAuth flows, `/call/`, `/mcp`, `/admin`, `/docs/api`. Names the sitemap. |
|
||||
| `/sitemap.xml` | **Generated**, not bundled — 80 of its URLs come from the catalog. Static pages take `lastmod` from their file's mtime, shelves from the newest mtime under `src/treg/catalog/`. |
|
||||
| `/catalog` | Server-rendered index: every shelf, grouped by category, with counts, `price_from` and provider names. |
|
||||
| `/catalog/<slug>` | One shelf: every capability, every endpoint, every price, as real text. |
|
||||
| `/docs` | Server-rendered API reference built from `app.openapi()`. |
|
||||
| `/docs/api` | FastAPI's Swagger UI, moved here and `Disallow`ed. ReDoc is off. |
|
||||
| `/media/og.png` | The 1200×630 social card, served by the pre-existing `/media` mount. |
|
||||
| `/catalog.css` | Shared skin for all three page types. |
|
||||
|
||||
`_page()` in `api.py` is the shell every server-rendered page goes through — it owns `<title>`, the
|
||||
meta description, the canonical, the og/twitter card and the JSON-LD, so a new page cannot ship
|
||||
missing them. That omission is exactly what left the landing bare.
|
||||
|
||||
## Things that will bite you
|
||||
|
||||
**`{BASE}`, never a hardcoded `treg.to`.** Every page is also served by self-hosted registries. A
|
||||
hardcoded canonical tells their crawler the real page lives on someone else's domain. `landing()`,
|
||||
`_legal_page()` and `tutorial_page()` all read-and-substitute for this reason — they were plain
|
||||
`FileResponse`s before. `tests/test_seo.py` asserts no response body leaks a literal `{BASE}`, and
|
||||
none leaks a hardcoded host when `public_url` is overridden.
|
||||
|
||||
**HEAD is widened after registration, and must not leak into the schema.** FastAPI's `APIRoute` pins
|
||||
`methods` to `{"GET"}` and never adds HEAD (unlike Starlette's plain `Route`), so every page 405'd on
|
||||
the probe crawlers send first. One loop at the bottom of `api.py` widens every GET-only route. But
|
||||
FastAPI derives one operation per (path, method), so that widening put **58 duplicate HEAD entries
|
||||
into `/openapi.json`**, each with a duplicate operation id. `_openapi_without_head()` narrows the
|
||||
widened routes for the duration of schema generation and puts them back. Only `/call/{rest}`, which
|
||||
declares HEAD itself, is documented with one.
|
||||
|
||||
**`/catalog/<slug>` sits in front of the JSON routes.** `/catalog/platforms`, `/catalog/search`,
|
||||
`/catalog/endpoints/…` and `/catalog/examples/…` keep matching only because they are registered
|
||||
first. `_CATALOG_RESERVED` refuses those names explicitly as a second guard, and the tests assert
|
||||
the JSON routes still answer `application/json` — if the page route ever swallows one, the dashboard
|
||||
and every installed CLI break at once.
|
||||
|
||||
**Structured data must match the visible page.** Google treats schema claiming something the page
|
||||
does not say as a violation, not a shortcut. The landing's `Offer` figures ($1.00 free, 0% markup)
|
||||
are asserted against the rendered HTML, and every FAQ question in `support.html`'s schema is
|
||||
asserted to appear in its body. Edit one, edit the other, same commit.
|
||||
|
||||
**Prices need `_usd_short`, not `%g`.** `%g` flips to scientific notation below `1e-4`, and a shelf
|
||||
advertising "from $1.2e-07 per call" reads as a bug. Anything under a hundredth of a cent renders as
|
||||
`<$0.0001` — which then has to be HTML-escaped at every use site, because that `<` is real markup.
|
||||
|
||||
**The sitemap is walked, not spot-checked.** `test_every_sitemap_url_answers_200` fetches what it
|
||||
publishes. Rename a route and the sitemap silently starts serving 404s to Google with nothing else
|
||||
failing.
|
||||
|
||||
**`catalog.css` is stamped with its mtime.** It is served with a real `max-age`, so without
|
||||
`?v=<mtime>` an edited skin keeps rendering from the browser's cache — the same trap `/tutorial.js`
|
||||
already guards.
|
||||
|
||||
## The social card
|
||||
|
||||
`assets/brand/og-card.html` is the **source**; `src/treg/web/media/og.png` is the render. Open the
|
||||
HTML at exactly 1200×630 in a headless browser and screenshot it. The provider favicons are fetched
|
||||
at render time and baked into the PNG, so the shipped card has no runtime network dependency.
|
||||
|
||||
Every brand on the card is a real provider — checked against `catalog_store.load()`, after an early
|
||||
draft showed Ahrefs, which treg does not carry. LinkedIn's mark is inlined because its Google s2
|
||||
favicon only resolves at 16px and falls back to a generic globe at 64.
|
||||
|
||||
Per-platform cards (`/media/og/<slug>.png`) are a deliberate follow-up. Until then every catalog page
|
||||
points at the shared one.
|
||||
|
||||
## Counts
|
||||
|
||||
`2,630 endpoints / 47 providers / 80 platforms`, from `catalog_store.load()`. The landing, `llms.txt`
|
||||
and the schema all state them and had drifted apart (2,617/42 and ~2,600/~48). Note the catalog index
|
||||
shows the **whole** catalog, not the sum of its tiles: a tile counts only its browse surface, so the
|
||||
account/utility endpoints — real inventory, listed on each shelf page — are excluded from tile counts
|
||||
by `catalog_store.HIDDEN_KINDS`.
|
||||
+570
-14
@@ -41,6 +41,7 @@ from pathlib import Path
|
||||
from fastapi import Cookie, Depends, FastAPI, Form, Header, HTTPException, Query, Request
|
||||
from fastapi.exception_handlers import http_exception_handler
|
||||
from fastapi.responses import FileResponse, HTMLResponse, JSONResponse, PlainTextResponse, RedirectResponse, Response, StreamingResponse
|
||||
from fastapi.routing import APIRoute
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from starlette.exceptions import HTTPException as StarletteHTTPException
|
||||
from pydantic import BaseModel
|
||||
@@ -157,7 +158,12 @@ async def lifespan(app: FastAPI):
|
||||
await app.state.http.aclose()
|
||||
|
||||
|
||||
app = FastAPI(title="tools-registry", version="0.0.1", lifespan=lifespan)
|
||||
# `/docs` is OURS — a server-rendered reference (see `docs_page`). FastAPI's stock Swagger UI moves
|
||||
# to `/docs/api`: it is a 1 KB JavaScript shell to anything that does not run scripts, it was titled
|
||||
# "tools-registry - Swagger UI" with FastAPI's own favicon, and it was the only thing we offered a
|
||||
# crawler looking for our API. ReDoc is off — two consoles for one schema is one too many.
|
||||
app = FastAPI(title="treg", version="0.0.1", lifespan=lifespan,
|
||||
docs_url="/docs/api", redoc_url=None)
|
||||
|
||||
|
||||
# The pre-treg.to hostnames must keep answering the API forever — every installed CLI, skill.md
|
||||
@@ -170,8 +176,12 @@ _LEGACY_HOSTS = set(LEGACY_PUBLIC_HOSTS)
|
||||
# Marketing pages — but only for ANONYMOUS visitors. A session cookie is host-scoped, so bouncing a
|
||||
# signed-in browser to the canonical host silently logs it out mid-flow (the invite confirmation,
|
||||
# for one, sets a legacy-host session and then lands on `/?invite_org=…`).
|
||||
# robots.txt and sitemap.xml join them for a search-engine reason rather than a marketing one: the
|
||||
# sitemap names canonical `public_url` URLs, and a sitemap whose own address is on a different host
|
||||
# than the URLs inside it is cross-submission — a crawler is entitled to ignore the lot. Redirecting
|
||||
# both means the legacy name resolves to one crawlable site, not a duplicate of it.
|
||||
_REDIRECT_PATHS = {"/", "/login", "/terms", "/privacy", "/support", "/contact", "/help",
|
||||
"/tutorial"}
|
||||
"/tutorial", "/robots.txt", "/sitemap.xml", "/catalog"}
|
||||
# The auth ENTRY points redirect unconditionally, and that is a correctness fix, not a marketing
|
||||
# one: each parks a host-scoped cookie and then continues on `public_url` — started on the legacy
|
||||
# host, the continuation never sees the cookie. /auth/github + /auth/google set the CSRF state
|
||||
@@ -428,9 +438,9 @@ def _provider_display(service: str) -> str:
|
||||
return p.display_name if p else service
|
||||
|
||||
|
||||
@app.get("/catalog/platforms", include_in_schema=False)
|
||||
async def catalog_platforms() -> dict:
|
||||
"""Open: the platform shelves of the endpoint catalog, busiest first."""
|
||||
def _platform_rows() -> list[dict]:
|
||||
"""The platform shelves, busiest first — one builder shared by the JSON route below and the
|
||||
server-rendered /catalog page, so the two can never disagree about what is on the shelf."""
|
||||
cat = catalog_store.load()
|
||||
rows = []
|
||||
for slug, plat in cat.platforms.items():
|
||||
@@ -464,10 +474,16 @@ async def catalog_platforms() -> dict:
|
||||
"providers": sorted({e["provider"] for e in eps}),
|
||||
})
|
||||
rows.sort(key=lambda r: (-r["endpoints"], r["slug"]))
|
||||
return {"platforms": rows, "generated_from": "catalog"}
|
||||
return rows
|
||||
|
||||
|
||||
@app.get("/catalog/platforms/{slug}", include_in_schema=False)
|
||||
@app.get("/catalog/platforms")
|
||||
async def catalog_platforms() -> dict:
|
||||
"""Open: the platform shelves of the endpoint catalog, busiest first."""
|
||||
return {"platforms": _platform_rows(), "generated_from": "catalog"}
|
||||
|
||||
|
||||
@app.get("/catalog/platforms/{slug}")
|
||||
async def catalog_platform(slug: str, include_hidden: int = 0) -> dict:
|
||||
"""Open: one platform's operations, grouped by capability so the same job across providers sits
|
||||
on one row — that grouping is what makes comparison (and a future failover router) possible.
|
||||
@@ -524,7 +540,7 @@ async def catalog_platform(slug: str, include_hidden: int = 0) -> dict:
|
||||
}
|
||||
|
||||
|
||||
@app.get("/catalog/search", include_in_schema=False)
|
||||
@app.get("/catalog/search")
|
||||
async def catalog_search(q: str = "", limit: int = 25) -> dict:
|
||||
"""Open: free-text search across the whole catalog — the DISCOVER half of the loop.
|
||||
|
||||
@@ -555,7 +571,7 @@ async def catalog_search(q: str = "", limit: int = 25) -> dict:
|
||||
return {"query": q, "count": len(results), "total": total, "results": results, "hints": hints}
|
||||
|
||||
|
||||
@app.get("/catalog/endpoints/{endpoint_id}", include_in_schema=False)
|
||||
@app.get("/catalog/endpoints/{endpoint_id}")
|
||||
async def catalog_endpoint(endpoint_id: str, db: AsyncSession = Depends(get_session)) -> dict:
|
||||
"""Open: everything about ONE endpoint — the INSPECT half of the loop.
|
||||
|
||||
@@ -618,6 +634,404 @@ async def catalog_example(endpoint_id: str) -> Response:
|
||||
return Response(content=path.read_bytes(), media_type="application/json")
|
||||
|
||||
|
||||
# ---- the crawlable catalog: /catalog and /catalog/<slug> -------------------------------------
|
||||
#
|
||||
# The JSON routes above are what agents and the dashboard read. These two render the SAME data as
|
||||
# server-side HTML, because until now none of it had a URL: the dashboard browses platforms through
|
||||
# hash routes (/app#platform/<slug>) behind a login, so ~2,600 endpoints across 80 shelves were
|
||||
# invisible to every crawler and every AI answer engine. No JavaScript here on purpose — the text IS
|
||||
# the product surface, and it has to be readable by something that will not run a script or click.
|
||||
#
|
||||
# `/catalog/<slug>` is registered after the JSON routes so /catalog/platforms, /catalog/search,
|
||||
# /catalog/endpoints/… and /catalog/examples/… keep matching first. Registration order alone is a
|
||||
# thin guarantee, so the reserved names are also refused explicitly below.
|
||||
_CATALOG_RESERVED = {"platforms", "search", "endpoints", "examples"}
|
||||
|
||||
_GH = "https://github.com/superdesigndev/treg"
|
||||
|
||||
|
||||
def _usd_short(usd: float) -> str:
|
||||
"""A dollar figure a person can read. `%g` flips to scientific notation below 1e-4, and a shelf
|
||||
advertising "from $1.2e-07 per call" reads as a bug rather than as a price — so anything under
|
||||
a hundredth of a cent is labelled as such instead."""
|
||||
if not usd:
|
||||
return "free"
|
||||
return "<$0.0001" if usd < 0.0001 else f"${usd:.3g}"
|
||||
|
||||
|
||||
def _price_label(cost: dict | None) -> str:
|
||||
"""A price in ONE currency, so rows down a page stay comparable. Mirrors `_cost_usd` in cli.py
|
||||
rather than importing it: pulling treg.cli into the server process costs ~200ms and drags the
|
||||
whole CLI in for one string (see `_treg_version`)."""
|
||||
if not isinstance(cost, dict):
|
||||
return ""
|
||||
usd = cost.get("usd")
|
||||
if usd is None:
|
||||
return "own account" # no rate published — never invent a dollar figure
|
||||
if not usd:
|
||||
return "free"
|
||||
unit = {"per_call": "call", "per_result": "result", "per_success": "success"}.get(
|
||||
cost.get("type"), "call")
|
||||
return f"{_usd_short(usd)}/{unit}"
|
||||
|
||||
|
||||
def _css_stamp() -> str:
|
||||
"""catalog.css's own mtime, stamped onto its URL. The stylesheet is served with a real max-age
|
||||
(it is static and every page pulls it), so without a stamp an edited skin keeps rendering from
|
||||
the browser's copy until the cache expires — the same trap `/tutorial.js` already guards."""
|
||||
f = _WEB_DIR / "catalog.css"
|
||||
try:
|
||||
return str(int(f.stat().st_mtime))
|
||||
except OSError:
|
||||
return "0"
|
||||
|
||||
|
||||
def _page(title: str, description: str, path: str, body: str, ld: list[dict],
|
||||
*, nav_current: str = "") -> HTMLResponse:
|
||||
"""The shared shell for every server-rendered page. One place that owns <title>, the meta
|
||||
description, the canonical, the og/twitter card and the JSON-LD, so a new page cannot ship
|
||||
without them — that omission is exactly what left the landing page bare for a year."""
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
t, d = _esc_html(title), _esc_html(description)
|
||||
url = base + path
|
||||
# `<` escaped to its \u form inside the JSON: a catalog label containing "</script>" would
|
||||
# otherwise close the block early and put the rest of the payload into the document as markup.
|
||||
# Still valid JSON, so parsers and Google's validator read it unchanged.
|
||||
blocks = "\n".join(
|
||||
'<script type="application/ld+json">'
|
||||
+ json.dumps(b, separators=(",", ":")).replace("<", "\\u003c")
|
||||
+ "</script>"
|
||||
for b in ld)
|
||||
def navlink(href: str, label: str, extra: str = "") -> str:
|
||||
cur = ' aria-current="page"' if href == nav_current else ""
|
||||
return f'<a href="{href}"{cur}{extra}>{label}</a>'
|
||||
return HTMLResponse(f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1"/>
|
||||
<title>{t}</title>
|
||||
<meta name="description" content="{d}"/>
|
||||
<link rel="canonical" href="{url}"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<meta property="og:type" content="website"/>
|
||||
<meta property="og:site_name" content="treg"/>
|
||||
<meta property="og:url" content="{url}"/>
|
||||
<meta property="og:title" content="{t}"/>
|
||||
<meta property="og:description" content="{d}"/>
|
||||
<meta property="og:image" content="{base}/media/og.png"/>
|
||||
<meta property="og:image:width" content="1200"/>
|
||||
<meta property="og:image:height" content="630"/>
|
||||
<meta property="og:image:alt" content="treg — one unified key for 2,600+ agent tools, priced per call"/>
|
||||
<meta name="twitter:card" content="summary_large_image"/>
|
||||
<meta name="twitter:title" content="{t}"/>
|
||||
<meta name="twitter:description" content="{d}"/>
|
||||
<meta name="twitter:image" content="{base}/media/og.png"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Geist+Pixel&family=Inter:wght@400;450;500;600;650;700&family=DM+Mono:ital,wght@0,400;0,500&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/catalog.css?v={_css_stamp()}"/>
|
||||
{blocks}
|
||||
</head>
|
||||
<body>
|
||||
<div class="navwrap"><nav class="nav">
|
||||
<a class="brand" href="/"><span class="glyph">▚</span> treg</a>
|
||||
<div class="links">
|
||||
{navlink("/catalog", "Catalog")}
|
||||
{navlink("/tutorial", "Tutorial")}
|
||||
{navlink("/docs", "API")}
|
||||
<a class="hidem" href="{_GH}" target="_blank" rel="noopener">GitHub ↗</a>
|
||||
<a class="candy" href="/app">Start free</a>
|
||||
</div>
|
||||
</nav></div>
|
||||
{body}
|
||||
<footer>
|
||||
<div class="foot-in">
|
||||
<div class="brand"><span class="glyph">▚</span> treg</div>
|
||||
<span style="font-family:var(--mono);font-size:12px">— 100% open source</span>
|
||||
<span class="sp"></span>
|
||||
<a href="/catalog">catalog</a><a href="/tutorial">docs</a><a href="/llms.txt">llms.txt</a
|
||||
><a href="{_GH}" target="_blank" rel="noopener">github ↗</a><a href="/docs">api</a
|
||||
><a href="/terms">terms</a><a href="/privacy">privacy</a>
|
||||
</div>
|
||||
</footer>
|
||||
</body>
|
||||
</html>""", headers={"Cache-Control": "public, max-age=600"})
|
||||
|
||||
|
||||
@app.get("/catalog", include_in_schema=False)
|
||||
async def catalog_index():
|
||||
"""Every platform shelf, grouped by category — the crawlable index of what an agent can call."""
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
rows = _platform_rows()
|
||||
# The WHOLE catalog, not the sum of the tiles: a tile counts only its browse surface, so the
|
||||
# account/utility endpoints (real inventory, listed on each shelf page) would go uncounted and
|
||||
# this page would quietly contradict the number on the landing.
|
||||
cat = catalog_store.load()
|
||||
total_eps = len(cat.endpoints)
|
||||
providers = sorted({e["provider"] for e in cat.endpoints})
|
||||
|
||||
# Categories in the order the busiest shelf in each appears, so the page opens on the deepest
|
||||
# inventory rather than on whichever category sorts first alphabetically.
|
||||
cats: dict[str, list[dict]] = {}
|
||||
for row in rows:
|
||||
cats.setdefault(row["category"], []).append(row)
|
||||
|
||||
sections = []
|
||||
for cat, items in cats.items():
|
||||
cards = []
|
||||
for r in items:
|
||||
price = _price_label(r["price_from"])
|
||||
vendors = ", ".join(_provider_display(p) for p in r["providers"])
|
||||
cards.append(
|
||||
f'<a class="pcard" href="/catalog/{_esc_html(r["slug"])}">'
|
||||
f'<h3>{_esc_html(r["label"])}</h3>'
|
||||
f'<p>{_esc_html(r["summary"])}</p>'
|
||||
f'<div class="meta"><b>{r["endpoints"]}</b> endpoints'
|
||||
f'<span class="dot">·</span><b>{r["capabilities"]}</b> capabilities</div>'
|
||||
+ (f'<div class="from">from {_esc_html(price)}</div>' if price else "")
|
||||
+ f'<div class="vendors">{_esc_html(vendors)}</div></a>')
|
||||
sections.append(f'<h2>{_esc_html(cat)}</h2><div class="grid">{"".join(cards)}</div>')
|
||||
|
||||
body = f"""<main class="wrap">
|
||||
<div class="phead">
|
||||
<span class="kicker">{total_eps:,} endpoints · {len(providers)} providers · <a href="{_GH}" target="_blank" rel="noopener">open source ↗</a></span>
|
||||
<h1>The tool catalog</h1>
|
||||
<p class="lede">Every endpoint your agent can call through one key — priced up front, billed per
|
||||
call, no provider signup. Pick a shelf to see what is on it and what each call costs.</p>
|
||||
<div class="facts">
|
||||
<span><b>{len(rows)}</b> platforms</span>
|
||||
<span><b>{total_eps:,}</b> endpoints</span>
|
||||
<span><b>{len(providers)}</b> providers</span>
|
||||
<span><b>$1.00</b> free on every new team</span>
|
||||
<span><b>0%</b> markup</span>
|
||||
</div>
|
||||
</div>
|
||||
<section class="cat">{"".join(sections)}</section>
|
||||
</main>"""
|
||||
|
||||
ld = [
|
||||
{"@context": "https://schema.org", "@type": "ItemList",
|
||||
"name": "treg tool catalog",
|
||||
"description": f"{total_eps} API endpoints across {len(rows)} platforms, callable through one key.",
|
||||
"numberOfItems": len(rows),
|
||||
"itemListElement": [
|
||||
{"@type": "ListItem", "position": i, "name": r["label"],
|
||||
"url": f"{base}/catalog/{r['slug']}"}
|
||||
for i, r in enumerate(rows, 1)]},
|
||||
{"@context": "https://schema.org", "@type": "BreadcrumbList", "itemListElement": [
|
||||
{"@type": "ListItem", "position": 1, "name": "treg", "item": base + "/"},
|
||||
{"@type": "ListItem", "position": 2, "name": "Catalog", "item": base + "/catalog"}]},
|
||||
]
|
||||
return _page(
|
||||
f"Tool catalog — {total_eps:,} API endpoints your agent can call | treg",
|
||||
f"Browse {total_eps:,} endpoints across {len(rows)} platforms and {len(providers)} providers "
|
||||
"— SEO, social, enrichment, ads and scraping data. One key, priced per call, no provider signup.",
|
||||
"/catalog", body, ld, nav_current="/catalog")
|
||||
|
||||
|
||||
@app.get("/catalog/{slug}", include_in_schema=False)
|
||||
async def catalog_page(slug: str):
|
||||
"""One platform shelf, rendered. Every capability, every endpoint, every price as real text."""
|
||||
if slug in _CATALOG_RESERVED:
|
||||
raise HTTPException(status_code=404, detail=f"unknown platform {slug!r}")
|
||||
detail = await catalog_platform(slug) # 404s for an unknown slug, same as the JSON route
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
plat = detail["platform"]
|
||||
label, category = plat["label"], plat["category"]
|
||||
row = next((r for r in _platform_rows() if r["slug"] == slug), None)
|
||||
summary = (row or {}).get("summary", "")
|
||||
caps = detail["capabilities"]
|
||||
eps = [e for cap in caps for e in cap["endpoints"]] + detail["extended"]
|
||||
prices = [c["usd"] for e in eps if isinstance(c := e.get("cost"), dict) and c.get("usd")]
|
||||
verified = len([e for e in eps if e.get("verified")])
|
||||
|
||||
blocks = []
|
||||
for cap in caps:
|
||||
rows_html = []
|
||||
for e in cap["endpoints"]:
|
||||
price = _price_label(e.get("cost"))
|
||||
tags = [f'<span class="tag">{_esc_html(e["provider_display"])}</span>']
|
||||
if e.get("verified"):
|
||||
tags.append('<span class="tag ok">live-verified</span>')
|
||||
tags.append(f'<code class="id">{_esc_html(e["id"])}</code>')
|
||||
rows_html.append(
|
||||
f'<div class="ep"><div class="eph"><span class="name">{_esc_html(e["name"])}</span>'
|
||||
+ (f'<span class="price">{_esc_html(price)}</span>' if price else "")
|
||||
+ f'</div><p>{_esc_html(e.get("summary") or "")}</p>'
|
||||
f'<div class="tags">{"".join(tags)}</div>'
|
||||
+ (f'<pre class="call">{_esc_html(e["call_template"])}</pre>'
|
||||
if e.get("call_template") else "")
|
||||
+ "</div>")
|
||||
blocks.append(
|
||||
f'<div class="cap"><h3>{_esc_html(cap["description"] or cap["id"])}'
|
||||
f'<code>{_esc_html(cap["id"])}</code></h3>'
|
||||
+ "".join(rows_html) + "</div>")
|
||||
|
||||
provs = "".join(
|
||||
f'<div class="prov"><b>{_esc_html(p["display_name"])}</b>'
|
||||
+ (f'<p>{_esc_html(p["note"])}</p>' if p.get("note") else "")
|
||||
+ "</div>"
|
||||
for p in detail["providers"].values())
|
||||
|
||||
cheapest = _usd_short(min(prices)) if prices else ""
|
||||
body = f"""<main class="wrap">
|
||||
<div class="phead">
|
||||
<div class="crumbs"><a href="/">treg</a> / <a href="/catalog">catalog</a> / {_esc_html(category)}</div>
|
||||
<h1>{_esc_html(label)}</h1>
|
||||
<p class="lede">{_esc_html(summary)}</p>
|
||||
<div class="facts">
|
||||
<span><b>{len(eps)}</b> endpoints</span>
|
||||
<span><b>{len(caps)}</b> capabilities</span>
|
||||
<span><b>{len(detail["providers"])}</b> providers</span>
|
||||
{f"<span>from <b>{_esc_html(cheapest)}</b> per call</span>" if cheapest else ""}
|
||||
{f"<span><b>{verified}</b> live-verified</span>" if verified else ""}
|
||||
</div>
|
||||
</div>
|
||||
<section class="cat">
|
||||
{"".join(blocks)}
|
||||
<h2>Providers on this shelf</h2>
|
||||
<div class="provs">{provs}</div>
|
||||
<p class="lede" style="margin-top:22px">Several providers can do the same job here. treg shows
|
||||
them side by side with measured success rates, speed and price — <b>choosing is yours</b>; there
|
||||
is no automatic routing or failover. <a href="/app#platform/{_esc_html(slug)}">Open this shelf in
|
||||
the dashboard</a> or <a href="/tutorial">see how a call works</a>.</p>
|
||||
</section>
|
||||
</main>"""
|
||||
|
||||
desc = (f"{len(eps)} {label.lower()} API endpoints from "
|
||||
f"{', '.join(p['display_name'] for p in list(detail['providers'].values())[:3])}"
|
||||
+ (f", from {cheapest} per call" if cheapest else "")
|
||||
+ ". Call them through one treg key — no provider signup.")
|
||||
ld = [
|
||||
{"@context": "https://schema.org", "@type": "ItemList",
|
||||
"name": f"{label} — API endpoints on treg",
|
||||
"numberOfItems": len(caps),
|
||||
"itemListElement": [
|
||||
{"@type": "ListItem", "position": i,
|
||||
"name": cap["description"] or cap["id"],
|
||||
"url": f"{base}/catalog/{slug}#{cap['id']}"}
|
||||
for i, cap in enumerate(caps, 1)]},
|
||||
{"@context": "https://schema.org", "@type": "BreadcrumbList", "itemListElement": [
|
||||
{"@type": "ListItem", "position": 1, "name": "treg", "item": base + "/"},
|
||||
{"@type": "ListItem", "position": 2, "name": "Catalog", "item": base + "/catalog"},
|
||||
{"@type": "ListItem", "position": 3, "name": label, "item": f"{base}/catalog/{slug}"}]},
|
||||
]
|
||||
return _page(f"{label} API — {len(eps)} endpoints, priced per call | treg",
|
||||
desc[:300], f"/catalog/{slug}", body, ld, nav_current="/catalog")
|
||||
|
||||
|
||||
@app.get("/catalog.css", include_in_schema=False)
|
||||
async def catalog_css():
|
||||
"""The shared skin for /catalog, /catalog/<slug> and /docs — the landing's tokens, one copy."""
|
||||
f = _WEB_DIR / "catalog.css"
|
||||
if not f.exists():
|
||||
raise HTTPException(status_code=404, detail="catalog.css not bundled")
|
||||
return FileResponse(f, media_type="text/css", headers={"Cache-Control": "public, max-age=600"})
|
||||
|
||||
|
||||
# ---- the API reference ------------------------------------------------------------------------
|
||||
# Prose first, because the schema cannot say the load-bearing part: what /call/ actually does. Kept
|
||||
# short and factual — the tutorial teaches, this page is the reference a reader lands on from search.
|
||||
_DOCS_INTRO = """
|
||||
<h2>How a call works</h2>
|
||||
<p>You make the <b>real upstream request</b> — the provider's own path, its own parameters, its own
|
||||
response. treg injects the credential server-side and relays the answer verbatim. Nothing here
|
||||
models a provider's API, which is why an upstream change does not break us and why the caller never
|
||||
holds a secret.</p>
|
||||
<pre class="call">curl -H "Authorization: Bearer $TREG_TOKEN" \\
|
||||
"{BASE}/call/moz.web.url.metrics"</pre>
|
||||
<p>Prefix any catalogued endpoint id with <code>/call/</code>. If your team has its own key for that
|
||||
provider, treg uses it and the call is <b>not metered</b>; otherwise eligible endpoints are served on
|
||||
treg's key and metered against your prepaid balance at the provider's own rate.</p>
|
||||
|
||||
<h2>Finding an endpoint</h2>
|
||||
<p>Search by what you want to <i>do</i>, not by vendor: <code>GET /catalog/search?q=backlinks</code>.
|
||||
When several providers can do the same job, <code>/catalog/platforms/{slug}</code> lists them side by
|
||||
side with measured success rate, speed and price. <b>Choosing is yours</b> — treg compares, but it
|
||||
does not route between providers automatically and does not fail over.</p>
|
||||
<p>The whole catalog is also browsable as pages: <a href="/catalog">/catalog</a>.</p>
|
||||
|
||||
<h2>Other ways in</h2>
|
||||
<p><a href="/llms.txt">/llms.txt</a> is the file to point a coding agent at — it teaches the whole
|
||||
protocol in one fetch. <code>curl -fsSL {BASE}/install.sh | sh</code> installs the CLI. The MCP
|
||||
endpoint is at <code>{BASE}/mcp</code>. An interactive console for everything below lives at
|
||||
<a href="/docs/api">/docs/api</a>.</p>
|
||||
|
||||
<h2>Endpoints</h2>
|
||||
<p>Authenticated requests carry <code>Authorization: Bearer <token></code> (or
|
||||
<code>X-Treg-Token</code>). The catalog routes are open and need no token.</p>
|
||||
"""
|
||||
|
||||
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def docs_page():
|
||||
"""The API reference, rendered server-side from the OpenAPI schema.
|
||||
|
||||
Replaces the stock Swagger UI at this path (now /docs/api), which was a script shell — the
|
||||
landing page linked "api" here and a crawler that followed it found an empty document.
|
||||
"""
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
schema = app.openapi()
|
||||
|
||||
def rank(path: str) -> tuple:
|
||||
"""The proxy first, then the catalog, then the rest alphabetically. Sorting purely by path
|
||||
opened the reference on /admin/* — super-admin plumbing, and the worst possible first
|
||||
impression of the API on a page built to be someone's search result."""
|
||||
return (0 if path.startswith("/call/") else 1 if path.startswith("/catalog") else 2, path)
|
||||
|
||||
# Auth travels the same way on every route; naming it on all 135 rows is noise, and the page
|
||||
# says it once above. `/admin/*` is super-admin only — still in openapi.json, not advertised here.
|
||||
_PLUMBING = {"x-treg-token", "treg_session", "authorization"}
|
||||
ops = []
|
||||
for path in sorted(schema.get("paths", {}), key=rank):
|
||||
if path.startswith("/admin"):
|
||||
continue
|
||||
for method, op in sorted(schema["paths"][path].items()):
|
||||
if method.lower() == "head": # implied by GET; see `_openapi_without_head`
|
||||
continue
|
||||
params = ", ".join(p["name"] for p in op.get("parameters", []) or []
|
||||
if p["name"].lower() not in _PLUMBING)
|
||||
summary = op.get("summary") or ""
|
||||
# FastAPI takes the description from the docstring; only the first paragraph belongs on
|
||||
# a reference index, and the rest is written for maintainers rather than callers.
|
||||
desc = (op.get("description") or "").strip().split("\n\n")[0].replace("\n", " ")
|
||||
ops.append(
|
||||
f'<div class="op"><div class="sig"><span class="verb">{_esc_html(method.upper())}</span>'
|
||||
f'<code>{_esc_html(path)}</code></div>'
|
||||
+ (f"<p>{_esc_html(summary or desc)}</p>" if (summary or desc) else "")
|
||||
+ (f'<div class="params">{_esc_html(params)}</div>' if params else "")
|
||||
+ "</div>")
|
||||
|
||||
body = f"""<main class="wrap">
|
||||
<div class="phead">
|
||||
<div class="crumbs"><a href="/">treg</a> / api</div>
|
||||
<h1>API reference</h1>
|
||||
<p class="lede">One base URL, one token. Call any of {len(ops)} documented operations, or proxy a
|
||||
real request to any of 2,630 catalogued provider endpoints through <code>/call/</code>.</p>
|
||||
<div class="facts">
|
||||
<span>base <b>{_esc_html(base)}</b></span>
|
||||
<span><b>Bearer</b> token auth</span>
|
||||
<span><a href="/openapi.json">openapi.json</a></span>
|
||||
<span><a href="/docs/api">interactive console</a></span>
|
||||
</div>
|
||||
</div>
|
||||
<section class="cat">
|
||||
<div class="prose">{_DOCS_INTRO.replace("{BASE}", _esc_html(base))}</div>
|
||||
{"".join(ops)}
|
||||
</section>
|
||||
</main>"""
|
||||
ld = [{"@context": "https://schema.org", "@type": "TechArticle",
|
||||
"headline": "treg API reference",
|
||||
"description": "How to call 2,630 provider API endpoints through one treg token.",
|
||||
"url": f"{base}/docs"}]
|
||||
return _page("API reference — call any tool through one endpoint | treg",
|
||||
"The treg HTTP API: proxy a real request to any of 2,630 catalogued provider "
|
||||
"endpoints through /call/, with the credential injected server-side. Plus the "
|
||||
"catalog, org, billing and tool-management routes.",
|
||||
"/docs", body, ld, nav_current="/docs")
|
||||
|
||||
|
||||
# ---- "the catalog doesn't have X" — tool requests -------------------------------------------
|
||||
TOOLREQ_HIT_NS = "toolreq"
|
||||
TOOLREQ_RATE_MAX = 10 # filings per IP per window
|
||||
@@ -1488,7 +1902,12 @@ async def landing(request: Request, treg_session: str = Cookie(default=""),
|
||||
if page.exists() and not request.query_params:
|
||||
if treg_session and await _user_from_session(treg_session, db):
|
||||
return RedirectResponse("/app", status_code=302)
|
||||
return FileResponse(page, headers={"Cache-Control": "no-cache"})
|
||||
# Read-and-substitute rather than a bare FileResponse: the canonical, og:url and og:image
|
||||
# are `{BASE}`-templated so they name the serving host. Hardcoded, a self-hosted registry
|
||||
# would tell crawlers its front page really lives on treg.to.
|
||||
html = page.read_text(encoding="utf-8").replace(
|
||||
"{BASE}", get_settings().public_url.rstrip("/"))
|
||||
return HTMLResponse(html, headers={"Cache-Control": "no-cache"})
|
||||
return await dashboard(request, treg_session, db)
|
||||
|
||||
|
||||
@@ -1589,6 +2008,85 @@ async def llms_txt():
|
||||
return PlainTextResponse(f.read_text(encoding="utf-8").replace("{BASE}", base), media_type="text/plain; charset=utf-8")
|
||||
|
||||
|
||||
@app.get("/robots.txt", include_in_schema=False)
|
||||
async def robots_txt():
|
||||
"""Crawler policy. `{BASE}`-templated like llms.txt, so a self-hosted registry advertises its own
|
||||
sitemap rather than treg.to's."""
|
||||
f = _WEB_DIR / "robots.txt"
|
||||
if not f.exists():
|
||||
raise HTTPException(status_code=404, detail="robots.txt not bundled")
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
return PlainTextResponse(f.read_text(encoding="utf-8").replace("{BASE}", base),
|
||||
media_type="text/plain; charset=utf-8",
|
||||
headers={"Cache-Control": "max-age=3600"})
|
||||
|
||||
|
||||
# The pages a crawler should know about. Everything here must answer 200 to a GET — a sitemap that
|
||||
# lists a redirect or a 404 is worse than no sitemap, so `tests/test_seo.py` walks every entry.
|
||||
# Deliberately absent: /contact and /help (alias URLs for the one support.html), /vendor-listing.md
|
||||
# (the text/plain twin of /vendor-listing), /login (302s to /app), /app* (authenticated SPA),
|
||||
# /connect-demo (noindex by design), and the shell installers.
|
||||
_SITEMAP_PAGES: tuple[tuple[str, str, str], ...] = (
|
||||
# (path, source file for lastmod — "" means use the catalog's, priority)
|
||||
("/", "landing.html", "1.0"),
|
||||
("/catalog", "", "0.9"),
|
||||
("/tutorial", "tutorial.html", "0.8"),
|
||||
("/docs", "", "0.7"),
|
||||
("/vendor-listing", "vendor-listing.md", "0.5"),
|
||||
("/support", "support.html", "0.4"),
|
||||
("/terms", "terms.html", "0.2"),
|
||||
("/privacy", "privacy.html", "0.2"),
|
||||
)
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _catalog_mtime() -> str:
|
||||
"""The newest mtime under the catalog directory, as a sitemap `lastmod` date. The catalog is
|
||||
read-only and changes only on deploy, so one scan per process is enough."""
|
||||
newest = 0.0
|
||||
for f in (Path(catalog_store.__file__).parent / "catalog").rglob("*.yaml"):
|
||||
try:
|
||||
newest = max(newest, f.stat().st_mtime)
|
||||
except OSError: # noqa: PERF203 -- a file vanishing mid-scan is not worth failing the sitemap
|
||||
continue
|
||||
return _iso_day(newest)
|
||||
|
||||
|
||||
def _iso_day(ts: float) -> str:
|
||||
return datetime.fromtimestamp(ts, tz=timezone.utc).date().isoformat() if ts else ""
|
||||
|
||||
|
||||
@app.get("/sitemap.xml", include_in_schema=False)
|
||||
async def sitemap_xml():
|
||||
"""Generated, not bundled: 80 of its URLs are the catalog's platform shelves, which move with the
|
||||
catalog rather than with a checked-in file. Every URL is absolute on `public_url` so a self-host
|
||||
publishes its own pages, and so the copy served on a legacy host still names the canonical one."""
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
cat_day = _catalog_mtime()
|
||||
out = ['<?xml version="1.0" encoding="UTF-8"?>',
|
||||
'<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">']
|
||||
|
||||
def add(path: str, lastmod: str, priority: str) -> None:
|
||||
out.append("<url>")
|
||||
out.append(f"<loc>{_esc_html(base + path)}</loc>")
|
||||
if lastmod:
|
||||
out.append(f"<lastmod>{lastmod}</lastmod>")
|
||||
out.append(f"<priority>{priority}</priority>")
|
||||
out.append("</url>")
|
||||
|
||||
for path, src, priority in _SITEMAP_PAGES:
|
||||
day = cat_day
|
||||
if src:
|
||||
f = _WEB_DIR / src
|
||||
day = _iso_day(f.stat().st_mtime) if f.exists() else ""
|
||||
add(path, day, priority)
|
||||
for row in _platform_rows():
|
||||
add(f"/catalog/{row['slug']}", cat_day, "0.6")
|
||||
out.append("</urlset>")
|
||||
return Response("\n".join(out), media_type="application/xml; charset=utf-8",
|
||||
headers={"Cache-Control": "max-age=3600"})
|
||||
|
||||
|
||||
@app.get("/install.sh", include_in_schema=False)
|
||||
async def install_sh():
|
||||
"""`curl -fsSL {BASE}/install.sh | sh` — installs the treg CLI and points it at this server.
|
||||
@@ -1655,10 +2153,16 @@ async def tutorial_access_md():
|
||||
|
||||
@app.get("/vendor-listing", include_in_schema=False)
|
||||
@app.get("/vendor-listing.md", include_in_schema=False)
|
||||
async def vendor_listing_md():
|
||||
async def vendor_listing_md(request: Request):
|
||||
"""Vendor listing instructions — what a vendor's coding agent reads before raising a PR that
|
||||
adds their API to the catalog. Linked from the dashboard's "List your API" modal."""
|
||||
return _serve_md("vendor-listing.md")
|
||||
resp = _serve_md("vendor-listing.md")
|
||||
# Two URLs, one document. `text/plain` cannot carry a <link rel=canonical>, so the duplicate is
|
||||
# suppressed with the header equivalent: /vendor-listing is the indexed one (it is what the
|
||||
# sitemap lists), /vendor-listing.md keeps serving agents and stays out of the index.
|
||||
if request.url.path.endswith(".md"):
|
||||
resp.headers["X-Robots-Tag"] = "noindex"
|
||||
return resp
|
||||
|
||||
|
||||
@app.get("/integrate.md", include_in_schema=False)
|
||||
@@ -1712,12 +2216,17 @@ async def legal_css():
|
||||
return FileResponse(f, media_type="text/css", headers={"Cache-Control": "no-cache"})
|
||||
|
||||
|
||||
def _legal_page(name: str) -> FileResponse:
|
||||
def _legal_page(name: str) -> HTMLResponse:
|
||||
page = _WEB_DIR / name
|
||||
if not page.exists():
|
||||
raise HTTPException(status_code=404, detail=f"{name} not bundled")
|
||||
# `{BASE}`-substituted rather than sent as a plain FileResponse, so each page's canonical and
|
||||
# og:url name the host actually serving it. A hardcoded treg.to would tell a self-hosted
|
||||
# registry's crawler that the real page lives on someone else's domain.
|
||||
base = get_settings().public_url.rstrip("/")
|
||||
html = page.read_text(encoding="utf-8").replace("{BASE}", base)
|
||||
# no-cache: a legal page must not be served stale after we publish an update.
|
||||
return FileResponse(page, headers={"Cache-Control": "no-cache"})
|
||||
return HTMLResponse(html, headers={"Cache-Control": "no-cache"})
|
||||
|
||||
|
||||
@app.get("/terms", include_in_schema=False)
|
||||
@@ -2449,6 +2958,7 @@ async def tutorial_page():
|
||||
stamp = int(js.stat().st_mtime) if js.exists() else 0 # tutorial.js's OWN mtime: _app_version()
|
||||
html = page.read_text(encoding="utf-8").replace( # hashes index.html and would not move
|
||||
'src="/tutorial.js"', f'src="/tutorial.js?v={stamp}"')
|
||||
html = html.replace("{BASE}", get_settings().public_url.rstrip("/")) # canonical + og:url
|
||||
return HTMLResponse(html, headers={"Cache-Control": "no-cache"})
|
||||
|
||||
|
||||
@@ -8827,6 +9337,52 @@ async def _bundle_view(bundle_id: int, db: AsyncSession) -> dict:
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# HEAD, everywhere GET is answered
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# FastAPI's APIRoute pins `methods` to {"GET"} and, unlike Starlette's plain Route, never adds HEAD
|
||||
# (fastapi/routing.py: `if methods is None: methods = ["GET"]`). So every page on the site answered
|
||||
# 405 to the HEAD probe that crawlers, link unfurlers and uptime checks send first — including `/`.
|
||||
# Widened once, here, rather than by editing ~120 decorators: this runs after every route is
|
||||
# registered, and only touches routes that are GET-only (a POST route keeps refusing HEAD, rightly).
|
||||
#
|
||||
# Sending the body is not a concern: ASGI servers drop it for HEAD per RFC 9110, and Starlette's
|
||||
# FileResponse already checks the scope method and sends headers only.
|
||||
_HEAD_WIDENED: list[APIRoute] = []
|
||||
for _route in app.routes:
|
||||
if isinstance(_route, APIRoute) and _route.methods == {"GET"}:
|
||||
_route.methods = {"GET", "HEAD"}
|
||||
_HEAD_WIDENED.append(_route)
|
||||
|
||||
|
||||
_fastapi_openapi = app.openapi
|
||||
|
||||
|
||||
def _openapi_without_head():
|
||||
"""The generated schema, minus the HEAD the loop above just added to every GET route.
|
||||
|
||||
Without this the widening leaks into a PUBLIC artifact: FastAPI derives one operation per
|
||||
(path, method), so /openapi.json grew 58 duplicate HEAD entries — each warning "Duplicate
|
||||
Operation ID" at generation and each doubling its operation on the /docs page. HEAD is a
|
||||
transport detail that HTTP already implies wherever GET is answered; it is not an API operation
|
||||
anyone reads about. So the routes are narrowed for the duration of generation and put back.
|
||||
Routes that declared HEAD themselves (the /call proxy) are untouched and still documented.
|
||||
"""
|
||||
if app.openapi_schema:
|
||||
return app.openapi_schema
|
||||
for r in _HEAD_WIDENED:
|
||||
r.methods = {"GET"}
|
||||
try:
|
||||
app.openapi_schema = _fastapi_openapi()
|
||||
finally:
|
||||
for r in _HEAD_WIDENED:
|
||||
r.methods = {"GET", "HEAD"}
|
||||
return app.openapi_schema
|
||||
|
||||
|
||||
app.openapi = _openapi_without_head
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
# The MCP front door
|
||||
# ---------------------------------------------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
/* ================================================================
|
||||
Skin for the server-rendered pages: /catalog, /catalog/<slug>, /docs.
|
||||
|
||||
Tokens lifted from landing.html's RESTYLE block so these pages read as
|
||||
the same product. Rules of the system, unchanged: no brand accent —
|
||||
emphasis is ink, colour is reserved for status; sans everywhere, mono
|
||||
only for identifiers, commands and counts; Geist Pixel on the h1 only;
|
||||
cards are surfaces (shadow, not border); layered shadows, quick eases.
|
||||
|
||||
These pages are the crawlable half of the catalog. Everything here has
|
||||
to survive with JavaScript off, so there is none — no reveal
|
||||
animations, no client-side filtering, nothing that hides content
|
||||
behind an interaction a crawler will not perform.
|
||||
================================================================ */
|
||||
:root{
|
||||
--bg:#f4f4f1; --surface:#fff; --panel:#fafaf9; --panel2:#f0f0ee;
|
||||
--ink:#1a1a1a; --muted:#7c7c7c; --muted2:#989898;
|
||||
--line:#2626231a; --line2:#25252233;
|
||||
--teal:#1a7da6;
|
||||
--green:#118453; --amber:#ba6603; --red:#c0362f;
|
||||
--inverse:#1a1a1a; --inverse-ink:#f8f8f7;
|
||||
--shadow-sm:0 1px 2px #00000014;
|
||||
--shadow-md:0 1px 2px -1px #0000000a,0 4px 6px -1px #0000000f;
|
||||
--shadow-lg:0 1px 2px -1px #0000000a,0 4px 6px -1px #0000000f,0 8px 16px #0000000a;
|
||||
--ease:cubic-bezier(.2,.72,.25,1);
|
||||
--r:15px; --rb:10px;
|
||||
--display:"Geist Pixel",Georgia,serif;
|
||||
--sans:"Suisse Intl","Inter","Segoe UI",system-ui,sans-serif;
|
||||
--mono:"DM Mono",ui-monospace,"SF Mono",Menlo,monospace;
|
||||
}
|
||||
*{box-sizing:border-box;min-width:0}
|
||||
html{scroll-behavior:smooth}
|
||||
html,body{overflow-x:clip}
|
||||
body{margin:0;background:var(--bg);color:var(--ink);font-family:var(--sans);
|
||||
font-size:15px;line-height:1.55;-webkit-font-smoothing:antialiased}
|
||||
a{color:var(--teal);text-decoration:none}
|
||||
a:hover{text-decoration:underline}
|
||||
:focus-visible{outline:0;box-shadow:0 0 0 2px var(--bg),0 0 0 3.5px var(--ink)}
|
||||
.wrap{max-width:1160px;margin:0 auto;padding:0 28px}
|
||||
|
||||
/* ---------- nav ---------- */
|
||||
.navwrap{position:sticky;top:0;z-index:50;padding:14px 20px;background:linear-gradient(var(--bg) 60%,transparent)}
|
||||
.nav{max-width:1160px;margin:0 auto;display:flex;align-items:center;gap:22px;
|
||||
background:rgba(255,255,255,.86);backdrop-filter:blur(14px);-webkit-backdrop-filter:blur(14px);
|
||||
border:1px solid var(--line);border-radius:14px;padding:10px 12px 10px 22px;
|
||||
box-shadow:var(--shadow-md)}
|
||||
.nav .brand{display:flex;align-items:center;gap:8px;font-family:var(--mono);font-weight:500;
|
||||
font-size:16px;color:var(--ink)}
|
||||
.nav .brand .glyph{width:22px;height:22px;border-radius:6px;background:var(--inverse);
|
||||
color:var(--inverse-ink);display:grid;place-items:center;font-size:13px}
|
||||
.nav .links{margin-left:auto;display:flex;align-items:center;gap:20px;font-size:14px}
|
||||
.nav .links a{color:var(--muted);font-weight:500}
|
||||
.nav .links a:hover{color:var(--ink);text-decoration:none}
|
||||
.nav .links a[aria-current]{color:var(--ink)}
|
||||
.candy{background:var(--inverse);color:var(--inverse-ink);border:0;border-radius:10px;
|
||||
padding:8px 15px;font-family:inherit;font-size:14px;font-weight:600;cursor:pointer;
|
||||
box-shadow:var(--shadow-md);transition:transform .16s var(--ease)}
|
||||
.candy:hover{transform:translateY(-1px);text-decoration:none;color:var(--inverse-ink)}
|
||||
|
||||
/* ---------- page head ---------- */
|
||||
.phead{padding:34px 0 26px}
|
||||
.kicker{display:inline-block;font-family:var(--mono);font-size:12.5px;color:var(--muted);
|
||||
letter-spacing:-.01em;margin-bottom:14px}
|
||||
.crumbs{font-family:var(--mono);font-size:12.5px;color:var(--muted2);margin-bottom:12px}
|
||||
.crumbs a{color:var(--muted)}
|
||||
h1{font-family:var(--display);font-weight:400;font-size:clamp(30px,4.4vw,50px);line-height:1.04;
|
||||
letter-spacing:-.005em;margin:0}
|
||||
.lede{font-size:17px;color:var(--muted);max-width:60ch;margin:16px 0 0}
|
||||
.facts{display:flex;flex-wrap:wrap;gap:8px 10px;margin-top:20px;font-family:var(--mono);font-size:12.5px}
|
||||
.facts span{background:var(--surface);border-radius:8px;padding:5px 11px;box-shadow:var(--shadow-sm);
|
||||
color:var(--muted)}
|
||||
.facts span b{color:var(--ink);font-weight:500}
|
||||
|
||||
/* ---------- category sections ---------- */
|
||||
.cat{padding:8px 0 30px}
|
||||
.cat h2{font-size:14px;font-weight:650;letter-spacing:.02em;text-transform:uppercase;
|
||||
color:var(--muted2);margin:26px 0 14px;padding-bottom:10px;border-bottom:1px solid var(--line)}
|
||||
.grid{display:grid;grid-template-columns:repeat(auto-fill,minmax(268px,1fr));gap:14px}
|
||||
|
||||
/* ---------- platform card ---------- */
|
||||
.pcard{display:flex;flex-direction:column;background:var(--surface);border-radius:var(--r);
|
||||
padding:17px 18px;box-shadow:var(--shadow-md);color:inherit;
|
||||
transition:transform .18s var(--ease),box-shadow .18s var(--ease)}
|
||||
.pcard:hover{transform:translateY(-2px);box-shadow:var(--shadow-lg);text-decoration:none}
|
||||
.pcard h3{margin:0;font-size:16.5px;font-weight:650;letter-spacing:-.01em}
|
||||
.pcard p{margin:7px 0 0;font-size:13.5px;color:var(--muted);line-height:1.5}
|
||||
/* Counts on their own line, price on the next. Right-aligning the price on the counts line fails
|
||||
both ways at this column width: `margin-left:auto` with wrapping on drops it to a line of its own
|
||||
anyway (leaving a dangling separator), and with wrapping off it runs past the card edge. */
|
||||
.pcard .meta{margin-top:auto;padding-top:14px;font-family:var(--mono);font-size:11.5px;
|
||||
color:var(--muted2);white-space:nowrap;overflow:hidden;text-overflow:ellipsis}
|
||||
.pcard .from{margin-top:3px;font-family:var(--mono);font-size:11.5px;color:var(--ink);
|
||||
font-weight:500;white-space:nowrap}
|
||||
.pcard .meta b{color:var(--ink);font-weight:500}
|
||||
.pcard .meta .dot{color:var(--line2);padding:0 4px}
|
||||
.pcard .vendors{margin-top:9px;font-family:var(--mono);font-size:11px;color:var(--muted2);
|
||||
overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
|
||||
/* ---------- capability sections on a platform page ---------- */
|
||||
.cap{background:var(--surface);border-radius:var(--r);box-shadow:var(--shadow-md);
|
||||
padding:18px 20px;margin-bottom:14px}
|
||||
.cap > h3{margin:0;font-size:16px;font-weight:650;letter-spacing:-.01em}
|
||||
.cap > h3 code{font-family:var(--mono);font-size:12px;font-weight:400;color:var(--muted2);
|
||||
margin-left:9px}
|
||||
.cap > p{margin:6px 0 0;font-size:13.5px;color:var(--muted)}
|
||||
|
||||
/* one endpoint */
|
||||
.ep{border-top:1px solid var(--line);padding:13px 0 3px;margin-top:13px}
|
||||
.ep:first-of-type{margin-top:14px}
|
||||
.eph{display:flex;align-items:baseline;gap:10px;flex-wrap:wrap}
|
||||
.eph .name{font-weight:600;font-size:14.5px}
|
||||
.eph .price{margin-left:auto;font-family:var(--mono);font-size:12.5px;font-weight:500;
|
||||
white-space:nowrap}
|
||||
.ep p{margin:5px 0 0;font-size:13.5px;color:var(--muted);line-height:1.5}
|
||||
.ep .tags{margin-top:8px;display:flex;flex-wrap:wrap;gap:6px;font-family:var(--mono);font-size:11px;
|
||||
color:var(--muted2);align-items:center}
|
||||
.tag{background:var(--panel2);border-radius:6px;padding:3px 8px}
|
||||
.tag.ok{background:#1184530f;color:var(--green)}
|
||||
.ep code.id{font-family:var(--mono);font-size:11.5px;color:var(--muted2)}
|
||||
pre.call{margin:9px 0 0;background:var(--panel);border-radius:var(--rb);padding:10px 12px;
|
||||
font-family:var(--mono);font-size:12px;color:var(--ink);overflow-x:auto;white-space:pre}
|
||||
|
||||
/* ---------- provider table ---------- */
|
||||
.provs{display:grid;grid-template-columns:repeat(auto-fill,minmax(240px,1fr));gap:12px;margin-top:12px}
|
||||
.prov{background:var(--surface);border-radius:var(--rb);padding:13px 15px;box-shadow:var(--shadow-sm)}
|
||||
.prov b{font-size:14.5px}
|
||||
.prov p{margin:5px 0 0;font-size:12.5px;color:var(--muted)}
|
||||
|
||||
/* ---------- docs page ---------- */
|
||||
.op{background:var(--surface);border-radius:var(--rb);padding:13px 16px;margin-bottom:9px;
|
||||
box-shadow:var(--shadow-sm)}
|
||||
.op .sig{display:flex;align-items:baseline;gap:11px;flex-wrap:wrap;font-family:var(--mono);font-size:13.5px}
|
||||
.op .verb{font-weight:500;color:var(--inverse-ink);background:var(--inverse);border-radius:5px;
|
||||
padding:2px 8px;font-size:11px;letter-spacing:.03em}
|
||||
.op p{margin:7px 0 0;font-size:13.5px;color:var(--muted);font-family:var(--sans)}
|
||||
.op .params{margin:8px 0 0;font-family:var(--mono);font-size:11.5px;color:var(--muted2)}
|
||||
.prose{max-width:72ch}
|
||||
.prose h2{font-size:20px;font-weight:650;letter-spacing:-.01em;margin:30px 0 10px}
|
||||
.prose p{margin:0 0 12px;color:var(--muted);font-size:14.5px}
|
||||
.prose code{font-family:var(--mono);font-size:13px;background:var(--panel2);border-radius:5px;
|
||||
padding:1px 6px;color:var(--ink)}
|
||||
|
||||
/* ---------- footer ---------- */
|
||||
footer{border-top:1px solid var(--line);margin-top:46px;padding:26px 0 40px}
|
||||
.foot-in{max-width:1160px;margin:0 auto;padding:0 28px;display:flex;align-items:center;gap:18px;
|
||||
flex-wrap:wrap;font-size:13px}
|
||||
.foot-in .brand{display:flex;align-items:center;gap:8px;font-family:var(--mono);font-weight:500}
|
||||
.foot-in .brand .glyph{width:20px;height:20px;border-radius:6px;background:var(--inverse);
|
||||
color:var(--inverse-ink);display:grid;place-items:center;font-size:12px}
|
||||
.foot-in .sp{margin-left:auto}
|
||||
.foot-in a{color:var(--muted)}
|
||||
|
||||
@media (max-width:640px){
|
||||
.wrap,.foot-in{padding:0 18px}
|
||||
.nav{padding:9px 10px 9px 16px;gap:12px}
|
||||
.nav .links{gap:13px}
|
||||
.nav .hidem{display:none}
|
||||
.eph .price{margin-left:0}
|
||||
.phead{padding:22px 0 18px}
|
||||
}
|
||||
@@ -4,6 +4,9 @@
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>treg</title>
|
||||
<!-- The dashboard is an authenticated app: every view needs a session and its routes are hash
|
||||
fragments. Nothing here is indexable, and a crawler that got in would index a sign-in prompt. -->
|
||||
<meta name="robots" content="noindex, follow"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
|
||||
@@ -4,9 +4,24 @@
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1"/>
|
||||
<title>treg — turn your coding agent into an SEO expert, a media buyer, an SDR</title>
|
||||
<meta name="description" content="Give your agent the tool catalog for the job. 2,617 endpoints across 42 providers — premium SEO, social, enrichment and ads data, pay as you go. Plus your own keys & skills, and an identity per agent."/>
|
||||
<link rel="canonical" href="https://treg.to/"/>
|
||||
<meta name="description" content="Give your agent the tool catalog for the job. 2,630 endpoints across 47 providers — premium SEO, social, enrichment and ads data, pay as you go. Plus your own keys & skills, and an identity per agent."/>
|
||||
<!-- {BASE}, not a hardcoded treg.to: this file is also served by every self-hosted registry, and a
|
||||
canonical pointing at someone else's domain tells a crawler the real page is not here. -->
|
||||
<link rel="canonical" href="{BASE}/"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<meta property="og:type" content="website"/>
|
||||
<meta property="og:site_name" content="treg"/>
|
||||
<meta property="og:url" content="{BASE}/"/>
|
||||
<meta property="og:title" content="treg — the tool catalog for your agent"/>
|
||||
<meta property="og:description" content="2,630 endpoints across 47 providers — SEO, social, enrichment and ads data your agent can call with one key, priced per call. Plus your own keys & skills."/>
|
||||
<meta property="og:image" content="{BASE}/media/og.png"/>
|
||||
<meta property="og:image:width" content="1200"/>
|
||||
<meta property="og:image:height" content="630"/>
|
||||
<meta property="og:image:alt" content="treg — one unified key for 2,600+ agent tools, priced per call, open source"/>
|
||||
<meta name="twitter:card" content="summary_large_image"/>
|
||||
<meta name="twitter:title" content="treg — the tool catalog for your agent"/>
|
||||
<meta name="twitter:description" content="2,630 endpoints across 47 providers, callable with one key and priced per call. No provider signup."/>
|
||||
<meta name="twitter:image" content="{BASE}/media/og.png"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Geist+Pixel&family=Inter:wght@400;450;500;600;650;700&family=DM+Mono:ital,wght@0,400;0,500;1,400&display=swap" rel="stylesheet">
|
||||
@@ -488,6 +503,51 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
}
|
||||
|
||||
</style>
|
||||
|
||||
<!-- Structured data. Every figure below is also visible on the page — the free credit in benefit 03,
|
||||
the 0% markup in the zeroband, the counts in the kicker and trustline. Schema that claims
|
||||
something the page does not say is a structured-data error, not a shortcut. -->
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "SoftwareApplication",
|
||||
"name": "treg",
|
||||
"url": "{BASE}/",
|
||||
"applicationCategory": "DeveloperApplication",
|
||||
"applicationSubCategory": "API gateway for AI agents",
|
||||
"operatingSystem": "Any (HTTP API, CLI, MCP)",
|
||||
"description": "The tool catalog for your agent: 2,630 API endpoints across 47 providers — SEO, social, enrichment, ads and scraping data — callable with one key and priced per call, with no provider signup. Bring your own API keys and skills alongside them.",
|
||||
"image": "{BASE}/media/og.png",
|
||||
"license": "https://www.gnu.org/licenses/agpl-3.0.html",
|
||||
"isAccessibleForFree": true,
|
||||
"featureList": [
|
||||
"2,630 curated API endpoints across 47 providers",
|
||||
"Credentials injected server-side — the caller never holds a key",
|
||||
"Per-call pricing shown before the call, with no markup",
|
||||
"Your own API keys and SKILL.md files as callable tools",
|
||||
"CLI, HTTP proxy and MCP interfaces"
|
||||
],
|
||||
"offers": {
|
||||
"@type": "Offer",
|
||||
"price": "0",
|
||||
"priceCurrency": "USD",
|
||||
"description": "$1.00 of free credit on every new team, then pay per call at the provider's own rate — treg adds 0% on top. No subscription.",
|
||||
"availability": "https://schema.org/InStock",
|
||||
"url": "{BASE}/"
|
||||
}
|
||||
}
|
||||
</script>
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "Organization",
|
||||
"name": "treg",
|
||||
"url": "{BASE}/",
|
||||
"logo": "{BASE}/media/og.png",
|
||||
"description": "The tool catalog for AI agents — one key for 2,630 API endpoints, plus your own keys and skills.",
|
||||
"sameAs": ["https://github.com/superdesigndev/treg"]
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -498,6 +558,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<div class="links">
|
||||
<a href="https://github.com/superdesigndev/treg" class="hidem navico" target="_blank" rel="noopener"><svg viewBox="0 0 16 16" aria-hidden="true"><path fill="currentColor" d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27s1.36.09 2 .27c1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.01 8.01 0 0 0 16 8c0-4.42-3.58-8-8-8z"/></svg>Repo</a>
|
||||
<a href="https://discord.gg/6mQYYfFMAn" class="hidem navico" target="_blank" rel="noopener"><svg viewBox="0 0 24 24" aria-hidden="true"><path fill="currentColor" d="M20.32 4.37a19.8 19.8 0 0 0-4.89-1.52.07.07 0 0 0-.08.04c-.21.38-.44.87-.6 1.25a18.3 18.3 0 0 0-5.49 0 12.6 12.6 0 0 0-.61-1.25.08.08 0 0 0-.08-.04 19.7 19.7 0 0 0-4.88 1.52.07.07 0 0 0-.03.03C.53 9.05-.32 13.58.1 18.06c0 .02.01.04.03.05a19.9 19.9 0 0 0 6 3.03.08.08 0 0 0 .08-.03c.46-.63.87-1.3 1.22-2a.08.08 0 0 0-.04-.11 13.1 13.1 0 0 1-1.87-.89.08.08 0 0 1-.01-.13c.13-.09.25-.19.37-.29a.07.07 0 0 1 .08-.01c3.93 1.79 8.18 1.79 12.06 0a.07.07 0 0 1 .08.01c.12.1.24.2.37.29a.08.08 0 0 1-.01.13c-.6.35-1.22.64-1.87.89a.08.08 0 0 0-.04.11c.36.7.77 1.36 1.22 2a.08.08 0 0 0 .08.03 19.8 19.8 0 0 0 6.02-3.03.08.08 0 0 0 .03-.05c.5-5.18-.84-9.67-3.55-13.66a.06.06 0 0 0-.03-.03zM8.02 15.33c-1.18 0-2.16-1.08-2.16-2.42s.96-2.42 2.16-2.42c1.21 0 2.18 1.1 2.16 2.42 0 1.34-.96 2.42-2.16 2.42zm7.97 0c-1.18 0-2.16-1.08-2.16-2.42s.96-2.42 2.16-2.42c1.21 0 2.18 1.1 2.16 2.42 0 1.34-.95 2.42-2.16 2.42z"/></svg>Community</a>
|
||||
<a href="/catalog">Catalog</a>
|
||||
<a onclick="openSignin()">Sign in</a>
|
||||
<button class="candy sm" onclick="openSignin()">Start free</button>
|
||||
</div>
|
||||
@@ -507,7 +568,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<!-- ================= HERO ================= -->
|
||||
<section class="hero">
|
||||
<div class="wrap">
|
||||
<span class="kicker rv">2,617 endpoints · 42 providers · <a href="https://github.com/superdesigndev/treg" target="_blank" rel="noopener">open source ↗</a></span>
|
||||
<span class="kicker rv"><a href="/catalog">2,630 endpoints · 47 providers</a> · <a href="https://github.com/superdesigndev/treg" target="_blank" rel="noopener">open source ↗</a></span>
|
||||
|
||||
<h1 class="hero-h1 rv">
|
||||
<span class="grad">OpenRouter for agent tools</span><br>
|
||||
@@ -559,7 +620,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<!-- ================= THE CATALOG — sub-sections only ================= -->
|
||||
<section class="roles" id="catalog">
|
||||
<div class="wrap">
|
||||
<div class="seclab rv">the catalog · 2,617 endpoints · one key</div>
|
||||
<div class="seclab rv">the catalog · 2,630 endpoints · one key</div>
|
||||
<div class="catwrap">
|
||||
<div class="pcat rv">Keyword & rank tracking</div>
|
||||
<div class="pgrid">
|
||||
@@ -662,7 +723,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
</div>
|
||||
</div>
|
||||
<div class="moreline rv">+2,500 more expert tools in the catalog<br>
|
||||
<button class="ghostbtn sm" onclick="openSignin()">Browse all 2,617 tools</button></div>
|
||||
<button class="ghostbtn sm" onclick="openSignin()">Browse all 2,630 tools</button></div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
@@ -677,7 +738,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<div class="bennum">01</div>
|
||||
<h3>One <i>unified key.</i></h3>
|
||||
<p>Forty-two providers, <b>one credential</b>. No per-provider signups, no key files scattered across machines — your agent holds a single treg token and every tool in the catalog answers to it.</p>
|
||||
<div class="benmeta"><span>one signup, 42 providers</span><span>revoke it in one place</span><span>works in any agent</span></div>
|
||||
<div class="benmeta"><span>one signup, 47 providers</span><span>revoke it in one place</span><span>works in any agent</span></div>
|
||||
</div>
|
||||
<div class="keywall">
|
||||
<div class="kw-key"><span class="kg">▚</span><code>trg_live_</code><span class="kdots">••••••••••••••••</span></div>
|
||||
@@ -739,7 +800,10 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<!-- 04 — 0% additional fee -->
|
||||
<div class="zeroband rv" id="zeroband">
|
||||
<h3>0% <i>additional fee.</i></h3>
|
||||
<div class="zeq"><span class="ps">$</span><code id="zeqtxt"></code><span class="zcaret"></span></div>
|
||||
<!-- The equation is spelled out here, not only in the typing animation below: this is the one
|
||||
line that states the pricing model, and a crawler (or a reader with JS off) saw an empty
|
||||
<code>. The script clears it on load and types it back in. -->
|
||||
<div class="zeq"><span class="ps">$</span><code id="zeqtxt">provider rate <b>+ $0.000 markup</b> = your price</code><span class="zcaret"></span></div>
|
||||
<div class="zsub">we earn on volume pricing with vendors — not on you</div>
|
||||
</div>
|
||||
|
||||
@@ -795,7 +859,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
</div>
|
||||
<div class="trustline rv">
|
||||
<span><b>100% open source</b> · AGPL</span>
|
||||
<span><b>2,617 endpoints</b> priced up front</span>
|
||||
<span><b>2,630 endpoints</b> priced up front</span>
|
||||
<span>credentials injected server-side, <b>never on a machine</b></span>
|
||||
</div>
|
||||
</div>
|
||||
@@ -806,7 +870,7 @@ footer{margin-top:150px;border-top:1px solid var(--line);position:relative;overf
|
||||
<div class="brand"><span class="glyph">▚</span> treg</div>
|
||||
<span style="font-family:var(--mono);font-size:12px">— 100% open source</span>
|
||||
<span class="sp"></span>
|
||||
<a href="/tutorial">docs</a><a href="/llms.txt">llms.txt</a><a href="https://github.com/superdesigndev/treg" target="_blank" rel="noopener">github ↗</a><a href="/docs">api</a><a href="/terms">terms</a><a href="/privacy">privacy</a>
|
||||
<a href="/catalog">catalog</a><a href="/tutorial">docs</a><a href="/llms.txt">llms.txt</a><a href="https://github.com/superdesigndev/treg" target="_blank" rel="noopener">github ↗</a><a href="/docs">api</a><a href="/terms">terms</a><a href="/privacy">privacy</a>
|
||||
</div>
|
||||
<div class="watermark">treg</div>
|
||||
</footer>
|
||||
@@ -1114,6 +1178,9 @@ function stopRotation(){ userTouched=true; clearInterval(roleTimer); }
|
||||
el.innerHTML=h;
|
||||
}
|
||||
if(REDUCED){ render(total); return; }
|
||||
render(0); // the markup ships with the equation already written out (so it survives with JS
|
||||
// off); clear it here, on load, so the type-in below starts from empty rather than
|
||||
// snapping from the full line back to one character when the band scrolls in.
|
||||
const io=new IntersectionObserver(es=>es.forEach(e=>{
|
||||
if(!e.isIntersecting) return;
|
||||
io.disconnect();
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# treg — the tool catalog for your agent
|
||||
|
||||
> **OpenRouter, but for agent tools instead of models.** Point an agent at ONE base URL with ONE
|
||||
> token and it can do the *job*: ~2,600 catalogued endpoints across ~48 providers (SEO and SERP
|
||||
> token and it can do the *job*: 2,630 catalogued endpoints across 47 providers (SEO and SERP
|
||||
> data, backlinks, social and trends, people and company enrichment, ads, scraping) — plus your
|
||||
> own team's keys, skills and CLIs. Every credential is injected **server-side**; nothing
|
||||
> sensitive lands on the agent's machine, and every call is audited.
|
||||
@@ -117,7 +117,7 @@ ledger.
|
||||
|
||||
## The catalog — tools you do not have a key for
|
||||
|
||||
Curated endpoints across ~48 providers, grouped by what they DO: keyword and rank tracking,
|
||||
Curated endpoints across 47 providers, grouped by what they DO: keyword and rank tracking,
|
||||
backlinks and authority, AI visibility, trending and discovery, publishing to socials, people and
|
||||
company enrichment, ads management and creative, measurement. Ask for the task; the catalog tells
|
||||
you which endpoints serve it, what each costs, and how you would be served. Exact live counts:
|
||||
@@ -474,7 +474,7 @@ skills` restrict it (`--dir` sets the location).
|
||||
8. **Report the outcome as a doorway, not an install log.** Nobody signed up to read which
|
||||
Python version uv picked. Open with ONE line about what they just gained, e.g.:
|
||||
|
||||
treg is live — your agent can now call 2,600+ tools across ~48 providers (market data, SEO &
|
||||
treg is live — your agent can now call 2,600+ tools across 47 providers (market data, SEO &
|
||||
keywords, social & trends, people/company enrichment, ads, scraping), with $1.00
|
||||
of free credit to spend.
|
||||
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 90 KiB |
@@ -5,6 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Treg Privacy Policy</title>
|
||||
<meta name="description" content="What Treg stores, why, who it's shared with, and how to get it deleted."/>
|
||||
<link rel="canonical" href="{BASE}/privacy"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
# treg — the tool catalog for your agent.
|
||||
# The catalog is meant to be found: /catalog and its per-platform shelves are open to every crawler.
|
||||
# What is disallowed below is either private, meaningless without a session, or costs money to call.
|
||||
|
||||
User-agent: *
|
||||
Allow: /
|
||||
|
||||
# The dashboard is an authenticated single-page app — every path under it needs a session, and its
|
||||
# views are hash routes a crawler cannot reach anyway.
|
||||
Disallow: /app
|
||||
Disallow: /login
|
||||
|
||||
# Auth and OAuth flows carry one-shot state in the URL. A crawled authorization link is, at best,
|
||||
# a burnt nonce.
|
||||
Disallow: /auth/
|
||||
Disallow: /oauth/
|
||||
Disallow: /connect-demo
|
||||
|
||||
# /call/ is the metered proxy: every request through it is a real upstream API call against a real
|
||||
# balance. Disallowed for cost, not for secrecy.
|
||||
Disallow: /call/
|
||||
Disallow: /mcp
|
||||
|
||||
Disallow: /admin
|
||||
|
||||
# The interactive API console. The same API is documented in server-rendered HTML at /docs, which
|
||||
# is the copy worth indexing.
|
||||
Disallow: /docs/api
|
||||
|
||||
# An agent looking for machine-readable onboarding wants this, not the sitemap:
|
||||
# {BASE}/llms.txt
|
||||
|
||||
Sitemap: {BASE}/sitemap.xml
|
||||
@@ -5,11 +5,67 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Treg Support</title>
|
||||
<meta name="description" content="How to get help with treg — the tool catalog for your agent."/>
|
||||
<!-- One page answers /support, /contact and /help. The canonical collapses all three into one URL
|
||||
so they cannot compete with each other in search. -->
|
||||
<link rel="canonical" href="{BASE}/support"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
<link href="https://fonts.googleapis.com/css2?family=Instrument+Serif:ital@0;1&family=Geist+Pixel&family=DM+Mono:ital,wght@0,400;0,500;1,400&display=swap" rel="stylesheet">
|
||||
<link rel="stylesheet" href="/legal.css"/>
|
||||
|
||||
<!-- FAQPage over section 02 below. The five questions and answers here are the ones ON the page,
|
||||
copied from it — Google treats FAQ schema that does not match the visible text as a violation,
|
||||
so if you edit a question in the body, edit it here in the same commit. -->
|
||||
<script type="application/ld+json">
|
||||
{
|
||||
"@context": "https://schema.org",
|
||||
"@type": "FAQPage",
|
||||
"url": "{BASE}/support",
|
||||
"mainEntity": [
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "I cannot sign in.",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "treg has three doors: GitHub, Google, and a one-time code sent to your email. If a code does not arrive, check spam for mail from no-reply@treg.to. Signing in for the first time creates the account — there is no separate registration."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "A call was refused for funds.",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Catalog endpoints that treg serves on its own key are metered against your team's prepaid balance. Run `treg balance` to see it, and `treg topup` to add more. A new team starts with $1.00 free, and there is a default ceiling of $5 per team per day so a runaway agent cannot spend without limit."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "I want my API keys removed.",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Deleting a tool or a secret in the dashboard removes the stored credential. To delete an entire team and everything in it, use `treg org delete <slug>`. If you would rather we did it, email us from the address on the account."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Something is charging more than I expected.",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "Every metered call is recorded with what it cost. `treg balance` shows the ledger. If a price looks wrong, tell us the endpoint id and we will check it against the provider's own rate — the catalog's prices are read from provider rate cards and from what we observe being charged, and both can drift."
|
||||
}
|
||||
},
|
||||
{
|
||||
"@type": "Question",
|
||||
"name": "Is my data used for anything else?",
|
||||
"acceptedAnswer": {
|
||||
"@type": "Answer",
|
||||
"text": "No. Credentials are encrypted at rest and used only to make the calls you or your team ask for. The privacy policy is the full answer."
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
</script>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Treg Terms of Service</title>
|
||||
<meta name="description" content="The terms that govern use of the hosted Treg service."/>
|
||||
<link rel="canonical" href="{BASE}/terms"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
<meta charset="utf-8"/>
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1"/>
|
||||
<title>Treg - Tutorial</title>
|
||||
<meta name="description" content="Learn treg in ten minutes: find a tool by what it does, call it through the proxy without holding a key, and bring your own keys and skills."/>
|
||||
<link rel="canonical" href="{BASE}/tutorial"/>
|
||||
<link rel="icon" type="image/svg+xml" href="/favicon.svg"/>
|
||||
<script src="/tutorial.js"></script>
|
||||
<style>
|
||||
|
||||
@@ -0,0 +1,279 @@
|
||||
"""Crawler-facing surfaces: robots.txt, sitemap.xml, HEAD, canonicals, and structured data.
|
||||
|
||||
These are easy to break silently — nothing in the app fails when a canonical goes stale or a
|
||||
sitemap starts listing a renamed route, and nobody notices until traffic does. So the sitemap test
|
||||
walks every URL it publishes rather than spot-checking, and the host tests assert on
|
||||
`public_url` rather than on the literal treg.to: a self-hosted registry must advertise itself.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from xml.etree import ElementTree as ET
|
||||
|
||||
import pytest
|
||||
from httpx import AsyncClient
|
||||
|
||||
from treg.config import get_settings
|
||||
|
||||
|
||||
SITEMAP_NS = "{http://www.sitemaps.org/schemas/sitemap/0.9}"
|
||||
|
||||
|
||||
def _base() -> str:
|
||||
return get_settings().public_url.rstrip("/")
|
||||
|
||||
|
||||
def _locs(xml: str) -> list[str]:
|
||||
return [e.text or "" for e in ET.fromstring(xml).iter(f"{SITEMAP_NS}loc")]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------- robots
|
||||
|
||||
async def test_robots_txt_is_served_and_names_the_sitemap(clients: AsyncClient):
|
||||
r = await clients.get("/robots.txt")
|
||||
assert r.status_code == 200, r.text
|
||||
assert r.headers["content-type"].startswith("text/plain")
|
||||
assert f"Sitemap: {_base()}/sitemap.xml" in r.text
|
||||
assert "User-agent: *" in r.text
|
||||
|
||||
|
||||
async def test_robots_txt_keeps_crawlers_out_of_what_costs_or_gates(clients: AsyncClient):
|
||||
"""The metered proxy and the authenticated app are the two that actually matter: one bills per
|
||||
request, the other has nothing to show a crawler."""
|
||||
body = (await clients.get("/robots.txt")).text
|
||||
for path in ("/app", "/call/", "/login", "/oauth/", "/docs/api"):
|
||||
assert f"Disallow: {path}" in body, path
|
||||
assert "Disallow: /catalog" not in body # the catalog is the whole point of indexing us
|
||||
|
||||
|
||||
async def test_robots_txt_has_no_unsubstituted_template(clients: AsyncClient):
|
||||
assert "{BASE}" not in (await clients.get("/robots.txt")).text
|
||||
|
||||
|
||||
# -------------------------------------------------------------------------------------- sitemap
|
||||
|
||||
async def test_sitemap_is_valid_xml_on_the_public_host(clients: AsyncClient):
|
||||
r = await clients.get("/sitemap.xml")
|
||||
assert r.status_code == 200, r.text
|
||||
assert "xml" in r.headers["content-type"]
|
||||
locs = _locs(r.text)
|
||||
assert len(locs) > 50, "the catalog shelves should dominate the sitemap"
|
||||
assert all(u.startswith(_base() + "/") or u == _base() + "/" for u in locs), locs[:3]
|
||||
|
||||
|
||||
async def test_sitemap_lists_the_catalog_shelves(clients: AsyncClient):
|
||||
locs = _locs((await clients.get("/sitemap.xml")).text)
|
||||
assert f"{_base()}/" in locs
|
||||
assert f"{_base()}/catalog" in locs
|
||||
assert any(u.startswith(f"{_base()}/catalog/") for u in locs)
|
||||
|
||||
|
||||
async def test_sitemap_omits_pages_that_would_not_answer_a_crawler(clients: AsyncClient):
|
||||
"""Each of these fails a crawl differently: /login redirects, /tool-requests is POST-only,
|
||||
/app needs a session, and /contact + /vendor-listing.md are duplicate URLs for pages already
|
||||
listed under their canonical name."""
|
||||
locs = set(_locs((await clients.get("/sitemap.xml")).text))
|
||||
for path in ("/login", "/tool-requests", "/app", "/contact", "/help",
|
||||
"/vendor-listing.md", "/connect-demo", "/install.sh", "/selfhost.sh"):
|
||||
assert f"{_base()}{path}" not in locs, path
|
||||
|
||||
|
||||
async def test_every_sitemap_url_answers_200(clients: AsyncClient):
|
||||
"""The test that earns its keep: rename a route and the sitemap starts publishing 404s, with
|
||||
nothing else in the suite noticing. Walks a sample of the catalog pages plus every static one,
|
||||
since 88 full renders would dominate the suite's runtime."""
|
||||
locs = _locs((await clients.get("/sitemap.xml")).text)
|
||||
static = [u for u in locs if not u.startswith(f"{_base()}/catalog/")]
|
||||
shelves = [u for u in locs if u.startswith(f"{_base()}/catalog/")][:5]
|
||||
for url in static + shelves:
|
||||
path = url[len(_base()):] or "/"
|
||||
r = await clients.get(path)
|
||||
assert r.status_code == 200, f"{path} -> {r.status_code} (listed in sitemap.xml)"
|
||||
|
||||
|
||||
async def test_sitemap_redirects_from_a_legacy_host(clients: AsyncClient):
|
||||
"""A sitemap served on treg.superdesign.dev but full of treg.to URLs is a cross-submission a
|
||||
crawler may discard wholesale. Send it to the canonical copy instead."""
|
||||
for path in ("/robots.txt", "/sitemap.xml"):
|
||||
r = await clients.get(path, headers={"Host": "treg.superdesign.dev"},
|
||||
follow_redirects=False)
|
||||
assert r.status_code == 301, path
|
||||
assert r.headers["location"] == f"{_base()}{path}"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------ HEAD
|
||||
|
||||
@pytest.mark.parametrize("path", ["/", "/tutorial", "/llms.txt", "/favicon.svg", "/support",
|
||||
"/robots.txt", "/sitemap.xml", "/catalog", "/meta"])
|
||||
async def test_head_is_answered_wherever_get_is(clients: AsyncClient, path: str):
|
||||
"""FastAPI's APIRoute never adds HEAD to a GET route, so every page 405'd on the probe crawlers
|
||||
and link unfurlers send first. api.py widens them all in one pass after registration."""
|
||||
r = await clients.head(path)
|
||||
assert r.status_code == 200, f"HEAD {path} -> {r.status_code}"
|
||||
assert r.content == b""
|
||||
|
||||
|
||||
async def test_head_is_still_refused_where_there_is_no_get(clients: AsyncClient):
|
||||
"""The widening must be surgical: a POST-only route keeps refusing HEAD."""
|
||||
assert (await clients.head("/tool-requests")).status_code == 405
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------- canonical
|
||||
|
||||
async def test_the_three_support_urls_share_one_canonical(clients: AsyncClient):
|
||||
"""/support, /contact and /help are one file (people guess differently). Without a canonical
|
||||
they are three URLs competing for the same page."""
|
||||
for path in ("/support", "/contact", "/help"):
|
||||
r = await clients.get(path)
|
||||
assert r.status_code == 200
|
||||
assert f'<link rel="canonical" href="{_base()}/support"/>' in r.text, path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path,canon", [("/terms", "/terms"), ("/privacy", "/privacy"),
|
||||
("/tutorial", "/tutorial")])
|
||||
async def test_pages_declare_their_canonical(clients: AsyncClient, path: str, canon: str):
|
||||
r = await clients.get(path)
|
||||
assert f'<link rel="canonical" href="{_base()}{canon}"/>' in r.text
|
||||
|
||||
|
||||
async def test_the_dashboard_is_noindex(clients: AsyncClient):
|
||||
r = await clients.get("/app")
|
||||
assert re.search(r'<meta name="robots" content="noindex', r.text)
|
||||
|
||||
|
||||
async def test_the_markdown_twin_of_vendor_listing_is_noindex(clients: AsyncClient):
|
||||
"""Two URLs, one document, and text/plain cannot carry a canonical — so the header does it."""
|
||||
assert (await clients.get("/vendor-listing.md")).headers.get("X-Robots-Tag") == "noindex"
|
||||
assert "X-Robots-Tag" not in (await clients.get("/vendor-listing")).headers
|
||||
|
||||
|
||||
# ------------------------------------------------------------------- catalog pages & structured data
|
||||
|
||||
async def test_the_catalog_index_lists_shelves_as_real_html(clients: AsyncClient):
|
||||
"""The whole point: 80 shelves that previously existed only as hash routes behind a login."""
|
||||
r = await clients.get("/catalog")
|
||||
assert r.status_code == 200
|
||||
assert r.text.count('class="pcard"') > 50
|
||||
assert 'href="/catalog/google"' in r.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", ["google", "web", "tiktok"])
|
||||
async def test_a_platform_page_renders_its_endpoints_as_text(clients: AsyncClient, slug: str):
|
||||
r = await clients.get(f"/catalog/{slug}")
|
||||
assert r.status_code == 200
|
||||
assert 'class="ep"' in r.text # endpoint rows, not an empty shell
|
||||
assert "treg call" in r.text # the call template is on the page
|
||||
|
||||
|
||||
async def test_the_json_catalog_routes_still_answer_json(clients: AsyncClient):
|
||||
"""`/catalog/<slug>` sits in front of these. Registration order keeps them matching first, and
|
||||
if that ever changes the dashboard and every CLI break at once."""
|
||||
for path in ("/catalog/platforms", "/catalog/platforms/google", "/catalog/search?q=backlinks",
|
||||
"/catalog/endpoints/moz.web.url.metrics"):
|
||||
r = await clients.get(path)
|
||||
assert r.status_code == 200, path
|
||||
assert r.headers["content-type"].startswith("application/json"), path
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", ["endpoints", "examples", "not-a-platform"])
|
||||
async def test_unknown_slugs_404(clients: AsyncClient, slug: str):
|
||||
"""`endpoints` and `examples` reach the page route (their JSON siblings need a trailing id), so
|
||||
only the reserved-word guard stops them rendering a nonsense shelf."""
|
||||
assert (await clients.get(f"/catalog/{slug}")).status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.parametrize("slug", ["platforms", "search"])
|
||||
async def test_reserved_slugs_never_render_the_html_page(clients: AsyncClient, slug: str):
|
||||
"""These two ARE valid URLs — the JSON routes registered before `/catalog/{slug}` claim them.
|
||||
What must never happen is the page route swallowing one and serving HTML to the dashboard."""
|
||||
r = await clients.get(f"/catalog/{slug}")
|
||||
assert r.headers["content-type"].startswith("application/json"), slug
|
||||
|
||||
|
||||
async def test_prices_never_render_in_scientific_notation(clients: AsyncClient):
|
||||
"""`%g` flips to exponent below 1e-4 and a shelf advertised "from $1.2e-07 per call", which
|
||||
reads as a bug rather than a price."""
|
||||
for slug in ("web", "google", "people"):
|
||||
assert not re.search(r"\$\d+(\.\d+)?e-\d+", (await clients.get(f"/catalog/{slug}")).text), slug
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path,types", [
|
||||
("/", {"SoftwareApplication", "Organization"}),
|
||||
("/catalog", {"ItemList", "BreadcrumbList"}),
|
||||
("/catalog/google", {"ItemList", "BreadcrumbList"}),
|
||||
("/support", {"FAQPage"}),
|
||||
])
|
||||
async def test_structured_data_parses_and_says_what_it_should(clients: AsyncClient, path, types):
|
||||
r = await clients.get(path)
|
||||
found = {json.loads(b)["@type"]
|
||||
for b in re.findall(r'application/ld\+json">(.*?)</script>', r.text, re.S)}
|
||||
assert types <= found, f"{path}: got {found}"
|
||||
|
||||
|
||||
async def test_the_landing_offer_matches_the_page(clients: AsyncClient):
|
||||
"""Schema that claims a price the page does not show is a structured-data violation, not a
|
||||
shortcut — so assert the free-credit figure appears in both."""
|
||||
r = await clients.get("/")
|
||||
ld = next(json.loads(b) for b in re.findall(r'application/ld\+json">(.*?)</script>', r.text, re.S)
|
||||
if json.loads(b)["@type"] == "SoftwareApplication")
|
||||
assert "$1.00 of free credit" in ld["offers"]["description"]
|
||||
assert "$1.00 free to start" in r.text # benefit 03, the visible claim
|
||||
assert "0%" in ld["offers"]["description"] and "0%" in r.text
|
||||
|
||||
|
||||
async def test_faq_schema_matches_the_visible_questions(clients: AsyncClient):
|
||||
r = await clients.get("/support")
|
||||
ld = next(json.loads(b) for b in re.findall(r'application/ld\+json">(.*?)</script>', r.text, re.S))
|
||||
for q in ld["mainEntity"]:
|
||||
assert f"<b>{q['name']}</b>" in r.text, f"schema asks {q['name']!r}, the page does not"
|
||||
|
||||
|
||||
async def test_every_page_carries_a_social_card(clients: AsyncClient):
|
||||
for path in ("/", "/catalog", "/catalog/google", "/docs"):
|
||||
body = (await clients.get(path)).text
|
||||
assert 'property="og:image"' in body and "/media/og.png" in body, path
|
||||
assert 'name="twitter:card" content="summary_large_image"' in body, path
|
||||
|
||||
|
||||
async def test_the_og_image_is_actually_served_at_the_right_size(clients: AsyncClient):
|
||||
"""Tags pointing at a 404 mean every shared link unfurls blank."""
|
||||
r = await clients.get("/media/og.png")
|
||||
assert r.status_code == 200
|
||||
assert r.headers["content-type"] == "image/png"
|
||||
# PNG header: width and height are big-endian uint32 at bytes 16..24
|
||||
width = int.from_bytes(r.content[16:20], "big")
|
||||
height = int.from_bytes(r.content[20:24], "big")
|
||||
assert (width, height) == (1200, 630), f"og.png is {width}x{height}, must be 1200x630"
|
||||
|
||||
|
||||
# ------------------------------------------------------------------------------------------ docs
|
||||
|
||||
async def test_docs_is_server_rendered_and_swagger_moved(clients: AsyncClient):
|
||||
"""/docs was a Swagger script shell — nothing for a crawler, and the landing linked to it."""
|
||||
r = await clients.get("/docs")
|
||||
assert r.status_code == 200
|
||||
assert "/call/{rest}" in r.text and "/catalog/search" in r.text
|
||||
assert "SwaggerUIBundle" not in r.text
|
||||
assert (await clients.get("/docs/api")).status_code == 200
|
||||
assert (await clients.get("/openapi.json")).status_code == 200
|
||||
|
||||
|
||||
async def test_docs_does_not_advertise_the_admin_api(clients: AsyncClient):
|
||||
assert "/admin/orgs" not in (await clients.get("/docs")).text
|
||||
|
||||
|
||||
async def test_widening_head_did_not_leak_into_the_public_schema(clients: AsyncClient):
|
||||
"""Adding HEAD to every GET route gave FastAPI a second operation per path — 58 duplicate
|
||||
entries in openapi.json, each with a duplicate operation id. Only the /call proxy, which
|
||||
declares HEAD itself, should have one."""
|
||||
paths = (await clients.get("/openapi.json")).json()["paths"]
|
||||
with_head = [p for p, ops in paths.items() if "head" in ops]
|
||||
assert with_head == ["/call/{rest}"], with_head
|
||||
|
||||
|
||||
async def test_no_page_ships_an_unsubstituted_base(clients: AsyncClient):
|
||||
"""`{BASE}` reaching a browser means a canonical or og:url is pointing at nothing."""
|
||||
for path in ("/", "/support", "/terms", "/privacy", "/tutorial", "/catalog"):
|
||||
assert "{BASE}" not in (await clients.get(path)).text, path
|
||||
Reference in New Issue
Block a user