feat(site): improve AI-crawler discoverability and page load performance (#3344)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

site/public/robots.txt

@@ -4,13 +4,31 @@ Allow: /
 User-agent: GPTBot
 Allow: /
 
-User-agent: Claude-Web
+User-agent: OAI-SearchBot
+Allow: /
+
+User-agent: ChatGPT-User
 Allow: /
 
 User-agent: ClaudeBot
 Allow: /
 
+User-agent: Claude-User
+Allow: /
+
+User-agent: Claude-SearchBot
+Allow: /
+
 User-agent: PerplexityBot
 Allow: /
 
+User-agent: Perplexity-User
+Allow: /
+
+User-agent: meta-externalagent
+Allow: /
+
+User-agent: Amazonbot
+Allow: /
+
 Sitemap: https://strandsagents.com/sitemap-index.xml

site/src/components/SiteScripts.astro

@@ -1,12 +1,23 @@
 ---
 /**
- * AWS Shortbread cookie consent and Analytics WebSDK initialization.
+ * Web font loading, AWS Shortbread cookie consent, and Analytics WebSDK initialization.
  * Included in both Head.astro (docs pages) and LandingLayout.astro (index page).
  */
 ---
 
+<!-- Web fonts: preconnect + a direct stylesheet link. Still render-blocking (deliberately,
+     to avoid FOUT), but drops the chained fetch a CSS @import in custom.css forced:
+     the font CSS request couldn't even start until the bundled stylesheet was parsed. -->
+<link rel="preconnect" href="https://fonts.googleapis.com" />
+<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
+<link
+  rel="stylesheet"
+  href="https://fonts.googleapis.com/css2?family=Figtree:wght@400;500;600;700&display=swap"
+/>
+
+<link rel="preconnect" href="https://prod.assets.shortbread.aws.dev" crossorigin />
 <link rel="stylesheet" href="https://prod.assets.shortbread.aws.dev/shortbread.css" />
-<script is:inline src="https://prod.assets.shortbread.aws.dev/shortbread.js"></script>
+<script is:inline defer src="https://prod.assets.shortbread.aws.dev/shortbread.js"></script>
 <script is:inline>
   /**
    * Shortbread Cookie Consent and AWS Analytics WebSDK Initialization
@@ -15,57 +26,59 @@
   ;(function () {
     'use strict'
 
-    const hostname = window.location.hostname
-    const isLocal = hostname === 'localhost' || hostname === '127.0.0.1'
-    const isCloudFrontPreview = hostname === 'd3ehv1nix5p99z.cloudfront.net'
-    const isProduction = hostname.endsWith('.strandsagents.com') || hostname === 'strandsagents.com'
+    function initShortbread() {
+      const hostname = window.location.hostname
+      const isLocal = hostname === 'localhost' || hostname === '127.0.0.1'
+      const isCloudFrontPreview = hostname === 'd3ehv1nix5p99z.cloudfront.net'
+      const isProduction = hostname.endsWith('.strandsagents.com') || hostname === 'strandsagents.com'
 
-    const domain = isLocal || isCloudFrontPreview ? hostname : '.strandsagents.com'
+      const domain = isLocal || isCloudFrontPreview ? hostname : '.strandsagents.com'
 
-    let webSdkStage, webSdkDomain
-    if (isProduction) {
-      webSdkStage = 'prod'
-      webSdkDomain = 'https://d0.m.awsstatic.com'
-    } else if (isLocal) {
-      webSdkStage = 'alpha'
-      webSdkDomain = 'https://da0.m.awsstatic.com'
-    } else {
-      webSdkStage = 'gamma'
-      webSdkDomain = 'https://dg0.m.awsstatic.com'
-    }
+      let webSdkStage, webSdkDomain
+      if (isProduction) {
+        webSdkStage = 'prod'
+        webSdkDomain = 'https://d0.m.awsstatic.com'
+      } else if (isLocal) {
+        webSdkStage = 'alpha'
+        webSdkDomain = 'https://da0.m.awsstatic.com'
+      } else {
+        webSdkStage = 'gamma'
+        webSdkDomain = 'https://dg0.m.awsstatic.com'
+      }
 
-    if (typeof AWSCShortbread === 'undefined') {
-      console.warn('Shortbread unavailable. Using essential cookies only.')
-      return
-    }
+      if (typeof AWSCShortbread === 'undefined') {
+        console.warn('Shortbread unavailable. Using essential cookies only.')
+        return
+      }
 
-    try {
-      const existingShortbreadEl = document.getElementById('awsccc-sb-ux-c')
-      if (existingShortbreadEl) existingShortbreadEl.remove()
+      try {
+        const existingShortbreadEl = document.getElementById('awsccc-sb-ux-c')
+        if (existingShortbreadEl) existingShortbreadEl.remove()
 
-      const shortbread = AWSCShortbread({
-        domain,
-        registry: {
-          GitHubRepoCache: { category: 'essential' },
-        },
-        language: AWSCShortbread.Locale?.getLocaleIdShortbread(
-          AWSCShortbread.Locale.getCurrentLocale()
-        ),
-      })
+        const shortbread = AWSCShortbread({
+          domain,
+          registry: {
+            GitHubRepoCache: { category: 'essential' },
+          },
+          language: AWSCShortbread.Locale?.getLocaleIdShortbread(
+            AWSCShortbread.Locale.getCurrentLocale()
+          ),
+        })
 
-      shortbread.checkForCookieConsent()
-      window.strandsShortbread = shortbread
+        shortbread.checkForCookieConsent()
+        window.strandsShortbread = shortbread
 
-      document.addEventListener('click', function (e) {
-        if (e.target.id === 'cookie-preferences') {
-          e.preventDefault()
-          shortbread.customizeCookies()
-        }
-      })
+        document.addEventListener('click', function (e) {
+          if (e.target.id === 'cookie-preferences') {
+            e.preventDefault()
+            shortbread.customizeCookies()
+          }
+        })
 
-      loadWebSDK(webSdkStage, webSdkDomain)
-    } catch (error) {
-      console.error('Shortbread error:', error)
+        loadWebSDK(webSdkStage, webSdkDomain)
+      } catch (error) {
+        console.error('Shortbread error:', error)
+      }
     }
 
     function loadWebSDK(stage, sdkDomain) {
@@ -80,5 +93,13 @@
         console.error('WebSDK loading error:', error)
       }
     }
+
+    // shortbread.js loads with `defer`, so AWSCShortbread is not defined at parse time.
+    // Deferred scripts are guaranteed to execute before DOMContentLoaded fires.
+    if (document.readyState === 'loading') {
+      document.addEventListener('DOMContentLoaded', initShortbread)
+    } else {
+      initShortbread()
+    }
   })()
 </script>

site/src/components/overrides/Head.astro

@@ -8,6 +8,7 @@ import { getCollection } from 'astro:content'
 import SiteScripts from '../SiteScripts.astro'
 import Analytics from '../Analytics.astro'
 import { baseSchemas } from '../../util/structured-data'
+import { getDocsIds } from '../../util/docs-ids'
 import { relatedUserGuideFor } from '../../util/related-docs'
 import { resolveLanguage, sourceLinkUrl } from '../../util/source-links'
 import type { SourceLink } from '../../content.config'
@@ -19,6 +20,13 @@ const route = Astro.locals.starlightRoute
 const base = import.meta.env.BASE_URL || '/'
 const faviconHref = base.endsWith('/') ? `${base}favicon.svg` : `${base}/favicon.svg`
 
+// Raw markdown twin served by src/pages/[...slug]/index.md.ts — one endpoint per docs
+// collection entry, at `/{entry.id}/index.md`. Only advertise it for routes backed by a
+// docs entry (blog/changelog pages flow through Starlight with synthetic entries).
+const docIds = await getDocsIds()
+const baseNoSlash = base.replace(/\/$/, '')
+const markdownHref = docIds.has(route.entry.id) ? `${baseNoSlash}/${route.entry.id}/index.md` : null
+
 // Build breadcrumb list from URL path
 const siteUrl = Astro.site?.origin || 'https://strandsagents.com'
 const pathSegments = Astro.url.pathname.replace(/^\/|\/$/g, '').split('/').filter(Boolean)
@@ -125,6 +133,9 @@ const structuredData = {
 <!-- Favicon - uses logo-auto.svg which adapts to light/dark mode via prefers-color-scheme -->
 <link rel="icon" type="image/svg+xml" href={faviconHref} />
 
+<!-- Raw markdown twin of this page, for LLMs and tooling -->
+{markdownHref && <link rel="alternate" type="text/markdown" href={markdownHref} />}
+
 <!-- AWS Shortbread cookie consent -->
 <SiteScripts />
 
@@ -135,48 +146,53 @@ const structuredData = {
 <script type="application/ld+json" set:html={JSON.stringify(structuredData)} />
 
 <script>
-  import mermaid from 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs'
+  // Mermaid is ~2MB from the CDN, so only load it on pages that actually have a diagram
+  if (document.querySelector('pre[data-language="mermaid"]')) {
+    const { default: mermaid } = await import(
+      /* @vite-ignore */ 'https://cdn.jsdelivr.net/npm/mermaid@11/dist/mermaid.esm.min.mjs'
+    )
 
-  // Transform Expressive Code mermaid blocks into mermaid.js format
-  document.querySelectorAll('pre[data-language="mermaid"]').forEach((pre) => {
-    const diagram = [...pre.querySelectorAll('code > .ec-line')].map((it) => it.textContent).join('\n')
-    const div = document.createElement('pre')
-    div.className = 'mermaid'
-    div.textContent = diagram
-    pre.replaceWith(div)
-  })
+    // Transform Expressive Code mermaid blocks into mermaid.js format
+    document.querySelectorAll('pre[data-language="mermaid"]').forEach((pre) => {
+      const diagram = [...pre.querySelectorAll('code > .ec-line')].map((it) => it.textContent).join('\n')
+      const div = document.createElement('pre')
+      div.className = 'mermaid'
+      div.textContent = diagram
+      pre.replaceWith(div)
+    })
 
-  const isDark = document.documentElement.dataset.theme === 'dark'
-  mermaid.initialize({ startOnLoad: false, theme: isDark ? 'dark' : 'neutral' })
+    const isDark = document.documentElement.dataset.theme === 'dark'
+    mermaid.initialize({ startOnLoad: false, theme: isDark ? 'dark' : 'neutral' })
 
-  // Only render diagrams in visible panels; defer hidden ones until shown
-  const hiddenDiagrams = new Set(
-    document.querySelectorAll('[role="tabpanel"][hidden] .mermaid')
-  )
-  const visibleDiagrams = [...document.querySelectorAll('.mermaid')].filter(
-    (el) => !hiddenDiagrams.has(el)
-  )
-  if (visibleDiagrams.length > 0) {
-    await mermaid.run({ nodes: visibleDiagrams })
-  }
-
-  // Render deferred diagrams once when their panel becomes visible
-  const observer = new MutationObserver((mutations) => {
-    for (const mutation of mutations) {
-      if (mutation.type === 'attributes' && mutation.attributeName === 'hidden') {
-        const panel = mutation.target as Element
-        if (panel.hasAttribute('hidden')) continue
-        const pending = [...panel.querySelectorAll('.mermaid')].filter(
-          (el) => hiddenDiagrams.has(el)
-        )
-        if (pending.length === 0) continue
-        pending.forEach((el) => hiddenDiagrams.delete(el))
-        mermaid.run({ nodes: pending })
-      }
+    // Only render diagrams in visible panels; defer hidden ones until shown
+    const hiddenDiagrams = new Set(
+      document.querySelectorAll('[role="tabpanel"][hidden] .mermaid')
+    )
+    const visibleDiagrams = [...document.querySelectorAll('.mermaid')].filter(
+      (el) => !hiddenDiagrams.has(el)
+    )
+    if (visibleDiagrams.length > 0) {
+      await mermaid.run({ nodes: visibleDiagrams })
     }
-  })
 
-  document.querySelectorAll('[role="tabpanel"]').forEach((panel) => {
-    observer.observe(panel, { attributes: true, attributeFilter: ['hidden'] })
-  })
+    // Render deferred diagrams once when their panel becomes visible
+    const observer = new MutationObserver((mutations) => {
+      for (const mutation of mutations) {
+        if (mutation.type === 'attributes' && mutation.attributeName === 'hidden') {
+          const panel = mutation.target as Element
+          if (panel.hasAttribute('hidden')) continue
+          const pending = [...panel.querySelectorAll('.mermaid')].filter(
+            (el) => hiddenDiagrams.has(el)
+          )
+          if (pending.length === 0) continue
+          pending.forEach((el) => hiddenDiagrams.delete(el))
+          mermaid.run({ nodes: pending })
+        }
+      }
+    })
+
+    document.querySelectorAll('[role="tabpanel"]').forEach((panel) => {
+      observer.observe(panel, { attributes: true, attributeFilter: ['hidden'] })
+    })
+  }
 </script>

site/src/pages/[...slug]/index.md.ts

@@ -9,7 +9,7 @@
  * their corresponding /index.md endpoints.
  *
  * Used by LLMs and tooling that need documentation in a machine-readable format.
- * See the /llms/ page for more information.
+ * See /docs/user-guide/build-with-ai/ for more information.
  */
 import type { APIRoute, GetStaticPaths } from 'astro'
 import { getCollection } from 'astro:content'

site/src/pages/llms.txt.ts

@@ -10,12 +10,25 @@ import path from 'node:path'
 // Sections to pull from sidebar (with their nav labels)
 const SIDEBAR_SECTIONS = ['Docs', 'Examples', 'Community']
 
+/**
+ * Format a llms.txt link line: `- [title](url): description`, omitting the
+ * `: description` suffix when no description is available. Internal whitespace
+ * (including newlines from folded YAML) collapses to single spaces so an entry
+ * can never span multiple lines of this machine-parsed format.
+ */
+function linkLine(label: string, url: string, description?: string, indent: string = ''): string {
+  const normalized = description?.replace(/\s+/g, ' ').trim()
+  const suffix = normalized ? `: ${normalized}` : ''
+  return `${indent}- [${label}](${url})${suffix}`
+}
+
 /**
  * Recursively extract links from sidebar items
  */
 function extractLinks(
   items: StarlightSidebarItem[],
   base: string,
+  descriptions: Map<string, string | undefined>,
   depth: number = 0
 ): string[] {
   const lines: string[] = []
@@ -26,7 +39,7 @@ function extractLinks(
       // Internal link
       const url = `${base}/${item.slug}/index.md`
       const label = item.label || item.slug.split('/').pop() || item.slug
-      lines.push(`${indent}- [${label}](${url})`)
+      lines.push(linkLine(label, url, descriptions.get(item.slug), indent))
     } else if ('link' in item && item.link) {
       // External link - skip or include as-is
       if (!item.link.startsWith('http')) {
@@ -35,7 +48,7 @@ function extractLinks(
     } else if ('items' in item && item.items) {
       // Group - add label and recurse
       lines.push(`${indent}- ${item.label}`)
-      lines.push(...extractLinks(item.items, base, depth + 1))
+      lines.push(...extractLinks(item.items, base, descriptions, depth + 1))
     }
   }
 
@@ -46,6 +59,12 @@ function buildLlmsTxt(docs: CollectionEntry<'docs'>[], sidebar: StarlightSidebar
   const base = getSiteOrigin() + getBase()
   const lines: string[] = []
 
+  // Frontmatter descriptions keyed by doc id, for annotating sidebar-derived
+  // links. Sidebar slugs live in the same namespace (sidebar.ts validates each
+  // against src/content files); if that ever diverges, a miss here degrades to
+  // a link without a description rather than an error.
+  const descriptions = new Map(docs.map((doc) => [doc.id, doc.data.description]))
+
   lines.push('# Strands Agents')
   lines.push('')
   lines.push('> Strands Agents is a simple yet powerful SDK that takes a model-driven approach to building and running AI agents. From simple conversational assistants to complex autonomous workflows, from local development to production deployment, Strands Agents scales with your needs.')
@@ -60,7 +79,7 @@ function buildLlmsTxt(docs: CollectionEntry<'docs'>[], sidebar: StarlightSidebar
     if (section && 'items' in section) {
       lines.push(`## ${sectionName}`)
       lines.push('')
-      lines.push(...extractLinks(section.items, base, 0))
+      lines.push(...extractLinks(section.items, base, descriptions, 0))
       lines.push('')
     }
   }
@@ -76,7 +95,7 @@ function buildLlmsTxt(docs: CollectionEntry<'docs'>[], sidebar: StarlightSidebar
     for (const doc of pythonApi) {
       const url = `${base}/${doc.id}/index.md`
       const title = doc.data.title || doc.id
-      lines.push(`- [${title}](${url})`)
+      lines.push(linkLine(title, url, doc.data.description))
     }
     lines.push('')
   }
@@ -87,7 +106,7 @@ function buildLlmsTxt(docs: CollectionEntry<'docs'>[], sidebar: StarlightSidebar
     for (const doc of typescriptApi) {
       const url = `${base}/${doc.id}/index.md`
       const title = doc.data.title || doc.id
-      lines.push(`- [${title}](${url})`)
+      lines.push(linkLine(title, url, doc.data.description))
     }
     lines.push('')
   }
@@ -99,7 +118,7 @@ function buildLlmsTxt(docs: CollectionEntry<'docs'>[], sidebar: StarlightSidebar
     lines.push('')
     for (const post of sorted) {
       const url = `${base}/blog/${post.id}/index.md`
-      lines.push(`- [${post.data.title}](${url})`)
+      lines.push(linkLine(post.data.title, url, post.data.description))
     }
     lines.push('')
   }

site/src/styles/custom.css

@@ -7,7 +7,9 @@
  * - Dark bg: #0E0E0E
  */
 
-@import url('https://fonts.googleapis.com/css2?family=Figtree:wght@400;500;600;700&display=swap');
+/* The Figtree web font is loaded via preconnect + <link rel="stylesheet"> in
+   src/components/SiteScripts.astro (included by Head.astro and LandingLayout.astro)
+   rather than an @import here, which would chain a render-blocking fetch. */
 
 /* Dark mode colors. */
 :root {

site/src/util/docs-ids.ts

@@ -0,0 +1,11 @@
+import { getCollection } from 'astro:content'
+
+// Built once per build, not once per page: Head.astro runs for every route
+// (~850 pages) and only needs membership checks against the docs ids.
+let cached: Promise<Set<string>> | undefined
+
+/** The set of docs content-collection ids, cached across pages within a build. */
+export function getDocsIds(): Promise<Set<string>> {
+  cached ??= getCollection('docs').then((docs) => new Set(docs.map((doc) => doc.id)))
+  return cached
+}