feat(pages): standardize web crawler policy (#939)

* feat: standardize web crawler policies

* js sitemap gen core

* route package

* robots.txt modifications & deploy & test

* chore: issues for OCR

* remove sitemap.xml

* feat: Apply suggestion from @lizhengfeng101

---------

Co-authored-by: Kite <254839944+lizhengfeng101@users.noreply.github.com>

.github/workflows/deploy-pages.yml

@@ -48,13 +48,14 @@ jobs:
           cp install.sh _site/install.sh
           cp install.ps1 _site/install.ps1
 
-      - name: Verify install scripts
+      - name: Verify deployment artifact
         run: |
           test -f _site/install.sh
           test -f _site/install.ps1
           sh -n _site/install.sh
           cmp install.sh _site/install.sh
           cmp install.ps1 _site/install.ps1
+          test -e _site/robots.txt
 
       - uses: actions/upload-pages-artifact@v5
         with:

pages/README.md

@@ -117,7 +117,7 @@ pages/
 │   ├── utils/           # Shared helpers
 │   ├── assets/          # Imported icons and images
 │   └── index.tsx        # Frontend entry point
-├── public/              # Static files copied verbatim into dist/
+├── public/              # Static files copied verbatim into dist/ & crawling policies
 ├── dist/                # Production build artifacts (generated, gitignored)
 ├── index.html           # HTML template used by HtmlWebpackPlugin
 ├── webpack.config.cjs   # Bundling + dev server config

pages/public/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Allow: /