User-agent: *
Allow: /
Allow: /knowledge/
Allow: /chat.html
Allow: /about.html
Allow: /terms.html
Allow: /privacy.html
Allow: /contact.html

Disallow: /api/
Disallow: /.well-known/

# 主流爬虫
User-agent: Baiduspider
Allow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: 360Spider
Allow: /

User-agent: Sogou web spider
Allow: /

User-agent: Yandex
Allow: /

# AI 爬虫（按需放行）
User-agent: GPTBot
Allow: /knowledge/
Disallow: /chat.html
Disallow: /api/

User-agent: ClaudeBot
Allow: /knowledge/
Disallow: /chat.html
Disallow: /api/

# 安全扫描器（拒绝）
User-agent: ZGrab
Disallow: /

User-agent: CensysInspect
Disallow: /

User-agent: l9scan
Disallow: /

Sitemap: https://shchenxingtech.com/sitemap.xml