User-agent: *
Allow: /

# 禁止抓取特定目录
Disallow: /admin/
Disallow: /private/
Disallow: /temp/
Disallow: /backup/
Disallow: /cgi-bin/
Disallow: /tmp/
Disallow: /logs/

# 允许抓取重要文件
Allow: /sitemap.xml
Allow: /robots.txt
Allow: /favicon.ico
Allow: /images/

# 爬虫延迟设置
Crawl-delay: 1

# Sitemap
Sitemap: https://falaiw.com/sitemap.xml

# 特定爬虫设置
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: Baiduspider
Allow: /
Crawl-delay: 2