# Ledger官网 Robots.txt
# https://www.zdfzbw.com/robots.txt

User-agent: *
Allow: /

# 禁止爬取的页面
Disallow: /about.html
Disallow: /contact.html
Disallow: /terms.html
Disallow: /privacy.html

# 禁止爬取的目录
Disallow: /admin/
Disallow: /api/
Disallow: /tmp/
Disallow: /*.json$

# 站点地图
Sitemap: https://www.zdfzbw.com/sitemap.xml

# 爬虫访问频率建议
Crawl-delay: 1

# 百度蜘蛛
User-agent: Baiduspider
Allow: /
Disallow: /about.html
Disallow: /contact.html
Disallow: /terms.html
Disallow: /privacy.html
Crawl-delay: 1

# 搜狗蜘蛛
User-agent: Sogou web spider
Allow: /
Disallow: /about.html
Disallow: /contact.html
Disallow: /terms.html
Disallow: /privacy.html
Crawl-delay: 1

# Google蜘蛛
User-agent: Googlebot
Allow: /
Disallow: /about.html
Disallow: /contact.html
Disallow: /terms.html
Disallow: /privacy.html