# www.robotstxt.org/
# www.google.com/support/webmasters/bin/answer.py?hl=en&answer=156449

User-agent: Slurp
Crawl-delay: 10

User-agent: TwitterBot
Crawl-delay: 10

User-agent: Baiduspider
Allow: /search/page/2
Allow: /search/page/3
Disallow: /search/page/

User-agent: Pinterest
Crawl-delay: 1

User-agent: msnbot
Crawl-delay: 1

User-agent: grapeshot
Disallow:

User-agent: *
Disallow: /app.php
Disallow: /pageviews.php
Disallow: /wp-admin
Disallow: /administration
Disallow: /logout
Disallow: /giveaway-form-submit
Disallow: /form-submit
Disallow: /bookmarks
Disallow: /firebase-subscribe
Disallow: /account
Disallow: /api
Disallow: /search-suggest
Disallow: /amp-disqus-embed
Disallow: /comments-section
Disallow: /hypeindex/graph
Disallow: /hypeindex/performance-graph
Disallow: /forum/
Disallow: /mailto
Disallow: /next-posts
Disallow: /.well-known
Disallow: /zh/next-posts
Disallow: /jp/next-posts
Disallow: /embed
Disallow: /_ajax
Disallow: /_fragment
Allow: /

# blocking bad bots
User-agent: rogerbot
Disallow: /
User-agent: mj12bot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: ahrefsbot
Disallow: /
User-agent: Linguee Bot
Disallow: /
User-agent: Domain Re-Animator Bot
Disallow: /
User-agent: MauiBot (crawler.feedback+wc@gmail.com)
Disallow: /
Disallow: /search
Disallow: /search/

# ChatGPT
User-agent: GPTBot
Disallow: /

# Google Bard
User-agent: Google-Extended
Disallow: /