User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /wp-admin/
Disallow: /wp-includes/

User-agent: msnbot
Crawl-delay: 5

User-agent: Slurp
Crawl-delay: 300

User-agent: Yahoo!-AdCrawler
Disallow: / 

User-agent: twiceler 
Disallow: /