User-agent: Googlebot
Allow: /

User-agent: bingbot
Crawl-delay: 60

User-agent: dotbot
Crawl-delay: 1000

User-agent: magpie-crawler
Disallow: /

User-agent: *
Disallow: /