User-agent: Googlebot Disallow: User-agent: Googlebot-Image Disallow: User-agent: MSNBot Disallow: User-agent: Slurp Disallow: User-agent: Teoma Disallow: User-agent: Scrubby Disallow: User-agent: Robozilla Disallow: User-agent: Nutch Disallow: User-agent: ia_archiver Disallow: User-agent: baiduspider Disallow: User-agent: yahoo-mmcrawler Disallow: User-agent: psbot Disallow: User-agent: asterias Disallow: User-agent: yahoo-blogs/v3.9 Disallow: User-agent: * Disallow: Disallow: /bbweblog/ # A capture bot, downloads gazillions of pages with no public benefit # http://www.webreaper.net/ User-agent: WebReaper Disallow: / User-agent: Teleport Disallow: / User-agent: TeleportPro Disallow: / # # Hits many times per second, not acceptable # http://www.nameprotect.com/botinfo.html User-agent: NPBot Disallow: /