#17 nov 09
User-agent: *
Disallow: /flvfiles
Disallow: /flvfiles2
Disallow: /flvfiles3
Disallow: /flvfiles4
Disallow: /flvplayer
Disallow: /flvpub
Disallow: /fmp3
Disallow: /ricola
Disallow: /xspf
Disallow: /404.asp
Crawl-delay: 30
Visit-time: 0600-0845
Request-rate: 1/5
User-agent: Googlebot
Disallow:
User-agent: cuil.com
Disallow:/
Crawl-delay: 5
User-agent: Slurp
Disallow: /
User-agent: bing
Disallow: /
User-agent: Inktomi Slurp
Disallow: /
User-agent: Ask Jeeves
Disallow: /
User-agent: boitho.com-dc
Disallow: /
User-agent: Busiverse
Disallow: /
User-agent: CazoodleBot
Disallow: /
User-agent: Exabot
Disallow: /
User-agent: Googlebot-Image
Disallow: /
User-agent: heritrix
Disallow: /
User-agent: IRLbot
Disallow: /
User-agent: Jyxobot
Disallow: /
#Disallow Majestic12.co.uk
User-agent: MJ12bot
Disallow: /
User-agent: Mirago-Test-Robot (http://www.miragorobot.com)
Disallow: /
User-agent: msnbot
Disallow: /*.gif$
UserAgent: nimblecrawler
Disallow: /
UserAgent: metadatalabs
Disallow: /
User-agent: psbot
Disallow: /
User-agent: Sirketce
Disallow: /
User-agent: Seekbot
Disallow: /
User-agent: semanticdiscovery
Disallow: /
User-agent: Sogou
Disallow: /
User-agent: sogou.com
Disallow: /
User-agent: Soso
Disallow: /
User-agent: Sosoimagespider
Disallow: /
User-agent: TinEye
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: twiceler
Disallow: /
User-agent: VoilaBot
Disallow: /
User-agent: Voila
Disallow: /
User-agent: WebAlta
Disallow: /
User-agent: Yahoo
Disallow: /
User-agent: Yahoo-MMCrawler
Disallow: /
User-agent: YodaoBot
Disallow: /
User-agent: Tarantula
Disallow: /
Disallow: / Yandex bot
Disallow: /
Disallow: / Yandex
Disallow: /
User-agent: zermelo
Disallow: /
User-agent: Tagoo
Disallow: /
User-agent: Wanadoo
Disallow: /
User-agent: BaiDuSpider
Disallow: /
User-agent: BSpider
Disallow: /
User-agent: alexa
Disallow: /
User-agent: The Python Robot
Disallow: /