# /robots.txt for info.webcrawler.com
User-agent: *
Disallow: /cgi/
Disallow: /*.taf$