# /robots.txt file for http://webcrawler.com/
# created 18mar02


User-agent: *
Disallow: /counters
Disallow: /utils
Disallow: /images
Disallow: /upload