# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism
# User-agent: Slurp (name, or * for any)
# Disallow: / (entire content of the site)
# Crawl-delay: 10 (time in minutes)

User-agent: *
Allow: / 
Crawl-delay: 1000