# /robots.txt file for http://webcrawler.com/
# mail webmaster@webcrawler.com for constructive criticism

User-agent: TurnitinBot
Disallow: /

User-agent: *
Disallow: /epost
Disallow: /gallery
Disallow: /gallery2
Disallow: /insults
Disallow: /fun
Crawl-delay: 10
Request-rate: 1/10        # maximum rate is one page every 10 seconds
Visit-time: 0100-0445     # only visit between 01:00 and 04:45 UTC (GMT)