# Robots.txt file 
# All robots will spider the domain
# User-Agent: Slurp
# Disallow: /
# go away



User-agent: ia_archiver
Disallow: /
Crawl-delay: 2000000

User-agent: *
Disallow:

User-agent: *
Disallow: /*.axd$

User-Agent: Spinn3r
Disallow: /

User-agent: *
Disallow: /search/