User-Agent: *
Allow: /
Disallow: /cgi-bin
Disallow: /lib
Disallow: /admin
Disallow: /stats
Disallow: /sendpage

# A capture bot, downloads gazillions of pages with no public benefit
# http://www.webreaper.net/
User-agent: WebReaper
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: HTTrack
Disallow: /

User-agent: wget
Disallow: /


User-agent: Yahoo Pipes 1.0
Disallow: /
User-agent: Yahoo Pipes 2.0
Disallow: /