User-agent: *
Disallow: /etc/
Disallow: /ftp/
Disallow: /tmp/
Disallow: /users/
Disallow: /var/
Disallow: /www/cgi-bin/
Disallow: /www/logs/
Disallow: /www/urchin5/
Disallow: /www/htdocs/persistent/
Disallow: /www/htdocs/logo/
Disallow: /www/htdocs/form/
Disallow: /www/htdocs/form/auth/
Disallow: /www/htdocs/files/

User-agent: msnbot
Crawl-delay: 10

User-agent: Teoma
Crawl-delay: 10

User-agent: Slurp
Crawl-delay: 10

User-agent: aipbot
Disallow: /

User-agent: BecomeBot
Disallow: /

User-agent: psbot
Disallow: /