User-agent: webcrawler
User-agent: *
Disalow: /aspdiagnostic
Disalow: /etc
Disalow: /mail
Disalow: /public_ftp
Disalow: /tmp
Disalow: /www
Disalow: /cgi-bin