user-agent: msnbot crawl-delay: 120 User-agent: * Disallow: /tmp Disallow: /logs Disallow: /webstats Disallow: /htdocs/webstats Disallow: /cgi-bin Disallow: /htdocs/itravel/admin # Wildcards are _not_ supported: instead of 'Disallow: /tmp/*' just say 'Disallow: /tmp/'. # You shouldn't put more than one path on a Disallow line (this may change in a future version of the spec)