User-agent: * # applies to all robots Disallow: /log/ Disallow: /cgi-bin/ Disallow: /admin/ Disallow: /client/