# Deny all robots that we do not specifically want to allow User-agent: * Disallow: / # Allow these robots only User-agent: googlebot Allow: / User-agent:* Crawl-delay:20