# General rules for all bots User-agent: * Crawl-delay: 10 # Disallow admin and sensitive areas Disallow: /admin/ Disallow: /upload/ Disallow: /resources/ Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /private/ # Allow everything else Allow: / # Block file types Disallow: /*.env$ Disallow: /*.log$ Disallow: /*.sql$