User-agent: * Disallow: /private/ Disallow: /secret/ Disallow: /admin/ Disallow: /cgi-bin/ Disallow: /tmp/ Disallow: /wp-admin/ Disallow: /wp-includes/ Disallow: /wp-content/plugins/ Disallow: /wp-content/themes/ Disallow: /readme.html Disallow: /license.txt Disallow: /xmlrpc.php Disallow: /wp- Disallow: /feed/ Disallow: /trackback/ Disallow: /page/ Disallow: /comments/ Disallow: /category/*/* Disallow: /tag/* Disallow: /author/* Disallow: /search/ Disallow: /?s= Disallow: /*.pdf$ Disallow: /*.doc$ Disallow: /*.xls$ Disallow: /*.ppt$ # Crawl delay for all bots Crawl-delay: 10 # Allow well-behaved bots User-agent: Googlebot User-agent: Bingbot User-agent: Slurp User-agent: DuckDuckBot Disallow: # Block known malicious bots User-agent: MJ12bot Disallow: / User-agent: AhrefsBot Disallow: / User-agent: SemrushBot Disallow: / User-agent: DotBot Disallow: / User-agent: YandexBot Disallow: / User-agent: Baiduspider Disallow: / User-agent: Sogou Disallow: / User-agent: Exabot Disallow: / User-agent: QQ Browser Disallow: / User-agent: BLEXBot Disallow: /