WordPress robots.txt

robots.txt

# robots.txt for WordPress / v1.01 2020-01-27 / Peeter Marvet
# v1.01 - changed Crawl-delay to max allowed by Yandex e.g 2.0
#
# Uncomment and specify hostname (sitemap can be on different domain):
#Sitemap: https://example.com/sitemap_index.xml
#
# User-agents are not cumulative, e.g if a bot finds matching group * is ignored.
User-agent: *
Crawl-delay: 2.0
# All requests with parameters (search, WooCommerce filters etc):
Disallow: /*?*

# Add everything to be blocked here:
User-agent: AhrefsBot
User-agent: AhrefsSiteAudit
User-agent: SemrushBot
User-agent: MJ12bot
Disallow: /