# =========================================
# Sitemap Declaration
# =========================================
Sitemap: https://portugalbiketours.com/sitemaps.xml

# =========================================
# General Rules for All User-Agents
# =========================================
User-agent: *
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php

# Block low-value and duplicate content
Disallow: /feed/
Disallow: /page/
Disallow: /search
Disallow: /tag/
Disallow: /author/

# Block all URLs with query parameters
Disallow: /*?*

# Block outdated yearly archives
Disallow: /2015/
Disallow: /2016/
Disallow: /2017/
Disallow: /2018/
Disallow: /2019/
Disallow: /2020/
Disallow: /2021/
Disallow: /2022/
Disallow: /2023/

# =========================================
# Bot-Specific Rules and Crawl Delay
# =========================================

# Allow Google AdSense bot
User-agent: Mediapartners-Google
Disallow:

# Allow Google Image bot
User-agent: Googlebot-Image
Disallow:

# Block Ahrefs and apply crawl delay
User-agent: AhrefsBot
Disallow: /
Crawl-delay: 10

# Block MJ12bot
User-agent: MJ12bot
Disallow: /

# Block Semrush bots and apply crawl delay
User-agent: SemrushBot
Disallow: /
Crawl-delay: 10

User-agent: SemrushBot-SA
Disallow: /
Crawl-delay: 10