# QEF Official — Hydrogen storefront
# robots.txt

# ============================================================
# Default rules — applies to all bots
# Strategy: canonical-only. Filter/sort/cursor/tracking variants are
# crawl-OPEN; canonical hints in <link> consolidate equity to clean URLs.
# Only truly non-public surfaces are disallowed below.
# ============================================================
User-agent: *

# App / internal surfaces (never indexable)
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/

# Collection URLs: crawl OPEN. Filters/sort/cursors share canonical with
# the clean collection URL — Google consolidates equity correctly.

# Blog tag union URLs (?+? combos) — duplicate content
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*

# Policies — thin content
Disallow: /policies/

# Search — query pages have no SEO value
Disallow: /search

# ============================================================
# Google AdsBot — needs to land on filtered URLs for ad QA
# ============================================================
User-agent: AdsBot-Google
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

# ============================================================
# Nutch — abusive scraper, fully blocked
# ============================================================
User-agent: Nutch
Disallow: /

# ============================================================
# 3rd-party SEO crawlers — rate-limited
# (Google ignores Crawl-delay; safe to leave these here.)
# ============================================================
User-agent: AhrefsBot
Crawl-delay: 10
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

User-agent: AhrefsSiteAudit
Crawl-delay: 10
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

User-agent: SemrushBot
Crawl-delay: 5
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

User-agent: MJ12bot
Crawl-delay: 10
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

User-agent: Pinterestbot
Crawl-delay: 1
Disallow: /api/
Disallow: /cart
Disallow: /account
Disallow: /discount/
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /policies/
Disallow: /search

# ============================================================
# SiteAuditBot — full access for site audit tooling
# ============================================================
User-agent: SiteAuditBot
Allow: /

# ============================================================
Sitemap: https://www.qefofficial.com/sitemap.xml