# AI/ML Crawlers
User-agent: GPTBot
Disallow: /
User-agent: ChatGPT-User
Disallow: /
User-agent: CCBot
Disallow: /
User-agent: anthropic-ai
Disallow: /
User-agent: Omgilibot
Disallow: /
User-agent: Omgili
Disallow: /
User-agent: Claude-Web
Disallow: /
User-agent: Claude-Web-Embed
Disallow: /
User-agent: Amazonbot
Disallow: /
User-agent: Bytespider
Disallow: /
User-agent: PerplexityBot
Disallow: /
User-agent: Diffbot
Disallow: /
User-agent: DeepSeekBot
Disallow: /
User-agent: DeepSeek
Disallow: /
User-agent: OAI-SearchBot
Disallow: /
User-agent: DuckAssistBot
Disallow: /

# ALLOW SHOP FOR OAI-Searchbot, ChatGPT-User, Perplexity
User-agent: OAI-Searchbot
Allow: /shop

User-agent: ChatGPT-User
Allow: /shop

User-agent: Perplexity
Allow: /shop

# Other Specific Crawlers
User-agent: discobot
Disallow: /
User-agent: dotbot
Disallow: /
User-agent: TurnitinBot
Disallow: /
User-agent: magpie-crawler
Disallow: /
User-agent: yacybot
Disallow: /
User-agent: Timpibot
Disallow: /
User-agent: Nutch
Disallow: /

# Grapeshot
User-agent: grapeshot
Disallow: /sitemap/

# Google Adsbot
User-agent: adsbot-google
Disallow: /checkouts/
Disallow: /checkout
Disallow: /carts
Disallow: /orders
Disallow: /63826591966/checkouts
Disallow: /63826591966/orders
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*

# Ahrefs Crawlers
User-agent: AhrefsBot
Crawl-delay: 10
User-agent: AhrefsSiteAudit
Crawl-delay: 10

Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /63826591966/checkouts
Disallow: /63826591966/orders
Disallow: /carts
Disallow: /account
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Allow: /search/
Disallow: /search/?*
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail

# Other Rate-Limited Crawlers
User-agent: MJ12bot
Crawl-Delay: 10
User-agent: Pinterest
Crawl-delay: 1

# All Other Crawlers
User-agent: *
Disallow: /*?*direction=
Disallow: /*?*cursor=
Disallow: /*?*jw_start=
Disallow: /*?*Size=
Disallow: /*?*dropID=
Disallow: /*?*Material=
Disallow: /*?*queryId=
Disallow: /core/
Disallow: /dex/
Disallow: /cgi-bin/
Disallow: /includes/
Disallow: /flash/
Disallow: /plugins/
Disallow: /system/
Disallow: /widgets/
Disallow: /assets/
Disallow: /blogs/wp-content/
Disallow: /blog-galleries/
Disallow: /blogs/
Disallow: /api/
Disallow: /static/js/
Disallow: /static/css/
Disallow: /js/
Disallow: /css/
Disallow: /tv/
Disallow: /search?*
Disallow: /admin
Disallow: /cart
Disallow: /orders
Disallow: /checkouts/
Disallow: /checkout
Disallow: /63826591966/checkouts
Disallow: /63826591966/orders
Disallow: /carts
Disallow: /account
Disallow: /blogs/*+*
Disallow: /blogs/*%2B*
Disallow: /blogs/*%2b*
Disallow: /*/blogs/*+*
Disallow: /*/blogs/*%2B*
Disallow: /*/blogs/*%2b*
Disallow: /*?*oseid=*
Disallow: /*preview_theme_id*
Disallow: /*preview_script_id*
Disallow: /policies/
Disallow: /*/*?*ls=*&ls=*
Disallow: /*/*?*ls%3D*%3Fls%3D*
Disallow: /*/*?*ls%3d*%3fls%3d*
Disallow: /search
Allow: /search/
Disallow: /search/?*
Disallow: /apple-app-site-association
Disallow: /.well-known/shopify/monorail
Disallow: /shop/drops/takashi-murakami-mlb-tokyo-series-2025

# Global Allow
Allow: /sitemap/

# Sitemaps
Sitemap: https://www.complex.com/shop/sitemap.xml
Sitemap: https://www.complex.com/sitemap/v3/news.xml
Sitemap: https://www.complex.com/sitemap/v3/articles.xml
Sitemap: https://www.complex.com/sitemap/v3/videos.xml
Sitemap: https://www.complex.com/sitemap/v3/pages.xml
Sitemap: https://www.complex.com/sitemap/v3/tags.xml
Sitemap: https://www.complex.com/sitemap/v3/authors.xml
Sitemap: https://www.complex.com/sitemap/v3/index.xml