# IP Market Robots.txt
# Last updated: 2026-01-29

User-agent: *
# Disallow LLM-specific files from search engine indexing
Disallow: /llms.txt
Disallow: /llms-full.txt

# Disallow sensitive application directories
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /storage
Disallow: /vendor
Disallow: /.env
Disallow: /.git
Disallow: /login
Disallow: /register
Disallow: /password
Disallow: /email
Disallow: /cart
Disallow: /checkout
Disallow: /my
Disallow: /settings

# Disallow search and filter pages with parameters
Disallow: /*?*sort=
Disallow: /*?*filter=
Disallow: /*?*page=

# Allow public assets
Allow: /css
Allow: /js
Allow: /img
Allow: /build
Allow: /sw-ipmarket.js

# Allow important marketing pages
Allow: /en/
Allow: /nl/
Allow: /de/
Allow: /es/
Allow: /fr/

# Crawl delay to prevent aggressive crawling
Crawl-delay: 1

# Sitemaps
Sitemap: https://ipmarket.io/sitemap.xml

# Google-specific rules
User-agent: Googlebot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /login
Disallow: /register
Crawl-delay: 0

# Bing-specific rules
User-agent: Bingbot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard
Disallow: /login
Disallow: /register
Crawl-delay: 1

# AI Crawlers - Allow for better AI understanding
User-agent: GPTBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: ChatGPT-User
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: Claude-Web
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: Anthropic-AI
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: PerplexityBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: Google-Extended
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: Applebot-Extended
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: CCBot
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

User-agent: cohere-ai
Allow: /
Disallow: /admin
Disallow: /api
Disallow: /dashboard

# Block aggressive/unwanted bots
User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Crawl-delay: 10