# Mantra Tech Fusion Robots.txt

User-agent: *
Allow: /

# Core Public Areas

Allow: /services/
Allow: /industries/
Allow: /locations/
Allow: /packages/
Allow: /tools/
Allow: /blog/
Allow: /contact-us/

# Block Sensitive Areas

Disallow: /admin/
Disallow: /storage/
Disallow: /cache/
Disallow: /logs/
Disallow: /includes/
Disallow: /templates/
Disallow: /vendor/
Disallow: /database/
Disallow: /scratch/
Disallow: /backups/

# Block Internal Search & Query Parameters

Disallow: /search
Disallow: /*?*
Allow: /*?page=

# AI Crawlers

User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: ClaudeBot
Allow: /

User-agent: PerplexityBot
Allow: /

User-agent: Google-Extended
Allow: /

User-agent: Applebot
Allow: /

User-agent: Bingbot
Allow: /

# Optional Crawl Delay (for aggressive bots only)

User-agent: AhrefsBot
Crawl-delay: 10

User-agent: SemrushBot
Crawl-delay: 10

User-agent: MJ12bot
Disallow: /

# Sitemap

Sitemap: https://mantratechfusion.com/sitemap.xml