# robots.txt for https://www.instepphysio.ca/
# Site Type: WordPress Healthcare Business Website
# Notes: Agency homepage
# Last updated: Sep 2026
# Maintained by LinkBuffer Studios — linkbufferstudios.com

User-agent: *
Allow: /
Allow: /llms.txt

# WordPress core - administration and utility paths
Disallow: /wp-admin/
Allow: /wp-admin/admin-ajax.php
Disallow: /wp-login.php
Disallow: /wp-register.php
Disallow: /xmlrpc.php
Disallow: /readme.html
Disallow: /license.txt

# WordPress frontend resources (wp-includes, themes, plugins,
# uploads and cache) are crawlable by default.
# Only temporary upgrade files remain excluded.
Disallow: /wp-content/upgrade/

# Block low-value WordPress paths
Disallow: /trackback/
Disallow: /*/trackback/
Disallow: /tag/
Disallow: /author/
Disallow: /?author=
Disallow: /comments/feed/

# Block internal search and tracking parameters
Disallow: /?s=
Disallow: /*?s=
Disallow: /*?*replytocom=
Disallow: /*?*share=
Disallow: /*?*print=
Disallow: /*?*utm_
Disallow: /*?*fbclid=
Disallow: /*?*gclid=
Disallow: /*?*msclkid=

# ============================================
# AI / LLM CRAWLERS - Explicitly allowed
# Existing crawler access policy preserved
# ============================================

# Amazon - Alexa / Amazon AI
User-agent: Amazonbot
Allow: /

# Anthropic - Claude
User-agent: ClaudeBot
Allow: /

User-agent: Claude-User
Allow: /

User-agent: Claude-SearchBot
Allow: /

# Apple - Siri / Apple Intelligence
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# ByteDance
User-agent: Bytespider
Allow: /

# Cohere
User-agent: cohere-ai
Allow: /

# Google
User-agent: Google-Extended
Allow: /

User-agent: GoogleOther
Allow: /

# Meta
User-agent: Meta-ExternalAgent
Allow: /

User-agent: FacebookBot
Allow: /

# OpenAI - ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Perplexity
User-agent: PerplexityBot
Allow: /

User-agent: Perplexity-User
Allow: /

# Major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

# Semrush
User-agent: SemrushBot
Allow: /

User-agent: SemrushBot-SA
Allow: /

# ============================================
# OTHER BOTS - Existing explicit blocks preserved
# ============================================
User-agent: AhrefsBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

User-agent: SeznamBot
Disallow: /

# ============================================
# SITEMAP & RESOURCES
# ============================================

Sitemap: https://www.instepphysio.ca/sitemap_index.xml

# LLM-readable site summary (if deployed at root)
# Reference: https://www.instepphysio.ca/llms.txt