# Baker Street Properties - crawler policy
#
# Search engines and AI crawlers are welcome. The AI-bot block shipped in
# April 2026 (v2.0.5) was deliberately reverted four days later (v2.0.6) to
# protect search visibility and traffic; the explicit allowances below codify
# that open posture so it is policy, not accident. Only the broker CRM under
# /backend/ is off limits.
#
# AI answer engines: a machine-readable guide to this site is published at
# https://www.baker-street.co.za/llms.txt

# --- AI retrieval and answer-engine crawlers (user-triggered fetches and
# --- AI-search indexing) ---

User-agent: OAI-SearchBot
Allow: /
Disallow: /backend/

User-agent: ChatGPT-User
Allow: /
Disallow: /backend/

User-agent: Claude-SearchBot
Allow: /
Disallow: /backend/

User-agent: Claude-User
Allow: /
Disallow: /backend/

User-agent: PerplexityBot
Allow: /
Disallow: /backend/

User-agent: Perplexity-User
Allow: /
Disallow: /backend/

# --- AI training crawlers ---

User-agent: GPTBot
Allow: /
Disallow: /backend/

User-agent: ClaudeBot
Allow: /
Disallow: /backend/

User-agent: CCBot
Allow: /
Disallow: /backend/

User-agent: meta-externalagent
Allow: /
Disallow: /backend/

User-agent: Amazonbot
Allow: /
Disallow: /backend/

# --- Everyone else ---

User-agent: *
Disallow: /backend/

Sitemap: https://www.baker-street.co.za/sitemap.xml