#
# Hello bot! There are some routes that need authentication. You should avoid them :)
#

User-agent: *
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview

# Faceted search on /app.
# The plain browse page (/app) and category facets stay crawlable. Sort order,
# keyword queries, price ranges and map/location params only produce
# near-duplicates of that same page, so crawling their permutations spends crawl
# budget that belongs to listing pages.
#
# NOTE: `page=` is deliberately NOT blocked. Search Console shows Google has
# found 115 products while the listings sitemap advertises only 43, so paginated
# browse is currently a real discovery path into the catalog. Revisit once the
# sitemap covers every listing (see docs/SEO_ANALYTICS_AUDIT.md §4.5 G, Wave 3).
# Param names verified against src/util/search.js and src/config/configSearch.js.
# Deliberately NOT blocked: pub_categoryLevel1 / pub_categoryLevel2 /
# pub_categoryLevel3 (and any other pub_* / meta_* facet) — single-facet category
# URLs are intended to become indexable landing pages in a later change.
Disallow: /app?*sort=
Disallow: /app?*keywords=
Disallow: /app?*price=
Disallow: /app?*dates=
Disallow: /app?*bounds=
Disallow: /app?*address=
Disallow: /app?*origin=
Disallow: /app?*mapSearch=

#
# AI crawlers.
#
# Pastel is a consumer marketplace and benefits from being visible on AI
# surfaces, so these crawlers are ALLOWED. They get their own group only so the
# same private/authenticated paths as above stay off limits — a named group makes
# a crawler ignore the "User-agent: *" group entirely, so the rules have to be
# repeated here.
#
# NOTE: flipping any of these to a blanket "Disallow: /" is a BUSINESS decision
# (AI training and AI-search visibility), not a technical one. Do not change it
# without an explicit product/business call.
#
User-agent: GPTBot
User-agent: ChatGPT-User
User-agent: OAI-SearchBot
User-agent: ClaudeBot
User-agent: Claude-User
User-agent: Claude-SearchBot
User-agent: PerplexityBot
User-agent: Google-Extended
User-agent: Applebot-Extended
User-agent: CCBot
User-agent: Bytespider
User-agent: meta-externalagent
Allow: /
Disallow: /profile-settings
Disallow: /l/new
Disallow: /l/*/checkout
Disallow: /l/*/draft
Disallow: /l/*/pending-approval
Disallow: /l/*/new
Disallow: /l/*/edit
Disallow: /inbox
Disallow: /order
Disallow: /sale
Disallow: /listings
Disallow: /account
Disallow: /reset-password
Disallow: /verify-email
Disallow: /preview

Sitemap: https://mypastel.com/sitemap-index.xml