# Ulvibaba.com - robots.txt
# Policy: crawl clean product/category pages; avoid search, filters and sort params.
# Updated: 2026-06-01

# --- Aggressive SEO/tool bots (block all) ---
User-agent: AhrefsBot
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: amazonbot
Disallow: /

User-agent: Sogou Spider
Disallow: /
User-agent: sogou spider
Disallow: /

# --- Social / preview bots (allow canonical pages, disallow params/search) ---
User-agent: FacebookExternalHit
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: Facebot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: meta-externalagent
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: Twitterbot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: LinkedInBot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: Slackbot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: Discordbot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: Pinterestbot
Disallow: /search
Disallow: /*?*
Allow: /

User-agent: WhatsApp
Disallow: /search
Disallow: /*?*
Allow: /

# --- Default (all other bots) ---
User-agent: *
Disallow: /ubadmin/
Disallow: /ajax/
Disallow: /cache/
Disallow: /tmp/
Disallow: /dosyalar/gecici/

# Search & parameterized URLs (do not crawl)
Disallow: /search
Disallow: /*?*q=
Disallow: /*?*filtre=
Disallow: /*?*redirect=*filtre
Disallow: /*?*siralama=
Disallow: /*?*attr=
Disallow: /*?*filter=
Disallow: /*?*sort=
Disallow: /*?*fiyat=
Disallow: /*?*beden=

# Non-indexable flows
Disallow: /*/yorumlar
Disallow: /*/benzer-urunler
Disallow: /sepet
Disallow: /giris
Disallow: /kayit
Disallow: /siparislerim
Disallow: /checkout
Disallow: /popup

# Sitemap
Sitemap: https://www.ulvibaba.com/sitemap.xml