# disallow 
User-agent: *
Disallow: /history
Disallow: /my_list
Disallow: /admin_*

User-agent: ICC-Crawler
User-agent: meta-externalagent
User-agent: Bingbot
User-agent: Applebot
User-agent: AmazonBot
User-agent: ClaudeBot
User-Agent: GPTBot
User-agent: AhrefsBot
User-agent: Qualys
User-agent: YandexBot
User-agent: PetalBot
User-agent: SeekportBot
User-agent: DataForSeoBot
User-Agent: barkrowler
User-Agent: DotBot
User-Agent: BLEXBot
User-Agent: ImagesiftBot
User-Agent: SemrushBot
User-Agent: MJ12bo
User-Agent: Barkrowler
Disallow: /buy_shop_search/area/*-*-*
Disallow: /buy_shop_search/route/*-*-*


# Crawl-delay
User-agent: *
Crawl-delay: 5

User-agent: ICC-Crawler
User-agent: meta-externalagent
User-agent: Bingbot
User-agent: Slurp
User-agent: ClaudeBot
User-Agent: GPTBot
User-agent: AmazonBot
User-agent: Applebot
User-agent: SeekportBot
User-agent: AhrefsBot
User-agent: DataForSeoBot
User-agent: barkrowler
User-Agent: DotBot
User-Agent: BLEXBot
User-Agent: ImagesiftBot
User-Agent: SemrushBot
User-Agent: MJ12bo
User-Agent: Barkrowler
Crawl-delay: 20