User-agent: Atomz/1.0
Disallow: 

# Algolia-Crawler-Verif: 9861C54A2B29EE1B

User-agent: Algolia Crawler
Allow: /

User-agent: AhrefsSiteAudit
Allow: /

User-agent: AhrefsBot
Allow: /

User-agent: *
Disallow: /*?*

User-agent: *
Disallow: /content/*.html


# Allow major search engines
User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: DuckDuckBot
Allow: /

User-agent: Yandex
Allow: /

User-agent: Applebot
Allow: /

# Block known AI crawlers
User-agent: GPTBot
Disallow: /

User-agent: CCBot
Disallow: /

User-agent: AnthropicBot
Disallow: /

User-agent: ClaudeBot
Disallow: /

User-agent: Claude-Web
Disallow: /

User-agent: Bytespider
Disallow: /

User-agent: ChatGPT-User
Disallow: /

User-agent: Google-Extended
Disallow: /

User-agent: Amazonbot
Disallow: /

User-agent: Applebot-Extended
Disallow: /

User-agent: PerplexityBot
Disallow: /

User-agent: Meta-ExternalFetcher
Disallow: /

User-agent: Diffbot
Disallow: /

User-agent: ImagesiftBot
Disallow: /

User-agent: Omgili
Disallow: /

User-agent: YouBot
Disallow: /

User-agent: PiplBot
Disallow: /

User-agent: Scrapy
Disallow: /

User-agent: OAI-SearchBot
Disallow: /


# Allow access to essential resources
Allow: /*.css$
Allow: /*.js$

Sitemap: https://www.hersheyland.com/sitemap.xml