User-Agent: AhrefsBot
Disallow: /

User-Agent: SemrushBot
Disallow: /

User-Agent: PetalBot
Disallow: /

User-Agent: TikTokSpider
Disallow: /

User-Agent: *
# Block private user areas
Disallow: /checkout/
Disallow: /my-board/
Disallow: /account/
Disallow: /mimi/
Disallow: /webhooks/

# Block duplicate content and parameters
Disallow: /*?
Disallow: /*&
Disallow: /*/search
Disallow: /*/filter

# Block development and testing URLs
Disallow: /dev/
Disallow: /test/
Disallow: /staging/

# Block low-value pages that consume crawl budget
Disallow: /static/
Disallow: /media/uploads/temp/
Disallow: /*/glossary/

# Allow important content
Allow: /en/
Allow: /fr/
Allow: /th/
Allow: /*/blog/*/
Allow: /*/new-hsk-*/

# Crawl delay to prevent overloading (based on 31,465 crawled but not indexed)
Crawl-delay: 1

# Sitemaps
Sitemap: https://www.hsktracker.com/sitemap.xml
Sitemap: https://www.hsktracker.com/sitemap-static.xml
Sitemap: https://www.hsktracker.com/sitemap-articles.xml
Sitemap: https://www.hsktracker.com/sitemap-characters.xml