User-agent: *
Allow: /
Disallow: /account
Disallow: /signin
Disallow: /register
Disallow: /forgot
Disallow: /reset
Disallow: /verify
Disallow: /newsroom
Disallow: /staff
Disallow: /messages
Disallow: /notifications
Disallow: /pay
Disallow: /*?sort=
Disallow: /*&sort=
Disallow: /news/*/react
Disallow: /news/*/readend
Sitemap: https://9ntyapi.com/sitemap.xml

# Site-copying and scraper bots (they only take, never send visitors)
User-agent: HTTrack
Disallow: /

User-agent: WebZIP
Disallow: /

User-agent: WebCopier
Disallow: /

User-agent: WebStripper
Disallow: /

User-agent: SiteSnagger
Disallow: /

User-agent: Offline Explorer
Disallow: /

User-agent: Teleport
Disallow: /

User-agent: TeleportPro
Disallow: /

User-agent: wget
Disallow: /

User-agent: EmailCollector
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: DotBot
Disallow: /

User-agent: BLEXBot
Disallow: /

User-agent: PetalBot
Disallow: /

User-agent: MegaIndex
Disallow: /