# robots.txt, ekstrabladet.dk

User-agent: msnbot
Crawl-delay: 1

User-agent: AhrefsBot
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: SemrushBot
Disallow: /

User-agent: Eyeotabot
Disallow: /

User-agent: SeznamBot
Disallow: /

User-agent: DataForSeoBot
Disallow: /

########################################################################################################################
### AI crawler reference                                                                                             ###
### The link below provides instructions to what kind of content can be used to train AI models on this website      ###
### https://ekstrabladet.dk/ai.txt                                                                                   ###
########################################################################################################################

# Common crawl
User-agent: CCBot
Disallow: /

# OpenAI (ChatGPT)
User-agent: GPTBot
Disallow: /

# OpenAI (ChatGPT realtime search)
User-agent: ChatGPT-User
Disallow: /

# Anthropic
User-agent: anthropic-ai
Disallow: /

User-agent: Google-Extended
Disallow: /

# adsense
User-agent: Mediapartners-Google
Disallow:

User-agent: *
Disallow: /ritzau_new/
Disallow: /ebtv/
Disallow: /component/
Disallow: /payflow/
Disallow: /bibliotek/
Disallow: /afstemninger/
Disallow: /backoffice/
Disallow: /EbNewsConfiguration/
Disallow: /side9/

Sitemap: https://ekstrabladet.dk/svc/sitemap/