# Dark Visitors robots.txt
# Consolidated with endpoint allow rules from website2control.properties
# This file blocks AI scrapers while allowing legitimate access to public endpoints

# robots.txt
# This blocks Amazonbot

User-agent: Amazonbot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Assistant
# https://darkvisitors.com/agents/cohere-ai

User-agent: cohere-ai
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/anthropic-ai

User-agent: anthropic-ai
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/ccbot

User-agent: CCBot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/facebookbot

User-agent: FacebookBot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/google-extended

User-agent: Google-Extended
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/gptbot

User-agent: GPTBot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

# AI Data Scraper
# https://darkvisitors.com/agents/omgilibot

User-agent: omgilibot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: omgili
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: PetalBot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: YandexBot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: Twitterbot
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: facebookexternalhit
Allow: /products*
Allow: /support*
Allow: /solutions*
Allow: /company*
Allow: /about*
Allow: /news*
Allow: /careers*
Allow: /contact*
Allow: /videos*
Allow: /download*
Allow: /partners*
Allow: /groupdiscount*
Allow: /platform*
Allow: /media*
Allow: /upubabout*
Allow: /purchase_faq*
Allow: /ios*
Allow: /ipad*
Allow: /ipad_signup*
Allow: /help*
Allow: /cookie_policy*
Allow: /contactus*
Allow: /$
Disallow: /

User-agent: *
Crawl-delay: 10
Disallow: /ddo
Disallow: /family
Disallow: /anesthesia
Disallow: /practice
Disallow: /pco
Disallow: /relief
Disallow: /evidence
Disallow: /medicine
Disallow: /pedscentral
Disallow: /nursingcentral
Disallow: /tabersonline
Disallow: /labdxtest
Disallow: /hopkins
Disallow: /5mcc
Disallow: /redbook
Disallow: /5minute/ub?cmd=fulltext
Disallow: /5minute/ub/fplogin
Disallow: /5minute/ub/citation
Disallow: /5minute/ub/print_abstract
Disallow: /medline/ebm/related
Disallow: /medline/ebm/*/*?start=
Disallow: /medline/medline?st=R
Disallow: /medline/medline?st=M
Disallow: /medline/medline?mesh
Disallow: /medline/mode=classic
Disallow: /medline/grapherence
Disallow: /medline/?st=R
Disallow: /medline/?st=M
Disallow: /medline/?mesh
Disallow: /medline/research/*?start=
Disallow: /medline/journal/*?start=
Disallow: /merckmanual/ub/view
Disallow: /merckmanual/view
Disallow: /harrietlane/ub/citation
Disallow: /harrietlane/ub/print_abstract
Disallow: /harrietlane/ub/print_citation
Disallow: /harrietlane/ub?cmd=fulltext
Disallow: /harrietlane2
Disallow: /washingtonmanual/ub/citation/
Disallow: /washingtonmanual/ub/print_abstract
Disallow: /washingtonmanual/ub/print_citation
Disallow: /washingtonmanual/ub?cmd=fulltext
Disallow: /products*dcode=
Disallow: /products/*/trial/
Disallow: /store/*/trial/
Disallow: /products/ub
Disallow: /products/*/demo
Disallow: /appstores/*
Disallow: /sitelicense
Disallow: /medl/
Disallow: /apis/
Disallow: /ascrs/
Disallow: /administration/
Disallow: /setup
Disallow: /account-update
Disallow: /*/setup
Disallow: /*/account-update
Disallow: /solutions/internal_grasp2
Disallow: /demo/
Disallow: /*/cookie-consent
Disallow: /cookie-consent

Sitemap: https://www.unboundmedicine.com/sitemap