# This file instructs all WWW robots NOT to index pages that begin 
# with the URLS listed.
User-agent: *
Disallow: /acquire
Disallow: /airpac
Disallow: /airwkst
Disallow: /articles
Disallow: /availlim
Disallow: /bookill
Disallow: /bookit
Disallow: /circhistlim
Disallow: /circpix
Disallow: /cisti_order
Disallow: /clearhist
Disallow: /documents
Disallow: /donate
Disallow: /extlang
Disallow: /feeds
Disallow: /ftlist
Disallow: /goto
Disallow: /iii
Disallow: /ill
Disallow: /illframe
Disallow: /indexsort
Disallow: /journill
Disallow: /kids
Disallow: /launch
Disallow: /logout
Disallow: /manage
Disallow: /manual
Disallow: /metafind
Disallow: /mfgo
Disallow: /netli
Disallow: /nonret
Disallow: /patroninfo
Disallow: /programs
Disallow: /record=
Disallow: /review
Disallow: /screens
Disallow: /search
Disallow: /selfreg
Disallow: /setlang
Disallow: /setscope
Disallow: /suggest
Disallow: /tmp
Disallow: /validate
Disallow: /VERIFYPATRON
Disallow: /VERSION
Disallow: /weblang
Disallow: /wm
Disallow: /xrecord=
Disallow: /z39
Disallow: /z39m

# For the WebBridge Google Scholar Extension. Allows googlebot_IA to crawl
# /screens
User-agent: Googlebot-IA
Disallow: /acquire
Disallow: /airpac
Disallow: /airwkst
Disallow: /articles
Disallow: /availlim
Disallow: /bookill
Disallow: /bookit
Disallow: /circhistlim
Disallow: /circpix
Disallow: /cisti_order
Disallow: /clearhist
Disallow: /documents
Disallow: /donate
Disallow: /extlang
Disallow: /feeds
Disallow: /ftlist
Disallow: /goto
Disallow: /iii
Disallow: /ill
Disallow: /illframe
Disallow: /indexsort
Disallow: /journill
Disallow: /kids
Disallow: /launch
Disallow: /logout
Disallow: /manage
Disallow: /manual
Disallow: /metafind
Disallow: /mfgo
Disallow: /netli
Disallow: /nonret
Disallow: /patroninfo
Disallow: /programs
Disallow: /record=
Disallow: /review
Disallow: /search
Disallow: /selfreg
Disallow: /setlang
Disallow: /setscope
Disallow: /suggest
Disallow: /tmp
Disallow: /validate
Disallow: /VERIFYPATRON
Disallow: /VERSION
Disallow: /weblang
Disallow: /wm
Disallow: /xrecord=
Disallow: /z39
Disallow: /z39m

# block Yandex crawler
User-agent: Yandex
Disallow: /
User-agent: Yandex
Disallow: /
# block BlexBot crawler
User-agent: BLEXBot
Disallow: /
# block AhrefsBot crawlers
User-agent: AhrefsBot
Disallow: /
##or delay .. delay 10 minutes.
rawl-Delay: 10
# block SemrushBo crawlers
User-agent: SemrushBo
Disallow: /
##or delay .. delay 10 minutes.
# block  DotBot crawlers
User-agent:  DotBot
Disallow: /

#To prevent FemtosearchBot from indexing your entire site, add the following to your website's robots.txt:
User-agent: FemtosearchBot
Disallow: *

#Alternatively, prevent FemtosearchBot from indexing specific locations only, for example /internal/, by adding the following to your website's robots.txt:
User-agent: FemtosearchBot
Disallow: /internal/