User-Agent: htdig-3.1.5
User-Agent: Harvest-NG
User-Agent: Harvest
Disallow: /
#Disallow: /cgi-bin/
#Disallow: /private_html/
#Disallow: /Liverpool/Private/
#Disallow: /pd-tutorials/
# Allow says Paul hagan (Corp Comme) - PDM 02/02/2009
#Disallow: /xjtlu/

User-Agent: W3C-checklink
User-Agent: Lycos
User-Agent: Lycos_Spider_(T-Rex)
User-Agent: Gulliver
User-Agent: Scooter
User-Agent: InfoSeek
User-Agent: Ultraseek
User-Agent: fido
User-Agent: NetMechanic
User-Agent: ArchitextSpider
User-Agent: Mirago
User-Agent: Slurp
User-Agent: Googlebot
User-Agent: googlebot
User-Agent: Google
User-Agent: WebCrawler
User-Agent: ia_archiver
User-Agent: gsa-crawler
User-Agent: msnbot
User-Agent: Connotate
User-Agent: Yandex
User-Agent: Funnelback 
Disallow: /cgi-bin/
Disallow: /private_html/
Disallow: /local_html/
Disallow: /Admin/
Disallow: /harvest/
Disallow: /Harvest/
Disallow: /Liverpool/Private/
Disallow: /pd-tutorials/
Disallow: /international/Clearing/
Disallow: /intranet/
Disallow: /Intranet/
Disallow: /wwwstats/
Disallow: /WWWstats/
#Google googling our google...
Disallow: /search/
#Disallow: /xjtlu/
Disallow: /2004/
Disallow: /2004site/

User-Agent: BioCrawler
Disallow: /
#Disallow: /a
#Disallow: /b
#Disallow: /ca
#Disallow: /cb
#Disallow: /cc
#Disallow: /cd
#Disallow: /ce
#Disallow: /cf
#Disallow: /cg
#Disallow: /ch
#Disallow: /cia
#Disallow: /cib
#Disallow: /cic
#Disallow: /cid
#Disallow: /cie
#Disallow: /cif
#Disallow: /cig
#Disallow: /cih
#Disallow: /cii
#Disallow: /cij
#Disallow: /cik
#Disallow: /cim
#Disallow: /cin
#Disallow: /cio
#Disallow: /cip
#Disallow: /ciq
#Disallow: /cir
#Disallow: /cis
#Disallow: /cit
#Disallow: /ciu
#Disallow: /civ
#Disallow: /ciw
#Disallow: /cix
#Disallow: /ciy
#Disallow: /ciz
#Disallow: /cj
#Disallow: /ck
#Disallow: /cl
#Disallow: /cm
#Disallow: /cn
#Disallow: /co
#Disallow: /cp
#Disallow: /cq
#Disallow: /cr
#Disallow: /cs
#Disallow: /ct
#Disallow: /cu
#Disallow: /cv
#Disallow: /cw
#Disallow: /cx
#Disallow: /cy
#Disallow: /cz
#Disallow: /d
#Disallow: /e
#Disallow: /f
#Disallow: /g
#Disallow: /h
#Disallow: /i
#Disallow: /j
#Disallow: /k
#Disallow: /l
#Disallow: /m
#Disallow: /n
#Disallow: /o
#Disallow: /p
#Disallow: /q
#Disallow: /r
#Disallow: /s
#Disallow: /t
#Disallow: /u
#Disallow: /v
#Disallow: /w
#Disallow: /x
#Disallow: /y
#Disallow: /z
#Disallow: /A
#Disallow: /B
#Disallow: /C
#Disallow: /D
#Disallow: /E
#Disallow: /F
#Disallow: /G
#Disallow: /H
#Disallow: /I
#Disallow: /J
#Disallow: /K
#Disallow: /L
#Disallow: /M
#Disallow: /N
#Disallow: /O
#Disallow: /P
#Disallow: /Q
#Disallow: /R
#Disallow: /S
#Disallow: /T
#Disallow: /U
#Disallow: /V
#Disallow: /W
#Disallow: /X
#Disallow: /Y
#Disallow: /z
#Disallow: /cgi-bin/
#Disallow: /private_html/
#Disallow: /local_html/
#Disallow: /Admin/
#Disallow: /harvest/
#Disallow: /Harvest/
#Disallow: /Liverpool/Private/
#Disallow: /pd-tutorials/

User-agent: Xenu Link Sleuth
Disallow: /

User-agent: SiteCheck-sitecrawl by Siteimprove.com
User-agent: LinkCheck by Siteimprove.com
#Disallow: /
Disallow: /cgi-bin/
Disallow: /private_html/
Disallow: /public_html/
Disallow: /local_html/
Disallow: /Admin/
Disallow: /harvest/
Disallow: /Harvest/
Disallow: /Liverpool/Private/
Disallow: /pd-tutorials/
Disallow: /info/
Disallow: /international/Clearing/
Disallow: /intranet/
Disallow: /Intranet/
Disallow: /wwwstats/
Disallow: /WWWstats/
Disallow: /WWWStats/
#Google googling our google...
Disallow: /search/
#Disallow: /xjtlu/
Disallow: /2004/
Disallow: /2004site/
Disallow: /~

User-agent: rogerbot
Allow: /integrative-biology/
Allow: /chemistry/
Allow: /engineering/
Allow: /study/
Allow: /ageing-and-chronic-disease/
Allow: /management/

# Allowing study/london for sitemap generation.
# JAG 13/03/2013
User-agent: *
Disallow: /