#
# robots.txt
#
# This file is to prevent the crawling and indexing of certain parts
# of your site by web crawlers and spiders run by sites like Yahoo!
# and Google. By telling these "robots" where not to go on your site,
# you save bandwidth and server resources.
#
# This file will be ignored unless it is at the root of your host:
# Used:    http://example.com/robots.txt
# Ignored: http://example.com/site/robots.txt
#
# For more information about the robots.txt standard, see:
# http://www.robotstxt.org/robotstxt.html

User-agent: *
# CSS, JS, Images
Allow: /core/*.css$
Allow: /core/*.css?
Allow: /core/*.js$
Allow: /core/*.js?
Allow: /core/*.gif
Allow: /core/*.jpg
Allow: /core/*.jpeg
Allow: /core/*.png
Allow: /core/*.svg
Allow: /profiles/*.css$
Allow: /profiles/*.css?
Allow: /profiles/*.js$
Allow: /profiles/*.js?
Allow: /profiles/*.gif
Allow: /profiles/*.jpg
Allow: /profiles/*.jpeg
Allow: /profiles/*.png
Allow: /profiles/*.svg
# Directories
Disallow: /core/
Disallow: /profiles/
# Files
Disallow: /README.txt
Disallow: /web.config
# Paths (clean URLs)
Disallow: /admin/
Disallow: /comment/reply/
Disallow: /filter/tips
Disallow: /node/add/
Disallow: /search/
Disallow: /user/register
Disallow: /user/password
Disallow: /user/login
Disallow: /user/logout
# Paths (no clean URLs)
Disallow: /index.php/admin/
Disallow: /index.php/comment/reply/
Disallow: /index.php/filter/tips
Disallow: /index.php/node/add/
Disallow: /index.php/search/
Disallow: /index.php/user/password
Disallow: /index.php/user/register
Disallow: /index.php/user/login
Disallow: /index.php/user/logout
Disallow: /node/*
# Resources
Disallow: /search-results
Disallow: /publications/search
Disallow: /healthlibrary/articles/document/document/*
Disallow: /healthlibrary/diagnosis-short/document/document/*
Disallow: /healthlibrary/diagnosis/document/document/*
Disallow: /healthlibrary/multimedia/document/document/*
Disallow: /healthlibrary/other-details/document/document/*
Disallow: /healthlibrary/shorts/document/document/*
Disallow: /healthlibrary/special/document/document/*
Disallow: /healthlibrary/surgical-details/document/document/*
Disallow: /healthlibrary/symptoms/document/document/*
Disallow: /healthlibrary/tests/document/document/*
Disallow: /healthlibraryajax/*
Disallow: /taxonomy/*
Disallow: /cs/Satellite
Disallow: /library/*
Disallow: /cancer/library/*
Disallow: /digestive/library/*
Disallow: /heart/library/*
Disallow: /morganstanley/library/*
Disallow: /neuroscience/library/*
Disallow: /pediatrics/healthlibrary/*
Disallow: /pediatrics/library/*
Disallow: /rehabmed/library/*
Disallow: /vascular/library/*
Disallow: /womens/library/*
Disallow: /rcollection
Disallow: /healthlibrary/focus/electronic-medical-and-health-records
Disallow: /marketing-web-tools/*

Crawl-delay: 10
Sitemap: https://www.nyp.org/sitemap.xml

User-agent: 360Spider
Disallow: /

User-agent: HTTPFetch Spider
Disallow: /

User-agent: discobot
Disallow: /

User-agent: Steeler
Disallow: /

User-agent: Linguee
Disallow: /

User-agent: Sogou
Disallow: /

User-agent: Sogou web spider
Disallow: /

User-agent: Sosospider
Disallow: /

User-agent: Sosospider+
Disallow: /

User-agent: YodaoBot
Disallow: /

User-agent: JikeSpider
Disallow: /

User-agent: Ezooms
Disallow: /

User-agent: MJ12bot
Disallow: /

User-agent: Yeti
Disallow: /

User-agent: Yandex
Disallow: /

User-agent: 008
Disallow: /

User-agent: Fess
Disallow: /

User-agent: Exabot
Disallow: /

User-agent: SkimBot
Disallow: /

User-agent: GlobalSpec Link Checker
Disallow: /

User-agent: HiScan
Disallow: /

User-agent: daumoa
Disallow: /

User-agent: SEOkicks-Robot
Disallow: /

User-agent: CovarioCSE
Disallow: /

User-agent: YisouSpider
Disallow: /

User-agent: ZumBot
Disallow: /

User-agent: AppCodesCrawler
Disallow: /

User-agent: AppCodes
Disallow: /

User-agent: EasouSpider
Disallow: /

User-agent: ScreenerBot
Disallow: /

User-agent: Slurp
Crawl-delay: 10

User-agent: bingbot
Crawl-delay: 10

User-agent: msnbot
Crawl-delay: 10

User-agent: msnbot-media
Crawl-delay: 10

User-agent: Baiduspider
Crawl-delay: 10

User-agent: heritrix
Crawl-delay: 10

User-agent: DotBot
Crawl-delay: 10