# Created 01-26-2010 NE
# Stop robot crawls of dynamic search pages

User-agent: Bender
Disallow: /wws/search.html
Disallow: /wws/link.html
Disallow: /wwsindexer/
Disallow: /multilingual/result-list/
Disallow: /wws/result-list/

User-agent: *
Disallow: /wws/search.html
Disallow: /wws/link.html
Disallow: /wwsindexer/
Disallow: /multilingual/result-list/
Disallow: /wws/result-list/
Disallow: /wws/desktop/en/ostiblue/service/link/track
Disallow: /wws/mobile/en/ostiblue/service/link/track

# 6-26-2012 NE adding site maps
Sitemap: http://worldwidescience.org/topicpages/sitemap-3.xml

User-agent: Grammarly
Crawl-delay: 1

User-agent: Slurp
Crawl-delay: 1

User-agent: Yandex
Crawl-delay: 1

User-agent: Baiduspider
Crawl-delay: 1

User-agent: SemrushBot
Crawl-delay: 1

User-agent: SemrushBot-SA
Crawl-delay: 1