#Website Sitemap
Sitemap: http://www.bestmaterials.com/sitemap.xml

User-agent: *
Disallow: /

User-agent: Googlebot
Allow: /

User-agent: Bingbot
Allow: /

User-agent: Slurp Bot
Allow: /

User-agent: AddSearchBot
Allow: /

# Stop Yandex from hogging bandwidth
User-agent: yandex bot 
Disallow: /

user-agent: AhrefsBot
disallow: /

# Crawlers Setup
User-agent: *
Crawl-delay: 10

Disallow: /_private/
Disallow: /aspnet_client/
Disallow: /ASPSecured/
Disallow: /bin/
Disallow: /CommonControls/
Disallow: /Controls/
Disallow: /cp/
Disallow: /db/
Disallow: /Download/
Disallow: /services/
Disallow: /sfaddons/
Disallow: /ssl/