User-Agent: * 
Allow: / 
Disallow: /App_Code
Disallow: /App_Data
Disallow: /App_Themes
Disallow: /App_WebReferences
Disallow: /Apps
Disallow: /aspnet_client
Disallow: /admin
Disallow: /buy
Disallow: /bin
Disallow: /cgi-bin
Disallow: /aspnet_client
Disallow: /temp
Disallow: /testsite
Disallow: /gastro
Disallow: /update
Disallow: /download/
Disallow: /downloads/
Disallow: /forums/AddPost.aspx
Disallow: /forums/PrintPost.aspx
Disallow: /playbook
Disallow: /sandbox-page
Disallow: /cookie-test-page
Disallow: /marketo-test-page

User-agent: dotbot 
Disallow: / 
User-agent: ScoutJet
Disallow: /