###############################
#
#
User-agent: *
#
# list folders robots are not allowed to index 
#
Disallow: /cgi-bin/
Disallow: /images/
Disallow: /assets/
#
# list specific files robots are not allowed to index
#
Disallow: /favicon.ico
Disallow: /*.js
Disallow: /*.css
Disallow: /*.xml
Disallow: /*.gif
Disallow: /*.jpeg
Disallow: /*.jpg
Disallow: /*.png
Disallow: /*.tif
Disallow: /*.swf
Disallow: /*.flv
# 
User-agent: psbot
Disallow: /
#
User-agent: Pingdom
Disallow: /
#
User-agent: Pingdom GIGRIB bot
Disallow: /
#
###############################