User-agent: *
Sitemap: http://get.sh
# disallow all files in these directories

Disallow: /logs/
Disallow: /cgi-bin/



User-agent: Googlebot
# disallow all files ending with these extensions
# Disallow: /*.php$
Disallow: /logs*
Disallow: /*.js$
Disallow: /*.inc$
Disallow: /*.css$
Disallow: /*.gz$
Disallow: /*.cgi$
Disallow: /*.wmv$
#Disallow: /*.png$
#Disallow: /*.gif$
#Disallow: /*.jpg$
Disallow: /*.cgi$
Disallow: /*.xhtml$
#Disallow: /*.php*

# allow google image bot to search all images
User-agent: Googlebot-Image
Allow: /*

# allow adsense bot on entire site
User-agent: Mediapartners-Google*
Disallow: /*?*
Allow: /*.php$
Allow: /*.js$
Allow: /*.inc$
Allow: /*.css$
Allow: /*.gz$
Allow: /*.cgi$
Allow: /*.wmv$
Allow: /*.cgi$
Allow: /*.xhtml$
Allow: /*.php*
Allow: /*.gif$
Allow: /*.jpg$
Allow: /*.png$

# disallow archiving site
User-agent: ia_archiver
Disallow: /

# disable duggmirror
User-agent: duggmirror
Disallow: /

# sonstoge Crawler

User-agent: Teleport*
Disallow: /
User-agent: Webwhacker*
Disallow: /
User-agent: Webzip*
Disallow: /
User-agent: Net Attache*
Disallow: /
User-agent: SiteSnagger*
Disallow: /
User-agent: HTTrack*
Disallow: /
User-agent: WebCapture*
Disallow: /
User-agent: WebSauger*
Disallow: /