# ===================================================================================
# robots.txt
# ===================================================================================
#
# Verify syntax at: http://tool.motoricerca.info/robots-checker.phtml
#
# Robots.txt org: http://www.robotstxt.org/
# Bots in the wild: http://www.botsvsbrowsers.com/
#
# Robots Database: http://www.robotstxt.org/db.html
# Generate your own robots.txt: http://www.mcanerin.com/EN/search-engine/robots-txt.asp
#
# Sitemap Protocol, if you want to impliment it: http://www.sitemaps.org/protocol.php
#
# About the Robots <META> tag
#
# You can use a special HTML <META> tag to tell robots:
# if they can or can't index the content of a page
# if they can or can't scan it for links to follow
#
# Valid values for the "CONTENT" attribute are: "INDEX", "NOINDEX", "FOLLOW", "NOFOLLOW".
#
# Multiple comma-separated values are allowed, but obviously only some combinations make sense.
# If no robots <META> tag exists, the default is "INDEX,FOLLOW".
#
# <META NAME="ROBOTS" CONTENT="NOINDEX, NOFOLLOW">  // Don't index or follow links
# <META NAME="ROBOTS" CONTENT="NOINDEX, FOLLOW">    // Don't index, follow links
# <META NAME="ROBOTS" CONTENT="INDEX, NOFOLLOW">    // Index, but don't follow links
#
# Also: <meta name="robots" content="noarchive">
#
#
# ======================================================================================
#
# Other useful tools: Online favicon generators
# http://www.freefavicon.com/
# http://www.favicon.cc/?action=import_request
#
User-agent: Googlebot  # Googlebot must follow the following...
#
Disallow: /rz*.txt       # CV's ingore SVP
Disallow: /rz*.htm       # CV's ingore SVP
Disallow: /rz*.html      # CV's ingore SVP
Disallow: /weblinks.htm  # webmaster pages ignore
Disallow: /web*.htm      # webmaster pages ignore  
Disallow: /*.png         # ignore PNGs
Disallow: /*.jpg         # ignore JPGs
Disallow: /*.jpeg        # ignore JPGs
Disallow: /*.gif         # ignore GIFs
#
#
#
User-agent: Googlebot-Image
Disallow: /*.gif
# The Image Googlebot should not index GIFs
#
#
User-agent: googlebot-mobile
Disallow: 
#
#
#
User-agent: MSNBot
Disallow: 
#
#
#
User-agent: Slurp
Disallow: 
#
#
#
User-agent: Teoma
Disallow: 
#
#
#
User-agent: twiceler
Disallow: 
#
#
#
User-agent: Gigabot
Disallow: 
#
#
#
User-agent: Scrubby
Disallow: 
#
#
#
User-agent: Robozilla
Disallow: 
#
#
#
User-agent: Nutch
Disallow: 
#
#
#
User-agent: ia_archiver
Disallow: 
#
#
#
User-agent: baiduspider
Disallow: 
#
#
#
User-agent: naverbot
Disallow: 
#
#
#
User-agent: voyager
Crawl-delay: 10
#
#
#
User-agent: yeti
Disallow: 
#
#
#
User-agent: yahoo-mmcrawler
Disallow: 
#
#
#
User-agent: psbot
Disallow: 
#
#
#
User-agent: asterias
Disallow: 
#
#
#
Crawl-delay: 10
Disallow: /cgi-bin/
#
#
#
# = End =
#