robots.txt snippet

robots.txt

# For documentation on how to use the robots.txt file see:
#
# - http://www.robotstxt.org/
# - https://support.google.com/webmasters/answer/156449
#
# To ban all spiders from the entire site uncomment the next two lines see
# http://www.robotstxt.org/wc/norobots.html:
#
# User-Agent: *
# Disallow: /
#
#
# User-agent: *
# Disallow: /css/
# Disallow: /img/
# Disallow: /js/
# Disallow: /styles/
# Allow: /
#