simbus82

# robots.txt for Magento 1.9.x / v0.1 2018-05-23 / Simone Bussoni / inspired by Peeter Marvet
# # based on:
# https://gist.github.com/petskratt/016c9dbf159a81b9d6aa
# http://inchoo.net/ecommerce/ultimate-magento-robots-txt-file-examples/
# http://www.byte.nl/blog/magento-robots-txt/
# https://astrio.net/blog/optimize-robots-txt-for-magento/
#
# comment and clone at https://gist.github.com/petskratt/016c9dbf159a81b9d6aa
# Keep in mind that by standard robots.txt should NOT contain empty lines, except between UA blocks!
#

simbus82

<h1>Testing display of HTML elements</h1>

<h2>This is 2nd level heading</h2>
<p>This is a test paragraph.</p>
<h3>This is 3rd level heading</h3>
<p>This is a test paragraph.</p>
<h4>This is 4th level heading</h4>
<p>This is a test paragraph.</p>
<h5>This is 5th level heading</h5>
<p>This is a test paragraph.</p>

simbus82

# If the Joomla site is installed within a folder 
# eg www.example.com/joomla/ then the robots.txt file 
# MUST be moved to the site root 
# eg www.example.com/robots.txt
# AND the joomla folder name MUST be prefixed to all of the
# paths. 
# eg the Disallow rule for the /administrator/ folder MUST 
# be changed to read 
# Disallow: /joomla/administrator/
#

simbus82

AddDefaultCharset UTF-8
ServerSignature Off
Options -Indexes
FileETag none

<IfModule mod_headers.c>
	Header unset ETag
	Header unset Cookie
	Header unset Set-Cookie
</IfModule>