Using Scrapy to find wrong escaped html code on a Rails 2.3 app with rails_xss plugin installed.

gistfile1.txt

from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from scrapy.http.request import Request

class HomeSpider(BaseSpider):
    name = "rails_xss"
    root = "http://0.0.0.0:3000/"
    start_urls = [root]

    def parse(self, response):
        hxs = HtmlXPathSelector(response)

        html_code = ['&', '"', '<', '>']
        for code in html_code:
            if (code in response.body):
                self.show_xss_bug(response, code) 

        # Crawling
        uris = [link for link in hxs.select('//a/@href').extract() if 'http' not in link]
        for site in uris:
            yield Request(self.root + site, self.parse)

    def show_xss_bug(self, response, test):
        index = response.body.index(test)
        print '='*150
        print 'RESPONSE URL: ', response.url
        print response.body[(index - 200):(index + 200)]
        print '='*150