Spider to crawl and entire site and save each page to a sqlite database for further parseing.

scrapy_to_sqlite_db.py

# -*- coding: utf-8 -*-
import scrapy
import dataset

# db = dataset.connect('sqlite:///:memory:')
db = dataset.connect('sqlite:///database.db')


class FullSiteSpider(scrapy.Spider):
    name = 'full_site'
    allowed_domains = ['allglassparts.com']
    start_urls = ['http://allglassparts.com/']

    def parse(self, response):

        prod_page = response.xpath('//div[@id="ProductDetails"]').extract_first()
        if prod_page is not None:
            db['pages'].insert({'page_source': response.text, 'url': response.url})

        for href in response.xpath('//a/@href').getall():
            yield scrapy.Request(response.urljoin(href), self.parse)