idealspider.py

import scrapy


class Spider(scrapy.Spider):

    name = 'loremipsum'
    start_urls = ('https://www.lipsum.com',)

    def parse(self, response):
        for lnk in response.links():
            yield scrapy.Request(lnk['url'], callback=self._parse2, meta=lnk)
            #yield response.follow(lnk['url'], callback=self._parse2, meta=lnk)
            #yield response.follow(lnk, callback=self._parse2, meta=lnk)

    def _parse2(self, response):
        for href in response.xpath('//a[class="product"]/@href').extract():
            yield response.follow(href, cookies={'foo': 'bar'})

        yield response.follow('relative/path.html', dont_filter=True)