shopify_theme_link_spider.py

import scrapy
import pickle

class ShopifyThemeLinkSpider(scrapy.Spider):

    name = 'shopifyspider'

    with open('theme_links.pkl', 'rb') as f:
      theme_links = pickle.load(f)

    start_urls = theme_links.keys()

    def parse(self, response):
        #for theme in response.css('.theme-preview-link'): # I couldn't figure out how to get the first element
        for theme in response.xpath("//a[contains(@class, 'theme-preview-link')][1]"):
            yield {"demo-url": f"https://{theme.css('::attr(data-demo-url)').get()}", 
                   "link" : response.url, #crawled page
                   "theme": self.theme_links[response.url]["theme"]} #theme from pickled file