Skip to content

Instantly share code, notes, and snippets.

@jluczak
Last active August 7, 2017 19:57
Show Gist options
  • Save jluczak/0b382010196e7f3bfd6e53b304640613 to your computer and use it in GitHub Desktop.
Save jluczak/0b382010196e7f3bfd6e53b304640613 to your computer and use it in GitHub Desktop.
Write a Scrapy parser for Human Talks
import scrapy
from human.items import HumanItem
class HumanSpider(scrapy.Spider):
start_urls = [
'http://humantalks.com/talks/'
]
name = 'human'
def parse(self, response):
for talk in response.css('.card-deck a'):
href = talk.css('a::attr(href)').extract_first()
full_url = response.urljoin(href)
yield scrapy.Request(full_url, callback=self.parse_book)
for href in response.css('a.page-link'):
yield response.follow(href, self.parse)
def parse_book(self, response):
title = response.css('h1::text').extract_first()
name = response.css('.speaker_name a::text').extract_first()
description = response.css('.offset-md-2 p::text').extract()
slide = response.css('.btn-primary::attr(href)').extract_first()
slide2 = response.css('.player iframe::attr(src)').extract_first()
video = response.css('.yt-uix-sessionlink a::attr(href)').extract_first()
yield HumanItem(title=title, description=description, name=name, slide2=slide2,slide=slide,video=video)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment