Skip to content

Instantly share code, notes, and snippets.

@jluczak
Created August 10, 2017 08:25
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save jluczak/b45db94680b40dc320ca10fb096b6198 to your computer and use it in GitHub Desktop.
Save jluczak/b45db94680b40dc320ca10fb096b6198 to your computer and use it in GitHub Desktop.
Parse PyCon speakers
import scrapy
from pycon.items import PyconItem
class HumanSpider(scrapy.Spider):
start_urls = [
'http://pyvideo.org/speakers.html'
]
name = 'pycon'
def parse(self, response):
for talk in response.css('.row a'):
href = talk.css('a::attr(href)').extract_first()
full_url = response.urljoin(href)
yield scrapy.Request(full_url, callback=self.parse_page)
def parse_page(self, response):
for link in response.css('div.container'):
href=link.css('a::attr(href)').extract_first()
full_url=response.urljoin(href)
yield response.follow(href, self.parse_book)
def parse_book(self, response):
title = response.css('h2.entry-title a::text').extract_first()
speaker = response.css('a.fn::text').extract_first()
description = response.css('.entry-content p::text').extract()
event = response.css('.details-content a::text').extract_first()
links = response.css('.details-content a::attr(href)').extract()
yield PyconItem(title=title, description=description, speaker=speaker, event=event,links=links)
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment