crawl-stackoverflow.py

# -*- coding: utf-8 -*-
import scrapy
import logging
from datetime import date, datetime
import sys



logfile = "logs/log"+date.today().strftime("%Y%m%d")+'.txt'
logging.basicConfig(filename=logfile, level=logging.DEBUG)
logging.getLogger().addHandler(logging.StreamHandler(sys.stdout))

class StackoverflowSpider(scrapy.Spider):
    name = 'stackoverflow'
    allowed_domains = ['stackoverflow.com']
    start_urls = ['https://stackoverflow.com/jobs?id=395682&v=true']

    def parse(self, response):
        listResults = response.xpath('//div[@class="listResults"]')
        
        companyLogoUrls = listResults.xpath("//div[@class='grid']/img/@src")
        for logoUrl in companyLogoUrls:
            logging.info(logoUrl.get())
        
        jobs = listResults.xpath("//div[@class='grid']/div/h2/a")
        for job in jobs:
            logging.info(job.xpath('@href').get())
            logging.info(job.xpath('text()').get())