xiuren image spider

download.py

#!/usr/bin/env python
# -*- encoding: utf-8 -*-
# Created on 2020-03-12 15:56:36
# Project: Girls

from pyspider.libs.base_handler import *
import os
import sys


reload(sys)
sys.setdefaultencoding("utf-8")

class Handler(BaseHandler):
    crawl_config = {
        'root_urls': [
            'http://www.xiuren.org/category/TuiGirl.html',
            #'http://www.xiuren.org/category/XiuRen.html'
        ],
        'download_path': os.path.expanduser('~/Downloads/') + 'Girls/',
        'index_page_detail_page_link': '#main > div > div > a',
        'detail_image_link': '#post > div.post > p > span > a',
        'detail_page_title': '#title > h1:nth-child(1)',
        'index_page_next_link': '#page > a.next'
    }

    def on_start(self):
        root_urls = self.crawl_config['root_urls']
        for each in root_urls:
            self.crawl(root_urls, callback=self.index_page)

    def index_page(self, response):
        category = response.url.split(".")[-2].split("/")[-1]
        for each in response.doc(self.crawl_config['index_page_detail_page_link']).items():
            self.crawl(each.attr.href, callback=self.detail_page,save={'category': category})
        next_link = response.doc(self.crawl_config['index_page_next_link']).attr.href
        if next_link:
            self.crawl(next_link, callback=self.index_page)

    def detail_page(self, response):
        category = response.save['category']
        detail_page_title = response.doc(self.crawl_config['detail_page_title']).text()
        for each in response.doc(self.crawl_config['detail_image_link']).items():
            file_url = each.attr.href
            file_name = detail_page_title + file_url.split("/")[-1]
            directory = self.crawl_config['download_path'] + category
            if os.path.exists(directory + '/' + file_name):
                print("file \"" + directory + '/' + file_name + "\" is existed, skip download")
                continue
            self.crawl(file_url, callback=self.save_img,
               save={'file_name': file_name, 'directory': directory})
        
    def save_img(self,response):
        directory = response.save["directory"]
        if not os.path.exists(directory):                         
            os.makedirs(directory)
        content = response.content
        file_path = directory + '/' + response.save["file_name"]
        f = open(file_path,"wb" )
        f.write(content)
        f.close()