我们在用到Scrapy下载图片时，框架已经内置了对图片下载的处理，但是只有磁盘存储和Amazon S3存储，而项目是准备采用又拍云进行图片存储，因此，就写了用存储类让ImagesPipeline存储图片到又拍云。

youpaiyun_scrapy

class UpYunStore(object):
    APP_NAME = None
    USERNAME = None
    PASSWORD = None
    TMP_PATH = None
    def __init__(self, uri):
        assert uri.startswith('http://')
        self.upyun = UpYun(self.APP_NAME, self.USERNAME, self.PASSWORD)
        self.prefix = '/' + uri.split('/')[-1]
 
    def stat_image(self, key, info):
        image_info = self.upyun.getFileInfo(self.prefix+'/'+key)
        last_modified = int(image_info['date'])
        checksum = image_info['size']
        return {'last_modified': last_modified, 'checksum': checksum}
 
    def persist_image(self, key, image, buf, info):
        tmp_path = os.path.join(self.TMP_PATH, 'tmp.jpg')
        image.save(tmp_path)
        data = open(tmp_path, 'rb')
        self.upyun.setContentMD5(md5file(data))
        # fiel secret
        self.upyun.setFileSecret('')
        result = self.upyun.writeFile(self.prefix+'/'+key, data, True)
        if not result:
            log.msg("Image: Upload image to Upyun Failed! %s" % (self.prefix+key))

class CoverImagesPipeline(ImagesPipeline):
    ImagesPipeline.STORE_SCHEMES['http'] = UpYunStore
    URL_PREFIX = None
 
    @classmethod
    def from_settings(cls, settings):
        upyun = cls.STORE_SCHEMES['http']
        upyun.APP_NAME = settings['UPYUN_APP_NAME']
        upyun.USERNAME = settings['UPYUN_USERNAME']
        upyun.PASSWORD = settings['UPYUN_PASSWORD']
        upyun.TMP_PATH = settings['TMP_PATH']
        cls.URL_PREFIX = settings['IMAGES_STORE']
        return super(CoverImagesPipeline, cls).from_settings(settings)
        image_info = self.upyun.getFileInfo(self.prefix+'/'+key)