laggardkernel

Create a base class RedisPipeline, whichever Pipeline inherits it get a redis connection. Access to the redis conn with self.redis_server.

Dependency: scrapy_redis.

# defaults.py
REDIS_ENCODING = "utf-8"
REDIS_FAILED_URLS_KEY = "%(spidername)s:failed_urls"

socrateslee

'''
京东联盟开放平台API的一个通用的Client封装。
京东联盟开放平台的文档详见 https://union.jd.com/openplatform
涉及到签名方法的文档见 https://union.jd.com/helpcenter/13246-13247-46301

Example:
    client = JdApiClient("<YOUR_APP_KEY>", "<YOUR_SECRET_KEY>")
    resp = client.call("jd.union.open.goods.promotiongoodsinfo.query",
                       {'skuIds':'12072066'})
    print(resp.json())

dawand

#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
   File name: download_apk.py
   Author: Dawand Sulaiman
   
   Download APK files from Google Play Store with Python
   This script scraps https://apkpure.com to get the apk download link
   Make sure you have BeautifulSoup and urllib libraries
"""

gbaman

# An example to get the remaining rate limit using the Github GraphQL API.

import requests

headers = {"Authorization": "Bearer YOUR API KEY"}


def run_query(query): # A simple function to use requests.post to make the API call. Note the json= section.
    request = requests.post('https://api.github.com/graphql', json={'query': query}, headers=headers)
    if request.status_code == 200:

miztiik

# Install deltarpm to reduce the size of download
yum install -y deltarpm \
&& yum install -y python-pip \
&& pip install --upgrade pip

# Scrapy Pre-Requisites
yum install -y gcc libffi-devel python-devel openssl-devel
yum install -y libxslt-devel libxml++-devel libxml2-devel
pip install lxml incremental

therg

import re
import random
import base64
from scrapy import log

class RandomProxy(object):
    def __init__(self, settings):
        self.proxy_list = settings.get('PROXY_LIST')
        fin = open(self.proxy_list)