获取vista看天下《天下焦点》链接

spider.py

import json
from concurrent.futures import ThreadPoolExecutor

import requests

pool = ThreadPoolExecutor(10)

sample = "http://v20.tp.wkread.com/index.php/WeiXin/Focus/getColumnList/id/103"


def get_content(uid):
    print("get {}".format(uid))
    base_url = "http://v20.tp.wkread.com/index.php/WeiXin/Focus/getColumnList/id/"
    url = "{}{}".format(base_url, uid)
    r = requests.get(url)

    if r.json():
        save(uid, r.json())


def save(uid, j):
    with open("content/{}.json".format(uid), 'w', encoding='u8') as f:
        f.write(json.dumps(j, ensure_ascii=False, indent=2))
        print("save {}".format(uid))


l = []
for i in range(1, 1033):
    p = pool.submit(get_content, i)
    l.append(p)

[i.result() for i in l]