s1wallpaper_extract_and_download.py

"""
提取 S1 壁纸战页面中的壁纸链接并下载

https://bbs.saraba1st.com/2b/thread-1579059-1-1.html

"""
import os
import pathlib
import re
import urllib3
from typing import List, Optional
import concurrent.futures

import click
import wget

import ipdb

PATTERN = "http://wx.*?\.jpg"

def extract_img_links(html: str, pattern: Optional[str]) -> List[str]:
    if pattern is None:
        pattern = PATTERN
    return re.findall(pattern, html)

def download_links(links: List[str], outdir: str, threads: int):
    def down_img(url):
        try:
            wget.download(url, out=outdir, bar=None)
            print(f"\"{url}\" download succees!")
        except:
            print(f"\"{url}\" download failed!")
    with concurrent.futures.ThreadPoolExecutor(max_workers=threads) as ex:
        ex.map(down_img, links)

@click.command()
@click.argument("input-html")
@click.option("--outdir", "-O",
    default=os.path.curdir,
    help="output directory")
@click.option("--pattern", "-p",
    help="target pattern(regular expression)")
@click.option("--threads", "-t",
    default=4,
    help="threads used to download images")
def _main(input_html, outdir, pattern, threads):
    with open(input_html, 'rb') as f:
        html = f.read().decode('utf-8')
        links = extract_img_links(html, pattern)
    #ipdb.set_trace()
    outpath = pathlib.Path(outdir)
    if not outpath.is_dir():
        outpath.mkdir()
    download_links(links, outdir, threads)

if __name__ == "__main__":
   _main()