批量下载文件.py

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright © 2021, All rights reserved.
# Author: Eric <wendachen@flickering.ai>

import requests
import os
import logging
from urllib import parse

level_num = 3  # 保存多级目录
dir_path = './data'  # 保存目录
urls_list = ['https://cdn.jsdelivr.net/gh/emn178/js-md5/build/md5.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-sha1/build/sha1.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/hi-base64/build/base64.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-htmlencode/build/htmlencode.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-md4/build/md4.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-sha512/build/sha512.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/hi-base32@v0.5.1/build/base32.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-sha256/build/sha256.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-crc/build/crc.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-sha3/build/sha3.min.js',
             'https://cdn.jsdelivr.net/gh/emn178/js-md2/build/md2.min.js']

if not os.path.isdir(dir_path):
    os.makedirs(dir_path)


def get_save_path(url):
    sub_dir = "/".join(parse.urlparse(url).path.split('/')[-level_num:-1])
    if not os.path.exists(os.path.join(dir_path, sub_dir)):
        os.makedirs(os.path.join(dir_path, sub_dir))
    return os.path.join(dir_path, sub_dir, os.path.basename(url))


def main():
    for url in urls_list:
        download_path = get_save_path(url)
        if os.path.exists(download_path):
            logging.warning(f"{download_path} 已经存在")
        print(download_path, end=" ")
        r = requests.get(url)
        with open(download_path, "wb") as f:
            f.write(r.content)
        print("done!")


if __name__ == '__main__':
    main()