get_wsgr_manifest.py

import os
import json
import requests
import re
import threading
import queue
from tqdm import tqdm
from pprint import pprint
from hashlib import md5


def md5sum(file):
    m = md5()
    a_file = open(file, 'rb')
    m.update(a_file.read())
    a_file.close()
    return m.hexdigest()


def check_file_md5(file_path, item):
    if not (md5sum(file_path) == item['md5']):
        print(file_path)
        return False
    else:
        return True


def download(file_path, item):
    m = re.split('/', item['name'])
    dir_path = folder_path + item['name'][:-len(m[-1])]
    try:
        if not os.path.exists(dir_path):
            os.makedirs(dir_path)
    except:
        pass
    finally:
        r = s.get(load_dict['packageUrl'] + item['name'])
        with open(folder_path + item['name'], "wb") as file:
            file.write(r.content)
        check_file_md5(file_path, item)


def main(item):
    try:
        file_path = folder_path + item['name']
        if not os.path.exists(file_path):
            download(file_path, item)
        elif not check_file_md5(file_path, item):
            download(file_path, item)
    except Exception as e:
        print(e)


def process_data(q):
    while not exitFlag:
        queueLock.acquire()
        if not workQueue.empty():
            item = q.get()
            queueLock.release()
            main(item)
        else:
            queueLock.release()


class myThread (threading.Thread):
    def __init__(self, threadID, name, q):
        threading.Thread.__init__(self)
        self.threadID = threadID
        self.name = name
        self.q = q

    def run(self):
        print("Start: " + self.name)
        process_data(self.q)
        print("End: " + self.name)


if __name__ == '__main__':
    exitFlag = 0

    s = requests.Session()
    folder_path = './data/'
    with open('./warshipgirlsr.manifest') as f:
        load_dict = json.load(f)
    threadList = [f"Thread-{i}" for i in range(100)]
    queueLock = threading.Lock()
    workQueue = queue.Queue()
    threadID = 0
    threads = []
    for tName in threadList:
        thread = myThread(threadID, tName, workQueue)
        thread.start()
        threads.append(thread)
        threadID += 1

    queueLock.acquire()
    for item in load_dict['hot']:
        workQueue.put(item)
    queueLock.release()

    while not workQueue.empty():
        pass

    exitFlag = 1

    for t in threads:
        t.join()
    print("End Of All")