FaceDownloader.py

import urllib3
import os, time
import threading, threadpool

def ImageSaver(root, url, f_name, f_type = ".JPG"):
    if not os.path.exists(root + f_name + f_type):
        http = urllib3.PoolManager(retries = 3)
        response = http.request('GET',url)
        if response.status == 200:
            with open(root + f_name + f_type,"wb") as f:
                f.write(response.data)
    print(f_name)

def numCreator(num):
    num_init = "000"
    if num < 10:
        return num_init[0:-1]+str(num)
    elif num < 100:
        return num_init[0]+str(num)
    else:
        return str(num)

def urlCreator(student_year, student_ID):
    url = "http://222.195.8.201/student/photo/"+str(student_year)+"/"+student_ID+".JPG"
    return url

def main():
    start_time = time.clock()
    grade = 6
    year_start = 2012
    major_num = 4
    major = 216
    thread_num = 200
    paras_list = []
    pool = threadpool.ThreadPool(thread_num,10)
    for student_year in range(year_start,year_start+grade):
        root = os.getcwd()+"/"+str(student_year)+"/"
        if not os.path.exists(root):
            os.mkdir(root)
        for student_major in range(major,major+major_num):
            for i in range(1000):
                student_ID = str(student_year)+str(student_major)+numCreator(i)
                paras = {'root': root, 'url': urlCreator(student_year,student_ID), 'f_name': student_ID}
                paras_list.append((None, paras))
    requests = threadpool.makeRequests(ImageSaver,paras_list)
    [pool.putRequest(req) for req in requests] 
    pool.wait()
    end_time = time.clock()
    print("Total Time:",end_time - start_time)

if __name__ == "__main__":
    main()