Answergarden downloader

app.py

#https://pastebin.fi/p/niOTvtFJu6BI/
import requests
import os
import time
from multiprocessing.dummy import Pool as ThreadPool

headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}

def download_file(url, local_filename):
    # NOTE the stream=True parameter below
    with requests.get(url, stream=True, headers=headers) as r:
        r.raise_for_status()
        with open(local_filename, 'wb') as f:
            for chunk in r.iter_content(chunk_size=8192): 
                # If you have chunk encoded response uncomment if
                # and set chunk_size parameter to None.
                #if chunk: 
                f.write(chunk)
    return local_filename

def download_garden(idx):
    idx = str(idx)

    os.mkdir('./gardens/' + idx)

    #Downlaod answers
    local_filename_question = './gardens/' + idx + '/question.txt' 
    local_filename_answers = './gardens/' + idx + '/answers.csv' 

    download_file('https://answergarden.ch/api/getquestion/' + idx, local_filename_question)
    download_file('https://answergarden.ch/api/csv/' + idx, local_filename_answers)
    print("Ladataan", idx)


if __name__ == '__main__':

    ids = []

    for i in range(1000000):
        ids.append(i)

    pool = ThreadPool(8)

    results = pool.map(download_garden, ids)

    pool.close()
    pool.join()