Skip to content

Instantly share code, notes, and snippets.

@laixintao
Last active May 20, 2021 05:56
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 0 You must be signed in to fork a gist
  • Save laixintao/7c6aa12c9e5e4ec96851e254f9ea2d7e to your computer and use it in GitHub Desktop.
Save laixintao/7c6aa12c9e5e4ec96851e254f9ea2d7e to your computer and use it in GitHub Desktop.
list.append is an atomic action, so no lock needed when multiple threads append on same list.
urls_to_download = ['baidu.com', 'taobao.com', 'zhihu.com', 'douban.com']
done_count = 0
done_count_lock = Lock()
def download(url):
requests.get(url)
with done_count_lock:
done_count += 1
print(f"currnet process: {done_count} / {len(urls_to_download)}")
def main():
with ThreadPoolExecutor(max_workers=2) as executor:
for url in urls_to_download:
future = executor.submit(download, url)
urls_to_download = ['baidu.com', 'taobao.com', 'zhihu.com', 'douban.com']
done_count = 0
done_urls = []
def download(url):
requests.get(url)
done_urls.append(url) # no lock needed
print(f"currnet process: {len(done_count)} / {len(urls_to_download)}")
def main():
with ThreadPoolExecutor(max_workers=2) as executor:
for url in urls_to_download:
future = executor.submit(download, url)
@uranusjr
Copy link

uranusjr commented May 20, 2021

completion_channel = queue.Queue()

def report_progress():
    done_count = 0
    while not completion_channel.get():
        done_count += 1
        print(f"currnet process: {done_count} / {len(urls_to_download)}")

def download_thing(url):
    requests.get(url)
    completion_channel.put(False)

def main():
    threading.Thread(target=report_progress).start()
    with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
        for url in urls_to_download:
            executor.submit(download_thing, url)
    completion_channel.put(True)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment