integrate youtube_dl with asyncio.

async_ydl.py

import asyncio
from datetime import datetime
from functools import partial
from concurrent.futures import ProcessPoolExecutor

from youtube_dl import YoutubeDL as YDL

PPE = ProcessPoolExecutor()


def extract_info(url, quiet=False):
    # use `quiet=True` to avoid noisy
    return YDL(dict(quiet=quiet)).extract_info(url, download=False)


async def flatten_urls(*urls, loop=None):
    print(f"trying to flat {len(urls)} URLs.")
    loop = asyncio.get_event_loop()
    futs = [loop.run_in_executor(PPE, extract_info, url) for url in urls]
    infos = await asyncio.gather(*futs)

    urlmatrix = [i.get('entries', [i]) for i in infos]
    return [u.get('webpage_url', "") for urls in urlmatrix for u in urls]


async def progress(*urls, loop=None):
    loop = loop or asyncio.get_event_loop()
    print(f"{len(urls)} URLs will be progress.")
    pure_urls = await flatten_urls(*urls, loop=loop)
    print(pure_urls)
    futs = [loop.run_in_executor(None, YDL().download, [url]) for url in pure_urls]
    return await asyncio.gather(*futs)


async def keep_noise():
    while True:
        await asyncio.sleep(1)
        print(f"Time is: {datetime.now()}", flush=True)


def main():
    u1 = ["https://youtu.be/-JNeBKlG0cI",
          "https://youtu.be/mVEItYOsXjM",
          "https://youtu.be/0JoMqP5UwQ8"]
    u2 = ["https://youtu.be/m3gLNa-fx_w"]
    u3 = ["https://www.youtube.com/watch?v=sm4BID47dTI&list=PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB"]
    loop = asyncio.get_event_loop()

    coros = [progress(*u1), progress(*u2), progress(*u3), keep_noise()]
    loop.run_until_complete(asyncio.gather(*coros))


if __name__ == '__main__':
    main()

example.log

3 URLs will be progress.
trying to flat 3 URLs.
1 URLs will be progress.
trying to flat 1 URLs.
1 URLs will be progress.
trying to flat 1 URLs.
[youtube] -JNeBKlG0cI: Downloading webpage
[youtube] 0JoMqP5UwQ8: Downloading webpage
[youtube] mVEItYOsXjM: Downloading webpage
Time is: 2020-03-07 20:17:22.442724
[youtube] -JNeBKlG0cI: Downloading video info webpage
[youtube] mVEItYOsXjM: Downloading video info webpage
[youtube] 0JoMqP5UwQ8: Downloading video info webpage
[youtube] m3gLNa-fx_w: Downloading webpage
Time is: 2020-03-07 20:17:23.442782
[youtube] m3gLNa-fx_w: Downloading video info webpage
[youtube:playlist] Downloading playlist PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB - add --no-playlist to just download video sm4BID47dTI
[youtube:playlist] PLFWyAXHl5a6ffjKkNGu27BEk1US055MzB: Downloading webpage
['https://www.youtube.com/watch?v=-JNeBKlG0cI', 'https://www.youtube.com/watch?v=mVEItYOsXjM', 'https://www.youtube.com/watch?v=0JoMqP5UwQ8']
[download] Downloading playlist: 【誰來晚餐 番外篇】
[youtube:playlist] playlist 【誰來晚餐 番外篇】: Downloading 7 videos
[download] Downloading video 1 of 7
[youtube] sm4BID47dTI: Downloading webpage
Time is: 2020-03-07 20:17:24.459840
[youtube] 0JoMqP5UwQ8: Downloading webpage
[youtube] -JNeBKlG0cI: Downloading webpage
[youtube] mVEItYOsXjM: Downloading webpage
[youtube] sm4BID47dTI: Downloading video info webpage
['https://www.youtube.com/watch?v=m3gLNa-fx_w']
[youtube] m3gLNa-fx_w: Downloading webpage
[youtube] mVEItYOsXjM: Downloading video info webpage
[youtube] 0JoMqP5UwQ8: Downloading video info webpage
[youtube] -JNeBKlG0cI: Downloading video info webpage
[youtube] m3gLNa-fx_w: Downloading video info webpage
Time is: 2020-03-07 20:17:25.465897
[download] Downloading video 2 of 7
[youtube] r5WApOc0WLI: Downloading webpage
[youtube] r5WApOc0WLI: Downloading video info webpage
[download] Destination: 台灣觀眾如何被媒體出賣？「媒體洗腦」完全破解！紅色滲透是啥？【記者真心話】Vol.2｜懶人包-mVEItYOsXjM.mp4
[download]   0.8% of 32.14MiB at  4.88MiB/s ETA 00:06         [download] Destination: 你也被媒體帶風向了嗎？ft.回覆酸民留言與頭版質疑【
記者真心話】Vol.3｜懶人包｜媒體識讀-0JoMqP5UwQ8.mp4
[download]   0.1% of 95.98MiB at  3.76MiB/s ETA 00:25 [download] Destination: 台灣媒體是怎麼爛掉的？七分鐘看懂媒體亂象五大關鍵！【記者真
心話】Vol.1｜懶人包--JNeBKlG0cI.mp4
[download]   0.0% of 93.69MiB at  4.88MiB/s ETA 00:19         Time is: 2020-03-07 20:17:26.465955
[download]   0.1% of 93.69MiB at  1.02MiB/s ETA 01:32         [download] Destination: 假新聞出沒！ 破解 #媒體與牠們的產地 ft.劣質媒體黑
名單【記者真心話】Vol.4｜懶人包｜媒體識讀-m3gLNa-fx_w.mp4
[download]   1.1% of 93.69MiB at  1.22MiB/s ETA 01:16         Time is: 2020-03-07 20:17:27.466012
[download] Downloading video 3 of 7
[youtube] W5oa03IOkzE: Downloading webpage
[download]  33.4% of 23.95MiB at  4.72MiB/s ETA 00:03         [youtube] W5oa03IOkzE: Downloading video info webpage
Time is: 2020-03-07 20:17:28.466069
[download]  37.3% of 32.14MiB at  3.95MiB/s ETA 00:05         Time is: 2020-03-07 20:17:29.466126
[download]  10.2% of 95.98MiB at  2.96MiB/s ETA 00:29 [download] Downloading video 4 of 7
[youtube] RxALj5QWsjo: Downloading webpage
[download]   8.5% of 93.69MiB at  2.00MiB/s ETA 00:42         Time is: 2020-03-07 20:17:30.466183
[download]  47.0% of 32.14MiB at  3.59MiB/s ETA 00:04         [youtube] RxALj5QWsjo: Downloading video info webpage
[download] 100% of 23.95MiB in 00:04
[download]  11.4% of 93.69MiB at  2.19MiB/s ETA 00:37         Time is: 2020-03-07 20:17:31.465241
[download]  15.8% of 95.98MiB at  2.99MiB/s ETA 00:26 [download] Downloading video 5 of 7
[youtube] aG2iEAs78V0: Downloading webpage
[download]  19.3% of 95.98MiB at  3.19MiB/s ETA 00:24 [download]  64.2% of 32.14MiB at  3.51MiB/s ETA 00:03         [youtube] aG2iEAs78V0: Downloading video info webpage
Time is: 2020-03-07 20:17:32.465298
[download]  23.1% of 93.69MiB at  3.12MiB/s ETA 00:23         Time is: 2020-03-07 20:17:33.465355
[download] Downloading video 6 of 7
[youtube] qO9J-WwUiEc: Downloading webpage
[download]  87.8% of 32.14MiB at  3.58MiB/s ETA 00:01         [youtube] qO9J-WwUiEc: Downloading video info webpage
Time is: 2020-03-07 20:17:34.465412
[download] 100% of 32.14MiB in 00:09
Time is: 2020-03-07 20:17:35.456469
[download] Downloading video 7 of 7
[youtube] YvVthtbNtEE: Downloading webpage
[download]  36.0% of 95.98MiB at  3.72MiB/s ETA 00:16 [youtube] YvVthtbNtEE: Downloading video info webpage
[download]  40.2% of 95.98MiB at  3.90MiB/s ETA 00:14 Time is: 2020-03-07 20:17:36.456526
[download]  44.3% of 95.98MiB at  4.06MiB/s ETA 00:13 [download] Finished downloading playlist: 【誰來晚餐 番外篇】
['https://www.youtube.com/watch?v=sm4BID47dTI', 'https://www.youtube.com/watch?v=r5WApOc0WLI', 'https://www.youtube.com/watch?v=W5oa03IOkzE', 'https://www.youtube.com/watch?v=RxALj5QWsjo', 'https://www.youtube.com/watch?v=aG2iEAs78V0', 'https://www.youtube.com/watch?v=qO9J-WwUiEc', 'https://www.youtube.com/watch?v=YvVthtbNtEE']
[youtube] sm4BID47dTI: Downloading webpage
[youtube] r5WApOc0WLI: Downloading webpage
[download]  48.8% of 93.69MiB at  4.22MiB/s ETA 00:11         [youtube] W5oa03IOkzE: Downloading webpage
[youtube] RxALj5QWsjo: Downloading webpage
[youtube] aG2iEAs78V0: Downloading webpage
[youtube] qO9J-WwUiEc: Downloading webpage
Time is: 2020-03-07 20:17:37.462584
[download]  48.5% of 95.98MiB at  4.19MiB/s ETA 00:11 [youtube] r5WApOc0WLI: Downloading video info webpage
[youtube] sm4BID47dTI: Downloading video info webpage
[youtube] W5oa03IOkzE: Downloading video info webpage
[download]  53.0% of 93.69MiB at  4.32MiB/s ETA 00:10         [youtube] RxALj5QWsjo: Downloading video info webpage
[youtube] qO9J-WwUiEc: Downloading video info webpage
[youtube] aG2iEAs78V0: Downloading video info webpage
[download]  52.7% of 95.98MiB at  4.32MiB/s ETA 00:10 Time is: 2020-03-07 20:17:38.462641
[download]  61.6% of 93.69MiB at  4.50MiB/s ETA 00:08         Time is: 2020-03-07 20:17:39.462698
[download]  65.8% of 93.69MiB at  4.59MiB/s ETA 00:06         [download] Destination: 【 誰來晚餐_番外篇 Ep1 】香港到底怎麼了？民主當飯吃
，吃飽繼續說！-r5WApOc0WLI.mp4
[download]   0.4% of 16.27MiB at  1.10MiB/s ETA 00:14         [download] Destination: 【 誰來晚餐_番外篇 Ep 4 】投你所好，好好投-aG2iEAs7
8V0.mp4
[download]   0.1% of 17.17MiB at 499.97KiB/s ETA 00:35        [download] Destination: 【 誰來晚餐_番外篇 Ep2 】浪浪別哭！毛小孩怎麼找幸福
？-W5oa03IOkzE.mp4
[download]   0.2% of 33.42MiB at 588.75KiB/s ETA 00:58[download] Destination: 【 誰來晚餐_番外篇 Ep3 】買房好還是租房好-RxALj5QWsjo.mp4
[download]   0.7% of 33.42MiB at 885.37KiB/s ETA 00:38[download] Destination: 【 誰來晚餐_番外篇 Ep5 】台北街頭有話要說-qO9J-WwUiEc.mp4
[download]   0.3% of 9.36MiB at 815.74KiB/s ETA 00:11Time is: 2020-03-07 20:17:40.462755
[download]   5.3% of 9.36MiB at  1.80MiB/s ETA 00:04 [download] Destination: 【 誰來晚餐_番外篇 Ep0 】 吃飽繼續說-sm4BID47dTI.mp4
[download]  24.6% of 16.27MiB at  2.72MiB/s ETA 00:04         Time is: 2020-03-07 20:17:41.462812ETA 00:06
[download]  40.8% of 16.27MiB at  2.70MiB/s ETA 00:03         Time is: 2020-03-07 20:17:42.462870
[download]  57.2% of 16.27MiB at  2.72MiB/s ETA 00:02         Time is: 2020-03-07 20:17:43.462927
[download]  93.3% of 9.36MiB at  2.22MiB/s ETA 00:00 Time is: 2020-03-07 20:17:44.462984
[download] 100% of 9.36MiB in 00:05
[youtube] YvVthtbNtEE: Downloading webpage
[download] 100% of 6.34MiB in 00:06
[download]  49.0% of 17.17MiB at  1.59MiB/s ETA 00:05         [youtube] YvVthtbNtEE: Downloading video info webpage
Time is: 2020-03-07 20:17:45.465041
[download] 100% of 16.27MiB in 00:07
Time is: 2020-03-07 20:17:46.467099
[download]  80.7% of 93.69MiB at  3.61MiB/s ETA 00:05         Time is: 2020-03-07 20:17:47.467156
[download] Destination: 【 誰來晚餐_番外篇 Ep6 】回家過年快樂嗎？-YvVthtbNtEE.mp4
[download] 100% of 17.17MiB in 00:08
[download]  23.1% of 8.66MiB at  3.48MiB/s ETA 00:01 Time is: 2020-03-07 20:17:48.478214
[download]  46.0% of 33.42MiB at  1.68MiB/s ETA 00:10 Time is: 2020-03-07 20:17:49.478271
[download] 100% of 8.66MiB in 00:02
[download]  53.0% of 33.42MiB at  1.77MiB/s ETA 00:08 Time is: 2020-03-07 20:17:50.489329
[download]  93.0% of 95.98MiB at  3.56MiB/s ETA 00:01 Time is: 2020-03-07 20:17:51.489386
[download]  97.1% of 95.98MiB at  3.60MiB/s ETA 00:00 Time is: 2020-03-07 20:17:52.489443
[download] 100% of 34.30MiB in 00:13
[download] 100% of 95.98MiB in 00:26
[download]  90.7% of 33.42MiB at  2.26MiB/s ETA 00:01 Time is: 2020-03-07 20:17:53.489500
[download] 100% of 33.42MiB in 00:14
[download] 100% of 93.69MiB in 00:27
Time is: 2020-03-07 20:17:54.501558
Time is: 2020-03-07 20:17:55.501615
Time is: 2020-03-07 20:17:56.501673

Great code, can you add progress hooks?