Skip to content

下载图片 in async (asyncio + ThreadPoolExecutor)

python
import asyncio
from concurrent.futures import ThreadPoolExecutor
import time
import random
import os
import requests

dir_d = 'd'
if not os.path.exists(dir_d):
    os.mkdir(dir_d)

def save_jpg(index, content):
    with open(os.path.join(dir_d, f'{index}.jpg'), 'wb') as fp:
        fp.write(content)

def work(task):
    try:
        index = task.get('index')
        url = task.get('url')
        content = requests.get(url=url, timeout=3).content
        save_jpg(index, content)
        print('finished', index)
    except Exception as e:
        pass
    return index

async def main():
    t = time.monotonic()
    loop = asyncio.get_running_loop()
    tasks = []
    with ThreadPoolExecutor(100) as pool:
        for index, url in enumerate(['http://wx2.sinaimg.cn/mw600/0089jzyPly1gqo2l5gwb7j31400u04qp.jpg']*2000):
            task = dict(index=index,url=url)
            tasks.append(loop.run_in_executor(pool, work, task))
    await asyncio.gather(*tasks, return_exceptions=True)
    print('task done', time.monotonic()-t)

asyncio.run(main())

Released under the MIT License.