下载图片 in async (asyncio + ThreadPoolExecutor)
python
import asyncio
from concurrent.futures import ThreadPoolExecutor
import time
import random
import os
import requests
dir_d = 'd'
if not os.path.exists(dir_d):
os.mkdir(dir_d)
def save_jpg(index, content):
with open(os.path.join(dir_d, f'{index}.jpg'), 'wb') as fp:
fp.write(content)
def work(task):
try:
index = task.get('index')
url = task.get('url')
content = requests.get(url=url, timeout=3).content
save_jpg(index, content)
print('finished', index)
except Exception as e:
pass
return index
async def main():
t = time.monotonic()
loop = asyncio.get_running_loop()
tasks = []
with ThreadPoolExecutor(100) as pool:
for index, url in enumerate(['http://wx2.sinaimg.cn/mw600/0089jzyPly1gqo2l5gwb7j31400u04qp.jpg']*2000):
task = dict(index=index,url=url)
tasks.append(loop.run_in_executor(pool, work, task))
await asyncio.gather(*tasks, return_exceptions=True)
print('task done', time.monotonic()-t)
asyncio.run(main())