Python 异步爬虫 aiohttp 示例

1、写在前面

之前一篇随笔记录了异步的个人理解 https://www.cnblogs.com/rainbow-tan/p/15081118.html

之前随笔异步都是以asyncio.sleep()来进行异步操作的演示，下面代码具体演示了一次异步爬虫

2、使用的异步爬虫库为 aiohttp

演示功能：

爬取 https://wall.alphacoders.com/ 中的小图片，进行批量下载，进行下载用时的对比

(1)先用一般的requests库进行爬虫演示，查看运行的时间

import os
import time

import requests
from bs4 import BeautifulSoup


def get_html(url):
    ret = requests.get(url)
    return ret


if __name__ == \'__main__\':
    index = 0
    start = time.time()
    response = get_html(\'https://wall.alphacoders.com/\')
    soup = BeautifulSoup(response.text, \'lxml\')
    boxgrids = soup.find_all(class_=\'boxgrid\')
    for boxgrid in boxgrids:
        img = boxgrid.find(\'a\').find(\'picture\').find(\'img\')
        link = img.attrs[\'src\']
        content = get_html(link).content
        picture_type = str(link).split(\'.\')[-1]
        index += 1
        path = os.path.abspath(\'imgs\')
        if not os.path.exists(path):
            os.makedirs(path)
        with open(\'{}/{}.{}\'.format(path, index, picture_type), \'wb\') as f:
            f.write(content)
    end = time.time()
    print(f\'下载完成{index}个图片,用时:{end - start}秒\')

运行

用时14秒，下载30个图片

 (2)使用异步库aiohttp下载

import asyncio
import os
import time

import aiohttp
from aiohttp import TCPConnector
from bs4 import BeautifulSoup


async def get_html(url):
    async with aiohttp.ClientSession(
            connector=TCPConnector(verify_ssl=False)) as session:
        async with session.get(url) as resp:
            text = await resp.text()
            soup = BeautifulSoup(text, \'lxml\')
            boxgrids = soup.find_all(class_=\'boxgrid\')
            links = []
            for boxgrid in boxgrids:
                img = boxgrid.find(\'a\').find(\'picture\').find(\'img\')
                link = img.attrs[\'src\']
                links.append(link)
            return links


async def write_file(url, index):
    async with aiohttp.ClientSession(
            connector=TCPConnector(verify_ssl=False)) as session:
        async with session.get(url) as resp:
            text = await resp.read()
            path = os.path.abspath(\'images\')
            if not os.path.exists(path):
                os.makedirs(path)
            with open(f\'{path}/{index}.{str(url).split(".")[-1]}\', \'wb\') as f:
                f.write(text)


if __name__ == \'__main__\':
    index = 0
    start = time.time()
    loop = asyncio.get_event_loop()
    task = loop.create_task(get_html(\'https://wall.alphacoders.com/\'))
    links = loop.run_until_complete(task)
    tasks = []
    for link in links:
        tasks.append(write_file(link, index))
        index += 1
    loop.run_until_complete(asyncio.gather(*tasks))
    end = time.time()
    print(f\'下载完成{index}个图片,用时:{end - start}秒\')

运行

下载30个图片，用时4秒

学习链接 ：

https://www.jianshu.com/p/20ca9daba85f

https://docs.aiohttp.org/en/stable/client_quickstart.html

https://juejin.cn/post/6857140761926828039 （这个未参考，但是看起来也很牛，收藏一下）