爬取卡通图片

因为想找个网站练练手，发现这个网站不错，所以对这个网站的部分图片进行了爬取，主要是卡通图片什么的，请直接查看代码。
这个还是初版，后面会有一个升级版多线程版。

# -*- coding: utf-8 -*-
# by wangcc
# mail:wangcc_sd@163.com

import requests
import sys
import io
import os
from bs4 import BeautifulSoup
import asyncio
import json

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding=\'utf8\')  ###改变标准输出的默认编码

headers = {
    \'User-agent\': \'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/60.0.3112.113 Safari/537.36\'}

async def get_url(queue,url):
    print(url)
    response  = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text,\'html.parser\')
    for line in range(len(soup.find_all(\'p\', class_=\'list_h\'))):
        divObj = soup.find_all(\'p\', class_=\'list_h\')[line]
        for i in divObj:
            #print(divObj)
            await queue.put(divObj)
    await asyncio.sleep(1)

def get_page(url):
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, \'html.parser\')
    soup_page = soup.find_all(\'a\', text="末页")
    try:
        page = str(str(soup_page).split(\'_\')[1]).split(\'.\')[0]
    except IndexError as e:
        print(e)
        return 0
    return page

def get_url_second(url):
    page = get_page(url)
    url_list = [url.split(\'.html\')[0] + \'_{}.html\'.format(i) for i in range(int(page)) if i>=2 ]
    url_list.append(url)
    for index in range(len(url_list)):
        url = url_list[index]
        #print(url)
        response = requests.get(url, headers=headers)
        soup = BeautifulSoup(response.text, \'html.parser\')
        one_jpg=soup.find_all(\'script\', type="application/ld+json")
        reversed_news_arr=[]
        for single_news in one_jpg:
            reversed_news_arr.append(single_news)
        script_info=(str(reversed_news_arr)[37:-10])
        script_dict = json.loads(script_info)
        id = script_dict["@id"]
        title = script_dict["title"]
        images = script_dict["images"][0]
        jpg_name = str(images).split(\'/\')[-1]
        jpg_name="./date/" + title +\'/\'+ jpg_name
        jpg_index = requests.get(images)
        with open(jpg_name, \'wb\')as jpg:
            jpg.write(jpg_index.content)
            jpg.close()



def dir_save(dir_name):
    path=\'./date\'
    if not os.path.exists(path+\'/\'+dir_name):
        os.mkdir(path+\'/\'+dir_name)

async def consumer(queue):
    while True:
        print(\'qsize--->\',queue.qsize())
        divObj = await queue.get()
        href=divObj.a.get(\'href\')
        title=divObj.a.get(\'title\')
        dir_save(title)
        url ="https://www.uumtu.com"+href
        get_url_second(url)
        continue
        await asyncio.sleep(1)



async def main():
    queue = asyncio.Queue()
    for i in range(1,50):
        #url=\'https://www.uumtu.com/meinv/list_{}.html\'.format(i)
        url = \'https://www.uumtu.com/katong/list_{}.html\'.format(i)
        print(url)
        producer_1 = asyncio.create_task(get_url(queue, url))
        consumer_1 = asyncio.create_task(consumer(queue))

if __name__ == \'__main__\':
    for i in range(931):
        pass
    asyncio.run(main())