爬虫之爬取豆瓣图书的评论

from urllib import request
from bs4 import BeautifulSoup as bs

#爬取豆瓣最受关注图书榜
resp = request.urlopen(\'https://book.douban.com/chart?subcat=I\')
html_data = resp.read().decode(\'utf-8\')

#转化为BeautifulSoup对象
soup = bs(html_data,\'html.parser\')

#搜索最受关注的图书列表
topchart_book =soup.find_all(\'ul\',class_=\'chart-dashed-list\')

#搜索列表中所有图书
topchart_book_list = topchart_book[0].find_all(\'li\',class_=\'media clearfix\')

#新建数组用于存放后续的数据
topchart_list = []

#遍历图书馆列表，从中过滤出我们所需的信息
for item in topchart_book_list:
    #新建字典用于存放我们的图书信息，之后可用class来存储
    topchart_dict = {}

    #搜索到具体信息的位置
    book_item = item.find(\'a\',class_=\'fleft\')

    #得到图书ID
    topchart_dict[\'id\'] = book_item[\'href\'].split(\'/\')[4]   

    #得到图书名称
    topchart_dict[\'name\'] = book_item.getText().replace(\'\t\',\'\').replace(\'\n\',\'\').replace(\' \',\'\')  #图书名字
    
    #将图书信息加入到数组中
    topchart_list.append(topchart_dict)
# print(topchart_list)

#拼接出图书对应的详情页
requrl = \'https://book.douban.com/subject/\'+topchart_list[0][\'id\']+\'/comments/hot\'+\'?\'+\'p-1\'

#爬取热门第一页中的评论信息
resp = request.urlopen(requrl)
html_data = resp.read().decode(\'utf-8\')
soup = bs(html_data,\'html.parser\')

#搜索到评论所在div
comment_div_lits = soup.find_all(\'div\',class_=\'comment\')

#新建数组用于存放评论信息
eachCommentList = []

for item in comment_div_lits:
    if item.find_all(\'p\')[0].string is not None:
        eachCommentList.append(item.find_all(\'p\')[0].string)
print(eachCommentList)