python网络爬虫设计————豆瓣小说爬取

选题的背景
为什么要选择此选题？要达到的数据分析的预期目标是什么？

小说是文化的一种载体,在古今中外的历史中都占据着重要的地位。人蒙对小说的需求也并不相同，如何在大量的小说里选取自己喜欢的小说，成为了问题，基于Python的数据爬虫技术是目前使用最广泛的方法之一，它能够以最快捷的方式展示用户体验数据，帮助读者进行小说选择。通过豆瓣小说爬取各种分类的小说。

 

数据来源：www.douban.com/tag/

 

Htmls 页面解析:

 

 

 

数据爬取与采集:

import importlib
import sys
import time
import urllib
from urllib.request import Request
import numpy as np
from bs4 import BeautifulSoup
from openpyxl import Workbook

importlib.reload(sys)
#sys.setdefaultencoding(\'utf8\')

 

#Some User Agents
hds=[{\'User-Agent\':\'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6\'},\
{\'User-Agent\':\'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11\'},\
{\'User-Agent\': \'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)\'}]

def book_spider(book_tag):
page_num=0;
book_list=[]
try_times=0

while(1):
#url=\'http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0\' # For Test
url=\'http://www.douban.com/tag/\'+urllib.parse.quote(book_tag)+\'/book?start=\'+str(page_num*15)
time.sleep(np.random.rand()*5)

#Last Version
try:
req = Request(url, headers=hds[page_num%len(hds)])
source_code = urllib.request.urlopen(req).read()

plain_text=str(source_code,\'utf-8\')

except (urllib.request.HTTPError, urllib.request.URLError) as e:
print (e)
continue

##Previous Version, IP is easy to be Forbidden
#source_code = requests.get(url)
#plain_text = source_code.text

soup = BeautifulSoup(plain_text)
list_soup = soup.find(\'div\', {\'class\': \'mod book-list\'})

try_times+=1;
if list_soup==None and try_times<200:
continue
elif list_soup==None or len(list_soup)<=1:
break # Break when no informatoin got after 200 times requesting

for book_info in list_soup.findAll(\'dd\'):
title = book_info.find(\'a\', {\'class\':\'title\'}).string.strip()
desc = book_info.find(\'div\', {\'class\':\'desc\'}).string.strip()
desc_list = desc.split(\'/\')
book_url = book_info.find(\'a\', {\'class\':\'title\'}).get(\'href\')

try:
author_info = \'作者/译者： \' + \'/\'.join(desc_list[0:-3])
except:
author_info =\'作者/译者： 暂无\'
try:
pub_info = \'出版信息： \' + \'/\'.join(desc_list[-3:])
except:
pub_info = \'出版信息： 暂无\'
try:
rating = book_info.find(\'span\', {\'class\':\'rating_nums\'}).string.strip()
except:
rating=\'0.0\'
try:
#people_num = book_info.findAll(\'span\')[2].string.strip()
people_num = get_people_num(book_url)
people_num = people_num.strip(\'人评价\')
except:
people_num =\'0\'

book_list.append([title,rating,people_num,author_info,pub_info])
try_times=0 #set 0 when got valid information
page_num+=1
print (\'Downloading Information From Page %d\' % page_num)
return book_list

def get_people_num(url):
#url=\'http://book.douban.com/subject/6082808/?from=tag_all\' # For Test
try:
req = urllib.Request(url, headers=hds[np.random.randint(0,len(hds))])
source_code = urllib.request.urlopen(req).read()
plain_text=str(source_code)
except (urllib.request.HTTPError, urllib.request.HTTPError) as e:
print (e)
soup = BeautifulSoup(plain_text)
people_num=soup.find(\'div\',{\'class\':\'rating_sum\'}).findAll(\'span\')[1].string.strip()
return people_num

def do_spider(book_tag_lists):
book_lists=[]
for book_tag in book_tag_lists:
book_list=book_spider(book_tag)
book_list=sorted(book_list,key=lambda x:x[1],reverse=True)
book_lists.append(book_list)
return book_lists

 

 

 

 

 数据清洗：

import pandas as pd

df = pd.DataFrame(pd.read_excel(r\'C:\Users\86591\book_list-文学.xlsx\'))


print(\'\n====各列是否有缺失值情况如下：====\')

#重复值处理
print(df.duplicated())

#统计空值
print(df.isnull().sum())  #返回0则没有空值

#缺失值处理
print(df[df.isnull().values==True]) #返回无缺失值

#用describe()命令显示描述性统计指标
print(df.describe())

 

 

柱状图：

plt.rcParams[\'axes.unicode_minus\']=False #用来正常显示负号
plt.bar(df.书名, df.评分, label="书名与评分柱状图")
plt.show()

 

 

 散点图：

def Scatter_point():

 plt.scatter(df.书名, df.评分, color=\'red\', s=25, marker="o")
 plt.xlabel("书名")
 plt.ylabel("评分")
 plt.title("书名与评分-散点图")
 plt.show()

Scatter_point()
#散点图

 

 

 

 

 

完整代码：

 

import importlib
import sys
import time
import urllib
from urllib.request import Request 
import numpy as np
from bs4 import BeautifulSoup
from openpyxl import Workbook

importlib.reload(sys)
#sys.setdefaultencoding(\'utf8\')



#Some User Agents
hds=[{\'User-Agent\':\'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6\'},\
{\'User-Agent\':\'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.12 Safari/535.11\'},\
{\'User-Agent\': \'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)\'}]


def book_spider(book_tag):
    page_num=0;
    book_list=[]
    try_times=0
    
    while(1):
        #url=\'http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0\' # For Test
        url=\'http://www.douban.com/tag/\'+urllib.parse.quote(book_tag)+\'/book?start=\'+str(page_num*15)
        time.sleep(np.random.rand()*5)
        
        #Last Version
        try:
            req = Request(url, headers=hds[page_num%len(hds)])
            source_code = urllib.request.urlopen(req).read()
            
            plain_text=str(source_code,\'utf-8\')   
            
        except (urllib.request.HTTPError, urllib.request.URLError) as e:
            print (e)
            continue
  
        ##Previous Version, IP is easy to be Forbidden
        #source_code = requests.get(url) 
        #plain_text = source_code.text  
        
        soup = BeautifulSoup(plain_text)
        list_soup = soup.find(\'div\', {\'class\': \'mod book-list\'})
        
        try_times+=1;
        if list_soup==None and try_times<200:
            continue
        elif list_soup==None or len(list_soup)<=1:
            break # Break when no informatoin got after 200 times requesting
        
        for book_info in list_soup.findAll(\'dd\'):
            title = book_info.find(\'a\', {\'class\':\'title\'}).string.strip()
            desc = book_info.find(\'div\', {\'class\':\'desc\'}).string.strip()
            desc_list = desc.split(\'/\')
            book_url = book_info.find(\'a\', {\'class\':\'title\'}).get(\'href\')
            
            try:
                author_info = \'作者/译者： \' + \'/\'.join(desc_list[0:-3])
            except:
                author_info =\'作者/译者： 暂无\'
            try:
                pub_info = \'出版信息： \' + \'/\'.join(desc_list[-3:])
            except:
                pub_info = \'出版信息： 暂无\'
            try:
                rating = book_info.find(\'span\', {\'class\':\'rating_nums\'}).string.strip()
            except:
                rating=\'0.0\'
            try:
                #people_num = book_info.findAll(\'span\')[2].string.strip()
                people_num = get_people_num(book_url)
                people_num = people_num.strip(\'人评价\')
            except:
                people_num =\'0\'
            
            book_list.append([title,rating,people_num,author_info,pub_info])
            try_times=0 #set 0 when got valid information
        page_num+=1
        print (\'Downloading Information From Page %d\' % page_num)
    return book_list


def get_people_num(url):
    #url=\'http://book.douban.com/subject/6082808/?from=tag_all\' # For Test
    try:
        req = urllib.Request(url, headers=hds[np.random.randint(0,len(hds))])
        source_code = urllib.request.urlopen(req).read()
        plain_text=str(source_code)   
    except (urllib.request.HTTPError, urllib.request.HTTPError) as e:
        print (e)
    soup = BeautifulSoup(plain_text)
    people_num=soup.find(\'div\',{\'class\':\'rating_sum\'}).findAll(\'span\')[1].string.strip()
    return people_num


def do_spider(book_tag_lists):
    book_lists=[]
    for book_tag in book_tag_lists:
        book_list=book_spider(book_tag)
        book_list=sorted(book_list,key=lambda x:x[1],reverse=True)
        book_lists.append(book_list)
    return book_lists


def print_book_lists_excel(book_lists,book_tag_lists):
    wb=Workbook(write_only =True)
    ws=[]
    for i in range(len(book_tag_lists)):
        ws.append(wb.create_sheet(title=book_tag_lists[i].encode().decode(\'utf-8\'))) #utf8->unicode
    for i in range(len(book_tag_lists)): 
        ws[i].append([\'序号\',\'书名\',\'评分\',\'作者\',\'出版社\'])
        count=1
        for bl in book_lists[i]:
            ws[i].append([count, bl[0],(bl[1]),bl[3],bl[4]])
            count+=1
    save_path=\'book_list\'
    for i in range(len(book_tag_lists)):
        save_path+=(\'-\'+book_tag_lists[i].encode().decode(\'utf-8\'))
    save_path+=\'.xlsx\'
    wb.save(save_path)




if __name__==\'__main__\':
    
    book_tag_lists = [\'文学\']
    book_lists=do_spider(book_tag_lists)
    print_book_lists_excel(book_lists,book_tag_lists)

总结：

小说使我们当前最普遍的娱乐方式之一，通过此课程设计完成通过小说标签对小说进行爬取，可以使人更容易找到自己感兴趣的小说。通过对·python的学习，可以让生活中的一些事情做起来变得更加轻松，这就是我们学习高级语言的原因，通过对python的学习，更加丰富了自己的知识。