爬虫初探-笔趣阁小说下载

# -*- coding: utf-8 -*-
"""
Created on Tue Dec  1 12:31:07 2020

@author: zhaolulu
"""
import pandas as pd
import requests
from lxml import etree


headers = {
    "User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.75 Safari/537.36"
}

def url_read(url):
    try:
        reponse = requests.get(url,headers=headers)
    except:
        print(\'failed\')
    return reponse.content.decode(\'utf-8\')

if __name__==\'__main__\':
    #笔趣阁小说网站
    url=\'http://www.xbiquge.la/\'
    text = url_read(url)
    print("============================================")
    selector=etree.HTML(text)
    #这个是主页上最新小说的url
    ret=selector.xpath(\'//*[@id="newscontent"]/div[1]/ul/li/span[2]/a//@href\')
    for note_url in ret:
        print(note_url)
    # 这选择了其中一条url 做测试
    #print(ret[0]) # http://www.xbiquge.la/62/62585/
    n_text = url_read(\'http://www.xbiquge.la/62/62585/\')
    n_html=etree.HTML(n_text)
    xpath_ret = n_html.xpath(\'//*[@id="list"]/dl/dd/a/@href\')
    index =0;
    for t_url in xpath_ret:
        #具体的章节内容
        f_url = \'http://www.xbiquge.la\'+t_url
        print(f_url)
        article = url_read(f_url)
        article_text=etree.HTML(article)
        article_detail=article_text.xpath(\'//*[@id="content"]/text()\')
        if len(article_detail) > 0:
            pd.Series(article_detail).to_csv(\'..\\book\\\'+str(index))
            index=index+1