爬虫百度贴吧

import requests
from bs4 import BeautifulSoup
import re

def getHTMLText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = \'utf-8\'
        return r.text
    except:
        return "Error"
    
def parsePage(html):
    infos = []
    soup = BeautifulSoup(html, \'lxml\')
    li_tags = soup.find_all(\'li\', class_=" j_thread_list clearfix")
    for li in li_tags:
        info = {}
        title = li.find(\'a\', class_="j_th_tit ").string
        link = \'http://tieba.baidu.com\' + li.find(\'a\', class_="j_th_tit ")[\'href\']
        author = li.find(\'span\', class_=re.compile("tb_icon_author "))[\'title\'].split(\':\')[-1].strip()
        time = li.find(\'span\', class_="pull-right is_show_create_time").string
        reply_num = li.find(\'span\', class_="threadlist_rep_num center_text").string
        info[\'title\'] = title
        info[\'author\'] = author
        info[\'time\'] = time
        info[\'reply_num\'] = reply_num
        info[\'link\'] = link
        infos.append(info)
    return infos


def ToFile(infos):
    with open(\'test.txt\', \'a+\', encoding=\'utf-8\') as f:
        for info in infos:
            f.write(\'标题：{} \t 发帖人：{} \t 发帖时间：{} \t 回复：{} \t 链接：{} \n\'.format(
                    info[\'title\'], info[\'author\'], info[\'time\'], info[\'reply_num\'], info[\'link\']))
    
def main(base_url, deep):
    url_list = []
    for i in range(deep):
        url_list.append(base_url + \'&pn=\' + str(50*i))
    for url in url_list:
        html = getHTMLText(url)
        infos = parsePage(html)
        ToFile(infos)
        
if __name__ == \'__main__\':
    base_url = \'http://tieba.baidu.com/f?kw=考研&ie=utf-8\'
    deep = 4
    main(base_url, deep)