豆瓣电影TOP250和书籍TOP250爬虫 - innerpeacez

java

豆瓣电影 TOP250 和书籍 TOP250 爬虫

最近开始玩 Python , 学习爬虫相关知识的时候，心血来潮，爬取了豆瓣电影TOP250 和书籍TOP250, 这里记录一下自己玩的过程。

电影 TOP250 爬虫

import requests
from bs4 import BeautifulSoup
import time


def getlist(list_url):
    time.sleep(2)
    res = requests.get(list_url)
    soup = BeautifulSoup(res.text, \'html.parser\')
    movie_list = soup.select(\'.grid_view li\')
    for m in movie_list:
        rank = m.select(\'em\')[0].text
        score = m.select(\'.rating_num\')[0].text
        title = m.select(\'.title\')[0].text
        direct = m.select(\'.info .bd p\')[0].text.strip()
        actor = \'\n主演:\'.join(direct.split(\'   主演:\'))
        director = \'年代:\'.join(actor.split(\'                           \'))
        if m.select(\'.inq\'):
            comments = m.select(\'.inq\')[0].text.strip()
        else:
            comments = \'None\'
        movie.append(
            \'排名: \' + rank + \'\n\'
            + \'评分: \' + score + \'\n\'
            + \'片名: \' + title + \'\n\'
            + director + \'\n\'
            + \'评论: \' + comments + \'\n\'
            + \'\n\')
    if soup.select(\'.next a\'):
        asoup = soup.select(\'.next a\')[0][\'href\']
        next_page = seed_url + asoup
        getlist(next_page)
    else:
        print(\'结束\')
    return movie


def write(movies):
    with open(\'movie.txt\', \'w\', encoding=\'utf8\') as m:
        for a in movies:
            m.write(a)


def main():
    write(getlist(seed_url))
    pass


if __name__ == \'__main__\':
    seed_url = \'https://movie.douban.com/top250\'
    movie = []
    main()

书籍 TOP250 爬虫

import bs4
import requests
import re
from bs4 import BeautifulSoup
from operator import itemgetter


def getHtmlText(url):
    try:
        r = requests.get(url)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return ""


def parserText(text, book_list):
    soup = BeautifulSoup(text, \'html.parser\')
    for table in soup(\'table\', {\'width\': \'100%\'}):
        if isinstance(table, bs4.element.Tag):
            tds = table.find(\'tr\')(\'td\')
            divs = tds[1](\'div\')
            content = {}
            for div in divs:
                if isinstance(div, bs4.element.Tag):
                    if div.find(\'a\'):
                        name = div.find(\'a\').attrs[\'title\']
                        content.update({"书名": name})
                    if div.select(\'.rating_nums\'):
                        score = div.select(\'.rating_nums\')[0].text
                        content.update({"评分": score})
                    if div.select(\'.pl\'):
                        people_num = div.select(\'.pl\')[0].text
                        regex = re.compile(r\'[\d]{1,10}\')
                        content.update({"评价人数": regex.findall(people_num)[0]})

            ps = tds[1](\'p\')
            for p in ps:
                if isinstance(p, bs4.element.Tag):
                    if p.attrs[\'class\'][0] == \'quote\':
                        description = p.find(\'span\').string
                        content.update({"介绍": description})
                    if p.attrs[\'class\'][0] == \'pl\':
                        author = p.string
                        content.update({"作者信息": author})

            book_list.append(content)

    next_books = soup.find(\'span\', {\'class\': \'next\'})
    if next_books.find(\'a\'):
        a = next_books.find(\'a\').attrs[\'href\']
        text = getHtmlText(a)
        parserText(text, books)

    return book_list


def sortedBookTop250(book_list):
    tmp = sorted(book_list, key=itemgetter(\'评分\'), reverse=True)
    for i in range(len(tmp)):
        tmp[i].update({"排名": i + 1})
    return tmp


def writeToFile(book_list):
    with open(\'good_books.txt\', \'w\', encoding=\'utf8\') as book_file:
        for book in book_list:
            for key, value in book.items():
                book_file.write(f\'{key}:{value}\n\')
            book_file.write(\'\n\')
    pass


def main():
    text = getHtmlText(seed_url)
    book_list = parserText(text, books)
    writeToFile(sortedBookTop250(book_list))
    pass


if __name__ == \'__main__\':
    seed_url = "https://book.douban.com/top250"
    books = []
    main()

总结

• 点击查看我的Github
• 点击查看我的个人Blog
• 日拱一卒，不期速成

以上直接贴出了代码，这是很简单的两段代码，主要用到了 requests 库和 beautifulsoup 库，需要的可以直接拿去，或者直接去我的 GIthub上拿 movies.txt 和 good_books.txt