爬取 豆瓣电影Top250

目标

学习爬虫，爬豆瓣榜单，获取爬取静态页面信息的能力

豆瓣电影 Top 250  https://movie.douban.com/top250

代码

import requests
from bs4 import BeautifulSoup
def getHTMLText(url):
    try:
        r = requests.get(url,timeout=30)
        r.raise_for_status()
        r.encoding = r.apparent_encoding
        return r.text
    except:
        return \'产生异常\'
    
if __name__ == \'__main__\':
    i = 0
    urls = [\'https://movie.douban.com/top250?start=\'+str(n)+\'&filter=\' for n in range(0,250,25)]
    for url in urls:
        r = getHTMLText(url)
        soup = BeautifulSoup(r,\'html.parser\')
        titles = soup.select(\'div.hd a\')
        rates = soup.select(\'span.rating_num\')
        pics = soup.select(\'img[width="100"]\')
        for title,rate,pic in zip(titles,rates,pics):
            data={\'title\':list(title.stripped_strings),
                  \'rate\':rate.get_text(),
                  \'pic\':pic.get(\'src\')}
            i+=1
            fileName=str(i)+\'_\'+data[\'title\'][0]+\' \'+data[\'rate\']+\'分.jpg\'
            pic1 = requests.get(data[\'pic\'])
            with open(\'G:\\test\\\'+fileName,\'wb\') as photo:
                photo.write(pic1.content)
            print(data) 

爬取结果