豆瓣Top250电影数据爬取学习

    def __init__(self, url, start_page, pages, page_size):
        """
        初始化
        @param url: 爬取主网址
        @param start_page: 起始页码
        @param pages: 总页码（截止页码）
        @param page_size: 每页的大小
        """
        self.url = url
        self.start_page = start_page
        self.pages = pages
        self.page_size = page_size
        self.data_info = []
        self.headers = {
            \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36\',
        }

    def get_one_page(self):
        """
        根据起始页码获取当前页面的所有电影
        :return:
        """
        # 如果当前页码小于0，异常退出
        if self.start_page < 0:
            return ""
        # 如果起始页面大于总页码数，退出
        if self.start_page > self.pages:
            return ""

        # 若当前页其实页码小于总页数，继续爬取数据
        while self.start_page<pages:
            # 根据每页数据条数确定起始下标
            start_number = self.start_page * self.page_size
            new_url = self.url + \'?start=\' + str(start_number) + \'&filter=\'
            print(\'正在爬取第 {0} 页数据\'.format(self.start_page+1))

            # 爬取当前页码的数据
            response = requests.get(url=new_url, headers=self.headers)
            # 解析数据
            self.get_per_movie(response.text)

            # 下一页
            self.start_page = self.start_page + 1

        # 将当前数据保存到数据库中
        self.data_to_mysql()

        return ""

    def get_per_movie(self, one_page_data):
        """
        解析每一页的每一个电影详细链接
        :param one_page_data:
        :return:
        """
        soup = BeautifulSoup(one_page_data, \'html.parser\')
        # 定位到每一个电影的 div （pic 标记的 div）
        soup_div_list = soup.find_all(class_="pic")
        # 遍历获取每一个 div 的电影详情链接
        for soup_div in soup_div_list:
            # 定位到每一个电影的 a 标签
            soup_a = soup_div.find_all(\'a\')[0]
            movie_href = soup_a.get(\'href\')
            print(movie_href)
            # 解析数据，获取当前页的 25 个电影详细链接
            self.get_movie_content(movie_href)

        return ""

    def get_movie_content(self, movie_detail_href):
        # 生成一个有序字典，保存影片结果
        movie_info = OrderedDict()
        \'\'\'爬取页面，获得详细数据\'\'\'
        response = requests.get(url=movie_detail_href, headers=self.headers)
        soup = BeautifulSoup(response.text, \'html.parser\')

        # 解析电影排名和名称
        movie_info[\'movie_rank\'] = soup.find_all(\'span\', class_=\'top250-no\')[0].string
        movie_info[\'movie_name\'] = soup.find_all(\'span\', property=\'v:itemreviewed\')[0].string

        # 定位到影片数据的 div
        soup_div = soup.find(id=\'info\')
        # 解析电影发布信息
        movie_info[\'movie_director\'] = self.get_mul_tag_info(soup_div.find_all(\'span\')[0].find_all(\'a\'))
        movie_info[\'movie_writer\'] = self.get_mul_tag_info(soup_div.find_all(\'span\')[3].find_all(\'a\'))
        movie_info[\'movie_starring\'] = self.get_mul_tag_info(soup_div.find_all(\'span\')[6].find_all(\'a\'))
        movie_info[\'movie_type\'] = self.get_mul_tag_info(soup_div.find_all(\'span\', property=\'v:genre\'))
        movie_info[\'movie_country\'] = soup_div.find(text=\'制片国家/地区:\').next_element.lstrip().rstrip()
        movie_info[\'movie_language\'] = soup_div.find(text=\'语言:\').next_element.lstrip().rstrip()
        movie_info[\'movie_release_date\'] = self.get_mul_tag_info(soup_div.find_all(\'span\', property=\'v:initialReleaseDate\'))
        movie_info[\'movie_run_time\'] = self.get_mul_tag_info(soup_div.find_all(\'span\', property=\'v:runtime\'))
        movie_info[\'movie_imdb_href\'] = soup_div.find(\'a\', target=\'_blank\')[\'href\']

        movie_second_name = \'\'
        try:
            movie_second_name = soup_div.find(text=\'又名:\').next_element.lstrip().rstrip()
        except AttributeError:
            print(\'{0} 没有又名\'.format(movie_info[\'movie_name\']))
            movie_info[\'movie_second_name\'] = movie_second_name

        # 获取总评分和总评价人数
        movie_info[\'movie_rating\'] = soup.find_all(\'strong\', property=\'v:average\')[0].string
        movie_info[\'movie_comments_user\'] = soup.find_all(\'span\', property=\'v:votes\')[0].string
        # 定位到影片星级评分占比的 div
        soup_div = soup.find(\'div\', class_="ratings-on-weight")
        # 获取每个星级的评分
        movie_info[\'movie_five_star_ratio\'] = soup_div.find_all(\'div\')[0].find(class_=\'rating_per\').string
        movie_info[\'movie_four_star_ratio\'] = soup_div.find_all(\'div\')[2].find(class_=\'rating_per\').string
        movie_info[\'movie_three_star_ratio\'] = soup_div.find_all(\'div\')[4].find(class_=\'rating_per\').string
        movie_info[\'movie_two_star_ratio\'] = soup_div.find_all(\'div\')[6].find(class_=\'rating_per\').string
        movie_info[\'movie_one_star_ratio\'] = soup_div.find_all(\'div\')[8].find(class_=\'rating_per\').string
        movie_info[\'movie_note\'] = \'\'

        print(\'movie_info:\',movie_info)
        # 保存当前影片信息
        self.data_info.append(movie_info)

    def data_to_mysql(self):
        """
        保存数据到数据库中
        @return:
        """
        # 获取数据并保存成 DataFrame
        df_data = pd.DataFrame(self.data_info)
        df_data.to_csv(\'E:/pythonob/data/movie_data/data_movie.csv\', encoding=\'utf-8\', index=False)
        # 导入数据到 mysql 中
        #df_data.to_sql(\'t_douban_movie_top_250\', self.pymysql_engine, index=False, if_exists=\'append\')

    def get_mul_tag_info(self, soup_span):
        """
        获取多个标签的结果合并在一个结果中返回，并用 / 分割
        :param soup_span:
        :type soup_span:
        :return:
        :rtype:
        """
        info = \'\'
        for second_span in soup_span:
            # 区分 href 和标签内容
            info = (\'\' if (info == \'\') else \'/\').join((info, second_span.string))

        return info


if __name__ == \'__main__\':
    url = \'https://movie.douban.com/top250\'
    start_page = 0
    pages = 10
    page_size = 25
    douban_movie = DouBanMovie(url, start_page, pages, page_size)
    douban_movie.get_one_page()




原文链接：https://zhuanlan.zhihu.com/p/97705654