猫眼电影网站页面布局整体采用静态页面，其TOP100榜单按照评分和评分人数从高到低综合排序取前100名，如下：
那么，如何爬取电影库中的经典电影呢？为此，本人特意对经典电影库进行了爬取，具体遇到的困难及解决的办法如下：
1、爬取内容：本次爬取维度有电影名称、电影类型、电影制片国家、电影时长/分钟、上映时间、上映地点、评分、评论/万条、票房/亿元、导演以及主演前三名，其中评分和评分人数要有数值，所遇到的问题：
1>票房数据单位混杂，需转换；
2>爬取维度存在部分无信息，为此采用try，避免程序中断，对不符合规则的网站进行网址保存《坏IP.csv》
2、页面请求及解析技术：采用requests库中get请求方式，解析采用lxml和xpath
3、避免反爬措施：
1>使用fake_usergent中的UserAgent类随机生成User-Agent，避免网站检测；
2>因页面关键信息（如评分、票房等）被网站采用动态自定义字体隐藏，为了破解获取关键信息，使用FontTransform对自定义字体进行破解
3>使用随机时间，延迟访问网站的频率
4、代码性能优化：
1>使用多线程和队列，提高爬取速度
2>存储方面：因数据量较小，采用csv存储，同时，为了减少磁盘IO，程序采用一次性存储所有爬取数据，提升效率；
程序主体如下：
import requests from lxml import etree from threading import Thread from queue import Queue from fake_useragent import UserAgent from font_transform import FontTransform import re import csv import os import time import random class MaoyanSpider: def __init__(self): self.url = \'https://maoyan.com/films?showType=3&offset={}\' self.q = Queue() self.headers = { "Referer": "https://maoyan.com/films?showType=3" } self.i = 0 self.j = 0 self.info_list = [] def get_url(self): # 目标链接入队列 for i in range(0, 1981, 30): url = self.url.format(i) self.headers[\'User-Agent\'] = UserAgent().random html = requests.get(url, headers=self.headers).content.decode(\'utf-8\') parse_html = etree.HTML(html) film_link_list = parse_html.xpath(\'//div[@class="movie-item"]/a/@href\') for link in film_link_list: self.q.put(\'https://maoyan.com\' + link) print(self.q.qsize()) def get_page(self): while True: if not self.q.empty(): self.headers[\'User-Agent\'] = UserAgent().random url = self.q.get() html = requests.get(url, headers=self.headers).content.decode(\'utf-8\') time.sleep(random.random()) self.transform_page(html, url) else: break def transform_page(self, html, url): # 源码中自定义字体转换 font_file = re.findall(r"url\(\'//vfile.meituan.net/colorstone/(.*?\.woff)\'", html)[0].strip() font = FontTransform().get_font(font_file) html = FontTransform().modify_data(font, html) self.get_data(html, url) def get_data(self, html, url): # 信息提取 parse_html = etree.HTML(html) term_base = parse_html.xpath(\'//div[@class="movie-stats-container"]/div[1]//text()\') if \'暂无\' not in term_base and \'用户评分\' in term_base: try: base = parse_html.xpath(\'//div[@class="celeInfo-right clearfix"]\') info_list = [] if base is not None: base = base[0] film_name = base.xpath(\'./div[1]/h3/text()\') if len(film_name) != 0: film_name = film_name[0].strip() else: film_name = \'无此信息\' film_type = base.xpath(\'./div[1]/ul/li[1]/text()\') if len(film_type) != 0: film_type = film_type[0].strip() else: film_type = \'无此信息\' film_country_lis = base.xpath(\'./div[1]/ul/li[2]/text()\') if len(film_country_lis) != 0: film_country = film_country_lis[0].strip().split(\'/\')[0].strip() film_length = film_country_lis[0].strip().split(\'/\')[1].strip()[:-2] else: film_country = \'无此信息\' film_length = \'无此信息\' lis = base.xpath(\'./div[1]/ul/li[3]/text()\') if len(lis) != 0: lis = lis[0].strip() film_release_time = re.findall(r\'[0-9|\-| |:]+\', lis)[0] film_release_place = re.findall(r\'[^0-9|\-| |:]+\', lis)[0] else: film_release_time = \'无此信息\' film_release_place = \'无此信息\' film_score = base.xpath(\'./div[3]/div[1]/div/span/span/text()\') if len(film_score) != 0: film_score = film_score[0].strip() else: film_score = \'无此信息\' film_coment_counts = base.xpath(\'./div[3]/div[1]/div/div/span/span/text()\') if len(film_coment_counts) != 0: film_coment_counts = film_coment_counts[0].strip() if \'万\' in film_coment_counts: film_coment_counts = film_coment_counts[:-1] else: film_coment_counts = str(float(film_coment_counts) / 10000) else: film_coment_counts = \'无此信息\' film_money = base.xpath(\'./div[3]/div[2]/div/span[1]/text()\') film_money_str = base.xpath(\'./div[3]/div[2]/div/span[2]/text()\') if len(film_money) != 0 and len(film_money_str) != 0: film_money = film_money[0].strip() if \'暂无\' not in film_money_str[0].strip() and \'万美元\' in film_money_str[0].strip(): film_money = str((float(film_money) * 6.88) / 10000) elif \'暂无\' not in film_money_str[0].strip() and \'万\' in film_money_str[0].strip(): film_money = str(float(film_money) / 10000) else: film_money = \'无此信息\' info_list = [ film_name, film_type, film_country, film_length, film_release_time, film_release_place, film_score, film_coment_counts, film_money] film_editor = parse_html.xpath(\'//div[@class="celebrity-container"]/div[1]/ul/li[1]/div/a/text()\') if film_editor != 0: film_editor = film_editor[0].strip() else: film_editor = \'无此信息\' info_list.append(film_editor) film_stars_list = parse_html.xpath(\'//div[@class="celebrity-container"]/div[2]/ul/li/div/a/text()\') if len(film_stars_list) != 0: for star in film_stars_list[:3]: info_list.append(star.strip()) if len(info_list) < 13: n = 13 - len(info_list) for i in range(0, n): info_list.append(\'无此信息\') else: info_list.append(\'无此信息\') info_list.append(\'无此信息\') info_list.append(\'无此信息\') self.info_list.append(info_list) except Exception as e: print(e) self.j += 1 print(self.j) print(url) with open(\'./坏IP.csv\', \'a\', encoding=\'utf-8\', newline=\'\') as f: f.write(url + \'\n\') def save_data(self): with open(\'./maoyan.csv\', \'a\', encoding=\'utf-8\', newline=\'\') as f: writer = csv.writer(f) writer.writerows(self.info_list) def main(self): if os.path.exists(\'./maoyan.csv\'): os.remove(\'./maoyan.csv\') with open(\'./maoyan.csv\', \'a\', encoding=\'utf-8\', newline=\'\') as f: writer = csv.writer(f) writer.writerow( [\'电影名称\', \'电影类型\', \'电影制片国家\', \'电影时长/分钟\', \'上映时间\', \'上映地点\', \'评分\', \'评论/万条\', \'票房/亿元\', \'导演\', \'主演1\', \'主演2\', \'主演3\']) self.get_url() t_list = [] # 创建线程 for i in range(0, 10): t = Thread(target=self.get_page) t_list.append(t) t.start() for j in t_list: j.join() self.save_data() if __name__ == \'__main__\': start = time.time() spider = MaoyanSpider() spider.main() end = time.time() print(\'共用时{:.2f}秒\'.format(end - start))
自定义字体转换程序如下：
import requests import os from fontTools.ttLib import TTFont from fake_useragent import UserAgent import random class FontTransform(object): def __init__(self): self.headers = { "Referer": "https://maoyan.com/films?showType=3" } # 因猫眼有动态自定义字体，且其字形坐标不变，因此可以先下载一个字体文件作为模板，对比前后坐标，找出真值 self.base_font = TTFont(\'./fonts/base.woff\') self.font_dict = {\'uniED99\': \'4\', \'uniE417\': \'1\', \'uniF623\': \'3\', \'uniEE70\': \'5\', \'uniF7E6\': \'2\', \'uniF7E7\': \'0\', \'uniEFA3\': \'7\', \'uniF77D\': \'6\', \'uniEEDF\': \'9\', \'uniF44E\': \'8\'} def get_html(self, url): self.headers[\'User-Agent\'] = UserAgent().random response = requests.get(url, headers=self.headers) return response.content def get_font(self, font_file): # 列出已下载文件 file_list = os.listdir(\'./fonts\') # 判断是否在已下载文件列表中 if font_file not in file_list: url = \'http://vfile.meituan.net/colorstone/\' + font_file new_file = self.get_html(url) with open(\'./fonts/\' + font_file, \'wb\') as f: f.write(new_file) font = TTFont(\'./fonts/\' + font_file) # font.saveXML(\'./fonts/\' + font_file[:-5] + \'.xml\') return font # 对比坐标找出关系，并对数据做转换 def modify_data(self, font, data): gly_list = font.getGlyphOrder() gly_list = gly_list[2:] for name in gly_list: uni = font[\'glyf\'][name] for k, v in self.font_dict.items(): # 判断与模板的坐标是否相同，相同则替换数据 if uni == self.base_font[\'glyf\'][k]: gly = name.replace(\'uni\', \'&#x\').lower() + \';\' if gly in data: data = data.replace(gly, v) # 返回替换后的字符串 return data # if __name__ == \'__main__\': # test = FontTransform() # test.modify_data()
具体程序和爬取结果见文件