xpath解析器和猪八戒网站爬取

豆瓣top205数据爬取

"""
1.先研究单页数据如何爬取
2.之后再研究多页数据如何爬取
3.最后可以考虑如何持久化保存
    ps:2 3两个步骤有时候是混合进行的
"""
import requests  # 请求库
from bs4 import BeautifulSoup  # 解析库
from openpyxl import Workbook
import time


wb = Workbook()
wb1 = wb.create_sheet(\'电影排行数据\', 0)
# 先创建表头数据
wb1.append([\'电影名称\', \'电影详情\', \'电影图标\', \'电影信息\', \'电影评分\', \'评价人数\', \'电影座右铭\'])

"""
研究多页规律
    https://movie.douban.com/top250?start=0&filter=
    https://movie.douban.com/top250?start=25&filter=
    https://movie.douban.com/top250?start=50&filter=
    https://movie.douban.com/top250?start=75&filter=
"""


def get_movie(url):
    res = requests.get(url,
                       # 豆瓣需要校验User-Agent
                       headers={
                           "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36"
                       }
                       )
    soup = BeautifulSoup(res.text, \'lxml\')
    # 分析标签规律
    li_list = soup.select("ol.grid_view>li")
    # print(len(li_list))  # 最好验证一下
    # 循环一个个li标签筛选所需内容
    for li in li_list:
        # 详情链接
        a_link = li.find(\'a\').get(\'href\')
        # 图标地址
        img_src = li.find(\'img\').get(\'src\')
        # 电影名称
        movie_title = li.find(\'span\').text
        # 主要信息
        info = li.select(\'div.bd>p\')[0].text
        # 评分
        num = li.find(\'span\', attrs={\'class\': \'rating_num\'}).text
        # 评价人数
        people_num = li.select(\'div.star span\')[-1].text
        # 电影座右铭
        quote_tag = li.select(\'p.quote span.inq\')
        \'\'\'并不是所有的电影都有该标签所以做判断\'\'\'
        # if quote_tag:
        #     quote = quote_tag[0].text
        # else:
        #     quote = \'暂无\'
        # 三元表达式 缩写
        quote = quote_tag[0].text if quote_tag else \'暂无\'
        wb1.append([movie_title, a_link, img_src, info, num, people_num, quote])


for i in range(0, 250, 25):
    base_url = \'https://movie.douban.com/top250?start=%s&filter=\'
    get_movie(base_url % i)
    time.sleep(1)

wb.save(r\'movieInfo.xlsx\')

 

解析库之Xpath解析器

必须掌握

# 效率很高 使用广泛

#1.导入xpath所在模块
from lxml import etree
# 2.将待匹配的 文本传入etree生成的一个对象
html = etree.HTML(doc)

xpath解析器主要功能

1.所有节点

a = html.xpath(\'//*\') 匹配所有标签

2.指定节点(结果是个列表)

a = html.xpath(\'//head\') 匹配所有的head标签

3.子节点,子孙节点

a = html.xpath(\'//div/a\') 匹配div标签内部所有的子a标签

a = html.xpath(\'//bady/a\') 匹配body标签内部所有的子a标签

a = html.xpath(\'//body//a\')

4.父节点

a = html.xpath(\'//body//a[@href = "image1.html"]\') 获取body内部所有href = image1.html的后代a

a = html.xpath(\'//body//a[@href ="image1.html"]/..\') ..表示查找上一级父标签

5.标签

a = html.xpath(\'//body//a[1]\') 从1开始取值

***xpath选择器中中括号内部可以放属性也可以放位置参数 从1开始***

a = html.xpath(\'//body//a[1]/parent::*\')

6. 文本获取

a = html.xpath(\'//body//a[@href="image1.html"]/text()\')

a =html.xpath(\'//body//a/text()\') 获取body内部所有a内部文本（一次性获取不需要循环）

7.属性获取

a = html.xpath(\'//body//a/@href\') 获取body内部所有后代a标签href属性值（一次性获取不需要循环）

a = html.xpath(\'//title/@id\') # 获取title标签id属性值

8.属性多值匹配

a = 标签有多个class类, 直接匹配就不可以了,需要用contains

a = html.xpath(\'//body//a[@class="li"]\')  写等号就表示等于 不是包含

要想用包含的写法

a = html.xpath(\'//body//a[contains(@class,"li")]/text()\') 关键字contains

9.多属性匹配

查找body标签内部所有class含有li或者name=items的a标签

a = html.xpath(\'//body//a[contains(@class,"li")or @name="items"]\')

查找body标签内部所有class含有li并且name=items的a标签的内部文本

a = html.xpath(\'//body//a[contains(@class,"li")and@name="items"]/text()\')

10.按序选择

取最后一个

a = html.xpath(\'a//[last()]/@href\')

位置小于3的

a = html.xpath(\'//a[position()<3]/@href\') position()关键字 用于定位

倒数第三个

a = html.xpath(\'//a[last()-2]/@href\')

了解即可

# 11 节点轴选择
# ancestor：祖先节点
# 使用了* 获取所有祖先节点
a = html.xpath(\'//a/ancestor::*\')
# # 获取祖先节点中的div
a = html.xpath(\'//a/ancestor::div\')
# attribute：属性值
a = html.xpath(\'//a[1]/attribute::*\')  # 查找a标签内部所有的属性值
# child：直接子节点
a = html.xpath(\'//a[1]/child::*\')
# descendant：所有子孙节点
a = html.xpath(\'//a[6]/descendant::*\')
# following:当前节点之后所有节点
a = html.xpath(\'//a[1]/following::*\')
a = html.xpath(\'//a[1]/following::*[1]/@href\')
# following-sibling:当前节点之后同级节点
a = html.xpath(\'//a[1]/following-sibling::*\')
a = html.xpath(\'//a[1]/following-sibling::a\')
a = html.xpath(\'//a[1]/following-sibling::*[2]/text()\')
a = html.xpath(\'//a[1]/following-sibling::*[2]/@href\')
print(a)

View Code

 xpath爬取猪八戒数据

import requests
from bs4 import BeautifulSoup
from lxml import etree

res = requests.get(\'https://shanghai.zbj.com/search/f/\',
                   params={\'kw\': \'python\'}
                   )
x_html = etree.HTML(res.text)
# 分析标签特征并书写xpath语法筛选
# 1.先查找所有的外部div标签
div_list = x_html.xpath(\'/html/body/div[6]/div/div/div[2]/div[5]/div[1]/div\')  # 利用浏览器自动生成
# div_list = x_html.xpath(\'//div[@class="new-service-wrap"]/div\')  # 自己写
# 2.循环获取每一个div标签
for div in div_list:
    price = div.xpath(\'.//span[@class="price"]/text()\')
    company_name = div.xpath(\'./div/div/a[1]/div[1]/p/text()\')
    order_num = div.xpath(\'./div/div/a[2]/div[2]/div[1]/span[2]/text()\')
    info = div.xpath(\'./div/div/a[2]/div[2]/div[2]/p/text()\')
    print(info)