selenium模块在爬虫中的应用

1. 相关概念

1. selenium模块

　　是一个基于浏览器自动化的模块

2. 与爬虫之间的关联

　　便捷的捕获到动态加载到的数据（可见即可得）

　　实现模拟登陆

3.环境安装

pip3 install selenium

简单演示

from selenium import webdriver
from time import sleep

# 后面是你的浏览器驱动位置，记得前面加r\'\',\'r\'是防止字符转义的
driver = webdriver.Chrome(r\'chromedriver.exe\')

# 用get打开百度页面
driver.get("http://www.baidu.com")

# 查找页面的“设置”选项，并进行点击
driver.find_elements_by_link_text(\'设置\')[0].click()
sleep(2)

# 打开设置后找到“搜索设置”选项，设置为每页显示50条
driver.find_elements_by_link_text(\'搜索设置\')[0].click()
sleep(2)

# 选中每页显示50条
m = driver.find_element_by_id(\'nr\')
sleep(2)
m.find_element_by_xpath(\'//*[@id="nr"]/option[3]\').click()
m.find_element_by_xpath(\'.//option[3]\').click()
sleep(2)

# 点击保存设置
driver.find_elements_by_class_name("prefpanelgo")[0].click()
sleep(2)

# 处理弹出的警告页面   确定accept() 和 取消dismiss()
driver.switch_to_alert().accept()
sleep(2)

# 找到百度的输入框，并输入 美女
driver.find_element_by_id(\'kw\').send_keys(\'美女\')
sleep(2)

# 点击搜索按钮
driver.find_element_by_id(\'su\').click()
sleep(2)

# 在打开的页面中找到“Selenium - 开源中国社区”，并打开这个页面
driver.find_elements_by_link_text(\'美女_百度图片\')[0].click()
sleep(3)

# 关闭浏览器
driver.quit()

2.基本使用

　　准备好某一款浏览器的驱动程序：http://chromedriver.storage.googleapis.com/index.html

　　版本的映射关系：https://blog.csdn.net/huilan_same/article/details/51896672

1. 访问京东网站,并搜索“苹果”

from time import sleep
from selenium import webdriver

bro = webdriver.Chrome(executable_path="chromedriver.exe")

# 录入路由地址
bro.get("https://www:jd.com/")
sleep(2)

# 进行标签定位
search_input = bro.find_element_by_id("key")

# 向搜索框中录入关键词
search_input.send_keys("苹果")

# 定位搜索按钮
btn = bro.find_element_by_xpath(\'//*[@id="search"]/div/div[2]/button\')

# 点击搜索按钮
btn.click()
sleep(2)

#执行js（滑动滚轮）
bro.execute_script(\'window.scrollTo(0,document.body.scrollHeight)\')
sleep(2)

# 获取页面的源码数据
page_text = bro.page_source
print(page_text)

# 退出访问
bro.quit()

2.selenium爬取动态加载的数据

from time import sleep
from selenium import webdriver
from lxml import etree

bro = webdriver.Chrome(executable_path="chromedriver.exe")

bro.get("http://125.35.6.84:81/xk/")
sleep(2)

page_text = bro.page_source
page_text_list = [page_text]

for i in range(3):
    bro.find_element_by_id("pageIto_next").click()  # 点击下一页
    sleep(2)
    page_text_list.append(bro.page_source)

for page_text in page_text_list:
    tree = etree.HTML(page_text)
    tree_list = tree.xpath(\'//ul[@id="gzlist"]/li\')
    for lis in tree_list:
        title = lis.xpath(\'./dl/@title\')[0]
        num = lis.xpath(\'./ol/@title\')[0]

sleep(2)
bro.quit()

3.动作链

　　一系列连续的动作

　　在实现标签定位时，如果发现定位的标签是存在于iframe标签中的，则在定位时必须执行一个固定的操作:bro.switch_to.frame(\'id\')

from selenium import webdriver
from time import sleep
from selenium.webdriver import ActionChains


bro = webdriver.Chrome(executable_path=\'chromedriver.exe\')

bro.get(\'https://www.runoob.com/try/try.php?filename=jqueryui-api-droppable\')
bro.switch_to.frame(\'iframeResult\')
div_tag = bro.find_element_by_id(\'draggable\')
# 拖动=点击+滑动
action = ActionChains(bro)
action.click_and_hold(div_tag)

for i in range(5):
    #perform让动作链立即执行
    action.move_by_offset(17,5).perform()
    sleep(0.5)
action.release()
sleep(3)
bro.quit()

4.模拟12306登录

超级鹰识别代码

# Cjy.py

import requests
from hashlib import md5

class Chaojiying_Client(object):

    def __init__(self, username, password, soft_id):
        self.username = username
        password =  password.encode(\'utf8\')
        self.password = md5(password).hexdigest()
        self.soft_id = soft_id
        self.base_params = {
            \'user\': self.username,
            \'pass2\': self.password,
            \'softid\': self.soft_id,
        }
        self.headers = {
            \'Connection\': \'Keep-Alive\',
            \'User-Agent\': \'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)\',
        }

    def PostPic(self, im, codetype):
        """
        im: 图片字节
        codetype: 题目类型 参考 http://www.chaojiying.com/price.html
        """
        params = {
            \'codetype\': codetype,
        }
        params.update(self.base_params)
        files = {\'userfile\': (\'ccc.jpg\', im)}
        r = requests.post(\'http://upload.chaojiying.net/Upload/Processing.php\', data=params, files=files, headers=self.headers)
        return r.json()

    def ReportError(self, im_id):
        """
        im_id:报错题目的图片ID
        """
        params = {
            \'id\': im_id,
        }
        params.update(self.base_params)
        r = requests.post(\'http://upload.chaojiying.net/Upload/ReportError.php\', data=params, headers=self.headers)
        return r.json()

模拟登陆

from selenium import webdriver
from time import sleep
from PIL import Image
from selenium.webdriver import ActionChains
from Cjy import Chaojiying_Client
from selenium.webdriver import ActionChains

bro = webdriver.Chrome(executable_path=\'chromedriver.exe\')
bro.get(\'https://kyfw.12306.cn/otn/login/init\')
sleep(5)
bro.save_screenshot(\'main.png\')

code_img_tag = bro.find_element_by_xpath(\'//*[@id="loginForm"]/div/ul[2]/li[4]/div/div/div[3]/img\')
location = code_img_tag.location
size = code_img_tag.size

# 裁剪的区域范围
rangle = (
int(location[\'x\']), int(location[\'y\']), int(location[\'x\'] + size[\'width\']), int(location[\'y\'] + size[\'height\']))

i = Image.open(\'./main.png\')
frame = i.crop(rangle)
frame.save(\'code.png\')


def get_text(imgPath, imgType):
    chaojiying = Chaojiying_Client(\'bobo328410948\', \'bobo328410948\', \'899370\')
    im = open(imgPath, \'rb\').read()
    return chaojiying.PostPic(im, imgType)[\'pic_str\']


# 55,70|267,133 ==[[55,70],[33,66]]
result = get_text(\'./code.png\', 9004)
all_list = []
if \'|\' in result:
    list_1 = result.split(\'|\')
    count_1 = len(list_1)
    for i in range(count_1):
        xy_list = []
        x = int(list_1[i].split(\',\')[0])
        y = int(list_1[i].split(\',\')[1])
        xy_list.append(x)
        xy_list.append(y)
        all_list.append(xy_list)
else:
    x = int(result.split(\',\')[0])
    y = int(result.split(\',\')[1])
    xy_list = []
    xy_list.append(x)
    xy_list.append(y)
    all_list.append(xy_list)
print(all_list)
# action = ActionChains(bro)
for a in all_list:
    x = a[0]
    y = a[1]
    ActionChains(bro).move_to_element_with_offset(code_img_tag, x, y).click().perform()
    sleep(1)

bro.find_element_by_id(\'username\').send_keys(\'123456\')
sleep(1)
bro.find_element_by_id(\'password\').send_keys(\'67890000000\')
sleep(1)
bro.find_element_by_id(\'loginSub\').click()

sleep(5)
bro.quit()

爬取梨视频

import requests
from lxml import etree
import re
headers = {
    \'User-Agent\':\'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36\'
}
url = \'https://www.pearvideo.com/category_1\'
page_text = requests.get(url,headers=headers).text
tree = etree.HTML(page_text)
li_list = tree.xpath(\'//*[@id="listvideoListUl"]/li\')
for li in li_list:
    detail_url = \'https://www.pearvideo.com/\'+li.xpath(\'./div/a/@href\')[0]
    title = li.xpath(\'./div/a/div[2]/text()\')[0]+\'.mp4\'
    detail_page_text = requests.get(detail_url,headers=headers).text
    ex = \'srcUrl="(.*?)",vdoUrl\'
    video_url = re.findall(ex,detail_page_text,re.S)[0]
    video_data = requests.get(video_url,headers=headers).content
    with open(title,\'wb\') as fp:
        fp.write(video_data)

5. 移动端数据的爬取

1. fiddler是一款抓包工具，代理服务器

　　- 青花瓷

　　- miteproxy

　　- 配置：让其可以抓取https协议的请求

　　- tools -> options -> https -> 安装证书

2. http：客户端和服务端进行数据交互的某种形式

　　- https：安全的http协议

　　- https的加密方式采用的是证书密钥加密。

3.步骤

1. 配置下fiddler的端口

2. 将手机和fiddler所在的电脑处在同一个网段下（pc开启wifi，手机连接）

3. 在手机中访问fiddler的ip+port：192.168.14.110:50816,在当前页面中点击对应的连接下载证书

4. 在手机中安装且信任证书

5. 设置手机网络的代理：开启代理==》fiddler对应pc端的ip地址和fiddler自己端口号

待续