引入
- 相关的门户网站在进行登录的时候，如果用户连续登录的次数超过3次或者5次的时候，就会在登录页中动态生成验证码。通过验证码达到分流和反爬的效果。
目录：
- 使用超级鹰打码平台识别验证码
知识点回顾
- session的创建方式
- session的作用
- proxies参数的作用
- 高匿，透明代理的区别
超级鹰打码平台处理验证码的实现流程：
- 1.对携带验证码的页面数据进行抓取
- 2.可以将页面数据中验证码进行解析，验证码图片下载到本地
- 3.可以将验证码图片提交给三方平台进行识别，返回验证码图片上的数据值
- 超级鹰打码平台：
- 1.在官网中进行注册（普通用户和开发者用户）
- 2.登录普通用户（查看余额），退出；
登录开发者用户：
- 1.实例代码的下载（开发文档-》各语言SDK例子下载-》Python实例下载http://www.chaojiying.com/api-14.html）import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode(\'utf8\') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { \'user\': self.username, \'pass2\': self.password, \'softid\': self.soft_id, } self.headers = { \'Connection\': \'Keep-Alive\', \'User-Agent\': \'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)\', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { \'codetype\': codetype, } params.update(self.base_params) files = {\'userfile\': (\'ccc.jpg\', im)} r = requests.post(\'http://upload.chaojiying.net/Upload/Processing.php\', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { \'id\': im_id, } params.update(self.base_params) r = requests.post(\'http://upload.chaojiying.net/Upload/ReportError.php\', data=params, headers=self.headers) return r.json() chaojiying = Chaojiying_Client(\'超级鹰用户名\', \'超级鹰用户名的密码\', \'911742\') #用户中心>>软件ID 生成一个替换 911742 im = open(r\'D:\others\下载.jpg\', \'rb\').read() #本地图片文件路径 来替换 a.jpg 有时WIN系统须要// code = chaojiying.PostPic(im, 9004)[\'pic_str\'].replace(\'|\', \',\') #9004 验证码类型 官方网站>>价格体系 print(\'code:\',code)
- 2.创建一个软件：软件ID http://www.chaojiying.com/apiuser/mysoft/-》添加新的软件
-3.使用示例代码中的源码文件中的代码进行修改，让其识别验证码图片中的数据值代码展示：
import requests from hashlib import md5 #调用了打码平台的相关的接口对指定的验证码图片进行识别，返回图片上的数据值 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode(\'utf8\') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { \'user\': self.username, \'pass2\': self.password, \'softid\': self.soft_id, } self.headers = { \'Connection\': \'Keep-Alive\', \'User-Agent\': \'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)\', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { \'codetype\': codetype, } params.update(self.base_params) files = {\'userfile\': (\'ccc.jpg\', im)} r = requests.post(\'http://upload.chaojiying.net/Upload/Processing.php\', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { \'id\': im_id, } params.update(self.base_params) r = requests.post(\'http://upload.chaojiying.net/Upload/ReportError.php\', data=params, headers=self.headers) return r.json() import requests from lxml import etree import json import time import re #1.对携带验证码的页面数据进行抓取 url = \'https://www.douban.com/accounts/login?source=movie\' headers = { \'User-Agent\': \'Mozilla/5.0 (Linux; Android 6.0; Nexus 5 Build/MRA58N) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Mobile Safari/537.36\' } page_text = requests.get(url=url,headers=headers).text #2.可以将页面数据中验证码进行解析，验证码图片下载到本地 tree = etree.HTML(page_text) codeImg_url = tree.xpath(\'//*[@id="captcha_image"]/@src\')[0] #获取了验证码图片对应的二进制数据值 code_img = requests.get(url=codeImg_url,headers=headers).content #获取capture_id \'<img id="captcha_image" src="https://www.douban.com/misc/captcha?id=AdC4WXGyiRuVJrP9q15mqIrt:en&size=s" alt="captcha" class="captcha_image">\' c_id = re.findall(\'<img id="captcha_image".*?id=(.*?)&.*?>\',page_text,re.S)[0] with open(\'./code.png\',\'wb\') as fp: fp.write(code_img) #获得了验证码图片上面的数据值 chaojiying = Chaojiying_Client(\'超级鹰用户名\', \'超级鹰用户名的密码\', \'911742\') #用户中心>>软件ID 生成一个替换 911742 im = open(\'code.png\', \'rb\').read() #本地图片文件路径 来替换 code.png 有时WIN系统须要// codeText = chaojiying.PostPic(im, 9101)[\'pic_str\'] #9101 验证码类型 官方网站>>价格体系 #进行登录操作 post = \'https://accounts.douban.com/login\' data = { "source": "movie", "redir": "https://movie.douban.com/", "form_email": "15027900535", "form_password": "bobo@15027900535", "captcha-solution":codeText, "captcha-id":c_id, "login": "登录", } print(c_id) login_text = requests.post(url=post,data=data,headers=headers).text with open(\'./login.html\',\'w\',encoding=\'utf-8\') as fp: fp.write(login_text)
模拟登录：爬取一些当前用户的相关数据信息
人人网的模拟登录（包含验证码处理及cookie）
import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode(\'utf8\') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { \'user\': self.username, \'pass2\': self.password, \'softid\': self.soft_id, } self.headers = { \'Connection\': \'Keep-Alive\', \'User-Agent\': \'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)\', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { \'codetype\': codetype, } params.update(self.base_params) files = {\'userfile\': (\'ccc.jpg\', im)} r = requests.post(\'http://upload.chaojiying.net/Upload/Processing.php\', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { \'id\': im_id, } params.update(self.base_params) r = requests.post(\'http://upload.chaojiying.net/Upload/ReportError.php\', data=params, headers=self.headers) return r.json() #人人网的模拟登录 import requests import urllib from lxml import etree #获取session对象 session = requests.Session() #将验证码图片进行下载 headers = { \'User-Agent\':\'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36\' } url = \'http://www.renren.com/\' page_text = requests.get(url=url,headers=headers).text tree = etree.HTML(page_text) code_img_url_list = tree.xpath(\'//*[@id="verifyPic_login"]/@src\') if len(code_img_url_list) == 0: data = { "email":"www.zhangbowudi@qq.com", "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1", "rkey":"9e75e8dc3457b14c55a74627fa64fb43", "f":"http%3A%2F%2Fwww.renren.com%2F289676607", } else: #识别验证码图片中的数据值 chaojiying = Chaojiying_Client(\'超级鹰用户名\', \'超级鹰用户名的密码\', \'911742\') #用户中心>>软件ID 生成一个替换 911742 im = open(\'code.jpg\', \'rb\').read() #本地图片文件路径 来替换 code.jpg 有时WIN系统须要// code_data = chaojiying.PostPic(im, 1902)[\'pic_str\'] #1902 验证码类型 官方网站>>价格体系 data = { "email":"www.zhangbowudi@qq.com", "icode":code_data, "origURL":"http://www.renren.com/home", "domain":"renren.com", "key_id":"1", "captcha_type":"web_login", "password":"4f0350f09aeffeef86307747218b214b0960bdf35e30811c0d611fe39db96ec1", "rkey":"9e75e8dc3457b14c55a74627fa64fb43", "f":"http%3A%2F%2Fwww.renren.com%2F289676607", } code_img_url = tree.xpath(\'//*[@id="verifyPic_login"]/@src\')[0] img_content = session.get(url=code_img_url,headers=headers).content with open(\'code.jpg\', \'wb\') as fp: fp.write(img_content) #模拟登录 login_url = \'http://www.renren.com/ajaxLogin/login?1=1&uniqueTimestamp=201914927558\' #该次请求产生的cookie会被自动存储到session对象中 session.post(url=login_url,data=data,headers=headers) url = \'http://www.renren.com/289676607/profile\' main_text = session.get(url=url,headers=headers).text with open(\'renren.html\',\'w\',encoding=\'utf-8\') as fp: fp.write(main_text)
import requests from hashlib import md5 class Chaojiying_Client(object): def __init__(self, username, password, soft_id): self.username = username password = password.encode(\'utf8\') self.password = md5(password).hexdigest() self.soft_id = soft_id self.base_params = { \'user\': self.username, \'pass2\': self.password, \'softid\': self.soft_id, } self.headers = { \'Connection\': \'Keep-Alive\', \'User-Agent\': \'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0)\', } def PostPic(self, im, codetype): """ im: 图片字节 codetype: 题目类型 参考 http://www.chaojiying.com/price.html """ params = { \'codetype\': codetype, } params.update(self.base_params) files = {\'userfile\': (\'ccc.jpg\', im)} r = requests.post(\'http://upload.chaojiying.net/Upload/Processing.php\', data=params, files=files, headers=self.headers) return r.json() def ReportError(self, im_id): """ im_id:报错题目的图片ID """ params = { \'id\': im_id, } params.update(self.base_params) r = requests.post(\'http://upload.chaojiying.net/Upload/ReportError.php\', data=params, headers=self.headers) return r.json() #模拟登录古诗文网 s = requests.Session() headers = { \'User-Agent\':\'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/72.0.3626.119 Safari/537.36\' } login_url = \'https://so.gushiwen.org/user/login.aspx?from=http://so.gushiwen.org/user/collect.aspx\' page_text = requests.get(url=login_url,headers=headers).text tree = etree.HTML(page_text) img_src = \'https://so.gushiwen.org\'+tree.xpath(\'//*[@id="imgCode"]/@src\')[0] img_data = s.get(url=img_src,headers=headers).content with open(\'./img.jpg\',\'wb\') as fp: fp.write(img_data) chaojiying = Chaojiying_Client(\'超级鹰用户名\', \'超级鹰用户名的密码\', \'911742\') #用户中心>>软件ID 生成一个替换 911742 im = open(\'img.jpg\', \'rb\').read() #本地图片文件路径 来替换 a.jpg 有时WIN系统须要// img_text = chaojiying.PostPic(im, 1902)[\'pic_str\'] #1902 验证码类型 官方网站>>价格体系 #模拟登录 url = \'https://so.gushiwen.org/user/login.aspx?from=http%3a%2f%2fso.gushiwen.org%2fuser%2fcollect.aspx\' data = { "__VIEWSTATE":"9AsGvh3Je/0pfxId7DYRUi258ayuEG4rrQ1Z3abBgLoDSOeAUatOZOrAIxudqiOauXpR9Zq+dmKJ28+AGjXYHaCZJTTtGgrEemBWI1ed7oS7kpB7Rm/4yma/+9Q=", "__VIEWSTATEGENERATOR":"C93BE1AE", "from":"http://so.gushiwen.org/user/collect.aspx", "email":"www.zhangbowudi@qq.com", "pwd":"bobo328410948", "code":img_text, "denglu":"登录", } page_text = s.post(url=url,headers=headers,data=data).text with open(\'./gushiwen.html\',\'w\',encoding=\'utf-8\') as fp: fp.write(page_text)