第二篇：请求库之requests,selenium

一、介绍

#介绍：使用requests可以模拟浏览器的请求，比起之前用到的urllib，requests模块的api更加便捷（本质就是封装了urllib3）

#注意：requests库发送请求将网页内容下载下来以后，并不会执行js代码，这需要我们自己分析目标站点然后发起新的request请求

#安装：pip3 install requests

#各种请求方式：常用的就是requests.get()和requests.post()
>>> import requests
>>> r = requests.get('https://api.github.com/events')
>>> r = requests.post('http://httpbin.org/post', data = {'key':'value'})
>>> r = requests.put('http://httpbin.org/put', data = {'key':'value'})
>>> r = requests.delete('http://httpbin.org/delete')
>>> r = requests.head('http://httpbin.org/get')
>>> r = requests.options('http://httpbin.org/get')

#建议在正式学习requests前，先熟悉下HTTP协议
http://www.cnblogs.com/linhaifeng/p/6266327.html

二、基于GET请求

1、基本请求

import requests
response=requests.get('http:/www.baidu.com/')
print(response.text)

2、带参数的GET请求->params

 1 #在请求头内将自己伪装成浏览器，否则百度不会正常返回页面内容
 2 import requests
 3 response=requests.get('https://www.baidu.com/s?wd=python&pn=1',
 4                       headers={
 5                         'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
 6                       })
 7 print(response.text)
 8 
 9 
10 #如果查询关键词是中文或者有其他特殊符号，则不得不进行url编码
11 from urllib.parse import urlencode
12 wb = "haiyan海燕"
13 encode_res = urlencode({"k":wb},encoding="utf-8")
14 print(encode_res)  #k=haiyan%E6%B5%B7%E7%87%95
15 keywords = encode_res.split("=")[1]  #haiyan%E6%B5%B7%E7%87%95
16 url = "https://www.baidu.com/s?wd=%s&pn=1"%(keywords)
17 # url = "https://www.baidu.com/s?"+encode_res
18 print(url)
19 # 然后拼接成url
20 response = requests.get(
21     url,
22     headers = {
23         "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.108 Safari/537.36",
24     }
25 )

自己拼接GET参数

 1 #上述操作可以用requests模块的一个params参数搞定，本质还是调用urlencode
 2 from urllib.parse import urlencode
 3 wd='海燕nnn'
 4 pn=1
 5 
 6 response=requests.get('https://www.baidu.com/s',
 7                       params={
 8                           'wd':wd,
 9                           'pn':pn
10                       },
11                       headers={
12                         'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36',
13                       })
14 res2=response.text
15 
16 #验证结果，打开a.html与b.html页面内容一样
17 with open('a.html','w',encoding='utf-8') as f:
18     f.write(res1) 
19 with open('b.html', 'w', encoding='utf-8') as f:
20     f.write(res2)
21 
22 params参数的使用

params参数的使用