import requests
import time
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
# kuaidaili 结果排序 默认 按响应速度(从快到慢)VIP 按最后检测时间(从近到远)VIP
"""
订单号*
查看我的订单 从网页提取
提取数量*
数量该怎么填?
所在地区
多个地区用英文逗号分隔 如何筛选地区?
排除地区
多个地区用英文逗号分隔 查看实时地区分布>>
筛选端口号
多个端口用英文逗号分隔 查看实时端口分布>>
排除端口号
多个端口用英文逗号分隔
筛选IP段
筛选以特定部分开头的IP
排除IP段
排除以特定部分开头的IP
运营商
无所谓 联通/网通 电信 移动 铁通 教育网 阿里云
匿名度
高匿名 匿名 透明 查看实时匿名度分布>>
响应速度
极速(<1秒) 快速(1~3秒) 慢速(>3秒) 查看实时速度分布>>
代理协议
全部 HTTP HTTPS (同时也支持HTTP) 查看实时协议分布>>
POST支持
全部 GET POST (同时也支持GET) 查看GET/POST分布>>
代理稳定性新
不筛选 稳定VIP 非常稳定SVIP 了解代理稳定性
结果排序
默认 按响应速度(从快到慢)VIP 按最后检测时间(从近到远)VIP
结果定制VIP
IP:Port 地区 匿名度 代理协议 响应速度(秒)
历史提取次数专业版把计数清零
提取结果包含的字段(顺序排列, 英文逗号分隔)
浏览器支持
谷歌浏览器(Chrome) IE / 360浏览器 火狐浏览器(Firefox) 其它浏览器 如何勾选?
安卓手机浏览器 iPhone手机浏览器 iPad浏览器 查看浏览器支持分布>>
IP去重
过滤今天提取过的IP
返回格式
文本 jsonVIP xmlVIP
结果分隔符
\r\n分隔 \n分隔 空格分隔 |分隔 自定义
"""
"""
结果排序 按响应速度(从快到慢)
匿名度 高匿名
代理协议 HTTP
#http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=30&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&an_ha=1&sp1=1&quality=1&sort=1&sep=2
"""
proxyip_d = {}
def gen_proxyip_dict_qps(
url_proxyip=\'http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=proxyip_num&b_pcchrome=1&b_pcie=1&b_pcff=1&protocol=1&method=2&an_ha=1&sp1=1&quality=1&sort=1&sep=2\',
proxyip_num=30, peroid_seconds=300):
global proxyip_d
url = url_proxyip.replace(\'proxyip_num\', str(proxyip_num))
# floating point number
# Return the current time in seconds since the Epoch.
current_time = time.time()
if current_time - proxyip_d[\'last_request_timestamp\'] > peroid_seconds:
try:
r = requests.get(url)
proxyip_d[\'last_request_timestamp\'] = int(time.time())
if r.status_code == 200:
# 不校验返回结果，假设合法且可用
l = r.text.split(\'\n\')
proxyip_d[\'ip_list\'] = [i for i in set(l)]
except Exception as e:
## log
s = \'%s%s\' % (\'\', e)
def fetch_one_proxyip(url_proxyip=\'http://dev.kuaidaili.com/api/getproxy/?orderid=123112322452122&num=proxyip_num\',
proxyip_num=30, peroid_seconds=300):
global proxyip_d
if \'ip_list\' not in proxyip_d:
proxyip_d[\'ip_list\'] = []
if \'last_request_timestamp\' not in proxyip_d:
proxyip_d[\'last_request_timestamp\'] = 0
gen_proxyip_dict_qps(url_proxyip=url_proxyip, proxyip_num=proxyip_num, peroid_seconds=peroid_seconds)
# 根据商家返回结果的排序规则，优先使用前ip
try:
r = proxyip_d[\'ip_list\'][0]
del proxyip_d[\'ip_list\'][0]
except Exception as e:
gen_proxyip_dict_qps(peroid_seconds=0)
r = \'localhost\'
print(e)
print(proxyip_d[\'ip_list\'])
print(r)
# r = \'localhost\'
return r
from selenium import webdriver
from bs4 import BeautifulSoup
import json
import time
from selenium.webdriver.common.proxy import *
f, s_l = \'biz0.txt\', []
with open(f, \'r\', encoding=\'utf-8\') as fo:
for i in fo:
s_l.append(i.replace(\'\n\', \'\'))
#browser = webdriver.Chrome().minimize_window()
# def spider_l_t_w(s_wd=\'长尾词\'):
# #global browser
# # browser = webdriver.Chrome()
#
# myproxyip = fetch_one_proxyip() # IP:PORT or HOST:PORT
#
# chrome_options = webdriver.ChromeOptions()
# chrome_options.add_argument(\'--proxy-server=%s\' % myproxyip)
#
# # chrome = webdriver.Chrome(chrome_options=chrome_options)
# # chrome_options = Options()
# # chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
# browser = webdriver.Chrome(chrome_options=chrome_options)
#
# url_seed = \'https://m.baidu.com/s?word=s_wd\'
# url_seed=\'http://www.zzyjschina.com.cn/shenzhen.html\'
# # url_seed = \'https://www.baidu.com/s?word=s_wd\'
#
# url_seed = url_seed.replace(\'s_wd\', s_wd)
# print(url_seed)
# #browser.set_window_size(30, 30)
#
# browser.get(url_seed)
# # time.sleep(2)
# # browser.minimize_window()
# rd = BeautifulSoup(browser.page_source, \'html.parser\').find_all(\'a\', class_=\'rw-item\')
# res_d_l = [{\'contents\': d.contents, \'href\': d.attrs[\'href\']} for d in rd]
# browser.quit()
# #browser.close()
# return res_d_l
def spider_l_t_w(s_wd=\'长尾词\'):
#global browser
# browser = webdriver.Chrome()
myproxyip = fetch_one_proxyip() # IP:PORT or HOST:PORT
# chrome_options = webdriver.Firefox()
# chrome_options.add_argument(\'--proxy-server=%s\' % myproxyip)
#
# # chrome = webdriver.Chrome(chrome_options=chrome_options)
# # chrome_options = Options()
# # chrome_options.add_experimental_option("mobileEmulation", mobile_emulation)
# browser = webdriver.Chrome(chrome_options=chrome_options)
myProxy =fetch_one_proxyip()
proxy = Proxy({
\'proxyType\': ProxyType.MANUAL,
\'httpProxy\': myProxy,
\'ftpProxy\': myProxy,
\'sslProxy\': myProxy,
\'noProxy\': \'\' # set this value as desired
})
browser = webdriver.Firefox(proxy=proxy)
print(browser)
url_seed = \'https://m.baidu.com/s?word=s_wd\'
# url_seed=\'http://www.zzyjschina.com.cn/shenzhen.html\'
# url_seed = \'https://www.baidu.com/s?word=s_wd\'
url_seed = url_seed.replace(\'s_wd\', s_wd)
print(url_seed)
#browser.set_window_size(30, 30)
browser.get(url_seed)
# time.sleep(2)
# browser.minimize_window()
rd = BeautifulSoup(browser.page_source, \'html.parser\').find_all(\'a\', class_=\'rw-item\')
res_d_l = [{\'contents\': d.contents, \'href\': d.attrs[\'href\']} for d in rd]
browser.quit()
#browser.close()
return res_d_l
save_c, save_limit, fw = 0, 30, open(\'biz0.res.txt\', \'a\', encoding=\'utf-8\')
for i in s_l:
try:
d = spider_l_t_w(i)
s = json.dumps(d, ensure_ascii=False)
except Exception as e:
s = e
s = \'%s\n%s\n\n\' % (i, s)
fw.write(s)
save_c += 1
if save_c == save_limit:
fw.close()
fw = open(\'biz0.res.txt\', \'a\', encoding=\'utf-8\')
d = 9
-- google 不能
-- firefox ok
myProxy = fetch_one_proxyip()
proxy = Proxy({
\'proxyType\': ProxyType.MANUAL,
\'httpProxy\': myProxy,
\'ftpProxy\': myProxy,
\'sslProxy\': myProxy,
\'noProxy\': \'\' # set this value as desired
})
browser = webdriver.Firefox(proxy=proxy)
print(browser)
url_seed = \'https://m.baidu.com/s?word=s_wd\'
url_seed = url_seed.replace(\'s_wd\', \'fddf\')
print(url_seed)
browser.set_window_size(30, 30)
try:
browser.get(url_seed)
except Exception as e:
print(\'---------\', e)