python获取代理IP - 啊峰哥

python获取代理IP

利用requests库获取代理,用Beautiful库解析网页筛选ip

# -*- coding: utf-8 -*-
import requests
from bs4 import BeautifulSoup
from threading import Thread

headers = {\'user-agent\': \'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0\'}


#定义获取IP函数
def get_ip():
   #写入txt
    write_ip = open(\'get_ip.txt\', \'w\')
    for page in range(1, 10):
        url = \'http://www.xicidaili.com/nn/%s\' % page
        r = requests.get(url, headers=headers,timeout=5)

        # 用beautifulsoup库解析网页
        soup = BeautifulSoup(r.content, \'lxml\')
        trs = soup.find(\'table\', id=\'ip_list\').find_all(\'tr\')

        for tr in trs[1:]:
            tds = tr.find_all(\'td\')
            ip = tds[1].text.strip()
            port = tds[2].text.strip()

            write_ip.write(\'%s\n\'%(ip+\':\'+port))
    write_ip.close()
    print(\'done\')

get_ip()