建立爬虫代理IP池

单线程构建爬虫代理IP池

#!/usr/bin/python3.5
# -*- coding:utf-8 -*-

import time
import tempfile
from lxml import etree
from urllib import request


user_agent = \'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0\'


def get_content(url):                   # 获取网页内容
    global user_agent
    headers = {\'User-Agent\': user_agent}
    req = request.Request(url=url, headers=headers)
    res = request.urlopen(req)
    return res.read().decode(\'utf-8\')

def get_info(tmp,content):              # 提取网页信息 / ip 端口
    ip_list = etree.HTML(content).xpath(\'//table[contains(@id,"ip_list")]/tr/td[2]/text()\')
    port_list = etree.HTML(content).xpath(\'//table[contains(@id,"ip_list")]/tr/td[3]/text()\')
    for i in range(0,len(ip_list)):
        out = u""
        out += u"" + ip_list[i]
        out += u":" + port_list[i]
        tmp.write((out + u"\n").encode(\'utf-8\'))          # 所有ip和端口号写入data文件

def verify_ip(ip,port,test_url):        # 验证 ip+port 有效性
    global user_agent
    headers = {\'User-Agent\': user_agent,\'Host\': \'www.12306.cn\',\'Referer\': \'http://www.12306.cn/\'}
    proxy = {\'http\':\'http://%s:%s\'%(ip,port)}
    print(proxy)

    proxy_handler = request.ProxyHandler(proxy)
    opener = request.build_opener(proxy_handler)
    request.install_opener(opener)

    req = request.Request(url=test_url,headers=headers)
    time.sleep(1)
    try:
        res = request.urlopen(req)
        time.sleep(2)
        content = res.read()
        if content:
            print(\'{0}:{1} is ok\'.format(ip,port))
            with open("proxy_info.txt", "a") as fd:       # 可用ip+port保存到proxy_info.txt文件中
                fd.write(ip + u":" + port + "\n")
        else:
            print(\'{0}:{1} is unavailable\'.format(ip,port))
    except request.URLError as e:
        print(e.reason)


def verify_ip2(ip,port,test_url):
    import requests
    try:
        response = requests.get(test_url,proxies={\'http\':\'http://{0}:{1}\'.format(ip,port)},timeout=2)
        # print(response.status_code)
    except Exception as e:
        print("{0}:{1} failed".format(ip,port),e)
    else:
        print("{0}:{1} is ok".format(ip,port))
        with open("proxy_info.txt", "a") as fd:  # 可用ip+port保存到proxy_info.txt文件中
            fd.write(ip + u":" + port + "\n")


if __name__ == \'__main__\':
    url = \'http://www.xicidaili.com/nn/\'
    test_url = "http://httpbin.org/"
    url_list = [ url + str(i) for i in range(1,2) ]
    tmp = tempfile.TemporaryFile()
    for url in url_list:
        content = get_content(url)
        time.sleep(2)
        get_info(tmp,content)

    tmp.seek(0)
    for item in tmp.readlines():
        item = item.decode(\'utf-8\')
        # verify_ip(item.split(u":")[0],item.split(u":")[1].strip(),test_url)
        verify_ip2(item.split(u":")[0],item.split(u":")[1].strip(),test_url)
    tmp.close()

使用线程池加快验证代理的速度

concurrent.futures.ThreadPoolExecutor

#!/usr/bin/python3.5
# -*- coding:utf-8 -*-

import time
import tempfile
from lxml import etree
from urllib import request
from concurrent.futures import ThreadPoolExecutor

user_agent = \'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.22 Safari/537.36 SE 2.X MetaSr 1.0\'
ip2port = []

def get_content(url):  # 获取网页内容
    global user_agent
    headers = {\'User-Agent\': user_agent}
    req = request.Request(url=url, headers=headers)
    res = request.urlopen(req)
    return res.read().decode(\'utf-8\')


def get_info(tmp, content):  # 提取网页信息 / ip 端口
    ip_list = etree.HTML(content).xpath(\'//table[contains(@id,"ip_list")]/tr/td[2]/text()\')
    port_list = etree.HTML(content).xpath(\'//table[contains(@id,"ip_list")]/tr/td[3]/text()\')
    for i in range(0, len(ip_list)):
        out = u""
        out += u"" + ip_list[i]
        out += u":" + port_list[i]
        tmp.write((out + u"\n").encode(\'utf-8\'))  # 所有ip和端口号写入data文件


def verify_ip(ip, port, url):
    ret = { \'code\':-1,\'ipport\':None }
    import requests
    try:
        response = requests.get(url, proxies={\'http\': \'http://{0}:{1}\'.format(ip, port)}, timeout=3)
        print(\'{}:{} --> {}\'.format(ip,port,response.status_code))
    except Exception as e:
        # print("{0}:{1} failed".format(ip, port), e)
        pass
    else:
        # print("{0}:{1} is ok".format(ip, port))
        if 200 == response.status_code:
            ret[\'code\'] = 0
            ret[\'ipport\'] = \'{0}:{1}\'.format(ip,port)
    finally:
        return ret

def callback(future):
    global ip2port
    ret = future.result()
    if 0 == ret[\'code\']:
        ip2port.append(ret[\'ipport\'])

if __name__ == \'__main__\':
    url = \'http://www.xicidaili.com/nn/\'
    verify_url = "http://httpbin.org/"
    url_list = [url + str(i) for i in range(1, 2)]
    tmp = tempfile.TemporaryFile()
    for url in url_list:
        content = get_content(url)
        time.sleep(2)
        get_info(tmp, content)

    print(\'原始数据下载完毕，开始构建代理池...\')

    tmp.seek(0)
    ipports = [ item.decode(\'utf-8\').strip().split(\':\') for item in tmp.readlines() ]
    tmp.close()

    pool = ThreadPoolExecutor(20)
    for ipport in ipports:
        ip,port = ipport
        v = pool.submit(verify_ip, ip, port, verify_url)
        v.add_done_callback(callback)
    pool.shutdown(wait=True)

    print(\'代理池构建完毕，共获得可用代理 {} 个\'.format(len(ip2port)))
    print(ip2port)

multiprocessing.dummy.Pool

import time
import requests
from lxml import etree
from requests.exceptions import RequestException
from multiprocessing.dummy import Pool as ThreadPool


available_proxies = []


def get_one_page(url):
    try:
        headers = {
            \'User-Agent\': \'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36\'
        }
        reponse = requests.get(url, headers=headers)
        if reponse.status_code == 200:
            return reponse.text
        return None
    except RequestException:
        return None

def get_one_parse(url):
    print(\'url: {}\'.format(url))
    ipports = []
    html = get_one_page(url)
    if html:
        html = etree.HTML(html)
        ips = html.xpath(\'.//*[@id="list"]/table/tbody//td[1]/text()\')
        ports = html.xpath(\'.//*[@id="list"]/table/tbody//td[2]/text()\')
        for (ip, port) in zip(ips, ports):
            ipports.append(\'{}:{}\'.format(ip, port))
        ipports = list(set(ipports))
        print(\'res: {}\'.format(ipports))
        return ipports
    return None

def fetch(all_proxies):
    url = \'https://www.kuaidaili.com/free/intr/{}/\'
    for i in range(1, 61):
        ret = get_one_parse(url.format(i))
        if ret:
            all_proxies.extend(ret)
        time.sleep(1)
    all_proxies = list(set(all_proxies))
    print(\'爬取了前60页，去重后共获得{}个代理\'.format(len(all_proxies)))

def save():
    with open(\'ip2port.txt\', \'a+\') as wf:
        for item in available_proxies:
            wf.write(item + \'\n\')
    print(\'{}个可用代理保存完毕\'.format(len(available_proxies)))

def sub_verify(item):
    proxy = {\'http\': \'http://{0}\'.format(item)}
    try:
        response = requests.get("http://httpbin.org/", proxies=proxy, timeout=3)
        if response.status_code == 200:
            print("{} is ok".format(item))
            available_proxies.append(item)
    except Exception as e:
        print("{} failed".format(item))

def verify(ipports):
    print(\'开始验证可用代理...\')
    pool = ThreadPool(20)
    pool.map(sub_verify, ipports)
    print(\'验证完毕，共获取可用代理 {} 个\'.format(len(available_proxies)))
    save()


if __name__ == "__main__":
    all_proxies = []
    fetch(all_proxies)
    print(all_proxies,len(all_proxies))
    ipports = list(map(lambda x: x.strip(), all_proxies))
    verify(ipports)