python_day06(ip代理池)

from urllib.request import Request, ProxyHandler
from urllib.request import build_opener
from bs4 import BeautifulSoup
import MySQLdb;
import redis
from urllib.request import urlopen
from lxml import etree
from lxml import etree
import re;
urlfront = "http://www.xicidaili.com"
url = "http://www.xicidaili.com/nn/1"
result = redis.Redis(host=\'127.0.0.1\', port=6379,db=0)

# def spider_IP(url):
# 获取整个页面
def get_allcode(url):
    # 设置代理IP
    proxy = {\'https\': \'110.73.0.45:8123\'}
    proxy_support = ProxyHandler(proxy);
    opener = build_opener(proxy_support)
    # 设置访问http协议头,模拟浏览器
    opener.addheaders = [
        (\'User-agent\', \'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6\')]
    r = opener.open(url)
    html = r.read().decode("UTF-8");
    # print(html)
    return str(html)
# lxml 方式 获取Ip
def find_ip(s):
    # s = get_allcode(url);
    selector = etree.HTML(s);
    links = selector.xpath(\'//tr[@class="odd"]/td/text()|//tr[@class=""]/td/text()\');
    ip=[]
    port=[]
    for link in links:
        # print(link)
        if \'-\' in link:
            # print()
            pass
        elif link.isdigit():
            port.append(link)
            # f.write(link + \'\n\');
        elif \'.\' in link:
            ip.append(link)
            # f.write(link + \':\');
    #  用redis 的 llist存 ip
    for i in range(len(ip)):
        # print(ip[i]+":"+port[i])
        ips=ip[i] + ":" + port[i]
        result.lpush(\'mylist\',ips)
def get_next_page(s):
    selecter = etree.HTML(s);
    link = selecter.xpath(\'//div[@class="pagination"]/a[@class="next_page"]/@href\');
    for i in link:
        if i == None:
            return None;
        return urlfront + i
def get_allcode_ip(url,ip):
    # 设置代理IP
    try:
        ip=str(ip, encoding="utf-8")# bytes与str相互转换
        timeout=5
        proxy = {\'http\':ip}
        proxy_support = ProxyHandler(proxy);
        opener = build_opener(proxy_support)
        # 设置访问http协议头,模拟浏览器
        opener.addheaders = [
            (\'User-agent\', \'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6\')]
        # 加运行超时
        r = opener.open(url,None,timeout)
        html = r.read().decode("UTF-8");
        print(\'+++++++++++++++\')
        # 将可用Ip放到redis的useable_ip中
        result.lpush(\'usable_ip\',ip)
        print(ip)
        print(\'+++++++++++++++\')
    except Exception as err:
        print(err)
while 1:
    print(url)
    s=get_allcode(url);
    url=get_next_page(s)
    print(url)
    if url==None:
        break
    find_ip(s)
    while 1:
        ip = result.lpop(\'mylist\')
        print(ip)
        if ip == None:
            break
        get_allcode_ip(url, ip)