【python】前程无忧51job岗位招聘信息爬虫程序，自动翻页，进阶版

数据挖掘的学习过程中一定绕不开的一个阶段性实践项目——前程无忧51job岗位招聘信息爬虫程序！

搞定这个之后可以尝试带有一定反爬机制的爬虫实践，比如需要登陆服务器才能进一步响应的网站，比如隔一段时间就弹出验证码的网站，有兴趣的可以去尝试一下哈！

# -*- coding:utf-8 -*-
import urllib
import re, codecs
import time, random
import requests
from lxml import html
from urllib import parse
 
key = \'python\'
key = parse.quote(parse.quote(key))
headers = {\'Host\': \'search.51job.com\',
           \'Upgrade-Insecure-Requests\': \'1\',
           \'User-Agent\': \'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36\'}
 
 
def get_links(page):
    url = \'http://search.51job.com/list/000000,000000,0000,00,9,99,\' + key + \',2,\' + str(page) + \'.html\'
    r = requests.get(url, headers, timeout=10)
    s = requests.session()
    s.keep_alive = False
    r.encoding = \'gbk\'
    reg = re.compile(r\'class="t1 ">.*? <a target="_blank" title=".*?" href="(.*?)".*? <span class="t2">.*?<span class="t4">(.*?)</span>\', re.S)
    links = re.findall(reg, r.text)
    return links
 
 
# 多页处理，下载到文件
def get_content(link,salary):
    r1 = requests.get(link, headers, timeout=10)
    s = requests.session()
    s.keep_alive = False
    r1.encoding = \'gbk\'
    t1 = html.fromstring(r1.text)
    l=[]
    try:
        job = t1.xpath(\'//div[@class="tHeader tHjob"]//h1/text()\')[0]
        company = t1.xpath(\'//p[@class="cname"]/a/text()\')[0]
        label = t1.xpath(\'//div[@class="t1"]/span/text()\')
        education = t1.xpath(\'//div[@class="cn"]/p[2]/text()\')[2]
        salary = salary
        area = t1.xpath(\'//div[@class="cn"]/p[2]/text()\')[0]
        companytype = \'Null\'
        workyear = t1.xpath(\'//div[@class="cn"]/p[2]/text()\')[1]
        describe = re.findall(re.compile(r\'<div class="bmsg job_msg inbox">(.*?)任职要求\', re.S), r1.text)
        require = re.findall(re.compile(r\'<div class="bmsg job_msg inbox">.*?任职要求(.*?)<div class="mt10">\', re.S),
                             r1.text)
        try:
            file = codecs.open(\'51job4.xls\', \'a+\',\'utf-8\')
            item = str(company) + \'\t\' + str(job) + \'\t\' + str(education) + \'\t\' + str(label) + \'\t\' + str(
                salary) + \'\t\' + str(companytype) + \'\t\' + str(workyear) + \'\t\' + str(area) + \'\t\' + str(
                workyear) + str(describe) + \'\t\' + str(require) + \'\n\'
            file.write(item)
            file.close()
            return True
        except Exception as e:
            print(e)
            return None
        # output=\'{},{},{},{},{},{},{},{}\n\'.format(company,job,education,label,salary,area,describe,require)
        # with open(\'51job.csv\', \'a+\', encoding=\'utf-8\') as f:
        # f.write(output)
    except:
        print(\'None\')
    print(l)
    return l
 
 
for i in range(1, 750):
    print(\'正在爬取第{}页信息\'.format(i))
    try:
        time.sleep(random.random()+random.randint(1,5))
        links = get_links(i)
        for link in links:
            url=link[0]
            salary=link[1]
            get_content(url,salary)
            time.sleep(random.random()+random.randint(0,1))
    except:
        continue
        print(\'有点问题\')