python爬虫爬取链家二手房信息

#coding=utf-8  
import requests
from fake_useragent import UserAgent
from bs4 import BeautifulSoup
import json
import csv
import time


# 构建请求头
userAgent = UserAgent()
headers = {
    \'user-agent\': userAgent .Chrome
}

# 声明一个列表存储字典
data_list = []


def start_spider(page):
    #设置重连次数
    requests.adapters.DEFAULT_RETRIES = 15
    s = requests.session()
    #设置连接活跃状态为False
    s.keep_alive = False
    #爬取的url,默认爬取的南京的链家房产信息
    url = \'https://nj.lianjia.com/ershoufang/pg{}/\'.format(page)
    # 请求url
    resp = requests.get(url, headers=headers,timeout=10)
    # 讲返回体转换成Beautiful
    soup = BeautifulSoup(resp.content, \'lxml\')
    # 筛选全部的li标签
    sellListContent = soup.select(\'.sellListContent li.LOGCLICKDATA\')
    # 循环遍历
    for sell in sellListContent:
        try:
            # 标题
            title = sell.select(\'div.title a\')[0].string
            # 先抓取全部的div信息，再针对每一条进行提取
            houseInfo = list(sell.select(\'div.houseInfo\')[0].stripped_strings)
            # 楼盘名字
            loupan = houseInfo[0]
            #对楼盘的信息进行分割
            info = houseInfo[0].split(\'|\')
            # 房子类型
            house_type = info[1].strip()
            # 面积大小
            area = info[2].strip()
            # 房间朝向
            toward = info[3].strip()
            # 装修类型
            renovation = info[4].strip()
            # 房屋地址
            positionInfo = \'\'.join(list(sell.select(\'div.positionInfo\')[0].stripped_strings))
            # 房屋总价
            totalPrice = \'\'.join(list(sell.select(\'div.totalPrice\')[0].stripped_strings))
            # 房屋单价
            unitPrice = list(sell.select(\'div.unitPrice\')[0].stripped_strings)[0]

            # 声明一个字典存储数据
            data_dict = {}
            data_dict[\'title\'] = title
            data_dict[\'loupan\'] = loupan
            data_dict[\'house_type\'] = house_type
            data_dict[\'area\'] = area
            data_dict[\'toward\'] = toward
            data_dict[\'renovation\'] = renovation
            data_dict[\'positionInfo\'] = positionInfo
            data_dict[\'totalPrice\'] = totalPrice
            data_dict[\'unitPrice\'] = unitPrice

            data_list.append(data_dict)
        except Exception as e:
            print(e)
            continue


def main():
    # 只爬取10页
    for page in range(1, 10):
        start_spider(page)
        time.sleep(3)

    # 将数据写入json文件
    with open(\'data_json.json\', \'a+\', encoding=\'utf-8\') as f:
        json.dump(data_list, f, ensure_ascii=False, indent=4)
    print(\'json文件写入完成\')

    # 将数据写入csv文件
    with open(\'./data_csv.csv\', \'w\', encoding=\'utf-8\', newline=\'\') as f:
        # 表头
        print(data_list)
        title = data_list[0].keys()
        # 创建writer对象
        writer = csv.DictWriter(f, title)
        # 写入表头
        writer.writeheader()
        # 批量写入数据
        writer.writerows(data_list)
    print(\'csv文件写入完成\')


if __name__ == \'__main__\':
    main()