爬取链家北京市二手房的链家编号

1、首先感谢肖祥博主的案例：https://www.cnblogs.com/xiao987334176/p/13739249.html

2、环境安装

（1）安装Anaconda:

　　下载地址：https://www.anaconda.com/products/individual

（2）安装：scrapy

（3）安装：Pycharm

（4）安装：Xpath helper

　　教程参考：

　　　　获取插件：https://blog.csdn.net/weixin_41010318/article/details/86472643

　　　　配置过程：https://www.cnblogs.com/pfeiliu/p/13483562.html

3、爬取过程

（1）网页分析：

　　网址：

　　　　第一页：https://bj.lianjia.com/ershoufang/

　　　　第二页：https://bj.lianjia.com/ershoufang/pg1/

　　　　末尾页：https://bj.lianjia.com/ershoufang/pg100/

　　利用xpath helper 分析出房源的链家编号：

（2）详细代码：

创建项目：

# 打开Pycharm，并打开Terminal，执行以下命令

scrapy startproject fang
cd fang
scrapy genspider lianjia bj.lianjia.com

在scrapy.cfg同级目录，创建bi.py，用于启动Scrapy项目，内容如下：

# ！/usr/bin/python3
# -*- coding: utf-8 -*-
# 在项目根目录下新建：bi.py
from scrapy.cmdline import execute
# 第三个参数是：爬虫程序名
# --nolog什么意思？
execute([\'scrapy\', \'crawl\', \'lianjia\', \'--nolog\'])

View Code

lianjia.py

# -*- coding: utf-8 -*-
import scrapy
from scrapy import Request  # 导入模块
import math
import re
from fang.items import FangItem
from selenium.webdriver import ChromeOptions
from selenium.webdriver import Chrome


class LianjiaSpider(scrapy.Spider):
    name = \'lianjia\'
    allowed_domains = [\'changde.lianjia.com\']
    base_url = "https://changde.lianjia.com/ershoufang/pg"

    # 实例化一个浏览器对象
    def __init__(self):
        # 防止网站识别Selenium代码
        # selenium启动配置参数接收是ChromeOptions类，创建方式如下：
        options = ChromeOptions()
        # 添加启动参数
        # 浏览器不提供可视化页面. linux下如果系统不支持可视化不加这条会启动失败
        options.add_argument("--headless")  # => 为Chrome配置无头模式
        # 添加实验性质的设置参数
        # 设置开发者模式启动，该模式下webdriver属性为正常值
        options.add_experimental_option(\'excludeSwitches\', [\'enable-automation\'])
        options.add_experimental_option(\'useAutomationExtension\', False)
        # 禁用浏览器弹窗
        self.browser = Chrome(options=options)
        # 最新解决navigator.webdriver=true的方法
        self.browser.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            "source": """
                               Object.defineProperty(navigator, \'webdriver\', {
                                 get: () => undefined
                               })
                             """
        })

        super().__init__()

    def start_requests(self):
        print("开始爬虫")
        self.base_url = "https://changde.lianjia.com/ershoufang/pg"
        url = self.base_url + str(2)
        print("url", url)
        # url = "https://news.163.com/"
        # 一个Request对象表示一个HTTP请求，它通常是在爬虫生成，并由下载执行，从而生成Response
        # url（string） - 此请求的网址
        # callback（callable） - 将使用此请求的响应（一旦下载）作为其第一个参数调用的函数。
        # 如果请求没有指定回调，parse()将使用spider的 方法。请注意，如果在处理期间引发异常，则会调用errback。
        # 仍有许多参数：method=\'GET\', headers, body, cookies, meta, encoding=\'utf-8\', priority=0, dont_filter=False, errback
        response = scrapy.Request(url=url, callback=self.parse_index)
        yield response

        # 整个爬虫结束后关闭浏览器
        def close(self, spider):
            print("关闭爬虫")
            self.browser.quit()

        # 访问主页的url, 拿到对应板块的response

    def parse_index(self, response):
        print("访问主页")
        # 获取分页
        # 查询条数
        ret_num = 3000
        # print("ret_num", ret_num, type(ret_num))
        # 计算分页，每一页30条
        jsfy = int(ret_num) / 30
        # 向上取整
        page_num = math.ceil(jsfy)
        # print("page_num",page_num)

        for n in range(2, page_num + 1):
            n += 1
            # 下一页url
            url = self.base_url + str(n)
            print("url", url)
            # 访问下一页，有返回时，调用self.parse_details方法
            yield scrapy.Request(url=url, callback=self.parse_details)

    def parse_details(self, response):
        # 获取页面中要抓取的信息在网页中位置节点
        node_list = response.xpath(\'//ul[@class="sellListContent"]/li/div[1]\')

        count = 0
        # 遍历节点，进入详情页，获取其他信息
        for node in node_list:
            count += 1
            try:
                # # 名称
                BeiJingFangYuan_Id = node.xpath(\'./div[@class="title"]/a/@href\').extract()

                if BeiJingFangYuan_Id:
                    BeiJingFangYuan_Id = BeiJingFangYuan_Id[0].strip()

                print("BeiJingFangYuan_Id", BeiJingFangYuan_Id)



                # item
                item = FangItem()
                item[\'ChangDeFangYuan_Id\'] = BeiJingFangYuan_Id
                yield item
            except Exception as e:
                print(e)

        print("本次爬取数据: %s条" % count)


    \'\'\'def parse(self, response):
        pass
    \'\'\'

items.py

# Define here the models for your scraped items
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/items.html

import scrapy

class FangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    BeiJingFangYuan_Id = scrapy.Field()
    pass

View Code

pipelines.py

# Define your item pipelines here
#
# Don\'t forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html


# useful for handling different item types with a single interface
from itemadapter import ItemAdapter
import json

class FangPipeline(object):
    def __init__(self):
        # python3保存文件 必须需要\'wb\' 保存为json格式
        # 为了方便，保存在一个json文件中。当然，也可以设置保存到数据库中。
        self.f = open("fang_pipline.json", \'wb\')

    # 在pipeline中写json文件
    def process_item(self, item, spider):
        # 读取item中的数据 并换行处理
        # json.dumps 序列化时对中文默认使用的ascii编码.想输出真正的中文需要指定ensure_ascii=False
        content = json.dumps(dict(item), ensure_ascii=False) + \',\n\'
        # f.write写入中文出错解决方法
        self.f.write(content.encode(\'utf=8\'))
        return item

    def close_spider(self, spider):
        # 关闭文件
        self.f.close()

View Code

修改settings.py，应用pipelines

# Scrapy settings for fang project
#
# For simplicity, this file contains only settings considered important or
# commonly used. You can find more settings consulting the documentation:
#
#     https://docs.scrapy.org/en/latest/topics/settings.html
#     https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#     https://docs.scrapy.org/en/latest/topics/spider-middleware.html

BOT_NAME = \'fang\'

SPIDER_MODULES = [\'fang.spiders\']
NEWSPIDER_MODULE = \'fang.spiders\'


# Crawl responsibly by identifying yourself (and your website) on the user-agent
#USER_AGENT = \'fang (+http://www.yourdomain.com)\'

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Configure maximum concurrent requests performed by Scrapy (default: 16)
#CONCURRENT_REQUESTS = 32

# Configure a delay for requests for the same website (default: 0)
# See https://docs.scrapy.org/en/latest/topics/settings.html#download-delay
# See also autothrottle settings and docs
#DOWNLOAD_DELAY = 3
# The download delay setting will honor only one of:
#CONCURRENT_REQUESTS_PER_DOMAIN = 16
#CONCURRENT_REQUESTS_PER_IP = 16

# Disable cookies (enabled by default)
#COOKIES_ENABLED = False

# Disable Telnet Console (enabled by default)
#TELNETCONSOLE_ENABLED = False

# Override the default request headers:
#DEFAULT_REQUEST_HEADERS = {
#   \'Accept\': \'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8\',
#   \'Accept-Language\': \'en\',
#}

# Enable or disable spider middlewares
# See https://docs.scrapy.org/en/latest/topics/spider-middleware.html
#SPIDER_MIDDLEWARES = {
#    \'fang.middlewares.FangSpiderMiddleware\': 543,
#}

# Enable or disable downloader middlewares
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html
#DOWNLOADER_MIDDLEWARES = {
#    \'fang.middlewares.FangDownloaderMiddleware\': 543,
#}

# Enable or disable extensions
# See https://docs.scrapy.org/en/latest/topics/extensions.html
#EXTENSIONS = {
#    \'scrapy.extensions.telnet.TelnetConsole\': None,
#}

# Configure item pipelines
# See https://docs.scrapy.org/en/latest/topics/item-pipeline.html
ITEM_PIPELINES = {
    \'fang.pipelines.FangPipeline\': 300,
}

# Enable and configure the AutoThrottle extension (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/autothrottle.html
#AUTOTHROTTLE_ENABLED = True
# The initial download delay
#AUTOTHROTTLE_START_DELAY = 5
# The maximum download delay to be set in case of high latencies
#AUTOTHROTTLE_MAX_DELAY = 60
# The average number of requests Scrapy should be sending in parallel to
# each remote server
#AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
# Enable showing throttling stats for every response received:
#AUTOTHROTTLE_DEBUG = False

# Enable and configure HTTP caching (disabled by default)
# See https://docs.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
#HTTPCACHE_ENABLED = True
#HTTPCACHE_EXPIRATION_SECS = 0
#HTTPCACHE_DIR = \'httpcache\'
#HTTPCACHE_IGNORE_HTTP_CODES = []
#HTTPCACHE_STORAGE = \'scrapy.extensions.httpcache.FilesystemCacheStorage\'

View Code

执行bi.py，启动爬虫项目，效果如下：