scrapy上海买房指南

Spider：

# -*- coding: utf-8 -*-
import scrapy
from scrapy_zhaopin.items import ScrapyHouseItem
from scrapy.http import Request


class MySpider(scrapy.Spider):
    name = "spiderhouse"
    allowed_domains = ["sh.lianjia.com"]
    start_urls = ["https://sh.lianjia.com/ershoufang/rs徐泾北城/"]

    def parse(self, response):

        for line in response.xpath(\'//*[contains(@log-mod,"list")]//li[contains(@class,"clear")]\'):
            item = ScrapyHouseItem()

            item[\'title\'] = line.xpath(\'//title/text()\').extract()[0].split("_")[0].replace("二手房房源", "")
            item[\'name\'] = line.xpath(\'.//*[@class="title"]/a/text()\').extract()
            item[\'address\'] = line.xpath(\'.//*[@class="positionInfo"]/a/text()\').extract()
            item[\'house_info\'] = line.xpath(\'.//*[@class="houseInfo"]/text()\').extract()
            item[\'price\'] = line.xpath(\'.//*[@class="totalPrice"]//span/text()\').extract()
            item[\'unit_price\'] = line.xpath(\'.//*[@class="unitPrice"]//span/text()\').extract()[0].replace("单价", "").replace("元/平米", "")
            yield item

        address_list = ["徐盈路", "徐泾镇", "华新镇", "嘉定北", "中山公园", "汇金路", "青浦新城", "爱博家园", "九亭",
                        "佘山", "泗泾", "洞泾", "赵巷"]

        for i in address_list:
            address_url = f\'https://sh.lianjia.com/ershoufang/rs{i}/\'
            yield Request(address_url, callback=self.parse)

        # if self.page < response.xpath(\'(//*[@class="pager-num"]//*[@class="num-iten"])[last()]/text()\'):
        #     self.page += 1
        #     page_url = self.page_url % self.page
        #     yield Request(page_url, callback=self.parse)