Scrapy如何复制内容

Question

我试图在 Tripadvisor 中抓取多家酒店的评论，并且能够抓取包含 150 次观察的数据，其中包含来自 30 家酒店的 150 条评论数据。 p>

但是，当我尝试添加 hotel_name 的新列并执行爬网时，酒店名称不会再次出现，并且观察次数减少到酒店数量为 30。如何使酒店名称重复到每个评论行的？

这是我正在使用的代码：

    import scrapy
from..items import ReviewItem
import re


class TripAdvisorReview(scrapy.Spider):
    name = "tripadvisor"
    start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]

    def parse(self, response):
        for href in response.css("div.listing_title a::attr('href')"):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_hotel)

    def parse_hotel(self, response):
        for info in response.css('div.page'):
            items = ReviewItem()
            hotel_names = info.css('._1mTlpMC3::text').extract()
            hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
            reviewer_names = info.css('._1r_My98y::text').extract()
            reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
            reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
            reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
            review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
            review_dates = [review_date.strip() for review_date in review_dates]
            review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
            review_stars = [review_star.strip() for review_star in review_stars]
            review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
            review_texts = [review_text.strip() for review_text in review_texts]
            #helpful_vote = info.css('._3kbymg8R::text').extract()
            result = zip(hotel_names, reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)

        for hotel_name, reviewer_name, review_date, review_text, review_star, reviewer_contribution in result:
            items['hotel_name'] = hotel_name
            items['reviewer_name'] = reviewer_name
            items['reviewer_contribution'] = reviewer_contribution
            items['review_date'] = review_date
            items['review_star'] = review_star
            items['review_text'] = review_text
            #items['helpful_vote'] = helpful_vote
            yield items

Answer 1

您的问题是 hotel_names 只有一个值，但其他元素有五个值 - 检查：

        print('hotel_names:', len(hotel_names))
        print('reviewer_names:', len(reviewer_names))
        print('review_dates:', len(review_dates))
        print('review_stars:', len(review_stars))
        print('review_texts:', len(review_texts))
        print('reviewer_contributions:', len(reviewer_contributions))

但是zip() 使用最短列表的长度来创建项目 - 所以它只创建一个项目。

您应该使用zip() 而不使用hotel_names，然后在每个项目中添加hotel_names[0]。

# without `hotel_names`
all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)

hotel_name = hotel_names[0]  # <-- manually get first hotel

# without `hotel_name`
for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
    #items = ReviewItem()
    items = dict()
    items['hotel_name'] = hotel_name  # <-- manually add first hotel
    items['reviewer_name'] = reviewer_name
    items['reviewer_contribution'] = reviewer_contribution
    items['review_date'] = review_date
    items['review_star'] = review_star
    items['review_text'] = review_text
    #items['helpful_vote'] = helpful_vote
    yield items

---

顺便说一句：还有其他问题 - review_texts 通常有超过 5 个项目（即 11 个项目），这意味着您使用错误的方法获取此文本。

当我检查 CSV 时，我看到它以 ... 为单独的评论。你必须改变它。

---

最少的工作代码。

您可以将所有代码放在一个文件中并在不创建项目的情况下运行它 - python script.py。这样每个人都可以测试它。

import scrapy
#from ..items import ReviewItem

class TripAdvisorReview(scrapy.Spider):
    
    name = "tripadvisor"
    start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]

    def parse(self, response):
        for href in response.css("div.listing_title a::attr('href')"):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_hotel)

    def parse_hotel(self, response):
        for info in response.css('div.page'):
            hotel_names = info.css('._1mTlpMC3::text').extract()
            hotel_names = [hotel_name.strip() for hotel_name in hotel_names]
            
            reviewer_names = info.css('._1r_My98y::text').extract()
            reviewer_names = [reviewer_name.strip() for reviewer_name in reviewer_names]
            reviewer_contributions = info.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract()
            reviewer_contributions = [reviewer_contribution.strip() for reviewer_contribution in reviewer_contributions]
            review_dates = info.xpath('//div[@class = "_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract()
            review_dates = [review_date.strip() for review_date in review_dates]
            review_stars = info.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract()
            review_stars = [review_star.strip() for review_star in review_stars]
            review_texts = info.css('#component_15 .cPQsENeY').css('::text').extract()
            review_texts = [review_text.strip() for review_text in review_texts]
            #helpful_vote = info.css('._3kbymg8R::text').extract()

            print('hotel_names:', len(hotel_names))
            print('reviewer_names:', len(reviewer_names))
            print('review_dates:', len(review_dates))
            print('review_stars:', len(review_stars))
            print('review_texts:', len(review_texts))
            print('reviewer_contributions:', len(reviewer_contributions))
            print('----')
            
            # without `hotel_names`
            all_reviews = zip(reviewer_names, review_dates, review_texts, review_stars, reviewer_contributions)

            hotel_name = hotel_names[0]  # <-- manually get first hotel

            # without `hotel_name`
            for reviewer_name, review_date, review_text, review_star, reviewer_contribution in all_reviews:
                #items = ReviewItem()
                items = dict()
                items['hotel_name'] = hotel_name  # <-- manually add first hotel
                items['reviewer_name'] = reviewer_name
                items['reviewer_contribution'] = reviewer_contribution
                items['review_date'] = review_date
                items['review_star'] = review_star
                items['review_text'] = review_text
                #items['helpful_vote'] = helpful_vote
                yield items
            
# --- run without project and save in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',
    # save in file CSV, JSON or XML
    'FEED_FORMAT': 'csv',     # csv, json, xml
    'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()

---

编辑：

我的版本没有zip()

首先我搜索所有reviews，然后我使用for-loop 分别处理每条评论。这样我可以控制text 并跳过... - 只是我在review 中只得到第一个text。

import scrapy
#from ..items import ReviewItem

class TripAdvisorReview(scrapy.Spider):
    
    name = "tripadvisor"
    start_urls = ["https://www.tripadvisor.co.uk/Hotels-g186217-England-Hotels.html"]

    def parse(self, response):
        for href in response.css("div.listing_title a::attr('href')"):
            url = response.urljoin(href.extract())
            yield scrapy.Request(url, callback=self.parse_hotel)

    def parse_hotel(self, response):
        hotel_name = response.css('#HEADING::text').extract_first().strip()
        print('hotel_name:', hotel_name)
        
        for review in response.xpath('.//div[@data-test-target="HR_CC_CARD"]'):
            name = review.css('._1r_My98y::text').extract_first().strip()
            contribution = review.css('._3fPsSAYi:nth-child(1) ._1fk70GUn , ._1TuWwpYf+ ._3fPsSAYi ._1fk70GUn').css('::text').extract_first().strip()
            date = review.xpath('.//div[@class="_2fxQ4TOx"]/span[contains(text(),"wrote a review")]/text()').extract_first().strip().replace('wrote a review ', '')
            stars = review.css('div.nf9vGX55 .ui_bubble_rating').xpath("@class").extract_first().strip().replace('ui_bubble_rating bubble_', '')
            text = review.xpath('.//div[@class="cPQsENeY"]//span/text()').extract_first().strip()

            #items = ReviewItem()
            items = dict()
            items['hotel_name'] = hotel_name  # <-- manually add first hotel
            items['reviewer_name'] = name
            items['reviewer_contribution'] = contribution
            items['review_date'] = date
            items['review_star'] = stars
            items['review_text'] = text

            yield items
            
# --- run without project and save in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',
    # save in file CSV, JSON or XML
    'FEED_FORMAT': 'csv',     # csv, json, xml
    'FEED_URI': 'output.csv', #
})
c.crawl(TripAdvisorReview)
c.start()