从嵌套解析方法中抓取重复的 JSON 输出

Question

我正在尝试从 Walmart.ca/grocery 商店抓取数据，以获得商品类中指定的以下输出模型。为此，我实现了三个嵌套的解析方法来检索：

1. 每页的产品列表（如果存在下一个按钮）
2. 可从 HTML 正文获得的相关信息（无法通过此直接关联 JSON 文件）
3. 处理库存可用性的 JSON 请求

我已设法检索页面本身上可用的基本信息，但是，请求 JSON 文件以获取库存可用性有点困难，因为在解析方法的最后一级重复输出。

能否请您查看我的代码行并为此提供一些解决方案？

import scrapy
import json
import re
from pprint import pprint
from product_scraping.items import Product


class CaWalmartBot(scrapy.Spider):
    name = 'walmart'
    allowed_domains = ['walmart.ca']
    start_urls = [
        'https://www.walmart.ca/en/grocery/fruits-vegetables/fruits/N-3852'
    ]
    stores = {'3106': ['43.656422', '-79.435567'], '3124': ['48.412997', '-89.239717']}

    def parse(self, response):
        for url in response.css('.product-link::attr(href)').getall():
            yield response.follow(url, callback=self.parse_first_level)
        next_page = response.css('#loadmore::attr(href)').get()
        if next_page is not None:
            yield response.follow(next_page, callback=self.parse)

    def parse_first_level(self, response):
        # """To Scrape: category, package"""
        data_1 = json.loads(response.xpath("/html/body/script[1]/text()").get()[27::][:-1])
        # """To Scrape: price, description, sku, image_url, store, name, brand"""
        data_2 = json.loads(response.css('.evlleax2 > script:nth-child(1)::text').get())
        url_link = response.xpath('/html/head/link[40]/@href').get()

        sku = data_2['sku']
        upc = data_1['entities']['skus'][sku]['upc']
        # url_json = 'https://www.walmart.ca/api/product-page/find-in-store?' \
        #            'latitude={}&longitude={}&lang=en&upc=' + upc[0]
        url_json = 'https://www.walmart.ca/api/product-page/find-in-store?' \
                   'latitude=48.412997&longitude=-89.239717&lang=en&upc=' + upc[0]

        args = {'data_1': data_1, 'data_2': data_2, 'url': url_link}

        yield scrapy.http.Request(url_json, callback=self.parse_second_level,
                              meta=args, dont_filter=True)

    def parse_second_level(self, response):
        item = Product()

        data_json = json.loads(response.body_as_unicode())
        pprint(data_json)
        data_1 = response.meta.get('data_1')
        data_2 = response.meta.get('data_2')
        url = response.meta.get('url')

        sku = data_2['sku']
        product_type = data_1['entities']['skus'][sku]['facets'][0]['value']
        category_1 = data_1['entities']['skus'][sku]['categories'][0]['hierarchy'][0]['displayName']['en']
        category_2 = data_1['entities']['skus'][sku]['categories'][0]['hierarchy'][1]['displayName']['en']
        category_3 = data_1['entities']['skus'][sku]['categories'][0]['hierarchy'][2]['displayName']['en']
        category = ' | '.join([category_3, category_2, category_1, product_type])
        upc = data_1['entities']['skus'][sku]['upc']
        package = data_1['entities']['skus'][sku]['description']
        description = data_2['description']
        name = data_2['name']
        brand = data_2['brand']['name']
        image_url = data_2['image']

        item['barcodes'] = upc
        item['store'] = re.findall(r'\.[\w]+\.', url)[0].replace('.', '').capitalize()
        item['category'] = category
        item['package'] = package
        item['url'] = url
        item['brand'] = brand
        item['image_url'] = image_url
        item['description'] = description.replace('<br>', '')
        item['sku'] = sku
        item['name'] = name

        branch = data_json['info'][0]['id']
        stock = data_json['info'][0]['availableToSellQty']
        price = data_json['info'][0]['sellPrice']

        item['branch'] = branch
        item['stock'] = stock
        item['price'] = price

        return item

项目类别是：

import scrapy


class Product(scrapy.Item):
    store = scrapy.Field()
    barcodes = scrapy.Field()
    sku = scrapy.Field()
    brand = scrapy.Field()
    name = scrapy.Field()
    description = scrapy.Field()
    package = scrapy.Field()
    image_url = scrapy.Field()
    product = scrapy.Field()
    category = scrapy.Field()
    url = scrapy.Field()
    branch = scrapy.Field()
    stock = scrapy.Field()
    price = scrapy.Field()

    pass

对于我抓取的每件商品，库存、价格和分支的输出都是相同的，对此有什么建议吗？ 谢谢！

Answer 1

在您的parse_first_level 方法中，当您通过回调parse_second_level 产生请求时，您指定了dont_filter=True。这会禁用 Scrapys 内置的重复过滤器。如果你这样做 yield scrapy.http.Request(url_json, callback=self.parse_second_level, meta=args) 你不应该得到重复。