如何使用 Scrapy 和 Playwright 抓取此页面？

Question

我已经处理了几天，但找不到解决方案。我的输出文件为空。

请帮帮我

settings.py 文件相应于文档

谢谢

import scrapy
from scrapy.utils.response import open_in_browser
from scrapy_playwright.page import PageMethod
from scrapy.selector import Selector

class ShopSpider(scrapy.Spider):
    name = 'shop'
    
    def start_requests(self):
        yield scrapy.Request(
            url='https://www.instacart.com/store/sprouts/collections/beef', 
            callback=self.parse, 
            meta={
                'playwright': True, 
                'playwright_page_methods': [
                    PageMethod('wait_for_selector', 'ul.css-qcn8wk-LockupBLarger > li:nth-child(16)')
                ], 
                'playwright_include_page': True
            }, 
            errback=self.close_page
        )


    async def parse(self, response):
        page = response.meta['playwright_page']
        
        for i in range(32, 67, 32):
            await page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
            # count = 16 * i
            await page.wait_for_selector(f'ul.css-qcn8wk-LockupBLarger > li:nth-child({i}) > div > div > div > a > div > div:nth-child(2) > h2 > span')
        s = scrapy.Selector(text=await page.content())
        await page.close()   
        for q in s.css('ul.css-qcn8wk-LockupBLarger > li > div > div > div > a > div > div:nth-child(2) > h2 > span::text').getall():
            yield {
                'title': q
                }

    async def close_page(self, failure):
        page = failure.request.meta["playwright_page"]
        await page.close()
        

Answer 1

您收到的错误消息是什么？您使用的是什么操作系统？ Playwright 最近因不在 Windows 上运行而臭名昭著。