Scrapy for eCommerce with Selenium Infinite Scroll，帮助返回值

Question

我对编程比较陌生。我自己完成了一些小项目，并开始使用 Scrapy 制作网络爬虫。我正在尝试为 Home Depot 制作刮板，但遇到了问题。这试图解决的问题是 Home Depot 网页有 javascript，只有当你向下滚动页面时才会加载，所以我添加了一些我发现的代码，它可以向下滚动页面以显示所有产品，以便它可以获取标题，评论计数和每个产品图块的价格。在添加此代码之前，它确实正确地抓取了产品信息；添加它之后，我最初遇到的问题是代码只能抓取结果的最后一页，所以我移动了一些东西。我认为作为新手，我只是不了解 Scrapy 中的对象以及如何传递信息，尤其是我试图让它返回 parse_product 中的值的 HTML。到目前为止，这确实打开了页面并转到下一页，但它不再抓取任何产品。我哪里错了？我已经为此苦苦挣扎了好几个小时，我正在上一门网络抓取课程，虽然我取得了一些成功，但如果我必须做一些稍微偏离课程的事情，这似乎是一场巨大的斗争。

import scrapy
import logging
from scrapy.utils.markup import remove_tags
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
from scrapy.selector import Selector
from time import sleep
from datetime import datetime

class HdSpider(scrapy.Spider):
    name = 'hd'
    allowed_domains = ['www.homedepot.com']
    start_urls = ['https://www.homedepot.com/b/Home-Decor-Artificial-Greenery-Artificial-Flowers/N-5yc1vZcf9y?Nao='] #Add %%Nao= to end of URL you got from search or category


    def parse(self, response):
        options = Options()
        chrome_path = which("chromedriver")
        driver = webdriver.Chrome(executable_path=chrome_path)#, chrome_options=options)
        p = 0 # The home depot URLs end in =24, =48 etc basically products are grouped 24 on a page so this is my way of getting the next page
        start_url = 'https://www.homedepot.com/b/Home-Decor-Artificial-Greenery-Artificial-Flowers/N-5yc1vZcf9y?Nao='

        while p < 25:
            driver.get(start_url + str(p))
            driver.set_window_size(1920, 1080)
            #sleep(2)
            scroll_pause_time = 1 
            screen_height = driver.execute_script("return window.screen.height;")   # get the screen height of the web
            i = 1

            while True: #this is the infinite scoll thing which reveals all javascript generated product tiles
                driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))  
                i += 1
                sleep(scroll_pause_time)
                scroll_height = driver.execute_script("return document.body.scrollHeight;")  
                if (screen_height) * i > scroll_height:
                    break
                self.html = driver.page_source    
            p = p + 24            

    def parse_product(self, response):
        resp = Selector(text=self.html)
        for products in resp.xpath("//div[@class='product-pod--padding']"):
            date = datetime.now().strftime("%m-%d-%y")
            brand = products.xpath("normalize-space(.//span[@class='product-pod__title__brand--bold']/text())").get() 
            title = products.xpath("normalize-space(.//span[@class='product-pod__title__product']/text())").get() 
            link = products.xpath(".//div//a//@href").get() 
            model = products.xpath("normalize-space(.//div[@class='product-pod__model'][2]/text())").get()
            review_count = products.xpath("normalize-space(.//span[@class='product-pod__ratings-count']/text())").get()
            price = products.xpath("normalize-space(.//div[@class='price-format__main-price']//span[2]/text())").get()
            yield {
               'Date scraped' : date,
               'Brand' : brand,
               'Title' : title,
               'Product Link' : "https://www.homedepot.com" + remove_tags(link),
               'Price' : "$" + price,
               'Model #' : model,
               'Review Count' : review_count
            }

Answer 1

我看不到你在哪里运行parse_product。它不会自动为您执行。除了像parse_product 和response 这样的功能之外，还不如在某些yield Requests(supage_url, parse_product) 中使用它来解析来自子页面的数据，而不是来自你在parse 中获得的页面。您应该将代码从parse_product 移动到parse，如下所示：

def parse(self, response):
    options = Options()
    chrome_path = which("chromedriver")
    driver = webdriver.Chrome(executable_path=chrome_path)#, chrome_options=options)
    driver.set_window_size(1920, 1080)

    p = 0 # The home depot URLs end in =24, =48 etc basically products are grouped 24 on a page so this is my way of getting the next page

    start_url = 'https://www.homedepot.com/b/Home-Decor-Artificial-Greenery-Artificial-Flowers/N-5yc1vZcf9y?Nao='

    scroll_pause_time = 1 
    screen_height = driver.execute_script("return window.screen.height;")   # get the screen height of the web

    while p < 25:
        driver.get(start_url + str(p))

        #sleep(2)
        i = 1

        # scrolling
        while True: #this is the infinite scoll thing which reveals all javascript generated product tiles
            driver.execute_script("window.scrollTo(0, {screen_height}*{i});".format(screen_height=screen_height, i=i))  
            i += 1
            sleep(scroll_pause_time)
            scroll_height = driver.execute_script("return document.body.scrollHeight;")  
            if (screen_height) * i > scroll_height:
                break

        # after scrolling
        self.html = driver.page_source    
        p = p + 24            
        resp = Selector(text=self.html)
        for products in resp.xpath("//div[@class='product-pod--padding']"):
            date = datetime.now().strftime("%m-%d-%y")
            brand = products.xpath("normalize-space(.//span[@class='product-pod__title__brand--bold']/text())").get() 
            title = products.xpath("normalize-space(.//span[@class='product-pod__title__product']/text())").get() 
            link = products.xpath(".//div//a//@href").get() 
            model = products.xpath("normalize-space(.//div[@class='product-pod__model'][2]/text())").get()
            review_count = products.xpath("normalize-space(.//span[@class='product-pod__ratings-count']/text())").get()
            price = products.xpath("normalize-space(.//div[@class='price-format__main-price']//span[2]/text())").get()
            yield {
               'Date scraped' : date,
               'Brand' : brand,
               'Title' : title,
               'Product Link' : "https://www.homedepot.com" + remove_tags(link),
               'Price' : "$" + price,
               'Model #' : model,
               'Review Count' : review_count
            }

但我会做其他更改 - 你使用 p = p + 24 但是当我在浏览器中检查页面时，我发现我需要 p = p + 48 来获取所有产品。而不是p = p + ...，我宁愿使用Selenium点击按钮>来获取下一页。

---

编辑：

我的版本有其他变化。

每个人都可以在不创建项目的情况下运行它。

#!/usr/bin/env python3

import scrapy
from scrapy.utils.markup import remove_tags
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from shutil import which
from scrapy.selector import Selector
from time import sleep
from datetime import datetime

class HdSpider(scrapy.Spider):

    name = 'hd'

    allowed_domains = ['www.homedepot.com']
    start_urls = ['https://www.homedepot.com/b/Home-Decor-Artificial-Greenery-Artificial-Flowers/N-5yc1vZcf9y?Nao='] #Add %%Nao= to end of URL you got from search or category

    def parse(self, response):
    
        options = Options()
        
        chrome_path = which("chromedriver")
        driver = webdriver.Chrome(executable_path=chrome_path) #, chrome_options=options)
        #driver.set_window_size(1920, 1080)
        print(dir(driver))
        driver.maximize_window()
        
        scroll_pause_time = 1 

        # loading first page
        start_url = 'https://www.homedepot.com/b/Home-Decor-Artificial-Greenery-Artificial-Flowers/N-5yc1vZcf9y?Nao=0'
        driver.get(start_url)

        screen_height = driver.execute_script("return window.screen.height;")   # get the screen height of the web

        #while True:        # all pages
        for _ in range(5):  # only 5 pages
        
            #sleep(scroll_pause_time)
            
            # scrolling page
            i = 1
            while True: #this is the infinite scoll thing which reveals all javascript generated product tiles
                driver.execute_script(f"window.scrollBy(0, {screen_height});")  
                sleep(scroll_pause_time)
                
                i += 1

                scroll_height = driver.execute_script("return document.body.scrollHeight;")  
                if screen_height * i > scroll_height:
                    break
        
            # after scrolling    
            resp = Selector(text=driver.page_source)
            
            for products in resp.xpath("//div[@class='product-pod--padding']"):
                date = datetime.now().strftime("%m-%d-%y")
                brand = products.xpath("normalize-space(.//span[@class='product-pod__title__brand--bold']/text())").get() 
                title = products.xpath("normalize-space(.//span[@class='product-pod__title__product']/text())").get() 
                link = products.xpath(".//div//a//@href").get() 
                model = products.xpath("normalize-space(.//div[@class='product-pod__model'][2]/text())").get()
                review_count = products.xpath("normalize-space(.//span[@class='product-pod__ratings-count']/text())").get()
                price = products.xpath("normalize-space(.//div[@class='price-format__main-price']//span[2]/text())").get()
                yield {
                   'Date scraped' : date,
                   'Brand' : brand,
                   'Title' : title,
                   'Product Link' : "https://www.homedepot.com" + remove_tags(link),
                   'Price' : "$" + price,
                   'Model #' : model,
                   'Review Count' : review_count
                }
            
            # click button `>` to load next page
            try:
                driver.find_element_by_xpath('//a[@aria-label="Next"]').click()
            except:
                break
                    
            
# --- run without project and save in `output.csv` ---

from scrapy.crawler import CrawlerProcess

c = CrawlerProcess({
    'USER_AGENT': 'Mozilla/5.0',
    # save in file CSV, JSON or XML
    'FEEDS': {'output.csv': {'format': 'csv'}},  # new in 2.1
})

c.crawl(HdSpider)
c.start()