使用 Selenium 和 BeautifulSoup 进行 Web 抓取返回空列表

Question

我正在尝试搜索网站以确定某些商品是否有货。我希望它搜索该网站，找出该项目是否“有货”并通知用户找到的信息。我可以让大部分代码运行，但这会不断返回空列表。 （我在 BeautifulSoup4 和 Selenium 中尝试了相同的结果。以下两个版本）。我错过了什么？任何帮助将不胜感激！

#selenium version
import os
import string
import subprocess
import traceback
import time
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.wait import WebDriverWait

def webscan():
    driver = webdriver.Chrome('/usr/lib/python3/dist-packages/selenium/webdriver/chrome/chromedriver')
    #driver.maximize_window()
    driver.get("https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11")

    #wait for the page to load fully

    time.sleep(5)

    status = driver.find_element_by_class('a-size-medium a-color-state')
    ps5_avail = status


    print(ps5_avail)
    if ps5_avail == "Currently unavailable":
        print("Sorry. Still not in stock...")
    elif ps5_avail == "Temporarily Out of Stock":
        print("Sorry. Still not in stock...")
    elif ps5_avail == "In stock soon.":
        print("sorr. Still not in stock...")
    else:
        print("Go NOW!!!!!")


    driver.close()


user_input = (input("Want a PS5?"))

if user_input == 'yes':
    webscan()
    print('scan is done')


####################################################################
#beautifulsoup version

import requests
from bs4 import BeautifulSoup

url = 'https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11'
res = requests.get(url)
html_page = res.content

soup = BeautifulSoup(html_page, 'html.parser')

text = soup.find_all(string='In Stock')

if text == True:
    print("Out of Stock")
else:
    print("Get it!")

Answer 1

您应该改用这个 css 选择器：

代码：

driver.get("https://www.amazon.com/GIGABYTE-GeForce-Windforce-Graphics-GV-N207SWF3OC-8GD/dp/B07WN6RVHH/ref=sr_1_11?crid=OOF3E7T9W54&dchild=1&keywords=graphics+card&qid=1630118746&sprefix=graphi%2Caps%2C190&sr=8-11")

#wait for the page to load fully

time.sleep(5)

lngth = len(driver.find_elements(By.CSS_SELECTOR, "span.a-size-medium.a-color-state"))
print("value of lngth", lngth)

try:
    if len(driver.find_elements(By.CSS_SELECTOR, "span.a-size-medium.a-color-state")) > 0:
        print("Inside if")
        status = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.CSS_SELECTOR, "div#availability span"))).text
        print(status)
    else:
        print("Looks like element was not found.")
except:
    print("Something went wrong")
    pass

进口：

from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC