selenium 试图抓取网站上的多个页面，但在转到下一页时出错

Question

我对硒很陌生

我当前的代码是这样的

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
        
shoe_links = []
page = 1
path = r"C:\Users\redacted\OneDrive\Desktop\chrome driver\chromedriver.exe"
driver = webdriver.Chrome(path)
driver.get("https://www.myntra.com")
search = driver.find_element_by_class_name("desktop-searchBar")
search.send_keys("jordan")
search.send_keys(Keys.RETURN)
try:
    mountRoot = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "mountRoot"))
    )
    while page != 6:
        item = mountRoot.find_element(By.XPATH, "//*[@id='desktopSearchResults']/div[2]/section/ul")
        liclass = item.find_elements_by_class_name("product-base")
        for i in range(len(liclass)):
            print(i)
            print(liclass[i].text)
            if 'shoe' in liclass[i].find_element_by_tag_name('a').get_attribute("href").lower():
                shoe_links.append(liclass[i].find_element_by_tag_name('a').get_attribute("href"))
            print("\n")
        for i in range(len(shoe_links)):
            print(shoe_links[i])
        next_link = mountRoot.find_element_by_link_text("Next")
        next_link.click()
        page = page + 1

finally:
    driver.quit()

此代码适用于第一页，但一旦进入下一页，它就会给我错误

C:\Users\redacted\AppData\Local\Programs\Python\Python39\lib\site-packages\selenium\webdriver\remote\webelement.py:264: UserWarning: find_element_by_* commands are deprecated. Please use find_element() instead
  warnings.warn("find_element_by_* commands are deprecated. Please use find_element() instead")
0Traceback (most recent call last):
  File "C:/Users/redacted/OneDrive/Desktop/Myntra.py", line 27, in <module>
    print(liclass[i].text)
  File "C:\Users\redacted\AppData\Local\Programs\Python\Python39\Lib\site-packages\selenium\webdriver\remote\webelement.py", line 76, in text
    return self._execute(Command.GET_ELEMENT_TEXT)['value']
  File "C:\Users\redacted\AppData\Local\Programs\Python\Python39\Lib\site-packages\selenium\webdriver\remote\webelement.py", line 693, in _execute
    return self._parent.execute(command, params)
  File "C:\Users\redacted\AppData\Local\Programs\Python\Python39\Lib\site-packages\selenium\webdriver\remote\webdriver.py", line 418, in execute
    self.error_handler.check_response(response)
  File "C:\Users\redacted\AppData\Local\Programs\Python\Python39\Lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 243, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.StaleElementReferenceException: Message: stale element reference: element is not attached to the page document
  (Session info: chrome=95.0.4638.54)
Stacktrace:
Backtrace:
    Ordinal0 [0x0083B9D3+2472403]
    Ordinal0 [0x007D6961+2058593]
    Ordinal0 [0x006E22B8+1057464]
    Ordinal0 [0x006E49E4+1067492]
    Ordinal0 [0x006E48AE+1067182]
    Ordinal0 [0x006E4B10+1067792]
    Ordinal0 [0x00708140+1212736]
    Ordinal0 [0x00725593+1332627]
    Ordinal0 [0x00703236+1192502]
    Ordinal0 [0x0072567A+1332858]
    Ordinal0 [0x0073495F+1395039]
    Ordinal0 [0x0072545B+1332315]
    Ordinal0 [0x00701FB4+1187764]
    Ordinal0 [0x00702E09+1191433]
    GetHandleVerifier [0x009C3396+1551078]
    GetHandleVerifier [0x00A6D53A+2247818]
    GetHandleVerifier [0x008C7FBB+521995]
    GetHandleVerifier [0x008C7079+518089]
    Ordinal0 [0x007DBD4D+2080077]
    Ordinal0 [0x007DFC18+2096152]
    Ordinal0 [0x007DFD52+2096466]
    Ordinal0 [0x007E94B1+2135217]
    BaseThreadInitThunk [0x757DFA29+25]
    RtlGetAppContainerNamedObjectPath [0x775B7A9E+286]
    RtlGetAppContainerNamedObjectPath [0x775B7A6E+238]

我用来指导我的检查元素是：https://i.stack.imgur.com/TOv1F.png

我认为问题来自 for 循环（我也使用了 while 循环并得到了相同的错误）

我的代码有什么问题？任何帮助将不胜感激。

Answer 1

等到加载微调器消失

from selenium import webdriver
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
        
shoe_links = []
page = 1
path = r"C:\Users\redacted\OneDrive\Desktop\chrome driver\chromedriver.exe"
driver = webdriver.Chrome(path)
driver.get("https://www.myntra.com")
search = driver.find_element_by_class_name("desktop-searchBar")
search.send_keys("jordan")
search.send_keys(Keys.RETURN)
try:
    mountRoot = WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.ID, "mountRoot"))
    )
    while page != 6:
        item = mountRoot.find_element(By.XPATH, "//*[@id='desktopSearchResults']/div[2]/section/ul")
        liclass = item.find_elements_by_class_name("product-base")
        for i in range(len(liclass)):
            print(i)
            print(liclass[i].text)
            if 'shoe' in liclass[i].find_element_by_tag_name('a').get_attribute("href").lower():
                shoe_links.append(liclass[i].find_element_by_tag_name('a').get_attribute("href"))
            print("\n")
        for i in range(len(shoe_links)):
            print(shoe_links[i])
        next_link = mountRoot.find_element_by_link_text("Next")
        next_link.click()
        page = page + 1
        WebDriverWait(driver, 10).until(
        EC.visibility_of_element_located((By.CLASS_NAME, "spinner-spinner")))
        WebDriverWait(driver, 10).until(
        EC.invisibility_of_element_located((By.CLASS_NAME, "spinner-spinner")))

finally:
    driver.quit()