使用 python 进行 Selenium Web 抓取 - 没有名称的任何下拉菜单

Question

我正在尝试弄清楚如何通过所有下拉菜单获取价格和变体名称。我可以通过名称属性来做到这一点，但不同的产品页面有不同的属性，所以这不是最佳做法。我使用 select by element id 完成了此代码，但收到以下错误。我也在试图弄清楚如何为此复制到 csv，但它似乎不起作用。

import csv
browser = webdriver.Chrome(executable_path='C:\Users\userman\PycharmProjects\seleniumTest\drivers\chromedriver.exe')
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
from selenium.webdriver.support.select import Select



from selenium import webdriver
import csv
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support.ui import Select

from selenium.webdriver.support.select import Select
from selenium.webdriver.support.select import Select
from selenium.webdriver import Chrome

import itertools
from pprint import pformat

url = "https://www.ebay.co.uk/itm/Pashmina-Scarf-100-Viscose-Plain-Wrap-Shawl-Stole-Scarf-Many-Colours-Available/252342060680"

browser.get(url)
optionones = len(browser.find_elements_by_xpath("//select[@id='msku-sel-1']/option"))
for colNum in range(optionones):
    #select color
    optiononeEle = browser.find_element_by_xpath("(//select[id='msku-sel-1']/option)[" + str(colNum+1) + "]")
    optionone = optiononeEle.text
    optiononeEle.click()

    # get the sizes
    sizes = len(browser.find_elements_by_xpath("//select[id='msku-sel-2']/option"))
    for sizeNum in range(sizes):
        # select optionone
        optiontwoEle = browser.find_element_by_xpath("(//select[id='msku-sel-2']/option)[" + str(sizeNum + 1) + "]")
        size = optiontwoEle.text
        optiontwoEle.click()
        price = browser.find_element_by_xpath("//span[@id='prcIsum']").text
        print ("optionone:" + optionone)
        print( "optiontwoEle:" + optiontwoEle)
        print("Price: "+ price)
        print ("----------------------------------------")

Traceback (most recent call last):
  File "C:/Users/userman/PycharmProjects/seleniumTest/test/test310.py", line 30, in <module>
    optiononeEle = browser.find_element_by_xpath("(//select[id='msku-sel-1']/option)[" + str(colNum+1) + "]")
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 394, in find_element_by_xpath
    return self.find_element(by=By.XPATH, value=xpath)
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 978, in find_element
    'value': value})['value']
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\webdriver.py", line 321, in execute
    self.error_handler.check_response(response)
  File "C:\Python27\lib\site-packages\selenium\webdriver\remote\errorhandler.py", line 242, in check_response
    raise exception_class(message, screen, stacktrace)
selenium.common.exceptions.NoSuchElementException: Message: no such element: Unable to locate element: {"method":"xpath","selector":"(//select[id='msku-sel-1']/option)[1]"}

Process finished with exit code 1

预期输出是选项 1 的名称（例如尺寸，但可以是任何东西）+ 变体类型名称（例如小），然后是价格。

Answer 1

为什么不找到这些下拉列表的数量，然后循环将每个下拉列表设置为一个变量并使用通常的语法来选择其中的选项？

number_of_dropdowns =  len(WebDriverWait(driver,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

for i in range(1, number_of_dropdowns + 1):
    current_dropdown = driver.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']")
    #then use usual syntax to work with select 

查看有关选项选择的大量答案：

Selenium - Python - drop-down menu option value

---

等待条件的导入：

from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

---

循环它们的示例：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select

urls = ['https://www.ebay.co.uk/itm/Pashmina-Scarf-100-Viscose-Plain-Wrap-Shawl-Stole-Scarf-Many-Colours-Available/252342060680']
d = webdriver.Chrome()

for url in urls:
    d.get(url)
    number_of_dropdowns =  len(WebDriverWait(d,10).until(EC.presence_of_all_elements_located((By.CSS_SELECTOR, "[id^='msku-sel-']"))))

    for i in range(1, number_of_dropdowns + 1):
        #then use usual syntax to work with select 
        select = Select(d.find_element_by_css_selector("[id^='msku-sel-" + str(i) + "']"))
        options = select.options
        for index in range(1, len(options) - 1): #avoid first index
            select.select_by_index(index)
            #print(select.first_selected_option.text)
            #your code to extract info per option goes here