Beautifulsoup 获取汤中未选择的元素

Question

我正在尝试抓取代码和数据中列出的网站，我选择了特定的 div/类，但它似乎返回任何以“产品”开头的内容。你能帮我处理这段代码吗？感谢您对 Python 新手的帮助。另外，如果这可以写得更好，我很乐意接受。

url="https://shop.coles.com.au/a/a-tas-regional-glenorchy/everything/browse/dairy--eggs-meals/milk-3796059?pageNumber="
for x in range(1,6):
turl=url+str(x)
driver.get(turl)
time.sleep(10)
soup=BeautifulSoup(driver.page_source,"html.parser")
data = soup.find_all("div",{"class":"product"})
t=driver.find_elements_by_xpath('.//span[@class = "product-brand"]')
count=0
maxC=int(len(t))
for item in data:
    if count < maxC and count>=0:
        if item.find("span", {"class":"product-brand"}) is not None:
            ProductText1=item.find("span", {"class":"product-brand"}).text
        else:
            ProductText1=""
        if item.find("span", {"class":"product-name"}) is not None:
            ProductText2=item.find("span", {"class":"product-name"}).text
        else:
            ProductText2=""
        if item.find("span", {"class":"package-size"}) is not None:
            size=item.find("span", {"class":"package-size"}).text
        else:
            size=""            
        if item.find("span", {"class":"package-price"}).text is not None:
            Price=item.find("span", {"class":"package-price"}).text
        else:
            Price=""
        if item.find("span", {"class":"product-qty"}).text is not None:
            Price1=item.find("span", {"class":"product-qty"}).text
        else:
            Price1=""
        if item.find("span", {"class":"product-price"}).text is not None:
            Price2=item.find("span", {"class":"product-qty"}).text
        else:
            Price2=""
        ProductText=str(ProductText1)+" "+str(ProductText2)
        writer.writerow([ProductText,Price2,size,Price])
    count+=1

Answer 1

您还可以考虑从 ajax 请求 url 中提取包含所有信息的 json。我将指定的项目读入数据框中，但您可以根据需要进行处理。

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from bs4 import BeautifulSoup
import json
import pandas as pd

url = 'https://shop.coles.com.au/online/a-tas-regional-glenorchy/dairy--eggs-meals/milk-3796059?tabType=everything&tabId=everything&personaliseSort=false&orderBy=20512_6&errorView=AjaxActionErrorResponse&requesttype=ajax&beginIndex=0'

d = webdriver.Chrome()
d.get(url)
WebDriverWait(d,10).until(EC.presence_of_element_located((By.CSS_SELECTOR, "[data-colrs-transformer]")))
soup = BeautifulSoup(d.page_source, 'lxml')
d.quit()

item = soup.select_one('[data-colrs-transformer]')
data= json.loads(item.text)
products = data['products']
items = [[product['m'] ,product['n'] ,product['a']['O3'][0],product['u2'], product['a']['A4'][0],product['p1']['o']] for product in products]
df = pd.DataFrame(items, columns = ['brand', 'productName', 'packageSize', 'packagePrice', 'productQty', 'productPrice'])
print(df)

---

JSON 结构示例：

---

计算结果的页数，从而计算 ajax 所需的偏移值

import math
itemsPerPage  = data['searchInfo']['pageSize']
totalItems = data['searchInfo']['totalCount']
numPages = math.ceil(totalItems/itemsPerPage)
offsetStart = 0
for i in range(2 ,numPages + 1):
    offsetStart += itemsPerPage
    print(offsetStart)