解析页面时出错 - Python

Question

我正在尝试使用 bs4 解析 g2g.com 页面。但没有什么对我有用。代码中一定有错误。可能是什么原因？提前致谢！

import requests
from bs4 import BeautifulSoup
link = "https://www.g2g.com/offer-11677180/Amnennar-TBC--FR----Alliance?service_id=lgc_service_1&brand_id=lgc_game_29076&region_id=ac3f85c1-7562-437e-b125-e89576b9a38e&sort=lowest_price&attributes=%257B%2522lgc_29076_server%2522%253A%255B%2522lgc_29076_server_40957%2522%255D%257D&online=online"
response = requests.get(link)
with open(path + str(fileName) + ".html", "w") as f:
    f.write(response.text)
with open(path + str(fileName) + ".html", "r") as f:
    html = f.read()
with open(path + str(fileName) + ".html", "w") as f:
    soup = BeautifulSoup(html, "html.parser")
    blockList = soup.find("div", {"id": "pre_checkout_sls_offer"})
    for i in blockList:
        block = blockList.find_all("div", {"class": "other_offer-desk-main-box other_offer-div-box"})
        for j in block:
            insideBlock = block.find_all("div", {"class": "flex-1 align-self"})
            for k in insideBlock:
                seller = insideBlock.find("div", class_="seller__name-detail")
                stock = insideBlock.find("div", class_="offers-bottom-attributes offer__content-lower-items")("span") # str(stock)[7:-8]
                price = insideBlock.find("span", class_="offer-price-amount")
                print(seller, stock, price)

Answer 1

您只是在处理JavaScript 网站，requests 库将无法为您呈现JS。因此，您必须使用真正的浏览器自动化，例如 selenium。

from selenium import webdriver
import pandas as pd
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.firefox.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC


def main(driver):
    try:
        driver.get("https://www.g2g.com/offer-11677180/Amnennar-TBC--FR----Alliance?service_id=lgc_service_1&brand_id=lgc_game_29076&region_id=ac3f85c1-7562-437e-b125-e89576b9a38e&sort=lowest_price&attributes=%257B%2522lgc_29076_server%2522%253A%255B%2522lgc_29076_server_40957%2522%255D%257D&online=online")

        elements = WebDriverWait(driver, 10).until(
            EC.presence_of_all_elements_located(
                (By.CLASS_NAME, "other_offer-desk-main-box.other_offer-div-box"))
        )

        goal = [(x.find_element_by_class_name("seller__name-detail").text,
                 x.find_elements_by_class_name(
                     "offers-bottom-attributes.offer__content-lower-items")[1].text,
                 x.find_element_by_class_name("offer-price-amount").text)
                for x in elements]
        return pd.DataFrame(goal, columns=["Seller", "Stock", "Price"])

    except TimeoutException:
        return "Unable to find desired information"

    finally:
        driver.quit()
        


if __name__ == "__main__":
    options = Options()
    options.headless = True
    driver = webdriver.Firefox(options=options)
    print(main(driver))

输出：

          Seller      Stock     Price
0      g2aseller      2,800  0.022841
1       kevinlei      3,000  0.022849
2           Eyse  3,318,614  0.023107
3   Meteorkasper        950   0.02313
4   fortwonasell      2,000   0.02315
5      VegoStore    195,780    0.0232
6         Thanku  4,959,200  0.023407
7     AlotofGold  7,807,751  0.023407
8       speedVIP  5,612,958   0.02344
9           Aeda    150,000   0.02376
10    HellenWong  9,786,442  0.023807
11          Good  9,252,414     0.024
12      BestGold    305,880     0.024
13         Happy  9,994,699     0.024
14   Moonbootica      1,150  0.024032
15     byefriend     37,030    0.0241
16          baby    714,645   0.02413
17         jaina  1,215,220   0.02419
18      yang9732     28,888    0.0242
19       SunRise  9,860,048  0.024228