Python Selenium 在某些页面上挂起

Question

当我尝试 scrape 跟随 url 时，selenium 大约 30 秒没有响应，然后在漂亮的汤线上发出错误，因为没有要解析的 html 代码。我的 selenium + chrome 设置适用于大多数网站，但不是这个：http://www.11st.co.kr/product/SellerProductDetail.tmall?method=getSellerProductDetail&prdNo=2609814501&trTypeCd=22&trCtgrNo=895019

我应该怎么做才能让它工作？

这是我的硒选项：

def GetSelenium():
    options = webdriver.ChromeOptions()
    options.add_argument("start-maximized")
    options.add_argument("--disable-infobars")
    options.add_argument('headless')
    options.add_argument('lang=ko_KR')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36')
    
    driver = webdriver.Chrome('/usr/bin/chromedriver',chrome_options=options)
    return driver

@api_view()
def crawlOthers(request, crawl_url):
    crawl_url = request.data["requestUrl"]
    print("URL:" + crawl_url)
    driver = GetSelenium()
    driver.get(crawl_url)
    html = driver.page_source
    driver.quit()
    bsObj = BeautifulSoup(html,'html.parser')

    #parsing deleted

Answer 1

我认为GetSelenium() 函数是错误的。我认为我们应该声明选项值并将内容放入chrome驱动程序并返回chrome驱动程序值。

我会附上下面的代码。

from selenium import webdriver
from bs4 import BeautifulSoup


def GetSelenium():
    options = webdriver.ChromeOptions()
    options.add_argument("start-maximized")
    options.add_argument("--disable-infobars")
    options.add_argument('headless')
    options.add_argument('lang=ko_KR')
    options.add_argument('--no-sandbox')
    options.add_argument('--disable-dev-shm-usage')
    options.add_argument('user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36')
    driver = webdriver.Chrome(chrome_options=options)

    return driver

def crawlOthers():
    url = 'http://www.11st.co.kr/product/SellerProductDetail.tmall?method=getSellerProductDetail&prdNo=2609814501&trTypeCd=22&trCtgrNo=895019'
    print("URL:" + url)
    driver = GetSelenium()
    driver.get(url)
    html = driver.page_source
    driver.quit()
    bsObj = BeautifulSoup(html,'html.parser')
    print(bsObj)

    #parsing deleted

if __name__ == '__main__':
    crawlOthers()