Python Selenium：向下滚动后如何获取更新的 HTML DOM？

Question

我正在访问一个实现视差滚动的page。我正在使用代码滚动底部，但 BeautifulSoup 它没有获取更新的 DOM。代码如下：

import requests
from bs4 import BeautifulSoup
from gensim.summarization import summarize

from selenium import webdriver
from datetime import datetime
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.keys import Keys
from time import sleep
import sys
import os
import xmltodict
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
import traceback
import random

driver = None
driver = webdriver.Firefox()
driver.maximize_window()
def fetch_links(tag):
    links = []
    url = 'https://steemit.com/trending/'+tag
    driver.get(url)
    html = driver.page_source
    sleep(4)

    soup = BeautifulSoup(html,'lxml')
    entries = soup.select('.entry-title > a')
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    sleep(5)
    entries = soup.select('.entry-title > a')
    for e in entries:
        if e['href'].strip() not in entries:
            links.append(e['href'])
    return links

Answer 1

您可能需要在窗口滚动后解析页面：

driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")

sleep(5)

soup = BeautifulSoup(driver.page_source, 'lxml')
entries = soup.select('.entry-title > a')