python selenium驱动程序不滚动以收集所有数据点

Question

我正在尝试从 EPA website 获取一些数据，不幸的是我无法捕获所有数据点，我推测这是由于滚动和等待标签变得可见的组合。但是我从昨天开始就一直在做这个，没有运气。

from selenium import webdriver
from bs4 import BeautifulSoup
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
from selenium.webdriver.common.keys import Keys
import numpy as np

options = webdriver.ChromeOptions()
path = '/Users/<user>/Applications/chromedriver'
options.set_headless(True) 

driver = webdriver.Chrome(chrome_options= options, executable_path=path)
url = 'https://edap.epa.gov/public/single/?appid=73b2b6a5-70c6-4820-b3fa-186ac094f10d&obj=b5bf280c-3488-4e46-84f6-58e2a0c34108&opt=noanimate%2Cnoselections&select=clearall'

driver.set_window_size(1920, 1080)
driver.get(url)



SCROLL_PAUSE_TIME = 0.5

# Get scroll height
last_height = driver.execute_script("return document.body.scrollHeight")

rin_data = []

while True:
    # Scroll down to bottom
    driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
    WebDriverWait(driver, 30).until(EC.presence_of_element_located((By.CLASS_NAME,"qv-st-value-overflow")))
    soup = BeautifulSoup(driver.page_source, "html.parser")
    tableURL = soup.select('.qv-st-value-overflow')
    for rin_val in tableURL:
        rin_data.append(rin_val.get_text())

    # Wait to load page
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll height and compare with last scroll height
    new_height = driver.execute_script("return document.body.scrollHeight")
    if new_height == last_height:
        break
    last_height = new_height

Answer 1

它使用Websocket 而不是Ajax 来获取数据，您需要滚动table[ng-style="tableStyles.content"] 而不是body，但它需要自定义滚动或使用mouse wheel 滚动。函数取自here

SCROLL_PAUSE_TIME = 2

driver.get(url)

# add mouse wheel function to the page
driver.execute_script('''
window.scrollTable = function() {
  var element = document.querySelector('table[ng-style="tableStyles.content"]')
  var box = element.getBoundingClientRect();
  var deltaY = box.height;
  var clientX = box.left + (box.width / 2);
  var clientY = box.top + (box.height / 2);
  var target = element.ownerDocument.elementFromPoint(clientX, clientY);

  for (var e = target; e; e = e.parentElement) {
    if (e === element) {
      target.dispatchEvent(new WheelEvent('wheel', {view: window, bubbles: true, cancelable: true, clientX: clientX, clientY: clientY, deltaY: deltaY}));
    }
  }
}
''')

rin_data = []

while True:
    WebDriverWait(driver, 30).until(
        EC.presence_of_element_located((By.CSS_SELECTOR, 'tr[class^="qv-st-data-row"]'))
    )
    last_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
    rows = driver.find_elements_by_css_selector('tr[class^="qv-st-data-row"]')
    for row in rows:
        rin_data.append(row.text)

    # Scroll down the table
    driver.execute_script('scrollTable()')
    # Wait to load content from Websocket, maybe need to increase
    time.sleep(SCROLL_PAUSE_TIME)

    # Calculate new scroll position and compare with last scroll position
    new_position = driver.find_element_by_css_selector(".scrollbar-thumb").get_attribute('style')
    if new_position == last_position:
        break

注意，在这种情况下你不需要使用BeautifulSoup