循环浏览不同的页面并在 Javascript 网站上抓取数据

Question

我想我已经接近了，但我不知道为什么我的代码没有按预期工作。我想从第一页抓取数据，然后单击next（箭头）按钮并移至下一页并执行相同操作，依此类推，直到next 箭头按钮变灰，此时驱动程序应该退出。任何帮助将非常感激。代码如下：

   
import selenium
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from bs4 import *
import time
import pandas as pd
import pickle
import html5lib


options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
chrome_driver_path = '/Users/Justin/Desktop/Python/chromedriver'
driver = webdriver.Chrome(executable_path=chrome_driver_path)
url = "https://cryptoli.st/lists/fixed-supply"
driver.get(url)
time.sleep(3)
page = driver.page_source

master_list = []

def get_next_page(url):
    while driver.find_element_by_xpath('/html/body/div/div/div/div[2]/div[1]/div[2]/div[2]/div[2]/ul/li[9]') != True:
        driver.find_elements_by_xpath(
            '/html/body/div/div/div/div[2]/div[1]/div[2]/div[2]/div[2]/ul/li[9]/a').click()
        soup = BeautifulSoup(page, 'html5lib')
        container = soup.find_all('div', attrs={
            'class': 'dataTables_scrollBody'})
        df = pd.read_html(str(container))
        dfs = df[0]
        page_next_data = dfs[['#', 'Name', 'Symbol', 'Max Supply', 'Summary', 'Price', 'Market Cap',
                              '24h Volume', '1h %', '24h %', '7d %', 'Circulation', 'Total Supply', 'Consensus Method']]
        return master_list.append(page_next_data)

    else:
        driver.quit()


def get_data(callback, url):
    global soup, container
    soup = BeautifulSoup(page, 'html5lib')
    container = soup.find_all('div', attrs={
        'class': 'dataTables_scrollBody'})
    df = pd.read_html(str(container))
    dfs = df[0]
    page_one_data = dfs[['#', 'Name', 'Symbol', 'Max Supply', 'Summary', 'Price', 'Market Cap',
                        '24h Volume', '1h %', '24h %', '7d %', 'Circulation', 'Total Supply', 'Consensus Method']]
    return master_list.append(page_one_data)
    return callback(args)


print(get_data(get_next_page, url))

这是它从第一页给我的结果，但它不会继续到下一页，也不会给我任何错误或任何东西。

    #          Name Symbol  ...  Circulation Total Supply               Consensus Method
0   1       Bitcoin    BTC  ...     18713700     18713700                  Proof of Work
1   4  Binance Coin    BNB  ...    153432897    169432897                            NaN
2   5       Cardano    ADA  ...  31948309441  45000000000       Delegated Proof of Stake
3   7           XRP    XRP  ...  46135372183  99990461026  Federated Byzantine Agreement
4  10  Bitcoin Cash    BCH  ...     18742750     18742750                  Proof of Work
5  11      Litecoin    LTC  ...     66752415     66752415                  Proof of Work
6  12     ChainLink   LINK  ...    428009554   1000000000                  Proof of Work

[7 rows x 14 columns]
(pyfinance) Justins-MacBook-Pro:Python-for-Finance-Repo-master Justin$ 

Answer 1

看起来你把它弄得太复杂了。您在一个函数中有两个 return 语句，并且您使用了函数范围内不存在的 args 变量。请参阅下面的改编和工作代码：

import selenium
from selenium.webdriver.chrome.options import Options
from selenium import webdriver
from bs4 import *
import time
import pandas as pd
import pickle
import html5lib

options = Options()
options.add_argument('--headless')
options.add_argument('--disable-gpu')
chrome_driver_path = '/usr/local/bin/chromedriver'
driver = webdriver.Chrome(executable_path=chrome_driver_path)
url = "https://cryptoli.st/lists/fixed-supply"
driver.get(url)
time.sleep(3)
page = driver.page_source
master_list = []

def get_next_page(url):
    proceed = True

    while driver.find_element_by_xpath('/html/body/div/div/div/div[2]/div[1]/div[2]/div[2]/div[2]/ul/li[9]/a') != True and proceed:        
        try:
            driver.find_elements_by_xpath('/html/body/div/div/div/div[2]/div[1]/div[2]/div[2]/div[2]/ul/li[9]/a')[0].click()
        except Exception:
            proceed = False

        soup = BeautifulSoup(page, 'html5lib')
        container = soup.find_all('div', attrs={'class': 'dataTables_scrollBody'})
        df = pd.read_html(str(container))
        dfs = df[0]
        page_next_data = dfs[['#', 'Name', 'Symbol', 'Max Supply', 'Summary', 'Price', 'Market Cap', '24h Volume', '1h %', '24h %', '7d %', 'Circulation', 'Total Supply', 'Consensus Method']]
        master_list.append(page_next_data)
    else:
        driver.quit()

def get_data(callback, url):
    global soup, container
    soup = BeautifulSoup(page, 'html5lib')
    container = soup.find_all('div', attrs={'class': 'dataTables_scrollBody'})
    df = pd.read_html(str(container))
    dfs = df[0]
    page_one_data = dfs[['#', 'Name', 'Symbol', 'Max Supply', 'Summary', 'Price', 'Market Cap', '24h Volume', '1h %', '24h %', '7d %', 'Circulation', 'Total Supply', 'Consensus Method']]
    master_list.append(page_one_data)
    
    return callback(url)

get_data(get_next_page, url)
print(master_list)

Answer 2

看来，即使是现在，您还没有测试过任何较小的代码片段。 看看你的逻辑：

while driver...
    driver....click()
    ...
    return master_list.append(...)

1. 在第一次迭代时退出函数。
2. 追加是就地操作；它总是返回None。

简而言之，您可以跳过循环，跳过点击和数据提取，只需将整个函数替换为正文

return None