如何使用 BeautifulSoup 将表格放入 Dataframe

Question

Beautifull Soup 似乎无法从表中检索信息。

我要做的是检索带有标题的表并将其保存到 pands 中的数据框。非常感谢任何帮助。

import requests
from bs4 import BeautifulSoup
import pandas as pd

# Create an URL object
url = 'xxxx'
# Create object page
page = requests.get(url)
soup = BeautifulSoup(page.content, "html5lib")
data = soup.find_all("table", id="cve_table", attrs={"class": "table"})
print(len(data))
headers = []
for body in data:
    print(body)
    for item in body:
        title = item.text
        print(title)
        headers.append(title)

print(headers)

我得到的只有这个：

<table class="table cell-border table-striped table-condensed table-hover" id="cve_table">
        <tbody></tbody>

    </table>
  
['\n        ', '', '\n\n    ']

Answer 1

看起来，表格是用javascript渲染的，所以当requests加载HTML页面时，表格是空的。在调查页面源时，可以看到，表格是使用最后一个脚本元素中调用的函数呈现的。此函数将呈现表格所需的数据结构作为参数。可提取如下：

import requests
from bs4 import BeautifulSoup
import pandas as pd
import re
import json
from collections import defaultdict

# Create an URL object
url = 'https://cve.rayvyn.net/rayvyn'

# Create object page
page = requests.get(url)
soup = BeautifulSoup(page.content, "html5lib")

dct = defaultdict(list)

script = soup.find(lambda tag: tag.name == "script" and "get_all_cve_data" in tag.text) 
if script:
  result = re.search('\((.*)\)', script.text)
  text = result.group(1)
  data = json.loads(json.loads(text))
  for row in data:
    dct['CVE ID'].append(row[0])
    dct['Feed'].append(row[1])
    dct['Date Modified'].append(row[2])
    dct['Description'].append(row[3])
    dct['Vector'].append(row[4])
    dct['Vendor'].append(row[5])
    dct['Product'].append(row[6])
    dct['Advisory Link'].append(row[7])
else:
  print('Script tag with function get_all_cve_data() not found')

df = pd.DataFrame(dct)
df

另一种方法是使用 selenium 框架：

import re
import json

from selenium import webdriver 
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait

from collections import defaultdict

# Create an URL object
url = 'https://cve.rayvyn.net/rayvyn'


# delay for selenium web driver wait
DELAY = 30

# create selenium driver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--no-sandbox')
driver = webdriver.Chrome('<<path to chromedriver.exe>>', options = chrome_options)

# open web page
driver.get(url)

script = WebDriverWait(driver, DELAY).until(EC.presence_of_element_located((By.XPATH, "//script[contains(text(), 'get_all_cve_data')]")))

dct = defaultdict(list)

if script:
  result = re.search('\((.*)\)', script.get_attribute('innerHTML'))
  text = result.group(1)
  data = json.loads(text)
  print(data)
else:
  print('Script tag with function get_all_cve_data() not found')
  
driver.quit()

data = json.loads(json.loads(text))
for row in data:
  dct['CVE ID'].append(row[0])
  dct['Feed'].append(row[1])
  dct['Date Modified'].append(row[2])
  dct['Description'].append(row[3])
  dct['Vector'].append(row[4])
  dct['Vendor'].append(row[5])
  dct['Product'].append(row[6])
  dct['Advisory Link'].append(row[7])

df = pd.DataFrame(dct)
df

请注意，使用 selenium 还需要 selenium webdriver（作为单独的可执行文件）。它将模拟浏览器行为并且（以及其他功能）将等待页面上的 javascript 代码被执行和 HTML 代码被呈现。