试图提取特定的表列

Question

我正在尝试从给定网站中提取特定列。我怎样才能从网站上只提取代码、标题和范围：https://training.gov.au/Organisation/Details/31102

在范围选项卡中，有资格表，我只想提取代码标题和范围，但要提取打印整个表的当前代码。有人可以帮帮我吗？

代码：

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import bs4 as bs4
from pathlib import Path
import os

option = webdriver.ChromeOptions()
option.add_argument("--incognito")
option.add_argument("--headless")
exec_path = "/Users/Downloads/MUR_scraping-master/Libraries/chromedriver"
browser = webdriver.Chrome(executable_path=exec_path, options=option)


course_links_file_path = Path(os.getcwd().replace('\\', '/'))
course_links_file_path = course_links_file_path.__str__() + '/links.txt'
course_links_file = open(course_links_file_path, 'r')

for each_url in course_links_file:
#print(each_url)
try:
    browser.get(each_url)
except:
    print(each_url)
    pass
pure_url = each_url.strip()
each_url = browser.page_source
delay_ = 15
soup = bs4.BeautifulSoup(each_url, 'lxml')


#browser.get("https://training.gov.au/Organisation/Details/31102")

# open the scope tab
browser.find_element_by_css_selector('a#detailsScopeTab').click()
# wait for the table to load
element = WebDriverWait(browser, 20).until(
EC.presence_of_element_located(
    (By.XPATH, '//div[@id="ScopeQualification"]//div[@class="t-pagesize-wrapper"]')))

# click on the button to have all rows at once
browser.find_element_by_xpath('//div[@id="ScopeQualification"]//div[@class="t-pagesize- 
wrapper"]/a[last()]').click()


# wait for the table loads the new data
class element_css_class_flashed(object):
def __init__(self, locator, css_class):
    self.locator = locator
    self.css_class = css_class
    self.was_on = False

def __call__(self, driver):
    element = driver.find_element(*self.locator)
    if self.css_class in element.get_attribute("class"):
        self.was_on = True
    elif self.was_on:
        return element
    else:
        return False

try:
wait = WebDriverWait(browser, 3)
element = wait.until(element_css_class_flashed(
    (By.XPATH, '//div[@id="ScopeQualification"]//div[@class="t-status"]/a'), 
    "t-loading"))
except:
# most likely the loading was too fast to detect
pass

soup = bs4.BeautifulSoup(browser.page_source, 'html.parser')
table = soup.select_one('div#ScopeQualification table')
all_rows = table.find_all('tr')
header_row = all_rows[0]
rows = all_rows[1:-1]

data = [[col.text for col in header_row.find_all('th')]]
for row in rows:
data_row = []
for col in row.find_all('td'):
    data_row.append(col.text)
data.append(data_row)

print(data)

Answer 1

似乎是一个 ajax 链接，您可以从中获取数据。我只是用 pandas 解析表格，然后稍微清理一下：

import pandas as pd
import requests
from bs4 import BeautifulSoup


# Get link to scope and strip off speacial id/code
url = 'https://training.gov.au/Organisation/Details/31102'
headers = {'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/87.0.4280.88 Safari/537.36'}
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')

obj = soup.find_all('a', {'class':'t-link'})
for each in obj:
    if 'AjaxDetailsLoadScope' in each['href']:
        link = each['href']
linkID = link.split('/')[-1]

# Get tables from that link
url = 'https://training.gov.au/Organisation/Details/{linkID}'.format(linkID=linkID)
dfs = pd.read_html(url)
dfs = [x for x in dfs if len(x) >= 10]
df = pd.concat(dfs)

# Clean up the table
df = df.iloc[:,[0,2]]
df.columns = ['Code', 'Extent']
df[['Code', 'Title']] = df['Code'].str.split(' - ', 1, expand=True)
df = df[~df.Code.str.contains("Refresh information")].reset_index(drop=True)

输出：

        Code              Extent                                                Title
0   BSB20115  Deliver and assess                           Certificate II in Business
1   BSB30415  Deliver and assess           Certificate III in Business Administration
2   BSB40215  Deliver and assess                           Certificate IV in Business
3   BSB40320  Deliver and assess  Certificate IV in Entrepreneurship and New Business
4   BSB40515  Deliver and assess            Certificate IV in Business Administration
5   BSB40520  Deliver and assess          Certificate IV in Leadership and Management
6   BSB40920  Deliver and assess        Certificate IV in Project Management Practice
7   BSB41015  Deliver and assess                    Certificate IV in Human Resources
8   BSB41419  Deliver and assess             Certificate IV in Work Health and Safety
9   BSB41515  Deliver and assess        Certificate IV in Project Management Practice
10  BSB10107  Deliver and assess                            Certificate I in Business
11  BSB10112  Deliver and assess                            Certificate I in Business
12  BSB20107  Deliver and assess                           Certificate II in Business
13  BSB20112  Deliver and assess                           Certificate II in Business
14  BSB20115  Deliver and assess                           Certificate II in Business
15  BSB20120  Deliver and assess                   Certificate II in Workplace Skills
16  BSB30120  Deliver and assess                          Certificate III in Business
17  BSB30407  Deliver and assess           Certificate III in Business Administration
18  BSB30412  Deliver and assess           Certificate III in Business Administration
19  BSB30415  Deliver and assess           Certificate III in Business Administration