如何在单次迭代中抓取多个标签？

Question

我在下面有一个完美运行的脚本 - 它访问 URL 列表中的每个 HREF 标记，然后返回关联的 p 标记信息。它将信息直接推送到 Google 表格。

我注意到，玩家“位置”不包括在内，因为它是一个 H2 标签而不是 p...我开始分别重做整个脚本以抓取这些（下面的第二个脚本）。

有没有一种方法可以重写第一个以包含一个添加这些 h2 标签（位置）的列？

工作（检索所有 P 标签）

  import requests
    from bs4 import BeautifulSoup
    
    import gspread
    gc = gspread.service_account(filename='creds.json')
    sh = gc.open_by_key('1DpasSS8yC1UX6WqAbkQ515BwEEjdDL-x74T0eTW8hLM')
    worksheet = sh.get_worksheet(3)
    # AddValue = ["Test", 25, "Test2"]
    # worksheet.insert_row(AddValue, 3)
    
    
    def get_links(url):
        data = []
        req_url = requests.get(url)
        soup = BeautifulSoup(req_url.content, "html.parser")
    
        for td in soup.find_all('td', {'data-th': 'Player'}):
            a_tag = td.a
            name = a_tag.text
            player_url = a_tag['href']
            print(f"Getting {name}")
    
            req_player_url = requests.get(
                f"https://basketball.realgm.com{player_url}")
            soup_player = BeautifulSoup(req_player_url.content, "html.parser")
            div_profile_box = soup_player.find("div", class_="profile-box")
            row = {"Name": name, "URL": player_url}
    
            for p in div_profile_box.find_all("p"):
                try:
                    key, value = p.get_text(strip=True).split(':', 1)
                    row[key.strip()] = value.strip()
                except:     # not all entries have values
                    pass
    
            data.append(row)
    
        return data
    
    
    urls = [
        'https://basketball.realgm.com/dleague/players/2022',
        'https://basketball.realgm.com/dleague/players/2021',
        'https://basketball.realgm.com/dleague/players/2020',
        'https://basketball.realgm.com/dleague/players/2019',
        'https://basketball.realgm.com/dleague/players/2018',
    ]
    
    
    res = []
    for url in urls:
        print(f"Getting: {url}")
        data = get_links(url)
        res = [*res, *data]
    
    if res != []:
        header = list(res[0].keys())
        values = [
            header, *[[e[k] if e.get(k) else "" for k in header] for e in res]]
        worksheet.append_rows(values, value_input_option="USER_ENTERED"

)

**NOT WORKING, BUT AN ATTEMPT TO GET POSITIONS:**

import requests
from bs4 import BeautifulSoup

import gspread
gc = gspread.service_account(filename='creds.json')
sh = gc.open_by_key('1DpasSS8yC1UX6WqAbkQ515BwEEjdDL-x74T0eTW8hLM')
worksheet = sh.get_worksheet(1)
# AddValue = ["Test", 25, "Test2"]
# worksheet.insert_row(AddValue, 3)


def get_links(url):
    data = []
    req_url = requests.get(url)
    soup = BeautifulSoup(req_url.content, "html.parser")

    for td in soup.find_all('td', {'data-th': 'Player'}):
        a_tag = td.a
        name = a_tag.text
        player_url = a_tag['href']
        print(f"Getting {name}")

        req_player_url = requests.get(
            f"https://basketball.realgm.com{player_url}")
        soup_player = BeautifulSoup(req_player_url.content, "html.parser")
        div_profile_box = soup_player.find("div", class_="profile-box")
        row = {"Name": name, "URL": player_url}

        for p in div_profile_box.find_all("h2"):
            try:
                p.get_text(strip=True)
            except:     # not all entries have values
                pass

        data.append(row)

    return data


urls = [
    'https://basketball.realgm.com/dleague/players/2022',
    # 'https://basketball.realgm.com/dleague/players/2021',
    # 'https://basketball.realgm.com/dleague/players/2020',
    # 'https://basketball.realgm.com/dleague/players/2019',
    # 'https://basketball.realgm.com/dleague/players/2018',
]


res = []
for url in urls:
    print(f"Getting: {url}")
    data = get_links(url)
    res = [*res, *data]

if res != []:
    header = list(res[0].keys())
    values = [
        header, *[[e[k] if e.get(k) else "" for k in header] for e in res]]
    worksheet.append_rows(values, value_input_option="USER_ENTERED")

Answer 1

有不同的选择：

• 使用name和url获取它find_next_sibling('td')

  pos = td.find_next_sibling('td').text
  

• 从详情页获取，直接调用

  row['pos_option2'] = div_profile_box.h2.span.text 
  

例子

请注意，出于演示目的，此示例在第一次迭代后中断，只需删除 break 即可获得所有结果

import requests
from bs4 import BeautifulSoup


def get_links(url):
    data = []
    req_url = requests.get(url)
    soup = BeautifulSoup(req_url.content, "html.parser")

    for td in soup.find_all('td', {'data-th': 'Player'}):
        a_tag = td.a
        name = a_tag.text
        player_url = a_tag['href']
        pos = td.find_next_sibling('td').text
        print(f"Getting {name}")

        req_player_url = requests.get(
            f"https://basketball.realgm.com{player_url}")
        soup_player = BeautifulSoup(req_player_url.content, "html.parser")
        div_profile_box = soup_player.find("div", class_="profile-box")
        row = {"Name": name, "URL": player_url, "pos_option1":pos}

        row['pos_option2'] = div_profile_box.h2.span.text
        for p in div_profile_box.find_all("p"):
            try:
                key, value = p.get_text(strip=True).split(':', 1)
                row[key.strip()] = value.strip()
            except:     # not all entries have values
                pass

        data.append(row)
        break
    return data

get_links('https://basketball.realgm.com/dleague/players/2022')

输出

Getting Darius Adams
[{'Name': 'Darius Adams',
  'URL': '/player/Darius-Adams/Summary/28720',
  'pos_option1': 'PG',
  'pos_option2': 'PG',
  'Current Team': 'Fujian',
  'Born': 'Apr 17, 1989(33 years old)',
  'Birthplace/Hometown': 'Decatur, Illinois',
  'Nationality': 'United States',
  'Height': '6-1 (185cm)Weight:165 (75kg)',
  'Current NBA Status': 'Unrestricted Free Agent',
  'Agent': 'Thaddeus Foucher,Joe Smith',
  'Draft Entry': '2011 NBA Draft',
  'Drafted': 'Undrafted',
  'Pre-Draft Team': 'Indianapolis(Sr)',
  'High School': 'MacArthur High School[Decatur, Illinois]'}]