Python HTML 解析器（未命名级别）

Question

我正在开发一个屏幕抓取工具，以从www.pro-football-reference.com 中提取足球统计数据。我目前正在从主要玩家的统计页面中抓取数据，然后按年份深入到他们的个人页面中。

我能够在我的第一批球员（四分卫，使用传球表）中成功实施这个过程。但是，当我尝试重新创建流程以获取运行数据时，我在数据框中接收到一个附加列，其值为“未命名：x_level_0”。这是我第一次使用 HTML 数据，所以我不确定我错过了什么，我只是假设它与四分卫的代码相同。

以下是 QB 代码示例和正确的数据框：

import requests
import urllib.request
import time
from bs4 import BeautifulSoup
import pandas as pd
from pandas import DataFrame
import lxml
import re
import csv
p = 1

url = 'https://www.pro-football-reference.com'
year = 2020
maxp = 300

#Passing Data
r = requests.get(url+ '/years/' + str(year) + '/passing.htm')
soup = BeautifulSoup(r.content, 'html.parser')
parsed_table = soup.find_all('table')[0]

results = soup.find(id='div_passing')
job_elems = results.find_all('tr')

df = []
LastNameList = []
FirstNameList = []


for i,row in enumerate(parsed_table.find_all('tr')[2:]):
        dat = row.find('td', attrs={'data-stat': 'player'})
        if dat != None:
            name = dat.a.get_text()
            print(name)
            stub = dat.a.get('href')
            

            #pos = row.find('td', attrs={'data-stat': 'fantasy_pos'}).get_text()
            #print(pos)

            # grab this players stats
            tdf = pd.read_html(url + stub)[1]
            for k,v in tdf.iterrows():
                #Scrape 2020 stats, if no 2020 stats move on
                try:
                    FindYear=re.search(".*2020.*",v['Year'])
                    if FindYear:
                        #If Year for stats is current year append data to dataframe

                        #get Name data
                        fullName = row.find('td', {'class':'left'})['csk']
                        findComma = fullName.find(',',0,len(fullName))
                        lName = fullName[0:findComma]
                        fName = fullName[findComma + 1:len(fullName)]
                        
                        LastNameList.append(lName)
                        FirstNameList.append(fName)
                        #get basic stats
                        df.append(v)
                except:
                    pass

此输出如下所示：

Philip Rivers
Year      2020
Age         39
Tm         IND
Pos         qb
No.         17
G            1
GS           1

以下是 RB 代码示例和错误的数据框：

import requests
import urllib.request
import time
from bs4 import BeautifulSoup
import pandas as pd
from pandas import DataFrame
import lxml
import re
import csv
p = 1


url = 'https://www.pro-football-reference.com'
year = 2020
maxp = 300

#Rushing Data
r = requests.get(url+ '/years/' + str(year) + '/rushing.htm')
soup = BeautifulSoup(r.content, 'html.parser')
parsed_table = soup.find_all('table')[0]

results = soup.find(id='div_rushing')
job_elems = results.find_all('tr')

df = []
LastNameList = []
FirstNameList = []


for i,row in enumerate(parsed_table.find_all('tr')[2:]):
        dat = row.find('td', attrs={'data-stat': 'player'})
        if dat != None:
            name = dat.a.get_text()
            print(name)
            stub = dat.a.get('href')
            print(stub)
            

            #pos = row.find('td', attrs={'data-stat': 'fantasy_pos'}).get_text()
            #print(pos)

            # grab this players stats
            tdf = pd.read_html(url + stub)[1]
            for k,v in tdf.iterrows():
                print(v)
                #Scrape 2020 stats, if no 2020 stats move on
                try:
                    FindYear=re.search(".*2020.*",v['Year'])
                    print('found 2020')
                    if FindYear:
                        #If Year for stats is current year append data to dataframe

                        #get Name data
                        fullName = row.find('td', {'class':'left'})['csk']
                        findComma = fullName.find(',',0,len(fullName))
                        lName = fullName[0:findComma]
                        fName = fullName[findComma + 1:len(fullName)]
                        
                        LastNameList.append(lName)
                        FirstNameList.append(fName)
                        #get basic stats
                        df.append(v)
                except:
                    pass

此输出如下所示：

Unnamed: 0_level_0   Year       2020
Unnamed: 1_level_0   Age          26
Unnamed: 2_level_0   Tm          TEN
Unnamed: 3_level_0   Pos          rb
Unnamed: 4_level_0   No.          22
Games                G             1
                     GS            1
Rushing              Rush         31
                     Yds         116
                     TD            0

从中提取此数据的示例 URL 是：https://www.pro-football-reference.com/players/J/JacoJo01.htm

它正在拉着冲和接收。在解析 HTML 时，我还需要注意什么？

我尝试将 index_col = 1 添加到我的 tdf = pd.read_html(url + stub)[1] 中。但是，这只是将两个值组合到一列中。

对此的任何意见将不胜感激。如果我可以提供任何进一步的信息，请告诉我。

谢谢

Answer 1

您可以尝试使用此代码解析为每个玩家传递的表格（现在我从 https://www.pro-football-reference.com/years/2020/passing.htm 获取玩家，但您可以将任何玩家 URL 传递给它：

import requests 
from bs4 import BeautifulSoup


def scrape_player(player_name, player_url, year="2020"):
    out = []

    soup = BeautifulSoup(requests.get(player_url).content, 'html.parser')

    row = soup.select_one('table#passing tr:has(th:contains("{}"))'.format(year))
    if row:
        tds = [player_name] + [t.text for t in row.select('th, td')]
        headers = ['Name'] + [th.text for th in row.find_previous('thead').select('th')]
        out.append(dict(zip(headers, tds)))

    return out

url = 'https://www.pro-football-reference.com/years/2020/passing.htm'
all_data = []
soup = BeautifulSoup(requests.get(url).content, 'html.parser')
for player in soup.select('table#passing [data-stat="player"] a'):
    print(player.text)
    for data in scrape_player(player.text, 'https://www.pro-football-reference.com' + player['href']):
        all_data.append(data)

df = pd.DataFrame(all_data)
df.to_csv('data.csv')
print(df)

创建这个 csv：

---

编辑：要解析 Rushing&Receiving，你可以使用这个脚本：

import requests 
from bs4 import BeautifulSoup, Comment


def scrape_player(player_name, player_url, year="2020"):
    out = []

    soup = BeautifulSoup(requests.get(player_url).content, 'html.parser')
    soup = BeautifulSoup(soup.select_one('#rushing_and_receiving_link').find_next(text=lambda t: isinstance(t, Comment)), 'html.parser')

    row = soup.select_one('table#rushing_and_receiving tr:has(th:contains("{}"))'.format(year))
    if row:
        tds = [player_name] + [t.text for t in row.select('th, td')]
        headers = ['Name'] + [th.text for th in row.find_previous('thead').select('tr')[-1].select('th')]
        out.append(dict(zip(headers, tds)))

    return out

url = 'https://www.pro-football-reference.com/years/2020/passing.htm'
all_data = []
soup = BeautifulSoup(requests.get(url).content, 'html.parser')
for player in soup.select('table#passing [data-stat="player"] a'):
    print(player.text)
    for data in scrape_player(player.text, 'https://www.pro-football-reference.com' + player['href']):
        all_data.append(data)

df = pd.DataFrame(all_data)
df.to_csv('data.csv')
print(df)

创建此 CSV：