R 中的解析与抓取

Question

我是 R 新手，对构建数据库的最有效方法有疑问。我想建立一个NFL统计数据库。这些统计数据在网络上的许多地方都很容易获得，但我发现最彻底的分析是在 Pro-Football-Reference (http://www.pro-football-reference.com/) 上。这将是面板数据，其中时间间隔是每个赛季的每周，我的观察是每场比赛中的每个球员，我的列是 Pro-Football-Reference 的所有表格中统计的统计数据 (http://www.pro-football-reference.com/boxscores/201702050atl.htm)。

我可以用类似这样的东西刮掉每个赛季每场比赛的每张桌子：

#PACKAGES
library(rvest)
library(XML)
page.201702050atl = read_html("http://www.pro-football-reference.com/boxscores/201702050atl.htm")
comments.201702050atl = page.201702050atl %>% html_nodes(xpath = "//comment()")
scoring.201702050atl = readHTMLTable("http://www.pro-football-reference.com/boxscores/201702050atl.htm", which = 2)
game.info.201702050atl = comments.201702050atl[17] %>% html_text() %>% read_html() %>% html_node("#game_info") %>% html_table()
officials.201702050atl = comments.201702050atl[21] %>% html_text() %>% read_html() %>% html_node("#officials") %>% html_table()
team.stats.201702050atl = comments.201702050atl[27] %>% html_text() %>% read_html() %>% html_node("#team_stats") %>% html_table()
scorebox.201702050atl = readHTMLTable("http://www.pro-football-reference.com/boxscores/201702050atl.htm", which = 1)
expected.points.201702050atl = comments.201702050atl[22] %>% html_text() %>% read_html() %>% html_node("#expected_points") %>% html_table()
player.offense.201702050atl = comments.201702050atl[31] %>% html_text() %>% read_html() %>% html_node("#player_offense") %>% html_table()
player.defense.201702050atl = comments.201702050atl[32] %>% html_text() %>% read_html() %>% html_node("#player_defense") %>% html_table()
returns.201702050atl = comments.201702050atl[33] %>% html_text() %>% read_html() %>% html_node("#returns") %>% html_table()
kicking.201702050atl = comments.201702050atl[34] %>% html_text() %>% read_html() %>% html_node("#kicking") %>% html_table()
home.starters.201702050atl = comments.201702050atl[35] %>% html_text() %>% read_html() %>% html_node("#home_starters") %>% html_table()
vis.starters.201702050atl = comments.201702050atl[36] %>% html_text() %>% read_html() %>% html_node("#vis_starters") %>% html_table()
home.snap.counts.201702050atl = comments.201702050atl[37] %>% html_text() %>% read_html() %>% html_node("#home_snap_counts") %>% html_table()
vis.snap.counts.201702050atl = comments.201702050atl[38] %>% html_text() %>% read_html() %>% html_node("#vis_snap_counts") %>% html_table()
targets.directions.201702050atl = comments.201702050atl[39] %>% html_text() %>% read_html() %>% html_node("#targets_directions") %>% html_table()
rush.directions.201702050atl = comments.201702050atl[40] %>% html_text() %>% read_html() %>% html_node("#rush_directions") %>% html_table()
pass.tackles.201702050atl = comments.201702050atl[41] %>% html_text() %>% read_html() %>% html_node("#pass_tackles") %>% html_table()
rush.tackles.201702050atl = comments.201702050atl[42] %>% html_text() %>% read_html() %>% html_node("#rush_tackles") %>% html_table()
home.drives.201702050atl = comments.201702050atl[43] %>% html_text() %>% read_html() %>% html_node("#home_drives") %>% html_table()
vis.drives.201702050atl = comments.201702050atl[44] %>% html_text() %>% read_html() %>% html_node("#vis_drives") %>% html_table()
pbp.201702050atl = comments.201702050atl[45] %>% html_text() %>% read_html() %>% html_node("#pbp") %>% html_table()

但是，每年清理 256 场游戏的每个抓取表所需的代码行数似乎表明可能存在一种更有效的方法。

NFL 在他们的游戏手册中正式记录统计数据 (http://www.nfl.com/liveupdate/gamecenter/57167/ATL_Gamebook.pdf)。由于诸如 Pro-Football-Reference 之类的网站包含官方游戏手册中未记录的统计数据，并且由于这样做所需的识别语言包含在游戏手册的 Play-by-P​​lay 中，我推断他们正在运行一个功能解析 Play-by-P​​lay 并统计他们的统计数据。尽管我是新手，但我以前从未在 R 中编写过函数或解析过任何东西；但是，我想我可以应用到每本游戏书中的一个功能将是一种比刮擦每个单独的桌子更有效的方法。我在正确的道路上吗？我不想在错误的方向上投入大量的精力。

由于游戏书是 PDF 文件，因此出现了另一个问题。 Play-by-P​​lays 以表格形式存在于其他网站上，但没有一个是完整的。我在这个网站上阅读了一些关于如何将 PDF 转换为文本的优秀教程

library(tm)

但是，出于我自己的目的，我还没有弄清楚。

将整个 PDF 转换为文本后，我是否只需识别 Play-by-P​​lay 部分，将其解析出来，然后从那里解析出每个统计信息？是否还有其他障碍让我因有限的经验而无法预见？

对于本网站来说，这可能太“初学者”了；但是，谁能把我安排在这里？或者，给我一个可以的资源？非常感谢您的帮助。

Answer 1

考虑通过将 html 表存储在所有 256 款游戏的不断增长的列表中，将您的一款游戏解析推广到所有游戏。以下是第 1 周的示例。

doc <- htmlParse(readLines("http://www.pro-football-reference.com/years/2016/week_1.htm"))

# EXTRACT ALL GAME PAGES
games <- xpathSApply(doc, "//a[text()='Final']/@href")

# FUNCTION TO BUILD HTML TABLE LIST
getwebdata <- function(url) {
    print(url)
    boxscoreurl <- paste0("http://www.pro-football-reference.com", url)
    page <- read_html(boxscoreurl)
    comments <- page %>% html_nodes(xpath = "//comment()")

    list(
      scoring = readHTMLTable(boxscoreurl, which = 2),
      game.info = comments[17] %>% html_text() %>% read_html() %>% html_node("#game_info") %>% html_table(),
      officials = comments[21] %>% html_text() %>% read_html() %>% html_node("#officials") %>% html_table(),
      team.stats = comments[27] %>% html_text() %>% read_html() %>% html_node("#team_stats") %>% html_table(),
      scorebox = readHTMLTable(boxscoreurl, which = 1),
      expected.points = comments[22] %>% html_text() %>% read_html() %>% html_node("#expected_points") %>% html_table(),
      player.offense = comments[31] %>% html_text() %>% read_html() %>% html_node("#player_offense") %>% html_table(),
      player.defense = comments[32] %>% html_text() %>% read_html() %>% html_node("#player_defense") %>% html_table(),
      returns = comments[33] %>% html_text() %>% read_html() %>% html_node("#returns") %>% html_table(),
      kicking = comments[34] %>% html_text() %>% read_html() %>% html_node("#kicking") %>% html_table(),
      home.starters = comments[35] %>% html_text() %>% read_html() %>% html_node("#home_starters") %>% html_table(),
      vis.starters = comments[36] %>% html_text() %>% read_html() %>% html_node("#vis_starters") %>% html_table(),
      home.snap.counts = comments[37] %>% html_text() %>% read_html() %>% html_node("#home_snap_counts") %>% html_table(),
      vis.snap.counts = comments[38] %>% html_text() %>% read_html() %>% html_node("#vis_snap_counts") %>% html_table(),
      targets.directions = comments[39] %>% html_text() %>% read_html() %>% html_node("#targets_directions") %>% html_table(),
      rush.directions = comments[40] %>% html_text() %>% read_html() %>% html_node("#rush_directions") %>% html_table(),
      pass.tackles = comments[41] %>% html_text() %>% read_html() %>% html_node("#pass_tackles") %>% html_table(),
      rush.tackles = comments[42] %>% html_text() %>% read_html() %>% html_node("#rush_tackles") %>% html_table(),
      home.drives = comments[43] %>% html_text() %>% read_html() %>% html_node("#home_drives") %>% html_table(),
      vis.drives = comments[44] %>% html_text() %>% read_html() %>% html_node("#vis_drives") %>% html_table(),
      pbp = comments[45] %>% html_text() %>% read_html() %>% html_node("#pbp") %>% html_table()
    )
}

# ALL WEEK ONE LIST OF HTML TABLE(S) DATA
week1datalist <- lapply(games, getwebdata)

# TRY/CATCH VERSION (ANY PARSING ERROR TO RETURN EMPTY LIST)
week1datalist <- lapply(games, function(g) {
   tryCatch({ return(getwebdata(g)) 
      }, error = function(e) return(list())
})

# NAME EACH LIST ELEMENT BY CORRESPONDING GAME
shortgames <- gsub("/", "", gsub(".htm", "", games))
week1datalist <- setNames(week1datalist, shortgames)

最终，您可以按名称引用某个游戏的特定统计数据表：

week1datalist$boxscores201609080den$scoring

week1datalist$boxscores201609110atl$game.info

此外，您可能需要在 lapply 中包含 tryCatch，因为某些页面可能不一致。