使用 rvest 在网页抓取时组合数据帧

Question

我有以下代码

 library('rvest')
 library('tidyverse')
 test_url <- c('http://www.citact.org/senator-brent-waltz-r-greenwood-
 district-36',
               'http://www.citact.org/senator-ron-grooms-r-new-albany-
 district-46',
               'http://www.citact.org/representative-mike-speedy-r-
 indianapolis-district-90')
 test <- lapply(test_url, function(i){
   web <- read_html(i)
   grades <- html_nodes(web, 'td')
   test_grades <- data.frame(one = (as.data.frame(html_text(grades), two = 
 'idk')))

    first <- as.data.frame(test_grades[2:11, ])
    second <- as.data.frame(test_grades[13:22, ])

    names(test_grades) <- names(test_grades)
   testing <- data.frame(c(first, second))
 })

 test_names <- lapply(test_url, function(i){
   web <- read_html(i)
   info <- html_nodes(web, 'h3')
    text_info <- html_text(info)

   names_test_df <- data_frame(member = text_info)
   names_test_df <- separate(names_test_df, col = member, c('Useless', 
  'Info'), sep = ': ')
    names_test_df <- separate(names_test_df, col = Info, c('names', 
 'District'), sep = ',')
   names_test_df <- separate(names_test_df, col = names, c('Position', 
  'First', 'Last', 'Party')
                        , sep = ' ')
   names_test_df <- separate(names_test_df, col = Party, c('Party','District 
 Name'), sep = '-')
 })

 y <- do.call(cbind.fill, c(list(do.call(rbind, test)), do.call(rbind, 
 test_names)))

这在收集所有信息并且没有错误的意义上有效，但问题在于我称为 y 的最终数据框。当我创建 y 时，数据框测试和数据框 test_names 不匹配。例如，测试中的某些成绩和年份与 test_names 中的正确候选人不匹配。有没有办法确保这些正确对应？我尝试在循环它们之前组合数据帧，但我没有成功。可能有更好的方法，这只是我最初的计划。

Answer 1

尽量避免在每第二行中重新整形和绑定和 do.call，这会使您的代码难以阅读，因此也很难找到错误。我继续并决定让它更简单一些。

person_xpath <- "//h1[contains(@class, 'title gutter')]"

url_tables <- lapply(test_url, function(x){
  # Scraping the information (note, connecting to URL once)
  page           <- read_html(x) 
  vote_outcome   <- t(html_table(page)[[1]])
  personal_info  <- html_nodes(page, xpath = person_xpath)
  personal_info  <- html_text(personal_info)

  # Remove some useless characters and split the string
  personal_info  <- gsub("\\(|\\) |,", "", personal_info)
  split_person   <- strsplit(personal_info, " ")[[1]]

  # Prepare the personal info for a cbind:
  dupe_info      <- sapply(split_person, rep, nrow(vote_outcome))
  if(ncol(dupe_info) == 7){
    dupe_info[,4] <- paste(dupe_info[,4], dupe_info[,5])
    dupe_info     <- dupe_info[,-5]
  }
  df             <- cbind(dupe_info, vote_outcome)[,-5]
  colnames(df)   <- c("Position", "First", "Last", "Party", 
                      "District", "Year", "Outcome")
  return(df)
})

url_tables[[1]]
#     Position  First   Last    Party         District Year    
# X1  "Senator" "Brent" "Waltz" "R-Greenwood" "36"     " "    
# X2  "Senator" "Brent" "Waltz" "R-Greenwood" "36"     "2008" 
# X3  "Senator" "Brent" "Waltz" "R-Greenwood" "36"     "2009" 
# X4  "Senator" "Brent" "Waltz" "R-Greenwood" "36"     "2010" 
# ..     ...      ...     ...       ...        ...       ...

# X1  "Pro-Consumer Voting Percentage"
# X2  "0%"                            
# X3  "57%"                           
# X4  "66%"                           
# ..   ...