在 R 中应用 vs For 循环

Question

我编写了以下代码来每天从门户网站中删除招标信息。

packages <- c('rvest', 'stringi', 'tidyverse','lubridate','dplyr')
purrr::walk(packages, library, character.only = TRUE, warn.conflicts = FALSE)
start_time <- proc.time()

要废弃的主页并获取记录总数。

data <- read_html('https://eprocure.gov.in/mmp/latestactivetenders')
total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
All_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
links_fair <- html_attr(links,'href')
links_fair <- links_fair[grep("tendersfullview",links_fair)]
All_tenders <- cbind(All_tenders,links_fair)

读取要获取的记录总数

Count_of_Recs_raw <- html_nodes(data, xpath = '//*[(@id = "edit-l-active-teners")]//div')
Count_of_Recs <- as.numeric(gsub("Total Tenders : ","",html_text(Count_of_Recs_raw[1])))

用于清理和处理日期和因素等数据字段的函数。

process_dates <- function(data){
    cols2date <- c('Bid.Submission.Closing.Date','epublished_date','document_download_start_date','bid_submission_start_date','bid_opening_date','document_download_end_date','bid_submission_end_date')
    date_processed_data <- data
    date_processed_data[cols2date] <- lapply(data[cols2date] , dmy_hm)
    return(date_processed_data)
}

clean_process_data <- function(data){
    cols2factor <- c('State.Name','product_category','pre_qualification','organisation_name','organisation_type','tender_type')
    clean_processed_data <- data
    clean_processed_data[cols2factor] <- lapply(data[cols2factor] , factor)
   #clean_processed_data <- process_dates(clean_processed_data)
    return(clean_processed_data)

}

下面的代码正是我的问题所在...

表报废从这里开始。第一页已经被报废以获取数据框的结构。

for (page_no in 2:round(Count_of_Recs/10)){
    closeAllConnections()
    on.exit(closeAllConnections())
    url_bit1 <- 'https://eprocure.gov.in/mmp/latestactivetenders/page='
    url <- paste(url_bit1, page_no, sep="")
    cat(page_no,"\t",proc.time() - start_time,"\n")
    data <- read_html(url)
    total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
    Page_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
    links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
    links_fair <- html_attr(links,'href')
    links_fair <- links_fair[grep("tendersfullview",links_fair)]
    Page_tenders <- cbind(Page_tenders,links_fair)
    All_tenders <- rbind(All_tenders,Page_tenders)
 }

这个 for 循环通常需要几个小时才能完成。 我正在寻找使用 apply 系列来获得良好的效果，以便节省时间。 该程序还负责获取和处理所有记录，然后再次处理每个单独的记录每次都报废一个全新的页面（此处未列出代码）....

我尝试了以下代码，但它没有给我想要的：

url_bit1 <- 'https://eprocure.gov.in/mmp/latestactivetenders/page='
read_page <- function(datain){
   closeAllConnections()
   on.exit(closeAllConnections())
   url <- paste(url_bit1, datain$S.No., sep="")
   cat(S.No.,"\t",proc.time() - start_time,"\n")
   data <- read_html(url)
   total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
   Page_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
   links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
   links_fair <- html_attr(links,'href')
   links_fair <- links_fair[grep("tendersfullview",links_fair)]
   Page_tenders <- cbind(Page_tenders,links_fair)
   All_tenders <- rbind(All_tenders,Page_tenders)
}

All_tenders <- sapply(All_tenders, FUN=read_page(All_tenders$S.No.))

欢迎任何建议、指导、建议、意见或帮助。我只使用 R 3-4 个月。我也知道 Python 在这个问题上相对于 R 的优势，但我倾向于 R 来解决这个问题。

Answer 1

您的 sapply 函数不正确。我对您的代码进行了一些编辑，并在样本大小 N = 50 上对其进行了测试。我们可能会使用 system.time() 来找出完成任务所需的时间。

“为”方法：

system.time(
  for (page_no in 1:50){
    closeAllConnections()
    on.exit(closeAllConnections())
    url_bit1 <- 'https://eprocure.gov.in/mmp/latestactivetenders/page='
    url <- paste(url_bit1, page_no, sep="")
    cat(page_no,"\t",proc.time() - start_time,"\n")
    data <- read_html(url)
    total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
    Page_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
    links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
    links_fair <- html_attr(links,'href')
    links_fair <- links_fair[grep("tendersfullview",links_fair)]
    Page_tenders <- cbind(Page_tenders,links_fair)
    All_tenders <- rbind(All_tenders,Page_tenders)
  }
)

#user  system elapsed 
# 50.15   81.26  132.73

“lapply”方法：

All_tenders = NULL
url_bit1 <- 'https://eprocure.gov.in/mmp/latestactivetenders/page='
read_page <- function(datain){
  closeAllConnections()
  on.exit(closeAllConnections())
  url <- paste(url_bit1, datain, sep="")
  cat(datain,"\t",proc.time() - start_time,"\n")
  data <- read_html(url)
  total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
  Page_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
  links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
  links_fair <- html_attr(links,'href')
  links_fair <- links_fair[grep("tendersfullview",links_fair)]
  Page_tenders <- cbind(Page_tenders,links_fair)
  All_tenders <- rbind(All_tenders,Page_tenders)
}

system.time(
  All_tenders <- lapply(1:50, function(x) read_page(x))
)
# user  system elapsed 
# 49.84   78.97  131.16

如果我们想把我们的结果放在一个数据框中，然后将 All_tenders 列表转换为一个数据框，如下所示：

All_tenders = do.call(rbind, lapply(All_tenders, data.frame, stringsAsFactors=FALSE)

原来 lapply 稍微快一点。

Answer 2

for 循环和 sapply 工作方式不同： - for 循环迭代地做事情：它们在第一个元素上进行计算，然后在第二个元素上...... - sapply 独立（并且以任何顺序）在元素列表上做一些事情。所以结果是独立构造的。

所以在你的 for 循环中，当你这样做时：

All_tenders <- rbind(All_tenders,Page_tenders)

All_tenders变量迭代递增。

在您的 sapply 函数中，它不起作用（因为它不知道其他元素的结果）。

所以你应该这样做：

url_bit1 <- 'https://eprocure.gov.in/mmp/latestactivetenders/page='
read_page <- function(datain){
   closeAllConnections()
   on.exit(closeAllConnections())
   url <- paste(url_bit1,  datain, sep="")
   cat(S.No.,"\t",proc.time() - start_time,"\n")
   data <- read_html(url)
   total_tenders_raw <- html_nodes(data,xpath = '//*[(@id = "table")]')
   Page_tenders <- data.frame(html_table(total_tenders_raw, header = TRUE))
   links <- html_nodes(data, xpath='//*[(@id = "table")] | //td | //a')
   links_fair <- html_attr(links,'href')
   links_fair <- links_fair[grep("tendersfullview",links_fair)]
   Page_tenders <- cbind(Page_tenders,links_fair)
   return(Page_tenders)
}

为每个页面返回一个结果并通过以下方式应用它：

All_tenders_tmp <- sapply(2:round(Count_of_Recs/10), FUN=read_page)

然后您的结果将是所有结果的列表，您可以将其与 data.table::rbindlist 例如合并。

我希望我很清楚。