使用 R、rvest 或 rcurl 抓取文本文件

Question

所以我有一个网站，https://ais.sbarc.org/logs_delimited/，它有一堆链接，每个链接中有 24 个链接，其中包含 .txt 文件。

我是 R 新手，但我可以通过一个链接循环将 24 个文本文件放入数据框中。但我不知道如何循环整个目录。

我可以使用 hours.list 循环 24 个链接，但是 year.list 和 trip.list 不起作用... 如果这与其他网络爬虫问题相似，或者我遗漏了一些非常简单的东西，我深表歉意，但我会很感激任何帮助

get_ais_text = function(ais_text){

    hours.list = c(0:23)
    hours.list_1 = sprintf('%02d', hours.list)

    year.list = c(2018:2022)
    year.list1 = sprintf('%d', year.list)

    trip.list = c(190101:191016)
    trip.list_1 = sprintf("%d", trip.list)

ais_text = tryCatch(    
lapply(paste0('https://ais.sbarc.org/logs_delimited/2019/190101/AIS_SBARC_190101-', hours.list_1,'.txt'),
                    function(url){
                      url %>% 
                        read_delim(";", col_names = sprintf("X%d", 1:25), col_types = ais_col_types)                   
                    }),
      error = function(e){NA}
    )
  DF = do.call(rbind.data.frame, ais_text)
  return(DF)
}

get_ais_text()

Answer 1

这是一个递归地工作以获取从主目录开始的所有链接的函数。请注意，运行需要一点时间：

library(xml2)
library(magrittr)
.get_link <- function(u){
  node <- xml2::read_html(u)
  hrefs <- xml2::xml_find_all(node, ".//a[not(contains(@href,'../'))]") %>% xml_attr("href")
  urls <- xml2::url_absolute(hrefs, xml_url(node))
  if(!all(tools::file_ext(urls) == "txt")){
    lapply(urls, .get_link)
  }else {
    return(urls)
  }
}

这基本上是从url 开始，然后阅读内容，使用xpath selector 查找任何链接<a...，上面写着“所有不是../ 的链接”即...不是最顶层的目录反向链接。然后，如果链接有更多链接，则循环并获取所有链接。如果我们有最终链接，即 .txt 文件，我们就完成了。

仅从 2018 年开始的作弊示例

a <- .get_link("https://ais.sbarc.org/logs_delimited/2018/")
> a[[1]][1:2]
[1] "https://ais.sbarc.org/logs_delimited/2018/180101/AIS_SBARC_180101-00.txt"
[2] "https://ais.sbarc.org/logs_delimited/2018/180101/AIS_SBARC_180101-01.txt"
> length(a)
[1] 365
> a[[365]][1:2]
[1] "https://ais.sbarc.org/logs_delimited/2018/181231/AIS_SBARC_181231-00.txt"
[2] "https://ais.sbarc.org/logs_delimited/2018/181231/AIS_SBARC_181231-01.txt"

您要做的只是从：https://ais.sbarc.org/logs_delimited/ 开始输入网址，然后添加类似data.table::fread 的内容来消化数据。我建议在单独的迭代中进行。这样的工作：

lapply(1:length(a), function(i){
    lapply(a[[i]], data.table::fread)
})

用于读取数据...

首先要注意的是有 11,636 个文件。有很多链接可以同时访问某人的服务器......所以我将抽样一些并展示如何做到这一点。我建议在您的电话中添加一个Sys.sleep...

# This gets all the urls
a <- .get_link("https://ais.sbarc.org/logs_delimited/")
# This unlists and gives us a unique array of the urls
b <- unique(unlist(a))
# I'm sampling b, but you would just use `b` instead of `b[...]`
a_dfs <- jsonlite::rbind_pages(lapply(b[sample(1:length(b), 20)], function(i){
    df <- data.table::fread(i, sep = ";") %>% as.data.frame()
    # Giving the file path for debug later if needed seems helpful
    df$file_path <- i
    df
}))

> a_dfs %>% head()
  17:00:00:165              24  0 338179477 LAUREN SEA        V8 V9   V15 V16 V17 V18 V19 V20 V21 V22 V23                                                                file_path   V1   V2 V3 V4
1 17:00:00:166     EUPHONY ACE 79     71.08          1 371618000  0 254.0 253  52   0   0   0   0   5  NA https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
2 17:00:01:607 SIMONE T BRUSCO 31     32.93          3 367593050 15 255.7  97  55   0   0   1   0 503   0 https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
3 17:00:01:626 POLARIS VOYAGER 89    148.80          1 311000112  0 150.0 151  53   0   0   0   0   0  22 https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
4 17:00:01:631         SPECTRE 60     25.31          1 367315630  5 265.1 511  55   0   0   1   0   2  20 https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
5 17:00:01:650          KEN EI 70     73.97          1 354162000  0 269.0 269  38   0   0   0   0   1  84 https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
6 17:00:02:866 HANNOVER BRIDGE 70     62.17          1 372104000  0 301.1 300  56   0   0   0   0   3   1 https://ais.sbarc.org/logs_delimited/2018/180113/AIS_SBARC_180113-17.txt <NA> <NA> NA NA
  V5 V6 V7 V10 V11 V12 V13 V14 02:00:00:489 338115994  1 37 SRTG0$ 10  7  4 17:00:00:798 BROADBILL 16.84 269   18 367077090 16.3 -119.981493 34.402530 264.3 511 40
1 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA
2 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA
3 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA
4 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA
5 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA
6 NA NA NA  NA  NA  NA  NA  NA         <NA>        NA NA NA     NA NA NA NA         <NA>      <NA>    NA  NA <NA>      <NA>   NA          NA        NA    NA  NA NA

显然需要做一些清洁工作.. 但我认为这就是你如何做到的。

编辑 2

我其实更喜欢这个，读入数据，然后拆分字符串并强制创建数据帧：

a_dfs <- rbind_pages(lapply(b[sample(1:length(b), 20)], function(i){
    raw <- readLines(i)
    str_matrix <- stringi::stri_split_regex(raw, "\\;", simplify = TRUE)
    as.data.frame(apply(str_matrix, 2, function(j){
        ifelse(!nchar(j), NA, j)
    })) %>% mutate(file_name = i)
}))

> a_dfs %>% head
            V1           V2 V3    V4    V5 V6 V7        V8 V9 V10  V11 V12         V13       V14   V15 V16 V17 V18 V19 V20 V21 V22  V23  V24  V25
1 09:59:57:746    STAR CARE 77 75.93   135  1  0 566341000  0   0 16.7   1 -118.839933 33.562167   321 322  50   0   0   0   0   6   19 <NA> <NA>
2 10:00:00:894     THALATTA 70 27.93 133.8  1  0 229710000  0 251 17.7   1 -119.366765 34.101742 283.9 282  55   0   0   0   0   7 <NA> <NA> <NA>
3 10:00:03:778   GULF GLORY 82 582.3   256  1  0 538007706  0   0 12.4   0 -129.345783 32.005983    87  86  54   0   0   0   0   2   20 <NA> <NA>
4 10:00:03:799    MAGPIE SW 70 68.59 123.4  1  0 352597000  0   0 10.9   0 -118.747970 33.789747 119.6 117  56   0   0   0   0   0   22 <NA> <NA>
5 10:00:09:152 CSL TECUMSEH 70 66.16 269.7  1  0 311056900  0  11   12   1 -120.846763 34.401482 105.8 106  56   0   0   0   0   6   21 <NA> <NA>
6 10:00:12:870    RANGER 85 60 31.39 117.9  1  0 367044250  0 128    0   1 -119.223133 34.162953   360 511  56   0   0   1   0   2   21 <NA> <NA>
                                                                 file_name  V26  V27
1 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>
2 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>
3 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>
4 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>
5 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>
6 https://ais.sbarc.org/logs_delimited/2018/180211/AIS_SBARC_180211-10.txt <NA> <NA>

Answer 2

这对我有用：

library(rvest)

crawler <- function(base_url) {

  get_links <- function(url) {
    read_html(url) %>% 
      html_nodes("a") %>% 
      html_attr("href") %>% 
      grep("../", ., fixed = TRUE, value = TRUE, invert = TRUE) %>% 
      url_absolute(url)
  }

  links <- base_url
  counter <- 1

  while (sum(grepl("txt$", links)) != length(links)) {
    links <- unlist(lapply(links, get_links))
    message("scraping level ", counter, " [", length(links), " links]")
    counter <- counter + 1
  }

  return(links)

}

txts <- crawler("https://ais.sbarc.org/logs_delimited/")

好像要放弃第 3 级了，但这只是因为要经过的链接太多了。

一旦你有了所有的 txt url，你就可以用它来读入文件：

library(dplyr)
library(data.table)

df <- lapply(txts, fread, fill = TRUE) %>% 
  rbindlist() %>% 
  as_tibble()

我肯定会分两步执行此操作，因为它会运行很长一段时间，并且保存中间结果（即链接）是有意义的。

如果需要，您也可以尝试并行运行（cl 是要使用的内核数）：

library(pbapply)             

df <- pblapply(txts[1:10], fread, fill = TRUE, cl = 3) %>% 
  rbindlist() %>% 
  as_tibble()

应该快一点，你也会得到一个不错的进度条。