在 download.file ... 无法打开 URL ... HTTP 状态为 '404 Not Found'

Question

感谢 StackOverflow，我能够使用以下代码在公共网站上下载一系列照片。

urls <- c("https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0090/13", 
"https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0089/13", 
"https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0088/13", 
"https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0087/13", 
"https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0086/13"
)

for (url in 1:length(urls)) {

  print(url)
  webpage <- html_session(urls[url])
  link.titles <- webpage %>% html_nodes("img")
  img.url <- link.titles %>% html_attr("src")

  for(j in 1:length(img.url)){

    download.file(img.url[j], paste0(url,'.',j,".jpg"), mode = "wb")
  }

}

但是，某些链接不包含照片，因此返回 HTTP 状态错误并停止下载过程。

所以，我想插入一个if 命令并告诉 R 忽略/绕过那些不包含照片或“404 Not Found”错误的页面。然而，问题是，我不知道什么函数或命令会代表没有图像或“404 Not Found”错误的页面。任何建议，将不胜感激。

Answer 1

library(purrr)
library(rvest)
library(httr)

urls <- c(
  "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0090/13", 
  "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0089/13", 
  "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0088/13", 
  "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0087/13", 
  "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0086/13"
)

sGET <- safely(GET)                                           # make a "safe" version of httr::GET

map(urls, read_html) %>%                                      # read each page
  map(html_nodes, "img") %>%                                  # extract img tags
  flatten() %>%                                               # convert to a simple list
  map_chr(html_attr, "src") %>%                               # extract the URL
  walk(~{                                                     # for each URL
    res <- sGET(.x)                                           # try to retrieve it
    if (!is.null(res$result)) {                               # if there were no fatal errors
      if (status_code(res$result) == 200) {                   # and, if found
        writeBin(content(res$result, as="raw"), basename(.x)) # save it to disk
      }
    }
  })

是一种替代的、实用的、“安全”的方式。

Answer 2

只需使用函数“try”：

urls <- c("https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0090/13", 
          "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0089/13", 
          "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0088/13", 
          "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0087/13", 
          "https://ec.europa.eu/consumers/consumers_safety/safety_products/rapex/alerts/?event=viewProduct&reference=A12/0086/13"
)

for (url in 1:length(urls)) {

  print(url)
  webpage <- html_session(urls[url])
  link.titles <- webpage %>% html_nodes("img")
  img.url <- link.titles %>% html_attr("src")

  for(j in 1:length(img.url)){

    try(download.file(img.url[j], paste0(url,'.',j,".jpg"), mode = "wb")
        ,silent = TRUE)

}

另外，还可以添加条件“if”：

for (url in 1:length(urls)) {

  print(url)
  webpage <- html_session(urls[url])
  link.titles <- webpage %>% html_nodes("img")
  img.url <- link.titles %>% html_attr("src")

  for(j in 1:length(img.url)){

    try_download <- try(
      download.file(img.url[j], paste0(url,'.',j,".jpg"), mode = "wb")
      ,silent = TRUE)

    if(is(try_download,"try-error")){
      print(paste0("ERROR: ", img.url[j]))
    }else{
      print(paste0("Downloaded: ", img.url[j]))
    }

}