记录未附加到 R 中的数据框中

Question

我有一个数据框 uuu_df，其中记录为网站链接

dim(uuu_df)
output
1950  1

uuu_df

1) http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=1&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs
2) http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=2&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs
3) http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=3&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs
.
.
.
1950) http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=>5&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=20-Crores&BudgetMax=20-Crores

在这里，我正在尝试使用数据框中的多个链接以及条件来抓取数据，即如果 html 属性的文本等于 "No Results Found!" 然后跳过该记录并继续下一个记录， 这是那个抓取的sn-p

UrlPage <- html("http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=2&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs")
ImgNode <- UrlPage %>% html_node("div.noResultHead")
u=ImgNode
u=as(u,"character")
u=paste("No",word(string = u, start = 4, end = 5),sep = " ")

这是我尝试过的

 wines=data.frame()
    url_test=c()
    UrlPage_test=c()
    u=c()
    ImgNode=c()

    for(i in 1:dim(uuu_df)[1]){

      url_test[i]=as.character(uuu_df[i,])
      UrlPage_test[i] <- html(url_test[i])
      ImgNode[i] <- UrlPage_test[i] %>% html_node("div.noResultHead")
      u[i]=ImgNode[i]
      u[i]=as(u[i],"character")
      u[i]=paste("No",word(string = u, start = 4, end = 5),sep = " ")

      if(u[i]=="No Results Found!") next
      {
        map_df(1:5, function(i) # here 1:5 is number of webpages of a website 
{

          # simple but effective progress indicator
          cat(".")

          pg <- read_html(sprintf(url_test, i))

          data.frame(wine=html_text(html_nodes(pg, ".agentNameh")),
                     excerpt=html_text(html_nodes(pg, ".postedOn")),
                     locality=html_text(html_nodes(pg,".localityFirst")),
                     society=html_text(html_nodes(pg,'.labValu .stop-propagation:nth-child(1)')),
                     stringsAsFactors=FALSE)

        }) -> wines

      }

但是 Wines 数据框给了我空行和列的空数据框 为什么它不能在其中追加行。 任何建议都会有所帮助。提前致谢

P.S: 可重现数据的 dput()

text1="http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom="
text2="1"
text3="&proptype="
text4="Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment"
text5="&cityName=Thane&BudgetMin="
text6="&BudgetMax="



bhk=c("1","2","3","4","5",">5")
budg_min=c("5-Lacs","10-Lacs","20-Lacs","30-Lacs","40-Lacs","50-Lacs","60-Lacs","70-Lacs","80-Lacs","90-Lacs","1-Crores","1.2-Crores","1.4-Crores","1.6-Crores","1.8-Crores","2-Crores","2.3-Crores","2.6-Crores","3-Crores","3.5-Crores","4-Crores","4.5-Crores","5-Crores","10-Crores","20-Crores")
budg_max=c("5-Lacs","10-Lacs","20-Lacs","30-Lacs","40-Lacs","50-Lacs","60-Lacs","70-Lacs","80-Lacs","90-Lacs","1-Crores","1.2-Crores","1.4-Crores","1.6-Crores","1.8-Crores","2-Crores","2.3-Crores","2.6-Crores","3-Crores","3.5-Crores","4-Crores","4.5-Crores","5-Crores","10-Crores","20-Crores")
eg <- expand.grid(bhk = bhk, budg_min = budg_min, budg_max = budg_max)
eg <- eg[as.integer(eg$budg_min) <= as.integer(eg$budg_max),]
uuu <- sprintf("%s%s%s%s%s%s%s%s", text1,eg[,1],text3,text4,text5,eg[,2],text6,eg[,3])
uuu_df=data.frame(Links=uuu)
dput(uuu_df)

Answer 1

您应该利用文档树来始终如一地找到您需要的元素并控制循环或矢量化函数的流程。在下面的示例中，我检查结果计数以确定是否有结果，然后单独解析每个节点以确保其一致。最后，您可以根据需要绑定它们。

旁注：llply 具有 .progress 参数，可以更优雅地处理您尝试使用 cat() 设计的进度指示器。

options(stringsAsFactors = FALSE)
library(plyr)
library(dplyr)
library(xml2)

uuu_df <- data.frame(x = c('http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=1&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs',
                           'http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=2&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=5-Lacs',
                           'http://www.magicbricks.com/property-for-sale/residential-real-estate?bedroom=3&proptype=Multistorey-Apartment,Builder-Floor-Apartment,Penthouse,Studio-Apartment&cityName=Thane&BudgetMin=5-Lacs&BudgetMax=90-Lacs'))

urlList <- llply(uuu_df[,1], function(url){     

  this_pg <- read_html(url)

  results_count <- this_pg %>% 
    xml_find_first(".//span[@id='resultCount']") %>% 
    xml_text() %>%
    as.integer()

  if(results_count > 0){

    cards <- this_pg %>% 
      xml_find_all('//div[@class="SRCard"]')

    df <- ldply(cards, .fun=function(x){
      y <- data.frame(wine = x %>% xml_find_first('.//span[@class="agentNameh"]') %>% xml_text(),
                      excerpt = x %>% xml_find_first('.//div[@class="postedOn"]') %>% xml_text(),
                      locality = x %>% xml_find_first('.//span[@class="localityFirst"]') %>% xml_text(),
                      society = x %>% xml_find_first('.//div[@class="labValu"]') %>% xml_text() %>% gsub('\\n', '', .))
      return(y)
    })

  } else {
    df <- NULL
  }

  return(df)   
}, .progress = 'text')
names(urlList) <- uuu_df[,1]

bind_rows(urlList)

Answer 2

考虑使用lapply 构建的一个大型列表，该列表遍历数据框的 url 列，而不是管理许多较小的向量：

urlList <- lapply(uuu_df[1,], function(url){     

    UrlPage <- html(as.character(url))
    ImgNode <- UrlPage %>% html_node("div.noResultHead")
    u <- paste("No", word(string = as(ImgNode, "character"), start=4, end=5), sep=" ")

    cat(".")        
    pg <- read_html(url)

    if(u!="No Results Found!") {
        df <- data.frame(wine=html_text(html_nodes(pg, ".agentNameh")),
                         excerpt=html_text(html_nodes(pg, ".postedOn")),
                         locality=html_text(html_nodes(pg,".localityFirst")),
                         society=html_text(html_nodes(pg,'.labValu .stop-propagation:nth-child(1)')),
                         stringsAsFactors=FALSE)
    } else {
        # ASSIGN EMPTY DATAFRAME (FOR CONSISTENT STRUCTURE)
        df <- data.frame(wine=c(), excerpt=c(), locality=c(), society=c())
    }
    # RETURN NAMED LIST
    return(list(UrlPage=UrlPage, ImgNode=ImgNode, u=u, df=df))    
})

# ROW BIND ONLY DATAFRAME ELEMENT FROM LIST
wines <- map_df(urlList, function(u) u$df)