大型数据库的数据重塑和聚合

Question

我有一个大型数据集，我想重塑和聚合，但遇到了一些困难。

例如，考虑以下情况，其中引号分隔数据库中的列。因子为year、habitat、site 和replicate。 species 和 biomass 字段是数据：

"year" "habitat" "site" "replicate" "species" "biomass"
 2010    inner       a          1       sp.1        10
 2010    inner       a          1       sp.3         6
 2010    inner       a          1       sp.1         5
 2010    inner       a          1       sp.2         8
 2010    inner       a          1       sp.4         4
 2010    inner       a          1       sp.5         7
 2010    inner       a          2       sp.3         5
 2010    inner       a          2       sp.2         6
 2010    inner       a          2       sp.5         2
 2010    inner       a          2       sp.1         5
 2010    inner       a          3       sp.4         5
 2010    inner       a          3       sp.3         4
 2010    inner       a          3       sp.6         8
 2010    inner       a          3       sp.2         5
 2010    outer       b          1       sp.1         6
 2010    outer       b          1       sp.3         9
 2010    outer       b          1       sp.3         3
 2010    outer       b          1       sp.2         6
 2010    outer       b          2       sp.5         4
 2010    outer       b          2       sp.1         5
 2010    outer       b          2       sp.1         7
 2010    outer       b          2       sp.2         5
 2010    outer       b          3       sp.4         2
 2010    outer       b          3       sp.6         5
 2010    outer       b          3       sp.2         4
 2010    outer       b          3       sp.1         4   

我想做的是在保留所有因素的同时重塑和汇总我的表格。因此，species by site 矩阵能够将因子和聚合保持在复制级别。我在下面包含了一张我想要的示例图片，其中数据使用 Excel 数据透视表填充了“总和”函数。 R 中是否有使用dplyr 和组的简单代码？ tapply 和 aggregate?

year    habitat site    replicate   sp.1    sp.2    sp.3    sp.4    sp.5    sp.6
2010    inner    a          1        15      8       6       4       7       0
2010    inner    a          3        0       5       4       5       0       8
2010    outer    b          1        6       6       12      0       0       0
2010    outer    b          2        12      5       0       0       4       0
2010    outer    b          3        4       4       0       2       0       5

Answer 1

首先，如果您有一个大型数据集，我建议您使用数据表。

我经常使用“BreakBuild”功能，但我无法在您的代码上进行测试。

library(data.table)
mydata<-data.table(read.table("mydata"))

aggmydata<- mydata[,list(biomass=sum(biomass)),by=list(year,habitat,site,replicate,species)]

BreakBuild <- function (df,BCol,IDCols,ValCols) {
  setkeyv(df,BCol)
  NewCols <- unique(df[[BCol]]); #Scan the column for unique entries.  Each entry gets a data table.
  ldt <- list(); #List that will hold a data table for each break.
  ColList <-c(IDCols,ValCols) # List of columns, does not include broken collumn.
  for (bframe in NewCols) {
    ldt[[bframe]] = df[bframe,ColList, with=FALSE] #Create and store a data table with columns from IDCols and VolCols.
    if(length(ValCols)>1){setnames(ldt[[bframe]], ValCols, paste(bframe,ValCols,sep="."))} #Prefix the Value columns with the name of the break.
    else {setnames(ldt[[bframe]], ValCols, bframe)}  #If there is only one Value Column, give it the name of the break.
  }
  Build<-Reduce(function(...) merge(...,by=IDCols,all=T),ldt)
  return(Build)
}

waggmydata<- BreakBuild(aggmydata,BCol="species",
                        IDCols=c("year","habitat","site","replicate"),
                        ValCols="biomass")

您也可以使用 reshape2 和 dcast，尽管我对 dcast 的语法不是很熟悉。会是这样的

library(reshape2)
waggmydata<-dcast.data.table(mydata,  year+habitat+site+replicate~ species, fun=sum)

如果您的数据集非常大（数百万行），那么使用 data.table 可能是值得的，尽管自从我上次尝试 dplyr 以来似乎取得了重大进展。

以下是测试的三种方法的基准测试结果。

require(data.table)
require(reshape2)
require(dplyr)
require(tidyr)
require(microbenchmark)
BreakBuild <- function (df,BCol,IDCols,ValCols) {
  setkeyv(df,BCol)
  NewCols <- unique(df[[BCol]]); #Scan the column for unique entries.  Each entry gets a data table.
  ldt <- list(); #List that will hold a data table for each break.
  ColList <-c(IDCols,ValCols) # List of columns, does not include broken collumn.
  for (bframe in NewCols) {
    ldt[[bframe]] = df[bframe,ColList, with=FALSE] #Create and store a data table with columns from IDCols and VolCols.
    if(length(ValCols)>1){setnames(ldt[[bframe]], ValCols, paste(bframe,ValCols,sep="."))} #Prefix the Value columns with the name of the break.
    else {setnames(ldt[[bframe]], ValCols, bframe)}  #If there is only one Value Column, give it the name of the break.
  }
  Build<-Reduce(function(...) merge(...,by=IDCols,all=T),ldt)
  return(Build)
}

mydata<-data.table(year=sample(c("2010","2011","2012"),1e6,replace=T),
                   habitat=sample(c("inner","outer"),1e6,replace=T),
                   site=sample(letters[1:15],1e6,replace=T),
                   replicate=sample(1:5,1e6,replace=T),
                   species=sample(paste("sp",1:20,sep="."),1e6,replace=T),
                   biomass=sample(1:30,1e6,replace=T))
dat1<- as.data.frame(mydata)


microbenchmark(
  DPLYR= {
    dat1%>% 
      group_by(year, habitat,site,replicate, species)%>%
      summarise(biomass=sum(biomass)) %>%
      spread(species, biomass,fill=0)
  },
  DATATABLE = {
    aggmydata<- mydata[,list(biomass=sum(biomass)),by=list(year,habitat,site,replicate,species)]
    waggmydata<- BreakBuild(aggmydata,BCol="species",
                            IDCols=c("year","habitat","site","replicate"),
                            ValCols="biomass")
  },
  DCAST.DATA.TABLE = {
    waggmydata<-dcast.data.table(mydata,  year+habitat+site+replicate~ species, fun=sum)
  }
  )

##             expr       min       lq   median       uq      max neval
##            DPLYR 168.26559 170.3902 171.9306 173.9712 189.5266   100
##        DATATABLE  97.21738 101.1543 103.5157 108.2527 125.9114   100
## DCAST.DATA.TABLE 184.58250 189.4021 192.0251 195.3731 242.9994   100

一旦您了解语法，我会说 DCAST.DATA.TABLE 是本示例最容易编写代码的方法。

Answer 2

使用dplyr（如果dat1 是数据集）

  library(dplyr)
  library(tidyr)

  dat1%>% 
  group_by(year, habitat,site,replicate, species)%>%
  summarise(biomass=sum(biomass)) %>%
  spread(species, biomass,fill=0)
  # Source: local data frame [6 x 10]

  #  year habitat site replicate sp.1 sp.2 sp.3 sp.4 sp.5 sp.6
  #1 2010   inner    a         1   15    8    6    4    7    0
  #2 2010   inner    a         2    5    6    5    0    2    0
  #3 2010   inner    a         3    0    5    4    5    0    8
  #4 2010   outer    b         1    6    6   12    0    0    0
  #5 2010   outer    b         2   12    5    0    0    4    0
  #6 2010   outer    b         3    4    4    0    2    0    5

顺便说一句：replicate 2 的栖息地 inner 未显示在预期输出中。