两个结构的猫：不同的字段

Question

我有多个 csv 文件

a.csv

field_a, field_b
111,     121
112,     122

b.csv

field_a, field_c
211,     231
212,     232

c.csv

field_a, field_b, field_c
311,     321,     331
312,     322,     332

我想将它们连接起来

输出.csv

field_a,field_b,field_c
111,    121,    NA
112,    122,    NA
211,    NA,     231
212,    NA,     232
311,    321,    331
312,    322,    332

我想用八度音来做这个。

到目前为止我做了什么：

a=csv2cell(a.csv)
A=cell2struct(a(2:end,:),a(1,:),1)

现在我正在寻找类似的东西

合并(A,B,C) 或者 vertcat(A,B,C)

但我没明白，所有字段都在输出中。

R 我这样做了：

 filelist<-list.files() 
 for (i in 1:length(filelist)) {
  datas[[i]]<-list(as.data.frame(read.csv(filelist[i])))
  merged <- merge(merged,datas[[i]], all=TRUE)}

但是 for 循环非常慢。所以我正在寻找一种将它们一次合并的可能性。

Answer 1

plyr 包中的rbind.fill 应该可以完美处理这个问题：

require(plyr)
rbind.fill(a,b,c)

#   field_a field_b field_c
# 1     111     121      NA
# 2     112     122      NA
# 3     211      NA     231
# 4     212      NA     232
# 5     311     321     331
# 6     312     322     332

Answer 2

我不确定 octave - 但在 Matlab 中我会使用 fieldnames 并设置函数。

在伪代码中是这样的：

all_fields = union of fieldnames(a), fieldnames(b) and fieldnames(c)
for each variable:
   missing_fields = setdiff(all_fields,fieldnames)
   add the missing fields
then join

Answer 3

我最后是怎么做到的：

使用 Octave (MATLAB)

% FileNames=readdir(pwd);
d=dir(pwd);

isDirIdx = [d.isdir];
names = {d.name};
FileNames = names(~isDirIdx);

for ii = 1:numel(FileNames)
  % Load csv to cell
  datas{ii}=csv2cell(FileNames{ii});
  %   Then I convert them to a struct
  Datas{ii}=cell2struct((datas{ii}(2:end,:)),[datas{ii}(1,:)],2);
  try fields=[fields, fieldnames(Datas{ii})'];% fails for the first loop, becauce 'fields' doesn't exist yet
  catch
   fields=[fieldnames(Datas{ii})'];  % create 'fields' in the first loop
  end
  Datalenght(ii)=numel(Datas{ii}(1));
end

cd(startdir)

for jj=1:numel(Datas)
  missing_fields{jj} = setdiff(fields,fieldnames(Datas{jj}));
  for kk=1:numel(missing_fields{jj})
    [Datas{jj}.(missing_fields{jj}{kk})]=deal(NaN);%*zeros(numel(datas{jj}(2:end,1)),1);)
  end
end

问题是，我没有看到将结构导出到 csv 的简单方法。所以我切换回 R。因为我没有足够的内存，我无法加载 r 中的所有文件并将它们导出为一个 csv。所以首先我将每个 netcdf 文件导出到具有完全相同值的 csv 中。然后我用 unix/gnu cat 命令将它们全部连接起来。

R：

# Converts all NetCDF (*.nc) in a folder to ASCII (csv)
# when there are more then one, all csv will have the same fields
# when there is a field missing in one NetCDF file, this scripts adds 'NA' Values

# it saves memory, because there is always only one NetCDF-File in the memory.

# Needs package RNetCDF:
# http://cran.r-project.org/web/packages/RNetCDF/index.html
# load package
library('RNetCDF')

# get list of all files to merge
filelist<-list.files() 

# initialise variable names
varnames_all<-{}
varnames_file<-list(filelist)

n_files<-length(filelist)
n_vars<-rep(NA,n_files) # initialise

# get variables-names of each NetCDF file
for (i in 1:n_files) {
  ncfile<-open.nc(filelist[i]) # open nc file
  print(paste(filelist[i],"opend!"))

  # get number of variable in the NetCDF
  n_vars[i]<-file.inq.nc(ncfile)$nvars 
  varnames="" # initialise and clear

  # read every variable name
  for (j in 0:(n_vars[i]-1)) {
    varnames[j]<-var.inq.nc(ncfile,j)$name
  }
  close.nc(ncfile)
  varnames_file[[i]]<-varnames # add to the list of all files
  varnames_all<-(c(varnames_all,varnames)) # concat to one array
}

varnames_all<-unique(varnames_all)  # take every varname only once
print("Existing variable names:")
print(varnames_all)

#initialise a data.frame for load the NetCDF
datas<-data.frame() 

for (i in 1:length(filelist)) {
  print(filelist[i])
  ncfile<-open.nc(filelist[i]) # open nc file
  print(paste("reading ", filelist[i], "...")) 
  datas<-as.data.frame(read.nc(ncfile))  #import data from ncfile as data frame
  close.nc(ncfile)

  #check witch variables are missing
  missing_vars<-setdiff(varnames_all,colnames(datas))

  # Add missing variables a colums with NA
  datas[missing_vars]<-NA
  print(paste("writing ", filelist[i], " to ", filelist[i],".csv ...", sep=""))

  #reorder colum in the same way as in the array varname_all
  datas<-datas[varnames_all] 

  # Write File
  write.csv(datas,file=paste(filelist[i],".csv", sep=""))

  # clear Memory
  rm(datas)
}

那么猫就直接向前了

#!/bin/bash
# Concatenate csv files, whitch have exactly the same fields  

## Change to the directory, from where the files is executed
path=$PWD
cd $path

if [ $# -gt 0 ]; then
  cd $1
fi

# get a list of all data files
datafile_list=$( ls )
read -a datafile_array <<< $datafile_list
echo "copying files ..."
echo "copying file:" ${datafile_array[0]}

cat < ./${datafile_array[0]} > ../outputCat.csv
for  (( i=1; i<${#datafile_array[@]}; i++))
  do
  echo "copying file" ${datafile_array[$i]}
  cat < ./${datafile_array[$i]} | tail -n+2 >> ../outputCat.csv
done