我正在处理 DTM，我想做 k-means、分层和 k-medoids 聚类。我想先规范化 DTM 吗？

Question

数据，AllBooks 有 8266 个变量的 590 个观测值。这是我的代码：

AllBooks = read_csv("AllBooks_baseline_DTM_Unlabelled.csv")
dtms = as.matrix(AllBooks)
dtms_freq = as.matrix(rowSums(dtms) / 8266)
dtms_freq1 = dtms_freq[order(dtms_freq),]
sd = sd(dtms_freq)
mean = mean(dtms_freq)

这告诉我我的平均值是：0.01242767 和我的标准。开发。是：0.01305608

因此，由于我的标准偏差较低，这意味着数据在文档大小方面的可变性较低。所以我不需要规范化 DTM？归一化是指使用 R 中的比例函数减去数据的平均值并除以标准偏差。

换句话说，我的大问题是：我应该什么时候标准化数据（特别是文档术语矩阵）以用于聚类目的？

这里有一点数据输出：

dput(head(AllBooks,10))
budding = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), enjoyer = c(0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0), needs = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), sittest = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), eclipsed = c(0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0), engagement = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 
    exuberant = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), abandons = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), well = c(0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0), cheerfulness = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 
    hatest = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), state = c(0, 0, 
    0, 0, 0, 0, 0, 0, 0, 0), stained = c(0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0), production = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), whitened = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), revered = c(0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0), developed = c(0, 0, 0, 2, 0, 0, 0, 0, 0, 0), 
    regarded = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), enactments = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), aromatical = c(0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0), admireth = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0
    ), foothold = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), shots = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), turner = c(0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0), inversion = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 
    lifeless = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), postponement = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), stout = c(0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0), taketh = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), kettle = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), erred = c(0, 0, 0, 0, 0, 0, 0, 
    0, 0, 0), thinkest = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), modern = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), reigned = c(0, 0, 0, 0, 0, 0, 
    0, 0, 0, 0), sparingly = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 
    visual = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), thoughts = c(0, 
    0, 0, 0, 0, 0, 0, 0, 0, 0), illumines = c(0, 0, 0, 0, 0, 
    0, 0, 0, 0, 0), attire = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0), 
    explains = c(0, 0, 0, 0, 0, 0, 0, 0, 0, 0)), class = c("tbl_df", 
"tbl", "data.frame"), row.names = c(NA, -10L))

您可以从链接查看完整数据：https://www.dropbox.com/s/p9v1y6oxith1prh/AllBooks_baseline_DTM_Unlabelled.csv?dl=0

Answer 1

您有一个稀疏数据集，其中大部分以零为主，因此标准差非常低。如果您的一些非零计数非常大，您可以对其进行缩放，例如，一些是 100，而另一些是 1 和 2。

在稀疏数据上使用 kmeans 可能不是一个好主意，因为您不太可能找到有意义的中心。可能有一些可用的选项，请查看this link on dimension reduction。还有基于图形的方法，例如this used in biology。

以下是聚类和可视化的简单方法：

x = read.csv("AllBooks_baseline_DTM_Unlabelled.csv")
# remove singleton columns
x = x[rowMeans(x)>0,colSums(x>0)>1]

在二进制距离上将其视为二进制和分层：

hc=hclust(dist(x,method="binary"),method="ward.D")
clus = cutree(hc,5)

计算 PCA 并可视化：

library(Rtsne)
library(ggplo2)

pca = prcomp(x,scale=TRUE,center=TRUE)
TS = Rtsne(pca$x[,1:30])
ggplot(data.frame(Dim1=TS$Y[,1],Dim2=TS$Y[,2],C=factor(clus)),
aes(x=Dim1,y=Dim2,col=C))+geom_point()

Cluster 5 似乎非常不同，它们的不同之处在于：

names(tail(sort(colMeans(x[clus==5,]) - colMeans(x[clus!=5,])),10))
 [1] "wisdom" "thee"   "lord"   "things" "god"    "hath"   "thou"   "man"   
 [9] "thy"    "shall"