如何在数据子集的组合上对 pandas 数据框应用统计测试（函数）

Question

我有类似于这个的数据框。

import pandas as pd
import string
import random
def generate_example_dataframe()-> pd.DataFrame:
    """
    This simple function will generate simple dataframe in long format
    """
    num = 20 # number of regions udsed in simulations
    subjects_num = 10
    random.seed(1)
    conditions = ["open", "closed"]
    groups = ["old", "young"]
    means = [1,1.5,1.25,1.75]
    regions = [f"region_{s}" for s in string.ascii_letters[:num]]
    subjects = [f"subject_{s}" for s in list(range(1, subjects_num))]

    list_of_dataframes = []
    for subject in subjects:
        for region in regions:
            lst = iter(means)
            for condition in conditions:
                for group in groups:
                    mean = next(lst)
                    values = mean + np.random.rand(num) + 0.2*random.random()
                    temp_df = pd.DataFrame({'region':[region] *num, 'group':[group] * num, 'condition':[condition] *num ,'subject':[subject] *num ,'values':values})
                    list_of_dataframes.append(temp_df)

    return pd.concat(list_of_dataframes)



# %% [markdown]
# Genereting sample dataframe is presented in the long format - one obe

# %%
df = generate_example_dataframe()
df.head(10).to_clipboard(sep=',', index=True)

这样的输出

,region,group,condition,subject,values
0,region_a,old,open,subject_1,1.4914914311214753
1,region_a,old,open,subject_1,1.9742822483723783
2,region_a,old,open,subject_1,1.0461147549953116
3,region_a,old,open,subject_1,1.9369465073938947
4,region_a,old,open,subject_1,1.817792271839675
5,region_a,old,open,subject_1,1.4272522367426221
6,region_a,old,open,subject_1,1.129423554333859
7,region_a,old,open,subject_1,1.9021298911486018
8,region_a,old,open,subject_1,1.950500304961099
9,region_a,old,open,subject_1,1.6832358513116206

我想对按区域、组和条件分隔的值进行简单的 t 检验。 （测试数 = 区域 x 组 x 条件） 最pythonic的方法是什么？ 我现在想的唯一方法是循环迭代这些变量的值并子集大数据框。

Answer 1

我还从其他人的答案中学习其他方法。我在下面做了一个这样的解决方案......

# grouping

df['grouping']=df['region']+"_"+df['group']+"_"+df['condition']

for i in df.grouping.unique():
    print(i)
    t='result_'+i
    locals()[t]=stats.ttest_1samp(df.loc[df['grouping']==i,'values'],0)

Answer 2

from scipy.stats import ttest_ind as ttest
from itertools import combinations

df = generate_example_dataframe()
grouped_dataframes = [frame for _, frame in df.groupby(['region', 'group', 'condition'])['values']]

# for the p value
results = [ttest(*comb).pvalue for comb in combinations(grouped_dataframes, 2)]
# for the statistic
results = [ttest(*comb).statistic for comb in combinations(grouped_dataframes, 2)]

df.groupby(['region', 'group', 'condition']) 将为您获取 region x group x condition 子集。

我不确定是否有针对子集的每个组合执行 t 检验的优化方法。如果我误解了想要的内容，请告诉我。

如果这是使用 ttest_1samp，您可以这样做：

from scipy.stats import ttest_1samp as ttest

array = np.stack(frame for _, frame in df.groupby(['region', 'group', 'condition'])['values'])
result = ttest(array, df['values'].mean(), axis=1)

Answer 3

如果我理解正确的话，我想这就是你想要的？

from scipy.stats import ttest_1samp
from itertools import product

pop_mean = df["values"].mean()
all_t_tests = [
    x for x in product(df.region.unique(), df.group.unique(), df.condition.unique())
]
all_t_test_groups = {
    "+".join(x): df.loc[
        (df.region == x[0]) & (df.group == x[1]) & (df.condition == x[2])
    ]["values"]
    for x in all_t_tests
}
all_t_test_values = {k: ttest_1samp(v, pop_mean) for k, v in all_t_test_groups.items()}
print(all_t_test_values)

输出：

{'region_a+old+open': Ttest_1sampResult(statistic=-15.269217874013226, pvalue=3.018447106624542e-34),...

编辑：在其他答案的基础上，您也可以使用 groupby 来做到这一点！尽管我喜欢 itertools，但可能会更整洁。

df = generate_example_dataframe()
pop_mean = df["values"].mean()
grouped_dfs = [x for x in df.groupby(['region', 'group', 'condition'])['values']]
all_t_test_groups = {
    "+".join(x[0]): x[1]
    for x in grouped_dfs
}
all_t_test_values = {k: ttest_1samp(v, pop_mean) for k, v in all_t_test_groups.items()}