python 交叉验证和train_test_split案例之间的评估指标值差异很大

Question

编写程序，使用支持向量Regression-SVR进行预测，首先将数据集拆分为训练数据集和测试数据集，测试数据集的比例为20%（案例1）；其次，使用交叉验证，将数据集分成5组进行预测（案例2），但是，使用相同的评估指标（R2，MAE，MSE）对两种方法进行评估，结果差异很大
程序如下：

dataset = pd.read_csv('Dataset/allGlassStraightThroughTube.csv')
tube_par = dataset.iloc[:, 3:8].values
tube_eff = dataset.iloc[:, -1:].values


# # form train dataset , test dataset
tube_par_X_train, tube_par_X_test, tube_eff_Y_train, tube_eff_Y_test = train_test_split(tube_par, tube_eff, random_state=33, test_size=0.2)

# normalize the data
sc_X = StandardScaler()
sc_Y = StandardScaler()
sc_tube_par_X_train = sc_X.fit_transform(tube_par_X_train)
sc_tube_par_X_test = sc_X.transform(tube_par_X_test)
sc_tube_eff_Y_train = sc_Y.fit_transform(tube_eff_Y_train)
sc_tube_eff_Y_test = sc_Y.transform(tube_eff_Y_test)
# fit rbf SVR to the sc_tube_par_X dataset
support_vector_regressor = SVR(kernel='rbf')
support_vector_regressor.fit(sc_tube_par_X_train, sc_tube_eff_Y_train)
#
# # predict new result according to the sc_tube_par_X Dataset
pre_sc_tube_eff_Y_test = support_vector_regressor.predict(sc_tube_par_X_test)

pre_tube_eff_Y_test = sc_Y.inverse_transform(pre_sc_tube_eff_Y_test)

# calculate the predict quality
print('R2-score value rbf SVR')
print(r2_score(sc_Y.inverse_transform(sc_tube_eff_Y_test), sc_Y.inverse_transform(pre_sc_tube_eff_Y_test)))
print('The mean squared error of rbf SVR is')
print(mean_squared_error(sc_Y.inverse_transform(sc_tube_eff_Y_test), sc_Y.inverse_transform(pre_sc_tube_eff_Y_test)))
print('The mean absolute error of rbf SVR is')
print(mean_absolute_error(sc_Y.inverse_transform(sc_tube_eff_Y_test), sc_Y.inverse_transform(pre_sc_tube_eff_Y_test)))


# normalize
sc_tube_par_X = sc_X.fit_transform(tube_par)
sc_tube_eff_Y = sc_Y.fit_transform(tube_eff)

scoring = ['r2','neg_mean_squared_error', 'neg_mean_absolute_error']
rbf_svr_regressor = SVR(kernel='rbf')
scores = cross_validate(rbf_svr_regressor, sc_tube_par_X, sc_tube_eff_Y, cv=5, scoring=scoring, return_train_score=False)

情况1，评价指标输出为：

R2-score value rbf SVR
0.6486074476528559
The mean squared error of rbf SVR is
0.00013501023459497165
The mean absolute error of rbf SVR is
0.007196636233830076

情况2，评价指标输出为：

R2-score
0.2621779727614816
test_neg_mean_squared_error
-0.6497292887710239
test_neg_mean_absolute_error
-0.5629408849740231

case 1 和 case 2 差别很大，请问一下原因和改正方法

Answer 1

斌。

我准备了一个小例子来看看使用交叉验证的结果如何变化。我建议您尝试在没有种子的情况下拆分数据，看看结果如何变化。

您会看到交叉验证结果几乎是一个独立于数据拆分的常数。

from sklearn import datasets
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split,cross_validate
#from sklearn.cross_validation import train_test_split
from sklearn.svm import SVR
from sklearn.linear_model import LinearRegression
from sklearn.metrics import r2_score,mean_squared_error,mean_absolute_error

import matplotlib.pyplot as plt


def print_metrics(real_y,predicted_y):
    # calculate the predict quality
    print('R2-score value         {:>8.4f}'.format(r2_score(real_y, predicted_y)))
    print('Mean  squared error is {:>8.4f}'.format(mean_squared_error(real_y, predicted_y)))
    print('Mean absolute error is {:>8.4f}\n\n'.format(mean_absolute_error(real_y, predicted_y)))

def show_plot(real_y,predicted_y):
    fig,ax = plt.subplots()
    ax.scatter(real_y,predicted_y,edgecolors=(0,0,0))
    ax.plot([real_y.min(),real_y.max()],[real_y.min(),real_y.max()],"k--",lw=4)
    ax.set_xlabel("Measured")
    ax.set_ylabel("Predicted")
    plt.show()


# dataset load
boston = datasets.load_boston()

#dataset info
# print(boston.keys())
# print(boston.DESCR)
# print(boston.data.shape)
# print(boston.feature_names)

# numpy_arrays
X = boston.data
Y = boston.target


# # form train dataset , test dataset
X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
#X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=33, test_size=0.2)
#X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=5, test_size=0.2)


# fit scalers
sc_X = StandardScaler().fit(X_train)

# standarizes X (train and test)
X_train = sc_X.transform(X_train)
X_test = sc_X.transform(X_test)



######################################################################
############################### SVR ##################################
######################################################################
support_vector_regressor = SVR(kernel='rbf')
support_vector_regressor.fit(X_train, Y_train)

predicted_Y = support_vector_regressor.predict(X_test)

print_metrics(predicted_Y,Y_test)

show_plot(predicted_Y,Y_test)

######################################################################
########################### LINEAR REGRESSOR #########################
######################################################################

lin_model = LinearRegression()
lin_model.fit(X_train, Y_train)

predicted_Y = lin_model.predict(X_test)

print_metrics(predicted_Y,Y_test)
show_plot(predicted_Y,Y_test)

######################################################################
######################### SVR + CROSS VALIDATION #####################
######################################################################
sc = StandardScaler().fit(X)
standarized_X = sc.transform(X)
scoring = ['r2','neg_mean_squared_error', 'neg_mean_absolute_error']
rbf_svr_regressor = SVR(kernel='rbf')
scores = cross_validate(rbf_svr_regressor, standarized_X, Y, cv=10, scoring=scoring, return_train_score=False)
print(scores["test_r2"].mean())
print(-1*(scores["test_neg_mean_squared_error"].mean()))
print(-1*(scores["test_neg_mean_absolute_error"].mean()))