mlr3，基准测试和嵌套重采样：如何从基准对象中提取调整模型以计算特征重要性答案

【问题标题】：mlr3, benchmarking and nested resampling: how to extract a tuned model from a benchmark object to calculate feature importancemlr3，基准测试和嵌套重采样：如何从基准对象中提取调整模型以计算特征重要性
【发布时间】：2021-12-18 00:02:39
【问题描述】：

我正在使用 mlr3 中的 benchmark() 函数来比较几种 ML 算法。其中之一是带有超参数调整的 XGB。因此，我有一个外部重采样来评估整体性能（保留样本）和一个内部重采样来进行超参数调整（5 折交叉验证）。除了对所有 ML 算法的准确度进行估计外，我还想看看调整后的 XGB 的特征重要性。为此，我必须访问调整后的模型（在基准对象内）。我不知道如何去做。 benchmark() 返回的对象是一个深度嵌套的列表，我不明白它的结构。

这个answer on stackoverflow 没有帮助我，因为它使用了不同的设置（管道中的学习者而不是基准对象）。

这个answer on github 对我没有帮助，因为它展示了如何一次提取有关基准测试的所有信息，而不是如何提取基准测试中一位学习者的一个（调整后的）模型。

下面是我用来执行嵌套重采样的代码。在基准测试之后，我想估计 here 描述的特征重要性，这需要访问调整后的 XGB 模型。

require(mlr3verse)

### Parameters

## Tuning

n_folds = 5

grid_search_resolution = 2

measure = msr("classif.acc")

task = tsk("iris")

# Messages mlr3
# https://stackoverflow.com/a/69336802/7219311
options("mlr3.debug" = TRUE)

### Set up hyperparameter tuning
# AutoTuner for the inner resampling

## inner-resampling design
inner_resampling = rsmp("cv", folds = n_folds)
terminator = trm("none")
 
## XGB: no Hyperparameter Tuning
xgb_no_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
set_threads(xgb_no_tuning, n = 6)

## XGB: AutoTuner
# Setting up Hyperparameter Tuning

xgb_learner_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
xgb_search_space = ps(nrounds = p_int(lower = 100, upper= 500),
                      max_depth = p_int(lower = 3, upper= 10),
                      colsample_bytree = p_dbl(lower = 0.6, upper = 1)
                  )
xgb_tuner = tnr("grid_search", resolution = grid_search_resolution)

# implicit parallelisation
set_threads(xgb_learner_tuning, n = 6)

xgb_tuned = AutoTuner$new(xgb_learner_tuning, inner_resampling, measure, terminator, xgb_tuner, xgb_search_space, store_tuning_instance = TRUE)

## Outer re-sampling: hold-out
outer_resampling = rsmp("holdout")
outer_resampling$instantiate(task)

bm_design = benchmark_grid(
  tasks = task,
  learners = c(lrn("classif.featureless"), 
               xgb_no_tuning,
               xgb_tuned 
  ),
  resamplings = outer_resampling
)

begin_time = Sys.time()
bmr = benchmark(bm_design, store_models = TRUE)
duration = Sys.time() - begin_time

print(duration)

## Results of benchmarking
benchmark_results = bmr$aggregate(measure)
print(benchmark_results)


## Overview

mlr3misc::map(as.data.table(bmr)$learner, "model")

## Detailed results

# Specification of learners
print(bmr$learners$learner)

解决方案

基于be-marc的cmets

require(mlr3verse)
require(mlr3tuning)
require(mlr3misc)

### Parameters

## Tuning

n_folds = 5

grid_search_resolution = 2

measure = msr("classif.acc")

task = tsk("iris")

# Messages mlr3
# https://stackoverflow.com/a/69336802/7219311
options("mlr3.debug" = TRUE)

### Set up hyperparameter tuning
# AutoTuner for the inner resampling

## inner-resampling design
inner_resampling = rsmp("cv", folds = n_folds)
terminator = trm("none")
 
## XGB: no Hyperparameter Tuning
xgb_no_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
set_threads(xgb_no_tuning, n = 6)

## XGB: AutoTuner
# Setting up Hyperparameter Tuning

xgb_learner_tuning = lrn("classif.xgboost", eval_metric = "mlogloss")
xgb_search_space = ps(nrounds = p_int(lower = 100, upper= 500),
                      max_depth = p_int(lower = 3, upper= 10),
                      colsample_bytree = p_dbl(lower = 0.6, upper = 1)
                  )
xgb_tuner = tnr("grid_search", resolution = grid_search_resolution)

# implicit parallelisation
set_threads(xgb_learner_tuning, n = 6)

xgb_tuned = AutoTuner$new(xgb_learner_tuning, inner_resampling, measure, terminator, xgb_tuner, xgb_search_space, store_tuning_instance = TRUE)

## Outer re-sampling: hold-out
outer_resampling = rsmp("holdout")
outer_resampling$instantiate(task)

bm_design = benchmark_grid(
  tasks = task,
  learners = c(lrn("classif.featureless"), 
               xgb_no_tuning,
               xgb_tuned 
  ),
  resamplings = outer_resampling
)

begin_time = Sys.time()
bmr = benchmark(bm_design, store_models = TRUE)
duration = Sys.time() - begin_time

print(duration)

## Results of benchmarking
benchmark_results = bmr$aggregate(measure)
print(benchmark_results)


## Overview

mlr3misc::map(as.data.table(bmr)$learner, "model")

## Detailed results

# Specification of learners
print(bmr$learners$learner)

## Feature Importance

# extract models from outer sampling
# https://stackoverflow.com/a/69828801

data = as.data.table(bmr)
outer_learners = map(data$learner, "learner")

xgb_tuned_model = outer_learners[[3]]

print(xgb_tuned_model)

# print feature importance 
# (presumably gain - mlr3 documentation not clear)
print(xgb_tuned_model$importance())

【问题讨论】：

标签： r xgboost feature-selection hyperparameters mlr3

【解决方案1】：

library(mlr3tuning)
library(mlr3learners)
library(mlr3misc)

learner = lrn("classif.xgboost", nrounds = to_tune(100, 500), eval_metric = "logloss")

at = AutoTuner$new(
  learner = learner,
  resampling = rsmp("cv", folds = 3),
  measure = msr("classif.ce"),
  terminator = trm("evals", n_evals = 5),
  tuner = tnr("random_search"),
  store_models = TRUE
)

design = benchmark_grid(task = tsk("pima"), learner = at, resampling = rsmp("cv", folds = 5))
bmr = benchmark(design, store_models = TRUE)

提取适合外循环的学习者

data = as.data.table(bmr)
outer_learners = map(data$learner, "learner")

提取适合内循环的学习者

archives = extract_inner_tuning_archives(bmr)
inner_learners = map(archives$resample_result, "learners")

【讨论】：

感谢您的回复！在我的示例中使用您的代码提取 outer_learners 返回三个列表，其中两个是 NULL，一个是 xgb 模型。是不是因为外部重采样只包含最好的模型（根据外部重采样）？
没有。外部重采样是 5 折交叉验证，结果为 5 AutoTuners。每个AutoTuner 获得完整数据集（外部重采样的训练集）的 4/5 来调整 xgboost 模型。每个AutoTuner 的最后一步是在 4/5 数据集上拟合具有优化超参数的 xgboost 模型。这个拟合的 xgboost 学习器存储在 AutoTuner$learner 中。您可以使用map(data$learner, "learner") 一次性提取所有 5 个调整过的 xgboost 模型。
由于您在示例中只使用了保留验证，因此您只获得了一个 xgboost 模型。 as.data.table(bmr)$learner 中的其他两个条目已经是学习者（classif.featureless 和未调整的classif.xgboost），因此它们没有学习者槽。 map(data$learner, "learner") 仅适用于 AutoTuners。您可以直接使用其他学习器来获取重要性值。