变压器在管道中初始化两次答案

【问题标题】：Transformer initialize twice in pipeline变压器在管道中初始化两次
【发布时间】：2017-04-09 14:42:53
【问题描述】：

我最近正在学习使用管道的 gridsearchCV。但是，我很确定我的管道模型没有问题，如下所示。

from sklearn.pipeline import Pipeline, FeatureUnion
from Transformers import TextTransformer, ColumnExtractor

a = TextTransformer('description', max_features=300)
b = TextTransformer('features', max_features=300)
c = TextTransformer('street_address', max_features = 300)
d = TextTransformer('display_address', max_features = 300)
pipeline = Pipeline([
        ('test', FeatureUnion
         ([
            ('description', a ), # can pass in either a pipeline
            #('features', b),
            #('street', c),
            #('display', d),
            #('lat_long', ColumnExtractor(['latitude', 'longitude']))
        ])),
    ('clf', SVC())
    ])

使用相同的代码：

这行得通

pipeline.fit(df,df['interest_level'])
pipeline.predict(df)

这失败了：

pg = {'clf__C': [0.1,0.2]}
grid = GridSearchCV(pipeline, param_grid = pg)
grid.fit(df, df['interest_level'])

我相信这不是我的管道问题，因为纯粹调用管道上的 fit 和 predict 工作正常。但是使用 gridsearch 它会抛出一个值错误。这对我来说没有任何意义。我很确定我也正确使用了 api

更新错误：

这段代码会失败

Transformer class 
class TextTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, column, max_features):
        self.tfidfVectorizer = TfidfVectorizer(use_idf=False, stop_words='english',
                                               tokenizer=self._custom_tokenizer, analyzer='word',
                                               max_features=max_features)
        self._vectorizer = None
        self._column = column

但是，如果我将 init 更改为此。它会工作

class TextTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, column, max_features):
        print 'column', column
        print 'init'
        self.tfidfVectorizer = TfidfVectorizer(use_idf=False, stop_words='english',
                                               tokenizer=self._custom_tokenizer, analyzer='word',
                                               max_features=max_features)
        self._vectorizer = None
        self._column = 'description'

问题是 TextTransformer 类被初始化了两次。碰巧第二次没有将参数传递给导致错误的列

变压器类

from sklearn.base import BaseEstimator, TransformerMixin
from bs4 import BeautifulSoup
from sklearn.feature_extraction.text import TfidfVectorizer
import nltk


class TextTransformer(BaseEstimator, TransformerMixin):
    def __init__(self, column, max_features):
        self.tfidfVectorizer = TfidfVectorizer(use_idf=False, stop_words='english',
                                               tokenizer=self._custom_tokenizer, analyzer='word',
                                               max_features=max_features)
        self._vectorizer = None
        self._column = column

    def _custom_tokenizer(self, string):
        # string = re.sub('^[\w]', '', string)
        tokens = nltk.word_tokenize(string)
        cleaned = [x if not x.isdigit() else '_NUM_' for x in tokens]
        return [str(x.encode('utf-8')) for x in cleaned if (x.isalpha() or x == '_NUM_')]

    def _clean_html_tags(self, content):
        return BeautifulSoup(content, 'lxml').text

    def fit(self, df, y = None):
        if self._column == 'features':
            df[self._column] = df[self._column].apply(lambda x : ' '.join(x))
        self._vectorizer = self.tfidfVectorizer.fit(df[self._column].apply(self._clean_html_tags))
        return self

    def transform(self, df, y = None):
        if self._column == 'features':
            df[self._column] = df[self._column].apply(lambda x : ' '.join(x))
        return self._vectorizer.transform(df[self._column])

class ColumnExtractor(BaseEstimator, TransformerMixin):

    def __init__(self, cols):
        self.cols = cols

    def transform(self, df, y = None):
        return df[self.cols].values

    def fit(self, X, y=None):
        return self

堆栈跟踪：

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-39-437510c295ef> in <module>()
     23     ('clf',SVC(probability = True))
     24     ])
---> 25 scores = cross_val_score(pipeline, df[['description','features','street_address','display_address','latitude', 'longitude']], df['interest_level'], cv=2)
     26 
     27 for train_index, test_index in skf.split(df, df['interest_level']):

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\model_selection\_validation.py in cross_val_score(estimator, X, y, groups, scoring, cv, n_jobs, verbose, fit_params, pre_dispatch)
    138                                               train, test, verbose, None,
    139                                               fit_params)
--> 140                       for train, test in cv_iter)
    141     return np.array(scores)[:, 0]
    142 

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self, iterable)
    756             # was dispatched. In particular this covers the edge
    757             # case of Parallel used with an exhausted iterator.
--> 758             while self.dispatch_one_batch(iterator):
    759                 self._iterating = True
    760             else:

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in dispatch_one_batch(self, iterator)
    606                 return False
    607             else:
--> 608                 self._dispatch(tasks)
    609                 return True
    610 

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in _dispatch(self, batch)
    569         dispatch_timestamp = time.time()
    570         cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
--> 571         job = self._backend.apply_async(batch, callback=cb)
    572         self._jobs.append(job)
    573 

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py in apply_async(self, func, callback)
    107     def apply_async(self, func, callback=None):
    108         """Schedule a func to be run"""
--> 109         result = ImmediateResult(func)
    110         if callback:
    111             callback(result)

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py in __init__(self, batch)
    324         # Don't delay the application, to avoid keeping the input
    325         # arguments in memory
--> 326         self.results = batch()
    327 
    328     def get(self):

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\model_selection\_validation.py in _fit_and_score(estimator, X, y, scorer, train, test, verbose, parameters, fit_params, return_train_score, return_parameters, return_n_test_samples, return_times, error_score)
    236             estimator.fit(X_train, **fit_params)
    237         else:
--> 238             estimator.fit(X_train, y_train, **fit_params)
    239 
    240     except Exception as e:

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\pipeline.py in fit(self, X, y, **fit_params)
    266             This estimator
    267         """
--> 268         Xt, fit_params = self._fit(X, y, **fit_params)
    269         if self._final_estimator is not None:
    270             self._final_estimator.fit(Xt, y, **fit_params)

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\pipeline.py in _fit(self, X, y, **fit_params)
    232                 pass
    233             elif hasattr(transform, "fit_transform"):
--> 234                 Xt = transform.fit_transform(Xt, y, **fit_params_steps[name])
    235             else:
    236                 Xt = transform.fit(Xt, y, **fit_params_steps[name]) \

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\pipeline.py in fit_transform(self, X, y, **fit_params)
    732             delayed(_fit_transform_one)(trans, name, weight, X, y,
    733                                         **fit_params)
--> 734             for name, trans, weight in self._iter())
    735 
    736         if not result:

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self, iterable)
    756             # was dispatched. In particular this covers the edge
    757             # case of Parallel used with an exhausted iterator.
--> 758             while self.dispatch_one_batch(iterator):
    759                 self._iterating = True
    760             else:

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in dispatch_one_batch(self, iterator)
    606                 return False
    607             else:
--> 608                 self._dispatch(tasks)
    609                 return True
    610 

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in _dispatch(self, batch)
    569         dispatch_timestamp = time.time()
    570         cb = BatchCompletionCallBack(dispatch_timestamp, len(batch), self)
--> 571         job = self._backend.apply_async(batch, callback=cb)
    572         self._jobs.append(job)
    573 

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py in apply_async(self, func, callback)
    107     def apply_async(self, func, callback=None):
    108         """Schedule a func to be run"""
--> 109         result = ImmediateResult(func)
    110         if callback:
    111             callback(result)

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\_parallel_backends.py in __init__(self, batch)
    324         # Don't delay the application, to avoid keeping the input
    325         # arguments in memory
--> 326         self.results = batch()
    327 
    328     def get(self):

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\externals\joblib\parallel.py in __call__(self)
    129 
    130     def __call__(self):
--> 131         return [func(*args, **kwargs) for func, args, kwargs in self.items]
    132 
    133     def __len__(self):

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\pipeline.py in _fit_transform_one(transformer, name, weight, X, y, **fit_params)
    575                        **fit_params):
    576     if hasattr(transformer, 'fit_transform'):
--> 577         res = transformer.fit_transform(X, y, **fit_params)
    578     else:
    579         res = transformer.fit(X, y, **fit_params).transform(X)

C:\ProgramData\Anaconda2\lib\site-packages\sklearn\base.py in fit_transform(self, X, y, **fit_params)
    495         else:
    496             # fit method of arity 2 (supervised transformation)
--> 497             return self.fit(X, y, **fit_params).transform(X)
    498 
    499 

<ipython-input-38-8d6ae99b7816> in fit(self, df, y)
     29         if self._column == 'features':
     30             df[self._column] = df[self._column].apply(lambda x : ' '.join(x))
---> 31         self._vectorizer = self.tfidfVectorizer.fit(df[self._column].apply(self._clean_html_tags))
     32         return self
     33 

C:\ProgramData\Anaconda2\lib\site-packages\pandas\core\frame.py in __getitem__(self, key)
   2057             return self._getitem_multilevel(key)
   2058         else:
-> 2059             return self._getitem_column(key)
   2060 
   2061     def _getitem_column(self, key):

C:\ProgramData\Anaconda2\lib\site-packages\pandas\core\frame.py in _getitem_column(self, key)
   2064         # get column
   2065         if self.columns.is_unique:
-> 2066             return self._get_item_cache(key)
   2067 
   2068         # duplicate columns & possible reduce dimensionality

C:\ProgramData\Anaconda2\lib\site-packages\pandas\core\generic.py in _get_item_cache(self, item)
   1384         res = cache.get(item)
   1385         if res is None:
-> 1386             values = self._data.get(item)
   1387             res = self._box_item_values(item, values)
   1388             cache[item] = res

C:\ProgramData\Anaconda2\lib\site-packages\pandas\core\internals.py in get(self, item, fastpath)
   3550                         loc = indexer.item()
   3551                     else:
-> 3552                         raise ValueError("cannot label index with a null key")
   3553 
   3554             return self.iget(loc, fastpath=fastpath)

ValueError: cannot label index with a null key

【问题讨论】：

嘿，你有没有机会输入一个重新创建问题的数据集？我觉得你的错误可能来自这部分代码： pipeline.fit(df,df['interest_level']) 你的 X 和 y 都将包含 'interest_level' 变量。虽然很难说
@plumbus_bouquet 但管道不会只占用我放入的相关列吗？是的，原始 df 包含兴趣级别列
啊@aceminer - 我明白你在说什么。我想我只是误读了它。您是否有机会附加一些重新创建错误的虚拟数据？
@plumbus_bouquet 我已经更新了我的问题。这似乎是一种奇怪的行为。我将错误范围缩小到这个
为更新干杯。听起来您对这个问题的了解比我多，而且我对 sklearn 的内部工作原理不是很熟悉（抱歉）。我所能推荐的是，如果你有一个解决方法然后使用它，也许在 sklearn github repo 上打开一个问题

标签： python pandas scikit-learn

【解决方案1】：

我知道这是旧的，但万一有人偶然发现：

问题出在TextTransformer 类中。特别是这一行：

self._column = column

这应该是

self.column = column

（不带下划线）。据我所知，GridSearchCV 在实例化转换器后使用set_params 函数。因此，如果您保存通过__int__ 传入的任何功能，您需要使用完全相同的名称。

【讨论】：