我想在 scikit-learn 中创建一个稳定的管道来预处理数据。我要完成的第一步是对数据框中不同列应用不同策略(即替换为均值、中位数或其他描述性统计数据)的 None
值进行估算。不过我
我开始将 SimpleImputer
转换器与 ColumnTransformer
一起使用。因为 SimpleImputer
返回 numpy 数组而不是 pandas 数据帧,所以我编写了一个新的转换器,它在底层使用 SimpleImputer
但将 pandas 列和索引添加回 numpy 数组。为什么我需要 pandas dataframe 回来?因为我看到我的管道是这样的:
pipeline = Pipeline([
('imputation', ImputationColumnTransformer),
('feature_encoding', EncodingColumnTransformer),
('model', MLModel)
])
如果没有列访问,特征编码的第二步就无法进行。
问题是,当我使用自定义转换器时,我总是会从内部 scikit-learn 验证代码中得到一些错误。
我创建了一个简单示例来展示我遇到的错误类型:
# Creating a toy dataset
m = np.random.randn(3, 3)
m[0, 1] = np.nan
m[2, 2] = np.nan
df = pd.DataFrame(m, columns=['a', 'b', 'c'])
class Imputer(BaseEstimator, TransformerMixin):
# This transformer returns dataframe instead of default ndarray
def __init__(self, ImputerCls, strategy):
self.imputer = ImputerCls(strategy=strategy)
def fit(self, X, y=None):
self.imputer.fit(X, y)
return self
def transform(self, X):
res = self.imputer.transform(X)
res = pd.DataFrame(res)
res.columns = X.columns
res.index = X.index
return res
imputation = ColumnTransformer([
('categorial_imputer', Imputer(SimpleImputer, strategy='most_frequent'), ['a']),
('numeric_imputer', Imputer(SimpleImputer, strategy='mean'), ['b', 'c'])
])
imputation.fit_transform(df)
我希望 pandas dataframe 保留所有列,但是我得到一个很长的回溯日志,我无法完全理解它来找到问题。似乎在某个阶段 ImputerCls
是 None。
---------------------------------------------------------------------------
TypeError Traceback (most recent call last)
<ipython-input-70-0ea27e638c36> in <module>
3 ('numeric_imputer', Imputer(SimpleImputer, strategy='most_frequent'), ['b', 'c'])
4 ])
----> 5 imputation.fit_transform(df)
~/anaconda3/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py in fit_transform(self, X, y)
466 self._validate_remainder(X)
467
--> 468 result = self._fit_transform(X, y, _fit_transform_one)
469
470 if not result:
~/anaconda3/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py in _fit_transform(self, X, y, func, fitted)
410 message=self._log_message(name, idx, len(transformers)))
411 for idx, (name, trans, column, weight) in enumerate(
--> 412 self._iter(fitted=fitted, replace_strings=True), 1))
413 except ValueError as e:
414 if "Expected 2D array, got 1D array instead" in str(e):
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in __call__(self, iterable)
919 # remaining jobs.
920 self._iterating = False
--> 921 if self.dispatch_one_batch(iterator):
922 self._iterating = self._original_iterator is not None
923
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in dispatch_one_batch(self, iterator)
752 tasks = BatchedCalls(itertools.islice(iterator, batch_size),
753 self._backend.get_nested_backend(),
--> 754 self._pickle_cache)
755 if len(tasks) == 0:
756 # No more tasks available in the iterator: tell caller to stop.
~/anaconda3/lib/python3.7/site-packages/joblib/parallel.py in __init__(self, iterator_slice, backend_and_jobs, pickle_cache)
208
209 def __init__(self, iterator_slice, backend_and_jobs, pickle_cache=None):
--> 210 self.items = list(iterator_slice)
211 self._size = len(self.items)
212 if isinstance(backend_and_jobs, tuple):
~/anaconda3/lib/python3.7/site-packages/sklearn/compose/_column_transformer.py in <genexpr>(.0)
409 message_clsname='ColumnTransformer',
410 message=self._log_message(name, idx, len(transformers)))
--> 411 for idx, (name, trans, column, weight) in enumerate(
412 self._iter(fitted=fitted, replace_strings=True), 1))
413 except ValueError as e:
~/anaconda3/lib/python3.7/site-packages/sklearn/base.py in clone(estimator, safe)
63 for name, param in new_object_params.items():
64 new_object_params[name] = clone(param, safe=False)
---> 65 new_object = klass(**new_object_params)
66 params_set = new_object.get_params(deep=False)
67
<ipython-input-57-a319579eaf68> in __init__(self, ImputerCls, strategy)
2 # This class returns dataframe instead of default ndarray
3 def __init__(self, ImputerCls, strategy):
----> 4 self.imputer = ImputerCls(strategy=strategy)
5
6 def fit(self, X, y=None):
TypeError: 'NoneType' object is not callable
最佳答案
我是这样操作的。我认为 Imputer 没有被实例化:
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.compose import ColumnTransformer
from sklearn.impute import SimpleImputer
import pandas as pd
import numpy as np
# Creating a toy dataset
m = np.random.randn(3, 3)
m[0, 1] = np.nan
m[2, 2] = np.nan
df = pd.DataFrame(m, columns=['a', 'b', 'c'])
class Imputer(BaseEstimator, TransformerMixin):
# This transformer returns dataframe instead of default ndarray
def __init__(self, imputer, strategy):
self.imputer = imputer
self.strategy = strategy
def fit(self, X, y=None):
self.imputer = self.imputer(strategy=self.strategy)
self.imputer.fit(X, y)
return self
def transform(self, X, *_):
return self.imputer.transform(X)
imputation = ColumnTransformer([
('categorial_imputer', Imputer(SimpleImputer, strategy='most_frequent'), ['a']),
('numeric_imputer', Imputer(SimpleImputer, strategy='mean'), ['b', 'c'])
])
df = pd.DataFrame(imputation.fit_transform(df), columns=df.columns, index=df.index)
就是这样!
关于python - scikit-learn 中 ColumnTransformer 的自定义转换器问题,我们在Stack Overflow上找到一个类似的问题: https://stackoverflow.com/questions/58248437/