Python 无法将 fit_generator 应用于具有多个输入的 keras 模型

Question

我有以下模型 - 这是具有 3 个输入的 LSTM + CNN。

我构建了这个生成器函数来使用 fit_generator 训练模型（基于此：https://stanford.edu/~shervine/blog/keras-how-to-generate-data-on-the-fly）：

class MultiInputDataGenerator(keras.utils.Sequence):
    'Generates data for Keras'

    def __init__(self, list_IDs, labels, shuffle=True):
        'Initialization'
        self.batch_size = 8
        self.labels = labels
        self.list_IDs = list_IDs
        self.n_classes = 5
        self.shuffle = shuffle
        self.on_epoch_end()

def __len__(self):
    'Denotes the number of batches per epoch'
    return int(np.floor(len(self.list_IDs) / self.batch_size))

def __getitem__(self, index):
    'Generate one batch of data'
    # Generate indexes of the batch
    indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]

    # Find list of IDs
    list_IDs_temp = [self.list_IDs[k] for k in indexes]

    # Generate data
    X, y = self.__data_generation(list_IDs_temp)

    return X, y

def on_epoch_end(self):
    'Updates indexes after each epoch'
    self.indexes = np.arange(len(self.list_IDs))
    if self.shuffle == True:
        np.random.shuffle(self.indexes)

def __data_generation(self, list_IDs_temp):
    'Generates data containing batch_size samples' # X : (n_samples, *dim, n_channels)
    # Initialization
    
    X = np.empty((self.batch_size, 1, 3), dtype=object)
    y = np.empty((self.batch_size), dtype=object)

    # Generate data
    for i, ID in enumerate(list_IDs_temp):
        X_id = []
        x_features = df.iloc[id][et_cols].values #ET_COLS are 14 columns so I get 1X14 here
        x_text = df.iloc[id].text_col #x_text is 1X768
        x_vid = df.iloc[id].frame_col #x_vid is (3,244,244)
        
        X_id.append(x_features)
        X_id.append(x_text) 
        X_id.append(x_vid)
           
        X[i,] = X_id
        y[i] = self.labels[ID]

    y_mat = tf.convert_to_tensor(pd.get_dummies(y))
    return X, y_mat


training_generator = MultiModelDataGenerator(generator_partition['train'], generator_labels)
validation_generator = MultiModelDataGenerator(generator_partition['val'], generator_labels)
net = build_LSTMCNN_net()
net.compile(keras.optimizers.Adam(0.001),'categorical_crossentropy',metrics=['acc'])
net.fit_generator(generator=training_generator,
                    validation_data=validation_generator,)
                    use_multiprocessing=True)#,    workers=6)

我得到了错误：

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-38-669153f703e6> in <module>()
      
      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,)
                          #use_multiprocessing=True)#,    workers=6)

/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
     96       dtype = dtypes.as_dtype(dtype).as_datatype_enum
     97   ctx.ensure_initialized()
---> 98   return ops.EagerTensor(value, ctx.device_name, dtype)
     99 
    100 

ValueError: Failed to convert a NumPy array to a Tensor (Unsupported object type numpy.ndarray).

我还尝试了几种变体，例如添加：

x_features = np.asarray(x_features).astype(object)
x_text = np.asarray(x_text).astype(object)
x_vid = np.asarray(x_text).astype(object)

或X[i,] = [X_id] 而不是X[i,] = X_id 但没有一个有效 知道如何解决这个问题吗？

已编辑：添加时：

astype(np.float32) 

和
tf.convert_to_tensor(X)

我得到错误： ValueError Traceback（最近一次调用最后一次） 在 ()

      net.fit_generator(generator=training_generator,
--->                      validation_data=validation_generator,
                          use_multiprocessing=True,    workers=6)


/usr/local/lib/python3.6/dist-packages/tensorflow/python/framework/constant_op.py in convert_to_eager_tensor(value, ctx, dtype)
            dtype = dtypes.as_dtype(dtype).as_datatype_enum
        ctx.ensure_initialized()
--->    return ops.EagerTensor(value, ctx.device_name, dtype)
  

Answer 1

在解决问题之前，让我们先总结一下您正在使用的数据集。根据您的描述，我创建了一个可能类似于您的示例DataFrame

import pandas as pd

dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,) 

# create an example DataFrame that I assume will be resemble yours 
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns 
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list 
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators 
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

如您所见，text 列是一个列表列，其中每个列表包含 768 个项目。 labels 列包含示例的标签，无论您使用 one-hot 编码还是其他类型的编码，只要它的形状与整个神经网络模型的输出层的形状匹配即可。 vids 列是 seeds 列，用于动态生成随机图像。

---

解决问题（基于上述数据集）

您可以将这种语法return {'feature':features,'text':text,'vid':vid},y 用于方法__getitem__，而不是堆叠三个输入数组。

为了解释这一点，让我们首先构建一个类似于你的玩具模型

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

这个模型最重要的是，我指定了三个输入层的名称

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

对于这个模型，你可以构造一个类似的生成器

# provide a seed for generating a random image 
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels 
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self, 
                 all_inds,labels, 
                 features,text,vid, 
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self): 
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y

如您所见，__getitem__ 方法返回一个字典 {'feature':features,'text':text,'vid':vid},y。字典的键匹配三个输入层的名称。此外，随机图像是动态生成的。

为了确保一切正常，您可以运行下面的脚本，

import numpy as np
import pandas as pd
from tensorflow import keras 
from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Model
from tensorflow.keras.layers import Input,Dense,Flatten,Add


# provide a seed for generating a random image
def fn2img(seed):
    np.random.seed(seed)
    # fake an image with three channels
    return np.random.randint(low=0,high=255,size=(3,244,244))


class MultiInputDataGenerator(keras.utils.Sequence):

    def __init__(self,
                 all_inds,labels,
                 features,text,vid,
                 shuffle=True):
        self.batch_size = 8
        self.labels = labels
        self.all_inds = all_inds
        self.shuffle = shuffle
        self.on_epoch_end()
        
        self.features = features
        self.text = text
        self.vid = vid

    def __len__(self):
        return int(np.floor(len(self.all_inds) / self.batch_size))


    def __getitem__(self,index):
        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]
        batch_indices = [self.all_inds[k] for k in indexes]
        features,text,vid,y = self.__data_generation(batch_indices)

        return {'feature':features,'text':text,'vid':vid},y

    def on_epoch_end(self):
        self.indexes = np.arange(len(self.all_inds))
        if self.shuffle == True:
            np.random.shuffle(self.indexes)

    def __data_generation(self,batch_indices):
        # Generate data
        features = self.features[batch_indices,:]
        # note that you need to stack the slice in order to reshape it to (num_samples,768)
        text = np.stack(self.text[batch_indices])
        # since batch_size is not a super large number, you can stack here
        vid = np.stack([fn2img(seed) for seed in self.vid[batch_indices]])
        y = self.labels[batch_indices]

        return features,text,vid,y


# fake a dataset
dataset_size = 500
train_idx,val_idx = train_test_split(range(dataset_size),test_size=0.2,)

# create an example DataFrame that I assume will be resemble yours
example_df = pd.DataFrame({'vids':np.random.randint(0,10000,dataset_size)})
# create feature columns
for ind in range(14): example_df['feature_%i' % ind] = np.random.rand(dataset_size)
# each cell contains a list
example_df['text'] = np.random.randint(dataset_size)
example_df['text'] = example_df['text'].astype('object')
for ind in range(dataset_size):example_df.at[ind,'text'] = np.random.rand(768).tolist()
# create the label column
example_df['label'] = np.random.randint(low=0,high=5,size=dataset_size)

# extract information from the dataframe, and create data generators
all_vids = example_df['vids'].values
feature_columns = ['feature_%i' % ind for ind in range(14)]
all_features = example_df[feature_columns].values
all_text = example_df['text'].values
all_labels = example_df['label'].values

training_generator = MultiInputDataGenerator(train_idx,all_labels,all_features,all_text,all_vids)

# create model
def features_part(x):
    y = Dense(14)(x)
    y = Dense(10,activation='linear')(y)
    return y

def text_part(x):
    y = Dense(768)(x)
    y = Dense(10,activation='linear')(y)
    return y

def vid_part(x):
    y = Flatten()(x)
    y = Dense(10,activation='linear')(y)
    return y

input_features = Input(shape=(14,),name='feature')
input_text = Input(shape=(768,),name='text')
input_vid = Input(shape=(3,244,244,),name='vid')

feature_block = features_part(input_features)
text_block = text_part(input_text)
vid_block = vid_part(input_vid)
added = Add()([feature_block,text_block,vid_block])
# you have five classes at the end of the day 
pred = Dense(1)(added)
# build model
model = Model(inputs=[input_features,input_text,input_vid],outputs=pred)
model.compile(loss='mae',optimizer='adam',metrics=['mae'])

model.fit_generator(generator=training_generator,epochs=10)

print(model.history.history)