为什么我的 GraphDef 实现超过了 TensorFlow 中的 2GB 限制？

Question

我正在使用tf.estimator API，并且我有以下model_fn 函数：

def model_fn(features, labels, mode, params):
    labels = tf.reshape(labels, (-1, 1))

    model = word2vec.create_model(features, params)

    if mode == tf.estimator.ModeKeys.PREDICT:
        return tf.estimator.EstimatorSpec(mode=mode, predictions=model)

    loss = sampled_softmax_loss.create_loss(model['softmax_w'],
                                            model['softmax_b'],
                                            model['relu_layer_1'],
                                            labels,
                                            params['softmax_sample'],
                                            params['vocabulary_size'])
    cost = tf.reduce_mean(loss)
    if mode == tf.estimator.ModeKeys.EVAL:
        return tf.estimator.EstimatorSpec(mode=mode, loss=cost)

    optimizer = adam_optimizer.create_optimizer(params['learning_rate'])
    train_operation = optimizer.minimize(cost)
    if mode == tf.estimator.ModeKeys.TRAIN:
        return tf.estimator.EstimatorSpec(mode=mode, loss=cost, train_op=train_operation)

    raise RuntimeError('Not a valid Mode value')

word2vec.create_model 函数如下所示。该函数返回一个包含网络有趣节点的 python 字典（例如嵌入矩阵、softmax 权重和训练偏差等）。

def create_model(features, hyper_params):
    model = {}
    vocabulary_size = hyper_params['vocabulary_size']
    hidden_size = hyper_params['hidden_size']
    feature_columns = hyper_params['feature_columns']

    with tf.variable_scope('word2vec'):
        # Creating the Embedding layer
        net = tf.feature_column.input_layer(features, feature_columns)
        # Creating the hidden layer
        net = dense_layer.create_layer(model, net, hidden_size)
        # Creating the SoftMax weight and bias variables to use in the sampled loss function 
        softmax_w = tf.Variable(tf.truncated_normal((vocabulary_size, hidden_size), stddev=0.1), name='softmax_weights')
        softmax_b = tf.Variable(tf.zeros(vocabulary_size), name='softmax_bias')

        model['softmax_w'] = softmax_w
        model['softmax_b'] = softmax_b

    return model

最后但并非最不重要的是，我的主要功能，我又使用tf.app.run(main) 命令运行：

def main():
    path = os.path.join('data', 'data.csv')
    (train_x, train_y), (test_x, test_y) = prepare_data.load_data(path, path, columns, columns[-1])

    vocabulary_size = len(train_x[columns[0]].unique())

    feature_columns = []
    for key in train_x.keys():
        item_id = tf.feature_column.categorical_column_with_identity(key=key, num_buckets=vocabulary_size)
        feature_columns.append(tf.feature_column.embedding_column(item_id, 512))

    classifier = tf.estimator.Estimator(
        model_fn=model_fn,
        params={
            'feature_columns': feature_columns,
            'vocabulary_size': vocabulary_size,
            'hidden_size': 256,
            'learning_rate': 0.001,
            'softmax_sample': 100,
        })

    print('Training the classifier...')
    classifier.train(input_fn=lambda: prepare_data.train_input_fn(train_x, train_y, 128), steps=2)

    print('Evaluating on test dataset...')
    eval_result = classifier.evaluate(input_fn=lambda: prepare_data.eval_input_fn(test_x, test_y, 128))

    print('Printing results...')
    print(eval_result)

当我运行它时，我得到一个ValueError: GraphDef cannot be larger than 2GB. 错误。这是为什么？我做错了什么？

下面是我的train_input_fn:

def train_input_fn(features, labels, batch_size):
    def gen():
        for f, l in zip(features, labels):
            yield f, l

    ds = tf.data.Dataset.from_generator(gen, (tf.int64, tf.int64), (tf.TensorShape([None]), tf.TensorShape([None])))
    ds = ds.repeat().batch(batch_size)
    feature, label = ds.make_one_shot_iterator().get_next()

    return {"Input": feature}, label

数据集是一个简单的 csv，如下所示：

    Input   Label
0   12600   838
1   12600   4558
2   12600   838
3   12600   4558
4   838     12600

Answer 1

Dataset.from_tensor_slices 将整个数据集添加到计算图中（参见details），因此最好使用Dataset.from_generator。我已经展示了一个如何使用 mnist 的示例：How to load MNIST via TensorFlow (including download)?