使用 Tensorflow 在 batch_norm 中无维度引发 ValueError答案

【问题标题】：None dimension raise ValueError in batch_norm with Tensorflow使用 Tensorflow 在 batch_norm 中无维度引发 ValueError
【发布时间】：2017-04-18 19:34:16
【问题描述】：

我在tensorflow 中实现了某种神经网络（GAN：Generative Adversarial Networks）。

在我决定在generator(z) 方法中添加以下批处理规范化层之前，它按预期工作（请参阅下面的完整代码）：

out = tf.contrib.layers.batch_norm(out, is_training=False)

当我收到以下错误时：

    G_sample = generator(Z)
  File "/Users/Florian/Documents/DeepLearning/tensorflow_stuff/tensorflow_stuff/DCGAN.py", line 84, in generator
    out = tf.contrib.layers.batch_norm(out, is_training=False)                                    
  File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/framework/python/ops/arg_scope.py", line 181, in func_with_args
    return func(*args, **current_args)
  File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/contrib/layers/python/layers/layers.py", line 551, in batch_norm
    outputs = layer.apply(inputs, training=is_training)
  File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 381, in apply
    return self.__call__(inputs, **kwargs)
  File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/base.py", line 328, in __call__
    self.build(input_shapes[0])
  File "/Users/Florian/anaconda2/lib/python2.7/site-packages/tensorflow/python/layers/normalization.py", line 143, in build
    input_shape)
ValueError: ('Input has undefined `axis` dimension. Input shape: ', TensorShape([Dimension(None), Dimension(None), Dimension(None), Dimension(None)]))

问题似乎是由于输入 out 的 [None, None, None, None] 形状造成的，但我不知道如何解决此问题。

这里是完整的代码：

from __future__ import division
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.contrib.layers import batch_norm 
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os




def leaky_relu(x, alpha):
    return tf.maximum(alpha * x, x)




def discriminator(x):

    with tf.variable_scope('discriminator', reuse=True):

        # conv_2D accepts shape (batch, height, width, channel) as input so
        # reshape it
        x = tf.reshape(x, shape=[-1, 28, 28, 1])
        out = tf.nn.conv2d(x, tf.get_variable('D_w_1'), strides=[1, 2, 2, 1], padding='SAME') 
        out = leaky_relu(out, alpha=0.2)
        #out = tf.nn.dropout(out, keep_prob=0.2)
        out = tf.nn.conv2d(out, tf.get_variable('D_w_2'), strides=[1, 2, 2, 1], padding='SAME') 
        out = leaky_relu(out, alpha=0.2)
        #out = tf.nn.dropout(out, keep_prob=0.2)

        # fully connected layer
        out = tf.reshape(out, shape=[-1, 7*7*128])
        D_logits = tf.matmul(out, tf.get_variable('D_w_fc_1'))
        #D_logits = tf.nn.sigmoid(D_logits)
        D_logits = leaky_relu(D_logits, alpha=0.2)

    return D_logits




def generator(z):

    with tf.variable_scope('generator', reuse=True):
        out = tf.matmul(z, tf.get_variable('G_w_fc_1'))
        out = tf.nn.relu(out)

        out = tf.reshape(out, shape=[-1, 7, 7, 128])

        out = tf.nn.conv2d_transpose(out, 
                                     tf.get_variable('G_w_deconv_1'), 
                                     output_shape=tf.stack([tf.shape(out)[0], 14, 14, 64]),
                                     strides=[1, 2, 2, 1],
                                     padding='SAME') 
        print(out.get_shape().as_list())
        out = tf.contrib.layers.batch_norm(out, is_training=False)                                    
        out = tf.nn.relu(out)

        out = tf.nn.conv2d_transpose(out, 
                                     tf.get_variable('G_w_deconv_2'), 
                                     output_shape=tf.stack([tf.shape(out)[0], 28, 28, 1]),
                                     strides=[1, 2, 2, 1],
                                     padding='SAME') 
        out = tf.nn.tanh(out)


    return out







def sample_Z(m, n):
    return np.random.uniform(-1., 1., size=[m, n])


def plot(samples):
    fig = plt.figure(figsize=(4, 4))
    gs = gridspec.GridSpec(4, 4)
    gs.update(wspace=0.05, hspace=0.05)

    for i, sample in enumerate(samples):
        ax = plt.subplot(gs[i])
        plt.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_aspect('equal')
        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')

    return fig


if __name__ == '__main__':


    mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)

    batch_size = 128
    # size of generator input
    Z_dim = 10 
    # batch within an epoch
    batches_per_epoch = int(np.floor(mnist.train.num_examples / batch_size))
    nb_epochs = 20

    # learning rate
    learning_rate = 0.00005 # 0.0002

    Z = tf.placeholder(tf.float32, [batch_size, Z_dim])
    X = tf.placeholder(tf.float32, [batch_size, 784])

    with tf.variable_scope('discriminator'):
        D_w_1 = tf.get_variable('D_w_1', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02))
        D_w_2 = tf.get_variable('D_w_2', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02))
        D_w_fc_1 = tf.get_variable('D_w_fc_1', initializer=tf.random_normal([7*7*128, 1], stddev=0.02)) 

    D_var_list = [D_w_1, D_w_2, D_w_fc_1]


    with tf.variable_scope('generator'):
        G_w_fc_1 = tf.get_variable('G_w_fc_1', initializer=tf.random_normal([Z_dim, 128*7*7], stddev=0.02))
        G_w_deconv_1 = tf.get_variable('G_w_deconv_1', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02))
        G_w_deconv_2 = tf.get_variable('G_w_deconv_2', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02))

    G_var_list = [G_w_fc_1, G_w_deconv_1, G_w_deconv_2]


    G_sample = generator(Z)
    D_logit_real = discriminator(X)
    D_logit_fake = discriminator(G_sample)


    # objective functions
    # discriminator aims at maximizing the probability of TRUE data (i.e. from the dataset) and minimizing the probability
    # of GENERATED/FAKE data:
    D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
    D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
    D_loss = D_loss_real + D_loss_fake

    # generator aims at maximizing the probability of GENERATED/FAKE data (i.e. fool the discriminator)
    G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))

    D_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(D_loss, var_list=D_var_list)
    # when optimizing generator, discriminator is kept fixed
    G_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(G_loss, var_list=G_var_list)


    with tf.Session() as sess:    

        sess.run(tf.global_variables_initializer())

        if not os.path.exists('out/'):
            os.makedirs('out/')

        for i_epoch in range(nb_epochs):

            G_loss_val = 0
            D_loss_val = 0

            for i_batch in range(batches_per_epoch):
                print('batch %i/%i' % (i_batch+1, batches_per_epoch))

                X_mb, _ = mnist.train.next_batch(batch_size)

                # train discriminator
                _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(batch_size, Z_dim)})
                D_loss_val += D_loss_curr

                # train generator
                _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(batch_size, Z_dim)})
                G_loss_val += G_loss_curr

                if i_batch % 50 == 0:
                    samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)})

                    fig = plot(samples)
                    plt.savefig('out/%i_%i.png' % (i_epoch, i_batch), bbox_inches='tight')
                    plt.close(fig)





            print('Iter: {}'.format(i_epoch))
            print('D loss: {:.4}'.format(D_loss))
            print('G_loss: {:.4}'.format(G_loss))

【问题讨论】：

添加 out.set_shape([out.get_shape().as_list()[0], 14, 14, 64] 似乎可行，但我不知道这是否是解决初始问题的正确方法。

标签： python tensorflow deep-learning

【解决方案1】：

如果您将[100, 14, 14, 64] 等常量形状作为output_shape 传递，conv2d_transpose 将返回具有正确形状集的张量。但是，如果您传入一个非常量张量（如果您事先不知道批量大小，则必须这样做），conv2d_transpose 假设它在运行图形之前无法知道形状，并返回一个施工期间全无形状。

理论上它可能已经意识到某些维度是恒定的，但目前还没有这样做。

您可以使用out.set_shape([None, 14, 14, 64]) 或out = tf.reshape(out, [-1, 14, 14, 64]) 解决此问题。批量维度的大小无需设置，batch_norm 不需要。

这在 tensorflow 问题 833 和 8972 上进行了讨论。

【讨论】：

【解决方案2】：

工作代码如下。代码中有一些小错误 - 可能来自您发布问题之前的测试 - 或者您还没有完全执行它，我的编辑用 #EDIT: 注明。您需要定义形状以使用批量标准化，如果您愿意，您可以提前执行此操作，但您的建议很好。我更喜欢通过使用 -1 out = tf.reshape(out, [-1, 14, 14, 64]) 来使用具有可变尺寸的 reshape。下面的代码适用于 TF > 1 和 python > 3.5。

from __future__ import division
from __future__ import division
from __future__ import print_function
import tensorflow as tf
from tensorflow.examples.tutorials.mnist import input_data
from tensorflow.contrib.layers import batch_norm 
import numpy as np
import matplotlib.pyplot as plt
import matplotlib.gridspec as gridspec
import os




def leaky_relu(x, alpha):
    return tf.maximum(alpha * x, x)




def discriminator(x):

    with tf.variable_scope('discriminator', reuse=True):

        # conv_2D accepts shape (batch, height, width, channel) as input so
        # reshape it
        x = tf.reshape(x, shape=[-1, 28, 28, 1])
        out = tf.nn.conv2d(x, tf.get_variable('D_w_1'), strides=[1, 2, 2, 1], padding='SAME') 
        out = leaky_relu(out, alpha=0.2)
        #out = tf.nn.dropout(out, keep_prob=0.2)
        out = tf.nn.conv2d(out, tf.get_variable('D_w_2'), strides=[1, 2, 2, 1], padding='SAME') 
        out = leaky_relu(out, alpha=0.2)
        #out = tf.nn.dropout(out, keep_prob=0.2)

        # fully connected layer
        out = tf.reshape(out, shape=[-1, 7*7*128])
        D_logits = tf.matmul(out, tf.get_variable('D_w_fc_1'))
        #D_logits = tf.nn.sigmoid(D_logits)
        D_logits = leaky_relu(D_logits, alpha=0.2)

    return D_logits




def generator(z):

    with tf.variable_scope('generator', reuse=True):
        out = tf.matmul(z, tf.get_variable('G_w_fc_1'))
        out = tf.nn.relu(out)

        out = tf.reshape(out, shape=[-1, 7, 7, 128])

        out = tf.nn.conv2d_transpose(out, 
                                     tf.get_variable('G_w_deconv_1'),
                                     output_shape=tf.stack([tf.shape(out)[0], 14, 14, 64]),
                                     strides=[1, 2, 2, 1],
                                     padding='SAME') 
        print(out.get_shape().as_list())
        out = tf.reshape(out, [-1, 14, 14, 64])   #EDIT: You need to define the shape for batch_norm

        #out.set_shape([out.get_shape().as_list()[0], 14, 14, 64])
        out = tf.contrib.layers.batch_norm(out, is_training=False)
        out = tf.nn.relu(out)

        out = tf.nn.conv2d_transpose(out, 
                                     tf.get_variable('G_w_deconv_2'), 
                                     output_shape=tf.stack([tf.shape(out)[0], 28, 28, 1]),
                                     strides=[1, 2, 2, 1],
                                     padding='SAME') 
        out = tf.nn.tanh(out)


    return out

def sample_Z(m, n):
    return np.random.uniform(-1., 1., size=[m, n])


def plot(samples):
    fig = plt.figure(figsize=(4, 4))
    gs = gridspec.GridSpec(12, 12)  #EDIT:  This wasn't large enough for the dataset.
    gs.update(wspace=0.05, hspace=0.05)

    for i, sample in enumerate(samples):
        ax = plt.subplot(gs[i])
        plt.axis('off')
        ax.set_xticklabels([])
        ax.set_yticklabels([])
        ax.set_aspect('equal')
        plt.imshow(sample.reshape(28, 28), cmap='Greys_r')

    return fig


if __name__ == '__main__':


    mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True)

    batch_size = 128
    # size of generator input
    Z_dim = 10 
    # batch within an epoch
    batches_per_epoch = int(np.floor(mnist.train.num_examples / batch_size))
    nb_epochs = 20

    # learning rate
    learning_rate = 0.00005 # 0.0002

    Z = tf.placeholder(tf.float32, [batch_size, Z_dim])
    X = tf.placeholder(tf.float32, [batch_size, 784])

    with tf.variable_scope('discriminator'):
        D_w_1 = tf.get_variable('D_w_1', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02))
        D_w_2 = tf.get_variable('D_w_2', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02))
        D_w_fc_1 = tf.get_variable('D_w_fc_1', initializer=tf.random_normal([7*7*128, 1], stddev=0.02)) 

    D_var_list = [D_w_1, D_w_2, D_w_fc_1]


    with tf.variable_scope('generator'):
        G_w_fc_1 = tf.get_variable('G_w_fc_1', initializer=tf.random_normal([Z_dim, 128*7*7], stddev=0.02))
        G_w_deconv_1 = tf.get_variable('G_w_deconv_1', initializer=tf.random_normal([5, 5, 64, 128], stddev=0.02))
        G_w_deconv_2 = tf.get_variable('G_w_deconv_2', initializer=tf.random_normal([5, 5, 1, 64], stddev=0.02))

    G_var_list = [G_w_fc_1, G_w_deconv_1, G_w_deconv_2]


    G_sample = generator(Z)
    D_logit_real = discriminator(X)
    D_logit_fake = discriminator(G_sample)


    # objective functions
    # discriminator aims at maximizing the probability of TRUE data (i.e. from the dataset) and minimizing the probability
    # of GENERATED/FAKE data:
    D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real)))
    D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake)))
    D_loss = D_loss_real + D_loss_fake

    # generator aims at maximizing the probability of GENERATED/FAKE data (i.e. fool the discriminator)
    G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake)))

    D_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(D_loss, var_list=D_var_list)
    # when optimizing generator, discriminator is kept fixed
    G_solver = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(G_loss, var_list=G_var_list)


    with tf.Session() as sess:    

        sess.run(tf.global_variables_initializer())

        if not os.path.exists('out/'):
            os.makedirs('out/')

        for i_epoch in range(nb_epochs):

            G_loss_val = 0
            D_loss_val = 0

            for i_batch in range(batches_per_epoch):
                print('batch %i/%i' % (i_batch+1, batches_per_epoch))

                X_mb, _ = mnist.train.next_batch(batch_size)

                # train discriminator
                _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(batch_size, Z_dim)})
                D_loss_val += D_loss_curr

                # train generator
                _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(batch_size, Z_dim)})
                G_loss_val += G_loss_curr

                if i_batch % 50 == 0:
                    samples = sess.run(G_sample, feed_dict={Z: sample_Z(batch_size, Z_dim)})  #EDIT: changed to batch_size to match the tensor

                    fig = plot(samples)
                    plt.savefig('out/%i_%i.png' % (i_epoch, i_batch), bbox_inches='tight')
                    plt.close(fig)

            print('Iter: {}'.format(i_epoch))
            print('D loss: {:.4}'.format(D_loss_curr)) #EDIT: You were trying to print the tensor.
            print('G_loss: {:.4}'.format(G_loss_curr))#EDIT: You were trying to print the tensor.

【讨论】：