ValueError：尝试在 tf.distribute.Strategy 范围内创建优化器槽变量

Question

请在下面找到将图像分类为 2 类的代码，我正在尝试使用 Kaggle TPU 执行该代码。你能帮忙解决这里的问题吗？ 为了使用 GPU，我已遵循 Kaggle 网站上的指南，但仍然没有运气。

下面是代码生成的错误栈

import tensorflow as tf
tpu = tf.distribute.cluster_resolver.TPUClusterResolver()
tf.config.experimental_connect_to_cluster(tpu)
tf.tpu.experimental.initialize_tpu_system(tpu)
tpu_strategy = tf.distribute.experimental.TPUStrategy(tpu)
print(tpu_strategy)
# save the final model to file
from keras.applications.vgg16 import VGG16
from keras.models import Model
from keras.layers import Dense
from keras.layers import Flatten
from keras.preprocessing.image import ImageDataGenerator

# define cnn model
def define_model():
    with tpu_strategy.scope():
        # load model
        model = VGG16(include_top=False, input_shape=(224, 224, 3))
        # mark loaded layers as not trainable
        for layer in model.layers:
            layer.trainable = False
        # add new classifier layers
        flat1 = Flatten()(model.layers[-1].output)
        class1 = Dense(128, activation='relu', kernel_initializer='he_uniform')(flat1)
        output = Dense(1, activation='sigmoid')(class1)
        # define new model
        model = Model(inputs=model.inputs, outputs=output)
        # compile model
        model.compile(optimizer=tf.keras.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True), 
                      loss='binary_crossentropy', metrics=['accuracy'])
        return model

# run the test harness for evaluating a model
def train():
    # define model
    model = define_model()
    # create data generator
    datagen = ImageDataGenerator(featurewise_center=True)
    # specify imagenet mean values for centering
    datagen.mean = [123.68, 116.779, 103.939]
    # prepare iterator
    train_it = datagen.flow_from_directory('/kaggle/working/train/',
                                           class_mode='binary', batch_size=64, target_size=(224, 224))
    # fit model
    model.fit_generator(train_it, steps_per_epoch=len(train_it), epochs=10, verbose=0)
    # save model
    model.save('final_model.h5')

# entry point, run the test harness
train()

错误堆栈：

Found 25000 images belonging to 2 classes.
---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-13-b7b93eb12fab> in <module>
     42 
     43 # entry point, run the test harness
---> 44 train()

<ipython-input-13-b7b93eb12fab> in train()
     37                                            class_mode='binary', batch_size=64, target_size=(224, 224))
     38     # fit model
---> 39     model.fit_generator(train_it, steps_per_epoch=len(train_it), epochs=10, verbose=0)
     40     # save model
     41     model.save('final_model.h5')

/opt/conda/lib/python3.7/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in fit_generator(self, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
   1730             use_multiprocessing=use_multiprocessing,
   1731             shuffle=shuffle,
-> 1732             initial_epoch=initial_epoch)
   1733 
   1734     @interfaces.legacy_generator_methods_support

/opt/conda/lib/python3.7/site-packages/keras/engine/training_generator.py in fit_generator(model, generator, steps_per_epoch, epochs, verbose, callbacks, validation_data, validation_steps, validation_freq, class_weight, max_queue_size, workers, use_multiprocessing, shuffle, initial_epoch)
     40 
     41     do_validation = bool(validation_data)
---> 42     model._make_train_function()
     43     if do_validation:
     44         model._make_test_function()

/opt/conda/lib/python3.7/site-packages/keras/engine/training.py in _make_train_function(self)
    314                     training_updates = self.optimizer.get_updates(
    315                         params=self._collected_trainable_weights,
--> 316                         loss=self.total_loss)
    317                 updates = self.updates + training_updates
    318 

/opt/conda/lib/python3.7/site-packages/keras/legacy/interfaces.py in wrapper(*args, **kwargs)
     89                 warnings.warn('Update your `' + object_name + '` call to the ' +
     90                               'Keras 2 API: ' + signature, stacklevel=2)
---> 91             return func(*args, **kwargs)
     92         wrapper._original_function = func
     93         return wrapper

/opt/conda/lib/python3.7/site-packages/keras/backend/tensorflow_backend.py in symbolic_fn_wrapper(*args, **kwargs)
     73         if _SYMBOLIC_SCOPE.value:
     74             with get_graph().as_default():
---> 75                 return func(*args, **kwargs)
     76         else:
     77             return func(*args, **kwargs)

/opt/conda/lib/python3.7/site-packages/keras/optimizers.py in get_updates(self, loss, params)
    760     def get_updates(self, loss, params):
    761         if isinstance(self.optimizer, tf.keras.optimizers.Optimizer):
--> 762             return self.optimizer.get_updates(loss, params)
    763         else:
    764             grads = self.optimizer.compute_gradients(loss, var_list=params)

/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py in get_updates(self, loss, params)
    507         if g is not None and v.dtype != dtypes.resource
    508     ])
--> 509     return [self.apply_gradients(grads_and_vars)]
    510 
    511   def _set_hyper(self, name, value):

/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py in apply_gradients(self, grads_and_vars, name)
    432         _ = self.iterations
    433         self._create_hypers()
--> 434         self._create_slots(var_list)
    435 
    436       if not grads_and_vars:

/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/gradient_descent.py in _create_slots(self, var_list)
     98     if self._momentum:
     99       for var in var_list:
--> 100         self.add_slot(var, "momentum")
    101 
    102   def _prepare_local(self, var_device, var_dtype, apply_state):

/opt/conda/lib/python3.7/site-packages/tensorflow_core/python/keras/optimizer_v2/optimizer_v2.py in add_slot(self, var, slot_name, initializer)
    590             "variables are created under the same strategy scope. This may "
    591             "happen if you're restoring from a checkpoint outside the scope"
--> 592             .format(strategy, var))
    593 
    594       with strategy.extended.colocate_vars_with(var):

ValueError: Trying to create optimizer slot variable under the scope for tf.distribute.Strategy (<tensorflow.python.distribute.distribute_lib._DefaultDistributionStrategy object at 0x7f3e60656050>), which is different from the scope used for the original variable (TPUMirroredVariable:{
  0 /job:worker/replica:0/task:0/device:TPU:0: <tf.Variable 'dense_17/kernel:0' shape=(25088, 128) dtype=float32, numpy=
array([[-0.01303554, -0.00389335,  0.00986688, ...,  0.01155722,
        -0.01016544,  0.00725855],
       [ 0.00431045, -0.01050912, -0.00490532, ..., -0.00075989,
         0.00883604,  0.01337762],
       [ 0.00195401,  0.01383564,  0.01248195, ..., -0.01159664,
         0.01150718,  0.00515156],
       ...,
       [-0.00288643,  0.00719406, -0.01046378, ...,  0.00476515,
         0.00593644,  0.00830421],
       [ 0.01492004, -0.00584323,  0.00321727, ..., -0.00236337,
        -0.01540608,  0.01260902],
       [-0.01198301,  0.00917004,  0.0068699 , ..., -0.00365373,
        -0.00087463,  0.01179958]], dtype=float32)>,
  1 /job:worker/replica:0/task:0/device:TPU:1: <tf.Variable 'dense_17/kernel/replica_1:0' shape=(25088, 128) dtype=float32, numpy=
array([[-0.01303554, -0.00389335,  0.00986688, ...,  0.01155722,
        -0.01016544,  0.00725855],
       [ 0.00431045, -0.01050912, -0.00490532, ..., -0.00075989,
         0.00883604,  0.01337762],
       [ 0.00195401,  0.01383564,  0.01248195, ..., -0.01159664,
         0.01150718,  0.00515156],
       ...,
       [-0.00288643,  0.00719406, -0.01046378, ...,  0.00476515,
         0.00593644,  0.00830421],
       [ 0.01492004, -0.00584323,  0.00321727, ..., -0.00236337,
        -0.01540608,  0.01260902],
       [-0.01198301,  0.00917004,  0.0068699 , ..., -0.00365373,
        -0.00087463,  0.01179958]], dtype=float32)>,
  7 /job:worker/replica:0/task:0/device:TPU:7: <tf.Variable 'dense_17/kernel/replica_7:0' shape=(25088, 128) dtype=float32, numpy=
array([[-0.01303554, -0.00389335,  0.00986688, ...,  0.01155722,
        -0.01016544,  0.00725855],
       [ 0.00431045, -0.01050912, -0.00490532, ..., -0.00075989,
         0.00883604,  0.01337762],
       [ 0.00195401,  0.01383564,  0.01248195, ..., -0.01159664,
         0.01150718,  0.00515156],
       ...,
       [-0.00288643,  0.00719406, -0.01046378, ...,  0.00476515,
         0.00593644,  0.00830421],
       [ 0.01492004, -0.00584323,  0.00321727, ..., -0.00236337,
        -0.01540608,  0.01260902],
       [-0.01198301,  0.00917004,  0.0068699 , ..., -0.00365373,
        -0.00087463,  0.01179958]], dtype=float32)>
}). Make sure the slot variables are created under the same strategy scope. This may happen if you're restoring from a checkpoint outside the scope

Answer 1

使用这个

def define_model():
    with tpu_strategy.scope():
        # load model
        model = VGG16(include_top=False, input_shape=(224, 224, 3))
        # mark loaded layers as not trainable
        for layer in model.layers:
            layer.trainable = False
        # add new classifier layers
        flat1 = Flatten()(model.layers[-1].output)
        class1 = Dense(128, activation='relu', kernel_initializer='he_uniform')(flat1)
        output = Dense(1, activation='sigmoid')(class1)
        # define new model
        model = Model(inputs=model.inputs, outputs=output)
        # compile model
    model.compile(optimizer=tf.keras.optimizers.SGD(lr=0.1, decay=1e-6, momentum=0.9, nesterov=True), 
                      loss='binary_crossentropy', metrics=['accuracy'])
   return model

Answer 2

错误看起来有点像这里描述的错误： https://github.com/tensorflow/tensorflow/issues/32561

查看您的错误跟踪，它（已编辑）在抱怨：

ValueError: 试图在作用域下创建优化器槽变量 用于 tf.distribute.Strategy，与用于的范围不同 原始变量 TPUMirroredVariable

所以看起来你在你的 train() 方法中调用的东西可能需要在 tpu_strategy 范围内。