基于补丁的图像训练并从图像中组合它们的概率

Question

首先，我实现了一个简单的VGG16 网络用于图像分类。

model = keras.applications.vgg16.VGG16(include_top = False,
                weights = None,
                input_shape = (32,32,3),
                pooling = 'max',
                classes = 10)

谁的输入形状是32 x 32。现在，我正在尝试实现patch-based neural network。主要思想是，从输入图像中，像这张图一样，提取4个图像patch，

最后训练提取的补丁图像（resizing to 32 x 32，因为它是我们模型的输入形状），结合它们的四个输出概率并找到最终的输出结果（使用归一化和 argmax）。像这样，

我该怎么做？

提前感谢您的帮助。

注意：

我猜使用lambda layer 是可能的。

我的简单VGG分类实现是here in Colab。

Answer 1

我使用 MNIST 数据集将每个图像作为 4 个带有 tf.image.extract_patches 的补丁，随后作为一个批次传递：

import tensorflow as tf
from tensorflow import keras as K
from tensorflow.keras.layers import Conv2D, Flatten, Dense, MaxPooling2D, Dropout
from tensorflow import nn as nn
from functools import partial
import matplotlib.pyplot as plt

(xtrain, ytrain), (xtest, ytest) = tf.keras.datasets.mnist.load_data()

train = tf.data.Dataset.from_tensor_slices((xtrain, ytrain))
test = tf.data.Dataset.from_tensor_slices((xtest, ytest))

patch_s = 18
stride = xtrain.shape[1] - patch_s

get_patches = lambda x, y: (tf.reshape(
    tf.image.extract_patches(
        images=tf.expand_dims(x[..., None], 0),
        sizes=[1, patch_s, patch_s, 1],
        strides=[1, stride, stride, 1],
        rates=[1, 1, 1, 1],
        padding='VALID'), (4, patch_s, patch_s, 1)), y)

train = train.map(get_patches)
test = test.map(get_patches)

fig = plt.figure()
plt.subplots_adjust(wspace=.1, hspace=.2)
images, labels = next(iter(train))
for index, image in enumerate(images):
    ax = plt.subplot(2, 2, index + 1)
    ax.set_xticks([])
    ax.set_yticks([])
    ax.imshow(image)
plt.show()

然后，在训练循环中，我得到了这 4 个输出中的每一个的损失：

def compute_loss(model, x, y, training):
  out = model(x=x, training=training)
  repeated_y = tf.repeat(tf.expand_dims(y, 0), repeats=4, axis=0)
  loss = loss_object(y_true=repeated_y, y_pred=out, from_logits=True)
  loss = tf.reduce_mean(loss, axis=0)
  return loss

然后我正在减少轴 0 的平均值以将所有概率合并在一起。这是完整的运行代码：

import tensorflow as tf
from tensorflow import keras as K
from tensorflow.keras.layers import Conv2D, Flatten, Dense, MaxPooling2D, Dropout
from tensorflow import nn as nn
from functools import partial
import matplotlib.pyplot as plt

(xtrain, ytrain), (xtest, ytest) = tf.keras.datasets.mnist.load_data()

train = tf.data.Dataset.from_tensor_slices((xtrain, ytrain))
test = tf.data.Dataset.from_tensor_slices((xtest, ytest))

patch_s = 18
stride = xtrain.shape[1] - patch_s

get_patches = lambda x, y: (tf.reshape(
    tf.image.extract_patches(
        images=tf.expand_dims(x[..., None], 0),
        sizes=[1, patch_s, patch_s, 1],
        strides=[1, stride, stride, 1],
        rates=[1, 1, 1, 1],
        padding='VALID'), (4, patch_s, patch_s, 1)), y)

train = train.map(get_patches)
test = test.map(get_patches)

fig = plt.figure()
plt.subplots_adjust(wspace=.1, hspace=.2)
images, labels = next(iter(train))
for index, image in enumerate(images):
    ax = plt.subplot(2, 2, index + 1)
    ax.set_xticks([])
    ax.set_yticks([])
    ax.imshow(image)
plt.show()

def prepare(inputs, targets):
    inputs = tf.divide(x=inputs, y=255)
    targets = tf.one_hot(indices=targets, depth=10)
    return inputs, targets

train = train.take(10_000).map(prepare)
test = test.take(10_00).map(prepare)

class MyCNN(K.Model):
    def __init__(self):
        super(MyCNN, self).__init__()
        Conv = partial(Conv2D, kernel_size=(3, 3), activation=nn.relu)
        MaxPool = partial(MaxPooling2D, pool_size=(2, 2))

        self.conv1 = Conv(filters=16)
        self.maxp1 = MaxPool()
        self.conv2 = Conv(filters=32)
        self.maxp2 = MaxPool()
        self.conv3 = Conv(filters=64)
        self.maxp3 = MaxPool()
        self.flatt = Flatten()
        self.dens1 = Dense(64, activation=nn.relu)
        self.drop1 = Dropout(.5)
        self.dens2 = Dense(10, activation=nn.softmax)

    def call(self, inputs, training=None, **kwargs):
        x = self.conv1(inputs)
        x = self.maxp1(x)
        x = self.conv2(x)
        x = self.maxp2(x)
        x = self.conv3(x)
        x = self.maxp3(x)
        x = self.flatt(x)
        x = self.dens1(x)
        x = self.drop1(x)
        x = self.dens2(x)
        return x

model = MyCNN()

loss_object = tf.losses.categorical_crossentropy

def compute_loss(model, x, y, training):
  out = model(inputs=x, training=training)
  repeated_y = tf.repeat(tf.expand_dims(y, 0), repeats=4, axis=0)
  loss = loss_object(y_true=repeated_y, y_pred=out, from_logits=True)
  loss = tf.reduce_mean(loss, axis=0)
  return loss

def get_grad(model, x, y):
    with tf.GradientTape() as tape:
        loss = compute_loss(model, x, y, training=False)
    return loss, tape.gradient(loss, model.trainable_variables)

optimizer = tf.optimizers.Adam()

verbose = "Epoch {:2d}" \
          " Loss: {:.3f} Acc: {:.3%} TLoss: {:.3f} TAcc: {:.3%}"

for epoch in range(1, 10 + 1):
    train_loss = tf.metrics.Mean()
    train_acc = tf.metrics.CategoricalAccuracy()
    test_loss = tf.metrics.Mean()
    test_acc = tf.metrics.CategoricalAccuracy()

    for x, y in train:
        loss_value, grads = get_grad(model, x, y)
        optimizer.apply_gradients(zip(grads, model.trainable_variables))
        train_loss.update_state(loss_value)
        train_acc.update_state(y, model(x, training=True))

    for x, y in test:
        loss_value, _ = get_grad(model, x, y)
        test_loss.update_state(loss_value)
        test_acc.update_state(y, model(x, training=False))

    print(verbose.format(epoch,
                         train_loss.result(),
                         train_acc.result(),
                         test_loss.result(),
                         test_acc.result()))

剧透警告：有这么小的补丁，效果不佳。制作大于 18/28 的补丁以获得更好的性能。