使用 Fast R-CNN 的多类目标检测和分类

Question

我正在尝试制作使用 Fast R-CNN 进行对象检测的模型 (VGG-16)。 简而言之，我想在图像上找到对象并将边界框放在对象所在的位置。

我已经尝试了多种方法来获得它，但我一直都会遇到一些错误，基本上大多数都是 RoiPoolingLayer 和损失函数。

你们能指导我做错了什么吗？

那么让我来介绍你吧：

这是我的自动取款机：

import pickle
import numpy
import tensorflow
from keras import Input, Model
from keras.initializers.initializers_v1 import RandomNormal
from keras.layers import Flatten, TimeDistributed, Dense, Dropout

from sklearn.preprocessing import LabelBinarizer
from tensorflow.keras.optimizers import Adam
from tensorflow.python.keras.regularizers import l2

from data import get_data, get_train_data
from rcnn.config import Config


import tensorflow as tf
from tensorflow.keras.layers import Layer


class RoiPoolingConv(Layer):

    def __init__(self, pool_size, **kwargs):
        self.pool_size = pool_size
        super(RoiPoolingConv, self).__init__(**kwargs)

    def build(self, input_shape):
        self.nb_channels = input_shape[0][3]
        super(RoiPoolingConv, self).build(input_shape)

    def compute_output_shape(self, input_shape):
        return None, None, self.pool_size, self.pool_size, self.nb_channels

    def crop_and_resize(self, image, boxes):
        box_ind = tf.range(tf.shape(boxes)[0])
        box_ind = tf.reshape(box_ind, (-1, 1))
        box_ind = tf.tile(box_ind, [1, tf.shape(boxes)[1]])
        boxes = tf.keras.backend.cast(
            tf.reshape(boxes, (-1, 4)), "float32"
        )
        box_ind = tf.reshape(box_ind, (1, -1))[0]
        result = tf.image.crop_and_resize(image, boxes, box_ind, [self.pool_size, self.pool_size])
        result = tf.reshape(result, (tf.shape(image)[0], -1, self.pool_size, self.pool_size, self.nb_channels))
        return result

    def call(self, x, mask=None):
        assert (len(x) == 2)
        img = x[0]
        rois = x[1]

        print(x)
        print(img)
        print(rois)

        x1 = rois[:, 0]
        y1 = rois[:, 1]
        x2 = rois[:, 2]
        y2 = rois[:, 3]

        boxes = tf.stack([y1, x1, y2, x2], axis=-1)
        print(boxes)
        rs = self.crop_and_resize(img, boxes)
        print(rs)
        return rs

    def get_config(self):
        config = {'pool_size': self.pool_size}
        base_config = super(RoiPoolingConv, self).get_config()
        return dict(list(base_config.items()) + list(config.items()))


PROPERTIES = Config()


def prepare_model(
        model_path="model\\FastRCNN.h5"
):
    roi_input = Input(shape=(None, 4), name="input_2")
    model_cnn = tensorflow.keras.applications.VGG16(
        include_top=True,
        weights='imagenet'
    )

    model_cnn.trainable = True
    x = model_cnn.layers[17].output
    x = RoiPoolingConv(7)([x, roi_input])
    x = TimeDistributed(Flatten())(x)

    softmaxhead = Dense(4096, activation='relu', kernel_initializer=RandomNormal(stddev=0.01), kernel_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))(x)
    softmaxhead = Dropout(0.5)(softmaxhead)
    softmaxhead = Dense(4096, activation='relu', kernel_initializer=RandomNormal(stddev=0.01), kernel_regularizer=l2(0.0005), bias_regularizer=l2(0.0005))(softmaxhead)
    softmaxhead = Dropout(0.5)(softmaxhead)
    softmaxhead = Dense(20, activation='softmax', kernel_initializer='zero', name='class_label')(softmaxhead)

    bboxhead = Dense(128, activation='relu')(x)
    bboxhead = Dense(64, activation='relu')(bboxhead)
    bboxhead = Dense(32, activation='relu')(bboxhead)
    bboxhead = Dense(4, activation='sigmoid', name='bounding_box')(bboxhead)

    model_final = Model(inputs=[model_cnn.input, roi_input], outputs=(bboxhead, softmaxhead))
    opt = Adam(learning_rate=0.0001)
    losses = {
        "class_label": PROPERTIES.CLASS_LABEL_LOSSES,
        "bounding_box": PROPERTIES.BOUNDING_BOX_LOSSES
    }
    lossWeights = {
        "class_label": PROPERTIES.LOSS_WEIGHTS,
        "bounding_box": PROPERTIES.LOSS_WEIGHTS
    }

    model_final.compile(
        loss=losses,
        optimizer=opt,
        metrics=["accuracy"],
        loss_weights=lossWeights
    )
    tensorflow.keras.utils.plot_model(
        model_final,
        "model.png",
        show_shapes=True,
        show_layer_names=False,
        rankdir='TB'
    )
    model_final.save(model_path)
    return model_final


def train_RCNN_VGG(path):
    # get voc data
    all_data, classes_count, class_mapping = get_data(path)
    tr_images, tr_labels_rois, tr_bboxes_rois, tr_bboxes_gt = get_train_data(all_data)
    #val_images, val_labels, val_bboxes = get_validation_data(all_data)

    # delete unnecessary data
    del classes_count
    del class_mapping
    del all_data

    # convert to numpy array
    tr_images = numpy.array(tr_images, dtype="float32")
    tr_bboxes_rois = numpy.array(tr_bboxes_rois, dtype="float32")
    tr_bboxes_gt = numpy.array(tr_bboxes_gt, dtype="float32")
    tr_labels_rois = numpy.array(tr_labels_rois)
    print(tr_images.shape)
    print(tr_bboxes_rois.shape)
    print(tr_bboxes_gt.shape)
    print(tr_labels_rois.shape)
    # same for validation data
    #val_images = numpy.array(val_images, dtype="float32")
    #val_bboxes = numpy.array(val_bboxes, dtype="float32")
    #val_labels = numpy.array(val_labels)

    # use label binarizer for signing which class/label if for image
    labelBinarizer = LabelBinarizer()
    tr_labels_rois = labelBinarizer.fit_transform(tr_labels_rois)
    #val_labels = labelBinarizer.fit_transform(val_labels)
    classes = len(labelBinarizer.classes_)

    # load model, provide number of classes
    #model_vgg = load_model_or_construct(classes)
    model_vgg = prepare_model()

    # define a dictionary to set the loss methods
    losses = {
        "class_label": PROPERTIES.CLASS_LABEL_LOSSES,
        "bounding_box": PROPERTIES.BOUNDING_BOX_LOSSES
    }

    # define a dictionary that specifies the weights per loss
    lossWeights = {
        "class_label": PROPERTIES.LOSS_WEIGHTS,
        "bounding_box": PROPERTIES.LOSS_WEIGHTS
    }

    # initialize the optimizer, compile the model, and show the model
    opt = Adam(learning_rate=PROPERTIES.LEARNING_RATE)
    model_vgg.compile(loss=losses, optimizer=opt, metrics=["accuracy"], loss_weights=lossWeights)

    # construct a dictionary for our target training outputs, for our target testing
    trainTargets = {
        "class_label": tr_labels_rois,
        "bounding_box": tr_bboxes_gt
    }
    #validationTargets = {
    #    "class_label": val_labels,
    #    "bounding_box": val_bboxes
    #}

    # train the network for bounding box regression and class label
    H = model_vgg.fit(
        [tr_images, tr_bboxes_rois], trainTargets,
    #    validation_data=(val_images, validationTargets),
        batch_size=PROPERTIES.BATCH_SIZE,
        epochs=PROPERTIES.EPOCHS,
        verbose=PROPERTIES.VERBOSE)

    # save model, print summary
    model_vgg.save(PROPERTIES.RCNN_MODEL_NAME, save_format=PROPERTIES.RCNN_MODEL_FORMAT)
    model_vgg.summary()

    # save binarizer
    f = open(PROPERTIES.BINARIZER_NAME, "wb")
    f.write(pickle.dumps(labelBinarizer))
    f.close()


if __name__ == '__main__':
    # load rcnn
    train_RCNN_VGG(PROPERTIES.DATASET_PATH)

我正在创建 RoiPooling 层，VGG-16 架构，加载预训练的权重，制作我自己的输出层，因为我有 20 个类（基于 2012 年的 VOC 数据），这就是为什么第一个输出有 20 个，第二个有 4 个- 边界框坐标的原因。

在训练方法中，您可以看到我正在打印我正在交付的数据的形状，它们是：

(1048, 224, 224, 3)
(1048, 4)
(1048, 4)
(1048,)

第一个，它是 1048 张 224x224 rgb 的图像 二、是为224x224准备的1048个rois坐标 第三，它是 1048 个 ground truth 的 bbox 第四，它是 1048 乘以 20 个标签。标签是这样的： [[0, 0, 0, 0, 0, 0, ... 1, 0, 0,](19 的零，和一个 1 - 正确的标签), [0, ....]]

我是基于这个：https://www.pyimagesearch.com/2020/10/12/multi-class-object-detection-and-bounding-box-regression-with-keras-tensorflow-and-deep-learning/

目前我有这个错误：

Traceback (most recent call last):
  File "C:\Users\Karol\anaconda3\lib\site-packages\keras\utils\traceback_utils.py", line 67, in error_handler
    raise e.with_traceback(filtered_tb) from None
  File "C:\Users\Karol\anaconda3\lib\site-packages\tensorflow\python\framework\func_graph.py", line 1129, in autograph_handler
    raise e.ag_error_metadata.to_exception(e)
ValueError: in user code:
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 878, in train_function  *
        return step_function(self, iterator)
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 867, in step_function  **
        outputs = model.distribute_strategy.run(run_step, args=(data,))
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 860, in run_step  **
        outputs = model.train_step(data)
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\training.py", line 809, in train_step
        loss = self.compiled_loss(
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\engine\compile_utils.py", line 201, in __call__
        loss_value = loss_obj(y_t, y_p, sample_weight=sw)
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 141, in __call__
        losses = call_fn(y_true, y_pred)
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 245, in call  **
        return ag_fn(y_true, y_pred, **self._fn_kwargs)
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\losses.py", line 1664, in categorical_crossentropy
        return backend.categorical_crossentropy(
    File "C:\Users\Karol\anaconda3\lib\site-packages\keras\backend.py", line 4994, in categorical_crossentropy
        target.shape.assert_is_compatible_with(output.shape)
    ValueError: Shapes (None, 20) and (None, None, 20) are incompatible
python-BaseException

所以，我的问题是：我错过了什么，我的预处理数据不正确吗？我正在尝试教我的模型识别 20 个课程，并指出该对象可能在图像上的位置。但我猜我必须提供错误的数据。

为了清楚起见，我对“类标签”和“边界框”使用分类交叉熵和平均精度。

也许我只是使用了错误的损失函数？ 请帮忙。

Answer 1

尝试改用损失tf.keras.losses.SparseCategoricalCrossEntropy，并确保您有一种热编码格式的标签，原因如下：

Getting a ValueError in tensorflow saying that my shapes are incompatible

Answer 2

RoiPooligLayer 的解释说 输入的形状必须是： [(batch_size, pooled_height, pooled_width, n_channels), 用于特征图 和 (batch_size, num_rois, 4)] 用于感兴趣区域 但是在您的工作中，您没有添加 batch_size 维度 试试这个：

model_cnn.trainable = True
x = model_cnn.layers[17].output
x = np.expand_dims(x, axis=0) 
x = RoiPoolingConv(7)([x, roi_input])
x = TimeDistributed(Flatten())(x)