使用 keras 密集模型的矩阵逆逼近

Question

我正在训练一个神经网络来计算 3x3 矩阵的逆矩阵。我正在使用具有 1 层和 9 个神经元的 Keras 密集模型。第一层的激活函数是“relu”，在输出层是线性的。我正在使用 10000 个行列式 1 的矩阵。我得到的结果不是很好（RMSE 有数百个）。我一直在尝试更多的层、更多的神经元和其他激活函数，但增益非常小。代码如下：

import numpy as np
from tensorflow.keras import Sequential
from tensorflow.keras.layers import Dense

def generator(nb_samples, matrix_size = 2, entries_range = (0,1), determinant = None):
    ''' 
    Generate nb_samples random matrices of size matrix_size with float 
    entries in interval entries_range and of determinant determinant 
    ''' 
    matrices = []
    if determinant:
        inverses = []
        for i in range(nb_samples):
            matrix = np.random.uniform(entries_range[0], entries_range[1], (matrix_size,matrix_size))
            matrix[0] *= determinant/np.linalg.det(matrix)
            matrices.append(matrix.reshape(matrix_size**2,))
            inverses.append(np.array(np.linalg.inv(matrix)).reshape(matrix_size**2,))
        return np.array(matrices), np.array(inverses)
    else:
        determinants = []
        for i in range(nb_samples):
            matrix = np.random.uniform(entries_range[0], entries_range[1], (matrix_size,matrix_size))
            determinants.append(np.array(np.linalg.det(matrix)).reshape(1,))
            matrices.append(matrix.reshape(matrix_size**2,))    
        return np.array(matrices), np.array(determinants)

### Select number of samples, matrix size and range of entries in matrices
nb_samples = 10000
matrix_size = 3
entries_range = (0, 100)
determinant = 1

### Generate random matrices and determinants
matrices, inverses = generator(nb_samples, matrix_size = matrix_size, entries_range = entries_range, determinant = determinant)

### Select number of layers and neurons
nb_hidden_layers = 1
nb_neurons = matrix_size**2
activation = 'relu'

### Create dense neural network with nb_hidden_layers hidden layers having nb_neurons neurons each
model = Sequential()
model.add(Dense(nb_neurons, input_dim = matrix_size**2, activation = activation))
for i in range(nb_hidden_layers):
    model.add(Dense(nb_neurons, activation = activation))
model.add(Dense(matrix_size**2))
model.compile(loss='mse', optimizer='adam')

### Train and save model using train size of 0.66
history = model.fit(matrices, inverses, epochs = 400, batch_size = 100, verbose = 0, validation_split = 0.33)

### Get validation loss from object 'history'
rmse = np.sqrt(history.history['val_loss'][-1])

### Print RMSE and parameter values
print('''
Validation RMSE: {}
Number of hidden layers: {}
Number of neurons: {}
Number of samples: {}
Matrices size: {}
Range of entries: {}
Determinant: {}
'''.format(rmse,nb_hidden_layers,nb_neurons,nb_samples,matrix_size,entries_range,determinant))

我在网上查了一下，似乎有论文处理逆矩阵逼近问题。但是，在更改模型之前，我想知道是否还有其他参数可以更改，这可能会对错误产生更大的影响。我希望有人可以提供一些见解。谢谢。

Answer 1

对于神经网络来说，反转 3x3 矩阵非常困难，因为它们往往不擅长乘法或除法激活。我无法让它与一个简单的密集网络一起工作，但是一个 7 层的 resnet 就可以了。它有数百万个权重，所以它需要超过 10000 个示例：我发现它完全记住了多达 100,000 个样本并且即使有 10,000,000 个样本也严重过度拟合，所以我只是连续生成样本并将每个样本在生成时馈送到网络一次.

import tensorflow as tf
import numpy as np

import matplotlib.pyplot as plt

#too_small_model = tf.keras.Sequential([
#    tf.keras.layers.Flatten(),
#    tf.keras.layers.Dense(1500, activation="relu"),
#    tf.keras.layers.Dense(1500, activation="relu"), 
#    tf.keras.layers.Dense(N * N),
#    tf.keras.layers.Reshape([ N, N])
#])

N = 3

inp = tf.keras.layers.Input(shape=[N, N])
x = tf.keras.layers.Flatten()(inp)
x = tf.keras.layers.Dense(128, activation="relu")(x)
for _ in range(7):
    skip = x
    for _ in range(4):
        y = tf.keras.layers.Dense(256, activation="relu")(x)
        x = tf.keras.layers.concatenate([x, y])
    #x = tf.keras.layers.BatchNormalization()(x)
    x = tf.keras.layers.Dense(128,
        kernel_initializer=tf.keras.initializers.Zeros(),
        bias_initializer=tf.keras.initializers.Zeros()
    )(x)
    x = skip + x
    #x = tf.keras.layers.BatchNormalization()(x)
x = tf.keras.layers.Dense(N * N)(x)
x = tf.keras.layers.Reshape([N, N])(x)
model2 = tf.keras.models.Model(inp, x)

model2.compile(loss="mean_squared_error", optimizer=tf.keras.optimizers.Adam(learning_rate=.00001))

for _ in range(5000):
    random_matrices = np.random.random((1000000, N, N)) * 4 - 2
    random_matrices = random_matrices[np.abs(np.linalg.det(random_matrices)) > .1]
    inverses = np.linalg.inv(random_matrices)
    inverses = inverses / 5. # normalize target values, large target values hamper training
    model2.fit(random_matrices, inverses, epochs=1, batch_size=1024)

zz = model2.predict(random_matrices[:10000])

plt.scatter(inverses[:10000], zz, s=.0001)

print(random_matrices[76] @ zz[76] * 5)