使用 NN 进行回归的 Numpy 实现

Question

我只使用NumPy 实现我自己的回归神经网络模型，当我在 m > 1 个样本上测试我的模型时，我得到了非常奇怪的结果（对于 m=1，它工作正常）..似乎模型崩溃并仅预测整个批次的特定值：

Input:
X [[ 7.62316802 -6.12433912]
 [ 1.11048966  4.97509421]]
Expected Output:
Y [[16.47952332 12.50288412]]
Model Output
y_hat [[10.42446234 10.42446234]]

知道什么可能导致这个问题吗？

我的代码：

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
from mpl_toolkits.mplot3d import Axes3D

# np.seterr(all=None, divide=None, over=None, under=None, invalid=None)

data_x = np.random.uniform(0, 10, size=(2, 1))
data_y = (2 * data_x).sum(axis=0, keepdims=True)

# data_y = data_x[0, :] ** 2 + data_x[1, :] ** 2
# data_y = data_y.reshape((1, -1))

# # fig = plt.figure()
# # ax = fig.add_subplot(111, projection='3d')
# # ax.scatter(data_x[0, :], data_x[1, :], data_y)
# # plt.show()

memory = dict()

nn_architecture = [
    {"input_dim": 2, "output_dim": 6, "activation": "sigmoid", "bias": True},
    {"input_dim": 6, "output_dim": 4, "activation": "sigmoid", "bias": True},
    {"input_dim": 4, "output_dim": 1, "activation": "relu", "bias": True}
]


def init_network_parameters(nn_architecture):
    parameters = []
    for idx, layer in enumerate(nn_architecture):
        layer_params = {}
        input_dim, output_dim, activation, bias = layer.values()
        W = np.random.uniform(0, 1, (output_dim, input_dim))
        B = np.zeros((output_dim, 1))
        if bias:
            B = np.ones((output_dim, 1))
        activation_func = identity
        backward_activation_func = identity_backward
        if activation is 'sigmoid':
            activation_func = sigmoid
            backward_activation_func = sigmoid_backward
        elif activation is 'relu':
            activation_func = relu
            backward_activation_func = relu_backward
        else:
            print(f"Activation function set to identity for layer {idx}")
        layer_params[f"W"] = W
        layer_params[f"B"] = B
        layer_params[f"activation"] = activation_func
        layer_params[f"backward_activation"] = backward_activation_func
        layer_params[f"bias"] = bias
        parameters.append(layer_params)
    return parameters


def identity(z):
    return z


def sigmoid(z):
    return np.clip(1 / (1 + np.exp(-z)), -100, 100)


def relu(z):
    output = np.array(z, copy=True)
    output[z <= 0] = 0
    return output


def identity_backward(z, dA):
    return dA


def sigmoid_backward(z, dA):
    return np.clip(z * (1-z) * dA, -100, 100)

def relu_backward(z, dA):
    output = np.ones(z.shape)
    output[z <= 0] = 0
    return output * dA


def forward_single_layer(prev_A, parameters, idx):
    W = parameters[f"W"]
    B = parameters[f"B"]
    activation = parameters[f"activation"]
    if parameters["bias"]:
        curr_Z = W.dot(prev_A) + B
    else:
        curr_Z = W.dot(prev_A)
    curr_A = activation(curr_Z)
    memory[f"Z{idx+1}"] = curr_Z
    memory[f"A{idx+1}"] = curr_A
    return curr_Z, curr_A


def forward(X, parameters):
    prev_A = X
    memory["A0"] = prev_A
    for idx, layer_params in enumerate(parameters):
        curr_Z, prev_A = forward_single_layer(prev_A=prev_A, parameters=layer_params, idx=idx)
    return prev_A


def criteria(y_hat, y):
    assert y_hat.shape == y.shape
    n = y_hat.shape[0]
    m = y_hat.shape[1]
    loss = np.sum(y_hat - y, axis=1) / m
    dA = (y_hat - y) / m
    return loss, dA


def backward_single_layer(prev_A, dA, curr_W, curr_Z, backward_activation, idx):
    m = prev_A.shape[1]
    dZ = backward_activation(z=curr_Z, dA=dA)

    dW = np.dot(dZ, prev_A.T) / m
    dB = np.sum(dZ, axis=1, keepdims=True) / m
    dA = np.dot(curr_W.T, dZ)
    return dA, dW, dB


def backpropagation(parameters, dA):
    grads = {}
    for idx in reversed(range(len(parameters))):
        layer = parameters[idx]
        prev_A = memory[f"A{idx}"]
        curr_Z = memory[f"Z{idx+1}"]
        curr_W = layer["W"]
        backward_activation = layer["backward_activation"]
        dA, dW, dB = backward_single_layer(prev_A, dA, curr_W, curr_Z, backward_activation, idx)
        grads[f"W{idx}"] = dW
        grads[f"B{idx}"] = dB
    return grads


def update_params(parameters, grads, lr=0.001):
    new_params = []

    for idx, layer in enumerate(parameters):
        layer["W"] -= lr*grads[f"W{idx}"]
        layer["B"] -= lr*grads[f"B{idx}"]
        new_params.append(layer)
    return new_params


X = np.random.uniform(-10, 10, (2, 2))
Y = 2*X[0, :] + X[1, :] ** 2
Y = Y.reshape((1, X.shape[1]))
parameters = init_network_parameters(nn_architecture)
n_epochs = 1000
lr = 0.01
loss_history = []
for i in range(n_epochs):
    y_hat = forward(X, parameters)
    loss, dA = criteria(y_hat, Y)
    loss_history.append(loss)
    grads = backpropagation(parameters, dA)
    parameters = update_params(parameters, grads, lr)
    if not i % 10:
        print(f"Epoch {i}/{n_epochs} loss={loss}")

print("X", X)
print("Y", Y)
print("y_hat", y_hat)

Answer 1

我的实现没有问题，只是过度拟合。 More information can be found here.