为什么我的神经网络在几个 epoch 之后权重极低？

Question

我刚开始学习神经网络，这是我的第一个。问题是我拥有的数据越多，在 2-3 个 epoch 之后权重就越低，这是不寻常的，这导致我的 NN 什么也学不到。

重播 在 DataSet 类中，搜索函数 CreateData 并将 nbofexample 更改为 20 之类的值，您将看到是否打印了它们在正常范围内的权重（在 -1 和 1 之间均匀分布），但是如果您将 nbofexample 设置为某个值像 200，然后仅在 2 或 3 个 epoch 之后，最后一层的大部分权重将与 0 非常接近，并且它们将在剩下的训练中停留在该区域。显然，这会导致 NN 失败。

顺便说一句，我的 NN 基本上是在分析 0 到 9 之间的数字数组除以 10 作为规范化来检查数组是否已排序。在下面的代码中我放了很多cmets代码可以很容易看懂。

可能有一个简单的解决方法，但我就是不明白:(

如果你想试试，这里是完整的代码：（它在 python 中）

import numpy as np
import time
import random
import time

#This class is only used for creating the data if needed
class DataSet():
    
    #check if sorted
    def checkPossibility(A):
        return sorted(A) == A

    #will be used later for more complex problems (taken from the faster answer of a coding challenge on LeetCode)
    #def checkPossibility(A):
    #    p = None
    #    for i in range(len(A) - 1):
    #        if A[i] > A[i+1]:
    #            if p is not None:
    #                return False
    #            p = i
    #    return (p is None or p == 0 or p == len(A)-2 or
    #            A[p-1] <= A[p+1] or A[p] <= A[p+2])
    

    #returns inputs and outputs using my poorly written algorithm
    def CreateData():
        
        #settings
        nbofchar=4
        nbofexample=200
        
        #initialize arrays
        inputs = [0]*nbofchar;
        output = [1]
        
        #handling dumbness
        if nbofexample>pow(10,nbofchar): 
            print("Too much data... resizing to max data")
            nbofexample=pow(10,nbofchar)
        elif nbofexample==0:
            print("You need examples to train! (Error nbofexample==0)")
        
        #if there is more than half of the max possible example being request, then create all possible examples and delete randomly until it's the requested size
        if nbofexample>pow(10,nbofchar)/2:
            
            #creating all possible examples
            for i in range(1,pow(10,nbofchar)): 
                new_ex = [int(a) for a in str(i)]
                while len(new_ex)<nbofchar:
                    new_ex=[0]+new_ex
                inputs = np.vstack((inputs,np.dot(new_ex,1/10)))  #normalization /10 so the value is between 0 and 1 ¯\_(ツ)_/¯
                output = np.vstack((output,[int(DataSet.checkPossibility(new_ex))]))
            
            #deleting     
            while len(inputs)>nbofexample:
                index = random.randint(0,len(inputs)-1)
                inputs = np.delete(inputs,index)
                output = np.delete(output,index)

            return inputs, output
        
        #if there is less than half (or half) then, create example randomly until it's the requested size
        else:
            i=1
            while i < nbofexample: 
                new_ex = [random.randint(0,9) for a in range(nbofchar)]
                if sum(np.any(inputs)==new_ex)==0:
                    i+=1
                    inputs = np.vstack((inputs,np.dot(new_ex,1/10)))    #normalization /10 so the value is between 0 and 1 ¯\_(ツ)_/¯
                    output = np.vstack((output,[int(DataSet.checkPossibility(new_ex))]))
            return inputs, output

#assigning weights to each layer
class NeuLayer():
    def __init__(self, nbofneuron, inputsperneuron):
        self.weight = 2 * np.random.random((inputsperneuron,nbofneuron))-1

#the actual neural network
class NeuNet():    

        def __init__(self, layers):
            self.layers = layers

        def _sigmoid(self, x):
            k = 1
            return 1 / (1+np.exp(-x/k))

        def _sigmoid_derivative(self, x):
            return x * (1-x)

        def train(self, training_set_inputs, training_set_outputs, nboftime):

            #debug
            timer1 = 0


            if len(self.layers)<2: return

            for iteration in range(nboftime):
                
                delta = [0] * len(self.layers)
                error = [0] * len(self.layers)
                outputlayers = self.think(training_set_inputs)
                
                #find deltas for each layer "i" (to be able to properly change weights)
                for i in range(len(self.layers)-1,-1,-1):
                    if i==len(self.layers)-1:
                        error[i] = training_set_outputs - outputlayers[i]                      
                    else:
                        error[i] = np.dot(delta[i+1],self.layers[i+1].weight.T)
                    delta[i] = error[i] * self._sigmoid_derivative(outputlayers[i])              


                #assign weigths for each layer "i"
                for i in range(len(self.layers)):
                   if i==0:
                       self.layers[0].weight += np.dot(training_set_inputs.T,delta[0])
                   else:
                       self.layers[i].weight += np.dot(outputlayers[i-1].T,delta[i])

                #display progression and the test result
                if Display_progression: 
                    if timer1<time.time():
                        timer1=time.time()+delay
                        value = ((iteration+1)/nboftime)*100
                        test_input = np.array([.1,.2,.1,.1])
                        print('%.2f'%value+"%     test_input = " + str(test_input) + "     test_output = "+ str(self.think(test_input)[-1]))

        #return output of each layer from an input
        def think(self, input):
            outforlayers = [None]*len(self.layers)
            outforlayer = input
            for i in range(len(self.layers)):
                outforlayer = self._sigmoid(np.dot(outforlayer, self.layers[i].weight))
                outforlayers[i] = outforlayer
            return outforlayers

#datamaker
creating_data=True
train = True

if creating_data:
    
    #creates files with inputs and their expected output
    print("Start creating data...")
    input, output = DataSet.CreateData();
    print("Data created!")
    file = open("data_input","wb")
    np.save(file, input)
    file.close;
    file = open("data_output","wb")
    np.save(file, output)
    file.close;

if train:

    default_data_set=False

    if default_data_set:
        #default training set
        inp_training = np.array([[0, 0, 0, 0, 0], [0.1, 0, 0, 0, 0], [0, 0.1, 0, 0, 0], [0.1, 0.1, 0, 0, 0], [0, 0, 0.1, 0, 0], [0.1, 0, 0.1, 0, 0], [0, 0.1, 0.1, 0, 0], [0.1, 0.1, 0.1, 0, 0],
                             [0, 0, 0, 0.1, 0], [0.1, 0, 0, 0.1, 0], [0, 0.1, 0, 0.1, 0], [0.1, 0.1, 0, 0.1, 0], [0, 0, 0.1, 0.1, 0], [0.1, 0, 0.1, 0.1, 0], [0, 0.1, 0.1, 0.1, 0], [0.1, 0.1, 0.1, 0.1, 0],
                             [0, 0, 0, 0, 0.1], [0.1, 0, 0, 0, 0.1], [0, 0.1, 0, 0, 0.1], [0.1, 0.1, 0, 0, 0.1], [0, 0, 0.1, 0, 0.1], [0.1, 0, 0.1, 0, 0.1], [0, 0.1, 0.1, 0, 0.1], [0.1, 0.1, 0.1, 0, 0.1],
                             [0, 0, 0, 0.1, 0.1], [0.1, 0, 0, 0.1, 0.1], [0, 0.1, 0, 0.1, 0.1], [0.1, 0.1, 0, 0.1, 0.1], [0, 0, 0.1, 0.1, 0.1], [0.1, 0, 0.1, 0.1, 0.1], [0, 0.1, 0.1, 0.1, 0.1], [0.1, 0.1, 0.1, 0.1, 0.1]])
        out_training = np.array([[0,0,0,0,0,0,0,1,
                             0,0,0,1,0,1,1,1,
                             0,0,0,1,0,1,1,1,
                             0,1,1,1,1,1,1,1]]).T

    else:
        print("Loading data files...")
        file = open("data_input","rb")
        inp_training = np.load(file)
        file.close;
        file = open("data_output","rb")
        out_training = np.load(file)
        file.close;
        print("Done reading from data files!")


    #debug
    Display_progression = True;
    delay = 1   #seconds

    #initialize
    np.random.seed(5)
    netlayer_input = NeuLayer(10,len(inp_training[0]))
    netlayer2 = NeuLayer(10,10)
    netlayer3 = NeuLayer(10,10)
    netlayer4 = NeuLayer(10,10)
    netlayer_out = NeuLayer(len(out_training[0]),10)
    All_layers = [netlayer_input,netlayer2,netlayer3,netlayer4,netlayer_out]
    brain = NeuNet(All_layers)

    #train
    print("Start training...")
    brain.train(inp_training, out_training, 100000)
    print("Done!")


    #final test
    outputfinal = brain.think(np.array([0,.1,.3,.7]))


    #output
    a = outputfinal[-1] #[-1] so we get the last layer's output(s)
    print(a)

注意 这是我第一次在 stackoverflow 上提问，如果我遗漏了这个问题的关键信息，请告诉我。

Answer 1

神经网络可能会受到称为 Vanishing Gradient Problem 的影响，这是由 Sigmoid 或 Tanh 等更经典的激活引起的。

用外行的话来说，基本上像 Sigmoid 和 Tanh 这样的激活确实会压缩输入，对吗？例如，sigmoid(10) 和 sigmoid(100) 分别是 0.9999 和 1。尽管输入发生了很大变化，但输出几乎没有变化——此时函数实际上是恒定的。在函数几乎恒定的情况下，它的导数趋于零（或非常小的值）。这些非常小的导数/梯度彼此相乘并实际上变为零，从而阻止您的模型学习任何东西 - 您的权重卡住并停止更新。

我建议您在自己的时间进一步阅读该主题。在几种解决方案中，解决此问题的一种方法是使用不同的激活方式，例如 ReLU。