【发布时间】：2021-01-26 13:33:11
【问题描述】：
我一直在研究一个神经网络类，以后可以将其变成我自己的库。这样做主要是为了更好地理解网络，我一直在阅读纯数学讲座中的所有公式，所以我可能有一些小细节错误。 （在开始之前我不知道该怎么做）
在这个网络中，我编写了一个普通的 SGD 算法，然后是一个动量算法（或者至少是我认为的）。
当我使用 SGD 在我的简单数据集上运行网络时，它运行良好，完全没有问题。但是如果我尝试使用带有动量的 SGD，网络根本不会学习，即使经过 10000 次迭代，损失也保持在 0.7 左右。
我来来回回，从很多地方引用公式，虽然我仍然怀疑我是否完全理解，但我觉得这绝对是我的代码的问题，但我无法弄清楚。我尝试了许多 alpha 和 lambda 值的组合，许多层和神经元的合理组合（特别是不止一个具有动量公式的隐藏层，但它也不适用于 1 层）。
我将发布完整网络的代码，因此，如果有人愿意快速浏览一下，看看是否有任何明显错误的地方，那将不胜感激。我觉得问题可能出在 updateweights() 函数中，因为这是大部分计算发生的地方，但也可能在 calcema() 函数中。
我已经尝试将权重更新公式从 W = W - (alpha * 偏导数) 更改为 W = W + ( alpha * PD) （并保持 PD 为正而不是使其为负），还尝试删除动量的正则化器更新公式，但实际上并没有产生任何影响。
我对此仍然很陌生，尽我所能，因此感谢任何反馈。
以下是输入文件中的示例：
in: 0.6 0.34 0.32 0.78
out: 1.0 0.0 0.0
in: 0.36 0.52 0.75 0.67
out: 1.0 0.0 0.0
in: 0.29 0.034 0.79 0.5
out: 0.0 1.0 0.0
in: 0.21 0.29 0.47 0.62
out: 0.0 1.0 0.0
in: 0.67 0.57 0.42 0.19
out: 0.0 1.0 0.0
in: 0.48 0.22 0.79 0.0096
out: 0.0 1.0 0.0
in: 0.75 0.48 0.61 0.67
out: 1.0 0.0 0.0
in: 0.41 0.96 0.65 0.074
out: 1.0 0.0 0.0
in: 0.19 0.88 0.68 0.1
out: 0.0 1.0 0.0
in: 0.9 0.89 0.95 0.45
out: 1.0 0.0 0.0
in: 0.71 0.58 0.95 0.013
out: 1.0 0.0 0.0
in: 0.66 0.043 0.073 0.98
out: 0.0 1.0 0.0
in: 0.12 0.37 0.2 0.22
out: 0.0 0.0 1.0
in: 0.11 0.38 0.54 0.64
out: 0.0 1.0 0.0
in: 0.42 0.81 0.94 0.98
out: 1.0 0.0 0.0
如果有人想要完整的输入文件，请告诉我，我只是不知道如何在此处发布文件，但我会找到方法。 所以我的具体问题是，当我使用带有动量的 SGD（或者我认为是带有动量的 SGD）时，我的网络根本没有学习并陷入 0.7 的损失……但如果我使用普通的 SGD，它可以完美运行.
代码：
#include <iostream>
#include <vector>
#include <iomanip>
#include <cmath>
#include <random>
#include <fstream>
#include <chrono>
#include <sstream>
#include <string>
#include <assert.h>
double Relu(double val)
{
if (val < 0) return 0.01 * (exp(val) - 1);
else return val;
}
double Reluderiv(double val)
{
if (val < 0) return Relu(val) + 0.01;
else return 1;
}
double randdist(double x, double y)
{
return sqrt(2.0 / (x + y));
}
int randomt(int x, int y)
{
std::random_device rd;
std::mt19937 mt(rd());
std::uniform_real_distribution<double> dist(x, y);
return round(dist(mt));
}
class INneuron
{
public:
double val{};
std::vector <double> weights{};
std::vector <double> weightderivs{};
std::vector <double> emavals{};
};
class HIDneuron
{
public:
double preactval{};
double actval{};
double actvalPD{};
double preactvalPD{};
std::vector <double> weights{};
std::vector <double> weightderivs{};
std::vector <double> emavals{};
double bias{};
double biasderiv{};
double biasema{};
};
class OUTneuron
{
public:
double preactval{};
double actval{};
double preactvalPD{};
double bias{};
double biasderiv{};
double biasema{};
};
class Net
{
public:
Net(int netdimensions, int hidlayers, int hidneurons, int outneurons, int inneurons, double lambda, double alpha)
{
NETDIMENSIONS = netdimensions; HIDLAYERS = hidlayers; HIDNEURONS = hidneurons; OUTNEURONS = outneurons; INNEURONS = inneurons; Lambda = lambda; Alpha = alpha;
}
void defineoptimizer(std::string optimizer);
void Feedforward(const std::vector <double>& invec);
void Backprop(const std::vector <double>& targets);
void Updateweights();
void printvalues(double totalloss);
void Initweights();
void softmax();
double regularize(double weight,std::string type);
double lossfunc(const std::vector <double>& target);
void calcema(int Layernum, int neuron, int weight, std::string layer, std::string BorW);
private:
INneuron Inn;
HIDneuron Hidn;
OUTneuron Outn;
std::vector <std::vector <HIDneuron>> Hidlayers{};
std::vector <INneuron> Inlayer{};
std::vector <OUTneuron> Outlayer{};
double NETDIMENSIONS{};
double HIDLAYERS{};
double HIDNEURONS{};
double OUTNEURONS{};
double INNEURONS{};
double Lambda{};
double Alpha{};
double loss{};
int optimizerformula{};
};
void Net::defineoptimizer(std::string optimizer)
{
if (optimizer == "ExpAvrg")
{
optimizerformula = 1;
}
else if (optimizer == "SGD")
{
optimizerformula = 2;
}
else if (optimizer == "Adam")
{
optimizerformula = 3;
}
else if (optimizer == "MinibatchSGD")
{
optimizerformula = 4;
}
else {
std::cout << "no optimizer matching description" << '\n';
abort();
}
}
double Net::regularize(double weight,std::string type)
{
if (type == "L1")
{
double absval{ weight };
/*if (weight < 0) absval = weight * -1;
else if (weight > 0 || weight == 0) absval = weight;
else;*/
if (absval > 0.0) return 1.0;
else if (absval < 0.0) return -1.0;
else if (absval == 0.0) return 0.0;
else return 2;
}
else if (type == "l2")
{
double absval{};
if (weight < 0.0) absval = weight * -1.0;
else absval = weight;
return (2.0 * absval);
}
else { std::cout << "no regularizer recognized" << '\n'; abort(); }
}
void Net::softmax()
{
double sum{};
for (size_t Osize = 0; Osize < Outlayer.size(); Osize++)
{
sum += exp(Outlayer[Osize].preactval);
}
for (size_t Osize = 0; Osize < Outlayer.size(); Osize++)
{
Outlayer[Osize].actval = exp(Outlayer[Osize].preactval) / sum;
}
}
void Net::Initweights()
{
unsigned seed = std::chrono::system_clock::now().time_since_epoch().count();
std::default_random_engine generator(seed);
std::normal_distribution<double> distribution(0.0, 1.0);
for (int WD = 0; WD < HIDLAYERS + 1; WD++)
{
if (WD == 0)
{
for (int WL = 0; WL < INNEURONS; WL++)
{
Inlayer.push_back(Inn);
for (int WK = 0; WK < HIDNEURONS; WK++)
{
double val = distribution(generator) * randdist(INNEURONS, HIDNEURONS);
Inlayer.back().weights.push_back(val);
Inlayer.back().weightderivs.push_back(0.0);
Inlayer.back().emavals.push_back(0.0);
}
}
}
else if (WD < HIDLAYERS && WD != 0)
{
Hidlayers.push_back(std::vector <HIDneuron>());
for (int WL = 0; WL < HIDNEURONS; WL++)
{
Hidlayers.back().push_back(Hidn);
for (int WK = 0; WK < HIDNEURONS; WK++)
{
double val = distribution(generator) * randdist(HIDNEURONS, HIDNEURONS);
Hidlayers.back().back().weights.push_back(val);
Hidlayers.back().back().weightderivs.push_back(0.0);
Hidlayers.back().back().emavals.push_back(0.0);
}
Hidlayers.back().back().bias = 0.0;
Hidlayers.back().back().biasderiv = 0.0;
Hidlayers.back().back().biasema = 0.0;
}
}
else if (WD == HIDLAYERS)
{
Hidlayers.push_back(std::vector <HIDneuron>());
for (int WL = 0; WL < HIDNEURONS; WL++)
{
Hidlayers.back().push_back(Hidn);
for (int WK = 0; WK < OUTNEURONS; WK++)
{
double val = distribution(generator) * randdist(HIDNEURONS, OUTNEURONS);
Hidlayers.back().back().weights.push_back(val);
Hidlayers.back().back().weightderivs.push_back(0.0);
Hidlayers.back().back().emavals.push_back(0.0);
}
Hidlayers.back().back().bias = 0.0;
Hidlayers.back().back().biasderiv = 0.0;
Hidlayers.back().back().biasema = 0.0;
}
}
}
for (int i = 0; i < OUTNEURONS; i++)
{
Outlayer.push_back(Outn);
Outlayer.back().bias = 0.0;
Outlayer.back().biasderiv = 0.0;
Outlayer.back().biasema = 0.0;
}
}
void Net::Feedforward(const std::vector <double>& invec)
{
for (size_t I = 0; I < Inlayer.size(); I++)
{
Inlayer[I].val = invec[I];
}
for (size_t h = 0; h < Hidlayers[0].size(); h++)
{
double preval = Hidlayers[0][h].bias;
for (size_t I = 0;I < Inlayer.size(); I++)
{
preval += Inlayer[I].val * Inlayer[I].weights[h];
}
Hidlayers[0][h].preactval = preval;
Hidlayers[0][h].actval = Relu(preval);
}
for (size_t H = 1; H < Hidlayers.size();H++)
{
size_t prevh = H - 1;
for (size_t h = 0; h < Hidlayers[H].size(); h++)
{
double preval = Hidlayers[H][h].bias;
for (size_t p = 0; p < Hidlayers[prevh].size(); p++)
{
preval += Hidlayers[prevh][p].actval * Hidlayers[prevh][p].weights[h];
}
Hidlayers[H][h].preactval = preval;
Hidlayers[H][h].actval = Relu(preval);
}
}
for (size_t O = 0; O < Outlayer.size(); O++)
{
size_t lhid = Hidlayers.size() - 1;
double preval = Outlayer[O].bias;
for (size_t h = 0; h < Hidlayers[lhid].size(); h++)
{
preval += Hidlayers[lhid][h].actval * Hidlayers[lhid][h].weights[O];
}
Outlayer[O].preactval = preval;
}
}
void Net::Backprop(const std::vector <double>& targets)
{
for (size_t O = 0; O < Outlayer.size(); O++)
{
double PDval{};
PDval = targets[O] - Outlayer[O].actval;
PDval = PDval * -1.0;
Outlayer[O].preactvalPD = PDval;
}
for (size_t H = Hidlayers.size(); H > 0; H--)
{
size_t Top = H;
size_t Current = H - 1;
for (size_t h = 0; h < Hidlayers[Current].size(); h++)
{
double actPD{};
double PreactPD{};
double biasPD{};
for (size_t hw = 0; hw < Hidlayers[Current][h].weights.size(); hw++)
{
double PDval{};
if (H == Hidlayers.size())
{
PDval = Outlayer[hw].preactvalPD * Hidlayers[Current][h].actval;
biasPD = Outlayer[hw].preactvalPD;
Outlayer[hw].biasderiv = biasPD;
actPD += Hidlayers[Current][h].weights[hw] * Outlayer[hw].preactvalPD;
calcema(0, hw, 0, "Outlayer", "Bias");
}
else
{
PDval = Hidlayers[Top][h].preactvalPD * Hidlayers[Current][h].actval;
actPD += Hidlayers[Current][h].weights[hw] * Hidlayers[Top][h].preactvalPD;
}
Hidlayers[Current][h].weightderivs[hw] = PDval;
calcema(Current, h, hw, "Hidlayer", "Weight");
}
if (H != Hidlayers.size())
{
biasPD = Hidlayers[Top][h].preactvalPD;
Hidlayers[Top][h].biasderiv = biasPD;
calcema(Top, h, 0, "Hidlayer", "Bias");
}
Hidlayers[Current][h].actvalPD = actPD;
PreactPD = Hidlayers[Current][h].actvalPD * Reluderiv(Hidlayers[Current][h].preactval);
Hidlayers[Current][h].preactvalPD = PreactPD;
actPD = 0;
}
}
for (size_t I = 0; I < Inlayer.size(); I++)
{
double PDval{};
for (size_t hw = 0; hw < Inlayer[I].weights.size(); hw++)
{
PDval = Hidlayers[0][hw].preactvalPD * Inlayer[I].val;
Inlayer[I].weightderivs[hw] = PDval;
double biasPD = Hidlayers[0][hw].preactvalPD;
Hidlayers[0][hw].biasderiv = biasPD;
}
}
}
//PROBABLE CULPRIT
void Net::Updateweights()
{
for (size_t I = 0; I < Inlayer.size(); I++)
{
double PD{};
for (size_t iw = 0; iw < Inlayer[I].weights.size(); iw++)
{
if (optimizerformula == 2)
{
PD = (Inlayer[I].weightderivs[iw] * -1.0) - (Lambda * regularize(Inlayer[I].weights[iw], "L1"));
Inlayer[I].weights[iw] = Inlayer[I].weights[iw] + (Alpha * PD);
}
else if (optimizerformula == 1)
{
PD = (Inlayer[I].emavals[iw] * -1.0) - (Lambda * regularize(Inlayer[I].weights[iw], "L1"));
Inlayer[I].weights[iw] = Inlayer[I].weights[iw] + (Alpha * PD);
}
}
}
for (size_t H = 0; H < Hidlayers.size(); H++)
{
for (size_t h = 0; h < Hidlayers[H].size(); h++)
{
double PD{};
for (size_t hw = 0; hw < Hidlayers[H][h].weights.size(); hw++)
{
if (optimizerformula == 2)
{
PD = (Hidlayers[H][h].weightderivs[hw] * -1.0) - (Lambda * regularize(Hidlayers[H][h].weights[hw], "L1"));
Hidlayers[H][h].weights[hw] = Hidlayers[H][h].weights[hw] + (Alpha * PD);
}
else if (optimizerformula == 1)
{
PD = (Hidlayers[H][h].emavals[hw] * -1.0) - (Lambda * regularize(Hidlayers[H][h].weights[hw], "L1"));
Hidlayers[H][h].weights[hw] = Hidlayers[H][h].weights[hw] + (Alpha * PD);
}
}
if (optimizerformula == 1)
{
PD = Hidlayers[H][h].biasema * -1.0;
Hidlayers[H][h].bias = Hidlayers[H][h].bias + (Alpha * PD);
}
else if (optimizerformula == 2)
{
PD = Hidlayers[H][h].biasderiv * -1.0;
Hidlayers[H][h].bias = Hidlayers[H][h].bias + (Alpha * PD);
}
}
}
for (size_t biases = 0; biases < Outlayer.size(); biases++)
{
if (optimizerformula == 2)
{
double PD = Outlayer[biases].biasderiv * -1.0;
Outlayer[biases].bias = Outlayer[biases].bias + (Alpha * PD);
}
else if (optimizerformula == 1)
{
double PD = Outlayer[biases].biasema * -1.0;
Outlayer[biases].bias = Outlayer[biases].bias + (Alpha * PD);
}
}
}
void Net::printvalues(double totalloss)
{
for (size_t Res = 0; Res < Outlayer.size(); Res++)
{
std::cout << Outlayer[Res].actval << " / ";
}
std::cout << '\n' << "loss = " << totalloss << '\n';
}
double Net::lossfunc(const std::vector <double>& target)
{
int pos{ -1 };
double val{};
for (size_t t = 0; t < target.size(); t++)
{
pos += 1;
if (target[t] > 0)
{
break;
}
}
val = -log(Outlayer[pos].actval);
return val;
}
//OTHER PROBABLE CULPRIT
void Net::calcema(int Layernum, int neuron, int weight, std::string layer, std::string BorW )
{
static double Beta{ 0.9 };
if (BorW == "Weight")
{
if (layer == "Inlayer")
{
Inlayer[neuron].emavals[weight] = (Beta * Inlayer[neuron].emavals[weight]) + ((1.0 - Beta) * Inlayer[neuron].weightderivs[weight]);
}
else if (layer == "Hidlayers")
{
Hidlayers[Layernum][neuron].emavals[weight] = (Beta * Hidlayers[Layernum][neuron].emavals[weight]) + ((1.0 - Beta) * Hidlayers[Layernum][neuron].weightderivs[weight]);
}
}
else if (BorW == "Bias")
{
if (layer == "Hidlayers")
{
Hidlayers[Layernum][neuron].biasema = (Beta * Hidlayers[Layernum][neuron].biasema) + ((1.0 - Beta) * Hidlayers[Layernum][neuron].biasderiv);
}
else if (layer == "Outlayer")
{
Outlayer[neuron].biasema = (Beta * Outlayer[neuron].biasema) + ((1.0 - Beta) * Outlayer[neuron].biasderiv);
}
}
}
int main()
{
std::vector <double> innums{};
std::vector <double> outnums{};
std::vector <std::string> INstrings{};
std::vector <std::string> OUTstrings{};
std::string nums{};
std::string in{};
std::string out{};
double totalloss{};
double loss{};
double single{};
int batchcount{0};
Net net(0, 2, 4, 3, 4, 0.0001, 0.006);
net.Initweights();
net.defineoptimizer("ExpAvrg");
std::ifstream file("N.txt");
while (file.is_open())
{
int count{ 0 };
while (file >> nums)
{
if (nums == "in:")
{
count += 1;
std::getline(file, in);
INstrings.push_back(in);
}
else if (nums == "out:")
{
count += 1;
std::getline(file, out);
OUTstrings.push_back(out);
}
else;
}
break;
}
for (int epoch = 0; epoch < 50000; epoch++)
{
int random = randomt(0, 99);
std::string invals = INstrings[random];
std::string outvals = OUTstrings[random];
std::stringstream in(invals);
std::stringstream out(outvals);
std::cout << "fetching" << '\n';
while (in >> single)
{
innums.push_back(single);
}
while (out >> single)
{
outnums.push_back(single);
}
std::cout << "epoch " << epoch << '\n';
std::cout << "In nums: " << '\n';
for (auto element : innums) std::cout << element << " / ";
std::cout << '\n' << "targets: " << '\n';
for (auto element : outnums) std::cout << element << " / ";
std::cout << '\n';
batchcount += 1;
net.Feedforward(innums);
net.softmax();
loss += net.lossfunc(outnums);
totalloss = loss / batchcount;
net.printvalues(totalloss);
net.Backprop(outnums);
net.Updateweights();
innums.clear();
outnums.clear();
}
std::cout << "in size: "<< INstrings.size() << '\n';
std::cout << "out size: " << OUTstrings.size() << '\n';
}
【问题讨论】：
-
该代码适用于复制和粘贴
标签： c++ neural-network