山东大学机器学习（实验三解读）——正则化

1. 正则化线性回归

x = load('ex3Linx.dat');
y = load('ex3Liny.dat');
plot(x,y,'ro','MarkerFaceColor','r');
hold on;
m = length(y);
x = [ones(m,1),x ,x.^2 ,x.^3 ,x.^4 ,x.^5 ];
L = eye(size(x,2),size(x,2));
L(1,1)=0;
lambda = [0;1;10];
x_val = linspace(-1,1,100)';
n = length(x_val);
xx = [ones(n,1),x_val ,x_val.^2 ,x_val.^3 ,x_val.^4 ,x_val.^5 ];
for i = 1:length(lambda)
    theta = (x'*x+lambda(i,1)*L)^-1*x'*y;
    plot(x_val,xx*theta,'-');
    hold on;
end
legend('Training data','\lambda=0','\lambda=1','\lambda=10');

绘制图片如下

观察图像，我们可以发现 $\lambda=0$ 的时候曲线会出现过拟合现象，而 $\lambda=0$ 对应的是非正则化的线性回归。对比 $\lambda=1$ 和 $\lambda=10$ ，会发现 $\lambda=10$ 时曲线偏离数据点比较严重，即发生了欠拟合现象。也就是 $\lambda$ 越大，曲线拟合效果越差，可以通过适当调整 $\lambda$ 的值来适应我们的数据点。

2. 正则化逻辑回归

因为当时老师没有上传map_feature.m文件，所以我就临时自己写了一个，如果你使用的是老师的，那么请修改相应的代码，因为老师上传的map_feature.m我还没有看。

%---------------------------map_feature.m---------------------------
function s  = map_feature(x,y,n)
for k=1:size(x,1)
    for i =1:n
        if(i == 1)
            s(k,1) = 1;
            s(k,2) = x(k,1);
            s(k,3) = y(k,1);
            m = 3;
        else
            for j=0:i
                m = m+1;
                s(k,m)=x(k,1)^(i-j)*y(k,1)^j;
            end
        end
    end
end
%---------------------------test.m---------------------------
x = load('ex3Logx.dat');
y = load('ex3Logy.dat');
% Find the indices for the 2 classes
pos = find ( y==1 ) ; neg = find ( y == 0 );
xx = x;
degree = 6;
x = map_feature(x(:,1),x(:,2),degree);
[m, n] = size(x);
lambda = [0;1;10];
L = eye(n,n);
L(1,1) = 0;
g=@(z) 1.0 ./ (1+exp(-z)); 
for k=1:length(lambda)
    theta = zeros(n,1);
    for i = 1:15
       %Calculate the hypothesis function
       z = x*theta;
       h = g(z);
       %Calculate the cost function
       J = -(1/m)*sum(y.*log(h)+(1-y).*log(1-h))+(lambda(k,1)/(2*m))*sum(theta(2:end).^2);
       %Calculate Hession matrix
       H = (1/m).*x'*diag(h)*diag(1-h)*x + (lambda(k,1)/m)*L;
       %Calculate gradient
       G = (lambda(k,1)/m).*theta; 
       G(1,1) = 0;
       J_delta = (1/m).*x'*(h-y) + G;
       %Update the value of theta
       theta = theta - H^(-1)*J_delta;
       store(i,k)=J;
       fprintf('J=%f\n',J);
    end
    % Define the ranges of the grid
    u = linspace(-1 , 1.5, 200);
    v = linspace(-1 , 1.5 ,200);
    %Initialize space for the values to be plotted
    b = zeros(length(u),length(v));
    % Evaluate z = theta*x over the grid
    for i = 1:length(u)
        for j = 1:length(v)
            b(i,j) = map_feature(u(i), v(j),degree)*theta;
        end
    end
    plot ( xx ( pos , 1 ) , xx ( pos , 2 ) , '+' );
    hold on;
    plot ( xx ( neg , 1 ) , xx ( neg , 2 ) , ' o ' );
    b = b';
    contour(u, v, b, [0, 0], 'LineWidth', 2);
    legend('y = 1', 'y = 0', 'Decision boundary');
    title(sprintf('\\lambda = %g', lambda(k,1)), 'FontSize', 10);
    figure;
end
 for i=1:length(lambda)
     plot(1:15,store(:,i),'o--','MarkerFaceColor', 'r');
     title(sprintf('\\lambda = %g', lambda(i,1)), 'FontSize', 10);
     legend('J(\theta)');
     if i == length(lambda)
         break;
     end
     figure;
 end