关于逻辑回归的问题

Question

我现在使用 OpenClassroom(http://openclassroom.stanford.edu/MainFolder/DocumentPage.php?course=DeepLearning&doc=exercises/ex4/ex4.html) 中的训练集来尝试 Logistic 回归，并且我只使用 LR，与使用 LR 和牛顿方法的页面不同。 以下是我的代码：

from numpy import *
import matplotlib.pyplot as plt

def loadDataSet():
    dataMat = []; labelMat = []
    frX = open('../ex4x.dat')
    frY = open('../ex4y.dat')
    for line1 in frX.readlines():
        lineArr1 = line1.strip().split()
        dataMat.append([1.0, float(lineArr1[0]), float(lineArr1[1])])

    for line2 in frY.readlines():
        lineArr2 = line2.strip().split()
        labelMat.append(float(lineArr2[0]))
    return dataMat,labelMat

def sigmoid(inX):
    return 1.0/(1+exp(-inX))

# def autoNorm(dataSet):
# #   newValue = (oldValue-min)/(max-min)
#     minVals = min(dataSet)
#     maxVals = max(dataSet)
#     ranges = list(map(lambda x: x[0]-x[1], zip(maxVals, minVals)))
#     normDataSet = zeros(shape(dataSet))
#     m,n = shape(dataSet)
#     normDataSet = list(map(lambda x: x[0]-x[1], zip(dataSet,tile(minVals, (m,1)))))
#     normDataSet = normDataSet/tile(ranges, (m,1))
#     return normDataSet, ranges, minVals

def gradDescent(dataMatIn, classLabels):
    x = mat(dataMatIn)
    y = mat(classLabels).transpose()
    m,n = shape(x)
    alpha = 0.001
    maxCycles = 100000
    theta = ones((n,1))
    for k in range(maxCycles):
        h = sigmoid(x*theta)
        error = h - y
        cost = -1*dot(log(h).T,y)-dot((1-y).T,log(1-h))
        print("Iteration %d | Cost: %f" % (k, cost))
        theta = theta - alpha * (x.transpose() * error /m)
    return theta

def plotBestFit(weights):
    dataMat,labelMat=loadDataSet()
    dataArr = array(dataMat)
    n = shape(dataArr)[0]
    xcord1 = []; ycord1 = []
    xcord2 = []; ycord2 = []
    for i in range(n):
        if int(labelMat[i])== 1:
            xcord1.append(dataArr[i,1]);ycord1.append(dataArr[i,2])
        else:
            xcord2.append(dataArr[i,1]);ycord2.append(dataArr[i,2])
    fig = plt.figure()
    ax = fig.add_subplot(111)
    ax.scatter(xcord1, ycord1, s=30, c='red', marker='s')
    ax.scatter(xcord2, ycord2, s=30, c='green')
    min_x = min(mat(dataMat)[:, 1])
    max_x = max(mat(dataMat)[:, 1])
    x = arange(min_x, max_x, 1)
    y = (-weights[0]-weights[1]*x)/weights[2]
    ax.plot(x, y)
    plt.xlabel('X1'); plt.ylabel('X2');
    plt.show()

dataMat, classLabel = loadDataSet()
weights = gradDescent(dataMat, classLabel)
print weights
plotBestFit(weights.getA())

这是我的问题： 1.我训练了100,000次，每次迭代都会打印错误，反正我没有看到它收敛，好吧，其实我在这里不确定。 2.我不知道matplotlib如何正确绘制分类器，当maxCycle为200,000时，我可以得到一个比较合理的分类器，而maxCyle为100,000，绘制似乎一点也不合理。

maxCycle is 100,000

更新代码：

count = 0
for i in range(80):
    result = sigmoid(dataMat[i] * weights)
    if result > 0.5:
        a = 1
    else:
        a = 0

    if float(a) != classLabel[i][0]:
        count += 1
errorRate = (float(count)/80)
print "error count is: %f, error rate is: %f" %(count,errorRate)

Answer 1

您的代码实际上很好！以下是一些备注：

1. 您使用全 1 初始化了 theta。在这个例子中我不会这样做。 sigmoid 函数的第一次调用将返回接近1 的值，因为theta 和x 的乘积给出了非常大的数字。 log(1 - h) 的计算可能会导致错误，因为 log 未在 0 中定义。我更喜欢用 0's 初始化 thetas。

2. 在计算成本函数时，您错过了除以m。算法没关系，但还是按照理论来比较好。

3. 绘制成本函数是个好主意，而不仅仅是打印它的值。可以非常清楚地看到正确的趋势。

4. 为了收敛，这个特定的例子需要更多的迭代。我在500.000 迭代中取得了不错的结果。

帖子已更新，请参阅下面的更新

这是我的情节：

如您所见，生成的分隔线与教程中显示的图非常匹配。

这是我的代码。它与您的略有不同，但它们非常相似。

import numpy as np
import matplotlib.pyplot as plt

def loadDataSet():
    dataMat = []; labelMat = []
    frX = open('../ex4x.dat')
    frY = open('../ex4y.dat')
    for line1 in frX.readlines():
        lineArr1 = line1.strip().split()
        dataMat.append([1.0, float(lineArr1[0]), float(lineArr1[1])])

    for line2 in frY.readlines():
        lineArr2 = line2.strip().split()
        labelMat.append([float(lineArr2[0])])
    return dataMat,labelMat

def sigmoid(inX):
    return 1.0/(1+np.exp(-inX))    

def gradDescent(dataMatIn, classLabels, alpha, maxCycles):
    x = np.mat(dataMatIn)
    y = np.mat(classLabels)
    m,n = np.shape(x)
    n = n - 1               #usually n is the number of features (without the 1's)

    theta = np.zeros((n+1,1))

    cost_history = []       #list to accumulate the cost values

    for k in range(maxCycles):

        h = sigmoid(x*theta)

        cost = ((-np.multiply(y, np.log(h)) -np.multiply(1-y, np.log(1-h))).sum(axis=0)/m)[0, 0]

        if ((k % 1000) == 0):
            cost_history.append(cost)   #on each 1000th iteration the cost is saved to a list

        grad = (x.transpose() * (h - y))/m

        theta = theta - alpha*grad

    plot_cost = 1 
    if (plot_cost == 1):
        plt.plot(cost_history)
        plt.title("Cost")
        plt.show()

    return theta   

def plotBestFit(dataMat, classLabel, weights):
    arrY = np.asarray(classLabel)
    arrX = np.asarray(dataMat)
    ind1 = np.where(arrY == 1)[0]
    ind0 = np.where(arrY == 0)[0]

    min_x1 = min(np.mat(dataMat)[:, 1])
    max_x1 = max(np.mat(dataMat)[:, 1])
    x1_val = np.arange(min_x1, max_x1, 1)
    x2_val = (-weights[0, 0]-weights[1, 0]*x1_val)/weights[2, 0]

    plt.scatter(arrX[ind1, 1], arrX[ind1, 2], s=30, c='red', marker='s')
    plt.scatter(arrX[ind0, 1], arrX[ind0, 2], s=30, c='blue', marker='s')
    plt.plot(x1_val, x2_val)
    plt.xlabel('X1', fontsize=18)
    plt.ylabel('X2', fontsize=18)
    plt.title("Separation border")
    plt.show()


dataMat, classLabel = loadDataSet()
weights = gradDescent(dataMat, classLabel, 0.0014, 500000) 

print(weights)
plotBestFit(dataMat, classLabel, weights)

更新

在阅读了您在帖子第一版的 cmets 中的问题后，我尝试优化代码以使用更少的迭代次数来实现成本函数的收敛。

确实，特征标准化创造了奇迹:)

仅经过 30 次迭代就获得了更好的结果！

这是新的地块：

由于标准化，您需要扩展每个新的测试示例，以便对其进行分类。

这是新代码。我更改了一些数据类型以避免不必要的数据类型转换。

import numpy as np
import matplotlib.pyplot as plt

def loadDataSet():
    dataMat = []; labelMat = []
    frX = open('../ex4x.dat')
    frY = open('../ex4y.dat')
    for line1 in frX.readlines():
        lineArr1 = line1.strip().split()
        dataMat.append([1.0, float(lineArr1[0]), float(lineArr1[1])])

    for line2 in frY.readlines():
        lineArr2 = line2.strip().split()
        labelMat.append([float(lineArr2[0])])

    return np.asarray(dataMat), np.asarray(labelMat)

def sigmoid(inX):
    return 1.0/(1+np.exp(-inX))    

def gradDescent(x, y, alpha, maxCycles):

    m,n = np.shape(x)
    n = n - 1               #usually n is the number of features (without the 1's)

    theta = np.zeros((n+1,1))

    cost_history = []       #list to accumulate the cost values
    cost_iter = []

    for k in range(maxCycles):

        h = sigmoid(np.dot(x, theta))

        cost = np.sum(-np.multiply(y, np.log(h)) -np.multiply(1-y, np.log(1-h)))/m


        cost_history.append(cost)   #on each 1000th iteration the cost is saved to a list
        cost_iter.append(k)

        grad = np.dot(x.transpose(), (h - y))/m

        theta = theta - alpha*grad

    plot_cost = 1 
    if (plot_cost == 1):
        plt.plot(cost_iter, cost_history)
        plt.title("Cost")
        plt.show()

    return theta   

def plotBestFit(arrX, arrY, weights):

    ind1 = np.where(arrY == 1)[0]
    ind0 = np.where(arrY == 0)[0]

    min_x1 = min(arrX[:, 1:2])
    max_x1 = max(arrX[:, 1:2])
    x1_val = np.arange(min_x1, max_x1, 0.1)
    x2_val = (-weights[0, 0]-weights[1, 0]*x1_val)/weights[2, 0]

    plt.scatter(arrX[ind1, 1], arrX[ind1, 2], s=30, c='red', marker='s')
    plt.scatter(arrX[ind0, 1], arrX[ind0, 2], s=30, c='blue', marker='s')
    plt.plot(x1_val, x2_val)
    plt.xlabel('X1', fontsize=18)
    plt.ylabel('X2', fontsize=18)
    plt.title("Separation border")
    plt.show()


dataMat, classLabel = loadDataSet()
m = np.shape(dataMat)[0]

#standardization
dataMatMean = np.mean(dataMat, axis=0)
dataMatStd = np.std(dataMat, axis=0)

dataMatMean_m = np.tile(dataMatMean, (m, 1))
dataMatStd_m = np.tile(dataMatStd, (m, 1))

dataMatStand = np.copy(dataMat)
dataMatStand[:, 1:3] = np.divide(  (dataMatStand[:, 1:3] - dataMatMean_m[:, 1:3]),   dataMatStd_m[:, 1:3])

weights = gradDescent(dataMatStand, classLabel, 1.0, 30) 

print(weights)
plotBestFit(dataMatStand, classLabel, weights)