综述

“子非鱼，焉知鱼之乐”

本文采用编译器：jupyter

逻辑回归方法是从线性回归方法发展过来的，通常解决的是分类问题，读者或许有这样一个疑问：既然是回归算法又么解决分类问题的呢？

道理其实很简单，在我们求出线性回归系数a，b之后，对于每一个输入的x值，模型都可以输出对应的y值，如果把输出值y限制在0到1的范围内，那么这个y就非常的像一个概率p，我们只用规定概率的不同取值范围对应不同的标记值，就可以把一个回归问题转化成分类问题。

这个转化过程就是介绍的重点，如图。

在回归问题中，输出的y取值范围为 负无穷到正无穷

回归问题要求输出结果值域为[0, 1]，故做如下变换：

定义Sigmoid函数：

01 Sigmoid函数

import numpy as npimport matplotlib.pyplot as pltdef sigmoid(t):return 1 / (1 + np.exp(-t))x = np.linspace(-10, 10, 500)y = sigmoid(x)​plt.plot(x, y)plt.show()

由上面的分析可得：

于是逻辑回归问题就可以转化成：对于给定的样本数据集X，y，找到参数使得可以最大程度获得样本数据集X对应的分类输出y

首先还是思考如何表示损失的函数：

没有公式解，只能用梯度下降法求解，过程如下（可跳过直接查看结果）：

前半部分：

后半部分：

02 实现逻辑回归

import numpy as npimport matplotlib.pyplot as pltfrom sklearn import datasets​iris = datasets.load_iris()X = iris.datay = iris.target# 逻辑回归解决的是二分类问题，所以只取两个类别X = X[y<2, :2]y = y[y<2]X.shape# Out[8]:# (100, 2)y.shape# Out[9]:# (100,)plt.scatter(X[y==0,0],X[y==0,1],color='red')plt.scatter(X[y==1,0],X[y==1,1],color='blue')plt.show()

使用逻辑回归

from playML.model_selection import train_test_split​X_train, X_test, y_train, y_test = train_test_split(X, y, seed=666)from playML.LogisticRegression import LogisticRegression​log_reg = LogisticRegression()log_reg.fit(X_train, y_train)# Out[12]:# LinearRegression()log_reg.score(X_test, y_test)# Out[14]:# 1.0log_reg.predict_proba(X_test)"""Out[15]:array([ 0.92972035, 0.98664939, 0.14852024, 0.17601199, 0.0369836 ,0.0186637 , 0.04936918, 0.99669244, 0.97993941, 0.74524655,0.04473194, 0.00339285, 0.26131273, 0.0369836 , 0.84192923,0.79892262, 0.82890209, 0.32358166, 0.06535323, 0.20735334])"""y_test # 概率大于0.5则判决为1，否则为0# Out[16]:# array([1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0])log_reg.predict(X_test)# Out[17]:# array([1, 1, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0, 0])

03 决策边界

# 根据模型计算由x1得到的x2def x2(x1):return (-log_reg.coef_[0] * x1 - log_reg.intercept_) / log_reg.coef_[1]# 计算决策边界x1_plot = np.linspace(4, 8, 100)x2_plot = x2(x1_plot)# 可以观察到有一个点分类错误，但当我们用测试数据集对模型测试时得分是1，所以该点在训练数据集中plt.scatter(X[y==0,0],X[y==0,1],color='red')plt.scatter(X[y==1,0],X[y==1,1],color='blue')plt.plot(x1_plot, x2_plot)​plt.show()

plt.scatter(X_test[y_test==0,0],X_test[y_test==0,1],color='red')plt.scatter(X_test[y_test==1,0],X_test[y_test==1,1],color='blue')plt.plot(x1_plot, x2_plot)​plt.show()

def plot_decision_boundary(model, axis):x0, x1 = np.meshgrid(np.linspace(axis[0], axis[1], int((axis[1]-axis[0])*100)).reshape(-1,1),np.linspace(axis[2], axis[3], int((axis[3]-axis[2])*100)).reshape(-1,1))X_new = np.c_[x0.ravel(), x1.ravel()]y_predict = model.predict(X_new)zz = y_predict.reshape(x0.shape)from matplotlib.colors import ListedColormapcustom_cmap = ListedColormap(['#EF9A9A', '#FFF59D','#90CAF9'])plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)plot_decision_boundary(log_reg, axis=[4, 7.5, 1.5, 4.5])plt.scatter(X[y==0,0],X[y==0,1])plt.scatter(X[y==1,0],X[y==1,1])​plt.show()

kNN的决策边界

from sklearn.neighbors import KNeighborsClassifier​knn_clf = KNeighborsClassifier()knn_clf.fit(X_train, y_train)"""Out[21]:KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',metric_params=None, n_jobs=1, n_neighbors=5, p=2,weights='uniform')"""knn_clf.score(X_test, y_test)# Out[22]:# 1.0plot_decision_boundary(knn_clf, axis=[4, 7.5, 1.5, 4.5])plt.scatter(X[y==0,0],X[y==0,1])plt.scatter(X[y==1,0],X[y==1,1])plt.show()

# 对有三个类别的数据进行分类，为了方便可视化取前两个特征knn_clf_all = KNeighborsClassifier()knn_clf_all.fit(iris.data[:,:2], iris.target)"""Out[24]:KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',metric_params=None, n_jobs=1, n_neighbors=5, p=2,weights='uniform')"""plot_decision_boundary(knn_clf_all, axis=[4, 8, 1.5, 4.5])plt.scatter(iris.data[iris.target==0,0],iris.data[iris.target==0,1])plt.scatter(iris.data[iris.target==1,0],iris.data[iris.target==1,1])plt.scatter(iris.data[iris.target==2,0],iris.data[iris.target==2,1])​plt.show()

# n_neighbors越小，边界越不规整knn_clf_all = KNeighborsClassifier(n_neighbors=50)knn_clf_all.fit(iris.data[:,:2], iris.target)"""Out[26]:KNeighborsClassifier(algorithm='auto', leaf_size=30, metric='minkowski',metric_params=None, n_jobs=1, n_neighbors=50, p=2,weights='uniform')"""plot_decision_boundary(knn_clf_all, axis=[4, 8, 1.5, 4.5])plt.scatter(iris.data[iris.target==0,0],iris.data[iris.target==0,1])plt.scatter(iris.data[iris.target==1,0],iris.data[iris.target==1,1])plt.scatter(iris.data[iris.target==2,0],iris.data[iris.target==2,1])​plt.show()

04 逻辑回归中添加多项式特征

import numpy as npimport matplotlib.pyplot as pltnp.random.seed(666)# 生成有两个特征的XX = np.random.normal(0, 1, size=(200, 2))# 特征y类似一个圆，半径小于1.5时置为1y = np.array(X[:,0]**2 + X[:,1]**2 < 1.5, dtype='int')plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

使用逻辑回归

from playML.LogisticRegression import LogisticRegressionlog_reg = LogisticRegression()log_reg.fit(X, y)# Out[5]:# LinearRegression()log_reg.score(X, y)# 分类准确度较低# Out[6]:# 0.60499999999999998def plot_decision_boundary(model, axis):x0, x1 = np.meshgrid(np.linspace(axis[0], axis[1], int((axis[1]-axis[0])*100)).reshape(-1,1),np.linspace(axis[2], axis[3], int((axis[3]-axis[2])*100)).reshape(-1,1))X_new = np.c_[x0.ravel(), x1.ravel()]y_predict = model.predict(X_new)zz = y_predict.reshape(x0.shape)from matplotlib.colors import ListedColormapcustom_cmap = ListedColormap(['#EF9A9A', '#FFF59D','#90CAF9'])plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)# 图示较低的分类准确度plot_decision_boundary(log_reg, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

from sklearn.pipeline import Pipelinefrom sklearn.preprocessing import PolynomialFeaturesfrom sklearn.preprocessing import StandardScaler​# 1.添加多项式项 2.归一化 3.逻辑回归(自己实现的类，符合“构造函数，fit,predict,score”等标准)def PolynomialLogisticRegression(degree):return Pipeline([('poly', PolynomialFeatures(degree=degree)),('std_scaler', StandardScaler()),('log_reg', LogisticRegression())])poly_log_reg = PolynomialLogisticRegression(degree=2)poly_log_reg.fit(X, y)"""Out[10]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=2, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LinearRegression())])"""poly_log_reg.score(X, y)# Out[11]:# 0.94999999999999996plot_decision_boundary(poly_log_reg, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

# 过拟合poly_log_reg2 = PolynomialLogisticRegression(degree=20)poly_log_reg2.fit(X, y)"""Out[14]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=20, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LinearRegression())])"""plot_decision_boundary(poly_log_reg2, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

在逻辑回归中使用正则化的方法如下：

05 scikit-learn中的逻辑回归、使用正则化

import numpy as npimport matplotlib.pyplot as plt​np.random.seed(666)X = np.random.normal(0, 1, size=(200, 2))y = np.array(X[:,0]**2 + X[:,1] < 1.5, dtype='int')for _ in range(20):y[np.random.randint(200)] = 1plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

from sklearn.model_selection import train_test_split​X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)

使用scikit-learn中的逻辑回归

from sklearn.linear_model import LogisticRegression​log_reg = LogisticRegression()log_reg.fit(X_train, y_train)"""Out[4]:LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False)"""log_reg.score(X_train, y_train)# Out[5]:# 0.79333333333333333def plot_decision_boundary(model, axis):x0, x1 = np.meshgrid(np.linspace(axis[0], axis[1], int((axis[1]-axis[0])*100)).reshape(-1,1),np.linspace(axis[2], axis[3], int((axis[3]-axis[2])*100)).reshape(-1,1))X_new = np.c_[x0.ravel(), x1.ravel()]y_predict = model.predict(X_new)zz = y_predict.reshape(x0.shape)from matplotlib.colors import ListedColormapcustom_cmap = ListedColormap(['#EF9A9A', '#FFF59D','#90CAF9'])plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)plot_decision_boundary(log_reg, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

from sklearn.pipeline import Pipelinefrom sklearn.preprocessing import PolynomialFeaturesfrom sklearn.preprocessing import StandardScaler​# 1.添加多项式项 2.归一化 3.逻辑回归(自己实现的类，符合“构造函数，fit,predict,score”等标准)def PolynomialLogisticRegression(degree):return Pipeline([('poly', PolynomialFeatures(degree=degree)),('std_scaler', StandardScaler()),('log_reg', LogisticRegression())])poly_log_reg = PolynomialLogisticRegression(degree=2)poly_log_reg.fit(X_train, y_train)"""Out[9]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=2, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False))])"""poly_log_reg.score(X_train, y_train)# Out[10]:# 0.91333333333333333# 泛化能力poly_log_reg.score(X_test, y_test)# Out[11]:# 0.93999999999999995plot_decision_boundary(poly_log_reg, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

poly_log_reg2 = PolynomialLogisticRegression(degree=20)poly_log_reg2.fit(X_train, y_train)"""Out[13]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=20, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False))])"""poly_log_reg2.score(X_train, y_train)# Out[14]:# 0.93999999999999995# 泛化能力，可以看出有一定的过拟合poly_log_reg2.score(X_test, y_test)# Out[15]:# 0.92000000000000004plot_decision_boundary(poly_log_reg2, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

# 重新定义管道，传入正则项系数Cdef PolynomialLogisticRegression(degree, C):return Pipeline([('poly', PolynomialFeatures(degree=degree)),('std_scaler', StandardScaler()),('log_reg', LogisticRegression(C=C))])poly_log_reg3 = PolynomialLogisticRegression(degree=20, C=0.1)poly_log_reg3.fit(X_train, y_train)"""Out[18]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=20, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False))])"""poly_log_reg3.score(X_train, y_train)# Out[19]:# 0.85333333333333339poly_log_reg3.score(X_test, y_test)# Out[20]:# 0.92000000000000004plot_decision_boundary(poly_log_reg3, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

加入正则项后更趋向于degree=2时的情况

from sklearn.pipeline import Pipelinefrom sklearn.preprocessing import PolynomialFeaturesfrom sklearn.preprocessing import StandardScaler​# 重新定义管道，传入正则项系数C，penaltydef PolynomialLogisticRegression(degree, C, penalty='12'):return Pipeline([('poly', PolynomialFeatures(degree=degree)),('std_scaler', StandardScaler()),('log_reg', LogisticRegression(C=C))])poly_log_reg4 = PolynomialLogisticRegression(degree=20, C=0.1, penalty='l1') # 使用L1正则项poly_log_reg4.fit(X_train, y_train)"""Out[23]:Pipeline(memory=None,steps=[('poly', PolynomialFeatures(degree=20, include_bias=True, interaction_only=False)), ('std_scaler', StandardScaler(copy=True, with_mean=True, with_std=True)), ('log_reg', LogisticRegression(C=0.1, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False))])"""poly_log_reg4.score(X_train, y_train)# Out[24]:# 0.85333333333333339poly_log_reg4.score(X_test, y_test)# Out[25]:# 0.92000000000000004plot_decision_boundary(poly_log_reg4, axis=[-4, 4, -4, 4])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.show()

以上所讨论的逻辑回归只适用于解决二分类问题，这显然是不符合现实所需要的，所以为了使其可以解决多分类问题，发明了OvR与OvO算法，二者实现原理不同。

OvR：每次选取一个类别与其他所有类别进行二分类任务

OvO：每次挑出两个类别进行二分类任务

OvO耗时更久，结果更准确。

06 OvR和OvO

import numpy as npimport matplotlib.pyplot as pltfrom sklearn import datasets​iris = datasets.load_iris()X = iris.data[:,:2]y = iris.targetfrom sklearn.model_selection import train_test_split​X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666) from sklearn.linear_model import LogisticRegression​log_reg = LogisticRegression()log_reg.fit(X_train, y_train)"""Out[3]:LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,penalty='l2', random_state=None, solver='liblinear', tol=0.0001,verbose=0, warm_start=False)"""log_reg.score(X_test, y_test)# Out[4]:# 0.65789473684210531def plot_decision_boundary(model, axis):x0, x1 = np.meshgrid(np.linspace(axis[0], axis[1], int((axis[1]-axis[0])*100)).reshape(-1,1),np.linspace(axis[2], axis[3], int((axis[3]-axis[2])*100)).reshape(-1,1))X_new = np.c_[x0.ravel(), x1.ravel()]y_predict = model.predict(X_new)zz = y_predict.reshape(x0.shape)from matplotlib.colors import ListedColormapcustom_cmap = ListedColormap(['#EF9A9A', '#FFF59D','#90CAF9'])plt.contourf(x0, x1, zz, linewidth=5, cmap=custom_cmap)plot_decision_boundary(log_reg, axis=[4, 8.5, 1.5, 4.5])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.scatter(X[y==2,0], X[y==2,1])plt.show()

# 使用OvO，注意更换计算方式'newton-cg'log_reg2 = LogisticRegression(multi_class='multinomial', solver='newton-cg')log_reg2.fit(X_train, y_train)log_reg2.score(X_test, y_test)# Out[8]:# 0.78947368421052633plot_decision_boundary(log_reg2, axis=[4, 8.5, 1.5, 4.5])plt.scatter(X[y==0,0], X[y==0,1])plt.scatter(X[y==1,0], X[y==1,1])plt.scatter(X[y==2,0], X[y==2,1])plt.show()

使用所有数据

X = iris.datay = iris.target​from sklearn.model_selection import train_test_split​X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666) log_reg = LogisticRegression()log_reg.fit(X_train, y_train)log_reg.score(X_test, y_test)# Out[11]:# 0.94736842105263153log_reg2 = LogisticRegression(multi_class='multinomial', solver='newton-cg')log_reg2.fit(X_train, y_train)log_reg2.score(X_test, y_test)# Out[12]:# 1.0

OvO and OvR

from sklearn.multiclass import OneVsRestClassifier​# 传入二分类器ovr = OneVsRestClassifier(log_reg)ovr.fit(X_train, y_train)ovr.score(X_test, y_test)# Out[13]:# 0.94736842105263153from sklearn.multiclass import OneVsOneClassifier​ovo = OneVsOneClassifier(log_reg)ovo.fit(X_train, y_train)ovo.score(X_test, y_test)# Out[14]:# 1.0