神经网络入门-用python实现一个两层神经网络并在CIFAR10数据集上调参，pythoncifar10,下面是我从cs231

神经网络入门-用python实现一个两层神经网络并在CIFAR10数据集上调参，pythoncifar10,下面是我从cs231

下面是我从cs231n上整理的神经网络的入门实现，麻雀虽小，五脏俱全，基本上神经网络涉及到的知识点都有在代码中体现。

理论看上千万遍，不如看一遍源码跑一跑。

源码上我已经加了很多注释，结合代码看一遍很容易理解。

最后可视化权重的图：

主文件，用来训练调参

two_layer_net.py

  1 # coding: utf-8  2   3 # 实现一个简单的神经网络并在CIFAR10上测试性能  4   5 import numpy as np  6 import matplotlib.pyplot as plt  7 from neural_net import TwoLayerNet  8 from data_utils import load_CIFAR10  9 from vis_utils import visualize_grid 10  11 def get_CIFAR10_data(num_training=49000, num_validation=1000, num_test=1000): 12     cifar10_dir = ‘cs231n/datasets/cifar-10-batches-py‘ 13     X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir) 14          15     # 采样 16     mask = list(range(num_training, num_training + num_validation)) 17     X_val = X_train[mask] 18     y_val = y_train[mask] 19     mask = list(range(num_training)) 20     X_train = X_train[mask] 21     y_train = y_train[mask] 22     mask = list(range(num_test)) 23     X_test = X_test[mask] 24     y_test = y_test[mask] 25  26     # 归一化操作：减去均值，使得数据以0为中心 27     mean_image = np.mean(X_train, axis=0) 28     X_train -= mean_image 29     X_val -= mean_image 30     X_test -= mean_image 31  32     X_train = X_train.reshape(num_training, -1) 33     X_val = X_val.reshape(num_validation, -1) 34     X_test = X_test.reshape(num_test, -1) 35  36     return X_train, y_train, X_val, y_val, X_test, y_test 37  38  39 X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data() 40 print(‘Train data shape: ‘, X_train.shape) 41 print(‘Train labels shape: ‘, y_train.shape) 42 print(‘Validation data shape: ‘, X_val.shape) 43 print(‘Validation labels shape: ‘, y_val.shape) 44 print(‘Test data shape: ‘, X_test.shape) 45 print(‘Test labels shape: ‘, y_test.shape) 46  47  48 #第一次训练 49 input_size = 32 * 32 * 3 50 hidden_size = 50 51 num_classes = 10 52 net = TwoLayerNet(input_size, hidden_size, num_classes) 53 stats = net.train(X_train, y_train, X_val, y_val, 54             num_iters=1000, batch_size=200, 55             learning_rate=1e-4, learning_rate_decay=0.95, 56             reg=0.25, verbose=True) 57 val_acc = (net.predict(X_val) == y_val).mean() 58 print(‘Validation accuracy: ‘, val_acc) 59  60 #效果不太理想，debug 61  62 # 先画一下loss和正确率的曲线看一看 63 plt.subplot(2, 1, 1) 64 plt.plot(stats[‘loss_history‘]) 65 plt.title(‘Loss history‘) 66 plt.xlabel(‘Iteration‘) 67 plt.ylabel(‘Loss‘) 68  69 plt.subplot(2, 1, 2) 70 plt.plot(stats[‘train_acc_history‘], label=‘train‘) 71 plt.plot(stats[‘val_acc_history‘], label=‘val‘) 72 plt.title(‘Classification accuracy history‘) 73 plt.xlabel(‘Epoch‘) 74 plt.ylabel(‘Clasification accuracy‘) 75 plt.show() 76  77  78  79 #可视化一下权重 80 def show_net_weights(net): 81     W1 = net.params[‘W1‘] 82     W1 = W1.reshape(32, 32, 3, -1).transpose(3, 0, 1, 2) 83     plt.imshow(visualize_grid(W1, padding=3).astype(‘uint8‘)) 84     plt.gca().axis(‘off‘) 85     plt.show() 86  87 show_net_weights(net) 88  89  90 #通过上面的曲线我们可以看到基本上loss还在线性下降，表示我们的loss下降的还不够。 91 #一方面，我们可以加大学习率使loss更加快速的下降，另一方面，也可以增加迭代的次数，让loss继续下降。 92 #还有，在训练集和验证集上的正确率没有明显差距，表明网络的容量可能不够，可以尝试增加网络的复杂度使之拥有更强的表达能力。 93  94  95  96 #下面是我调出来的参数，实际上选了很久 ,在测试集上的正确率在55%左右 97 hidden_size = 150#[50,70,100,130] 98 learning_rates = 1e-3#np.array([0.5,1,1.5])*1e-3 99 regularization_strengths = 0.2#[0.1,0.2,0.3]100 best_net = None101 results = {}102 best_val_acc = 0103 104 105 for hs in hidden_size:106     for lr in learning_rates:107         for reg in regularization_strengths:108             109             net = TwoLayerNet(input_size, hs, num_classes)110             # Train the network111             stats = net.train(X_train, y_train, X_val, y_val,112             num_iters=3000, batch_size=200,113             learning_rate=lr, learning_rate_decay=0.95,114             reg= reg, verbose=False)115             val_acc = (net.predict(X_val) == y_val).mean()116             if val_acc > best_val_acc:117                 best_val_acc = val_acc118                 best_net = net         119             results[(hs,lr,reg)] = val_acc120             121             plt.subplot(2, 1, 1)122             plt.plot(stats[‘loss_history‘])123             plt.title(‘Loss history‘)124             plt.xlabel(‘Iteration‘)125             plt.ylabel(‘Loss‘)126 127             plt.subplot(2, 1, 2)128             plt.plot(stats[‘train_acc_history‘], label=‘train‘)129             plt.plot(stats[‘val_acc_history‘], label=‘val‘)130             plt.title(‘Classification accuracy history‘)131             plt.xlabel(‘Epoch‘)132             plt.ylabel(‘Clasification accuracy‘)133             plt.show()134 135 136 for hs,lr, reg in sorted(results):137     val_acc = results[(hs, lr, reg)]138     print (‘hs %d lr %e reg %e val accuracy: %f‘ % (hs, lr, reg,  val_acc))139     140 print (‘best validation accuracy achieved during cross-validation: %f‘ % best_val_acc)141 142 143 show_net_weights(best_net)144 test_acc = (best_net.predict(X_test) == y_test).mean()145 print(‘Test accuracy: ‘, test_acc)

View Code

定义神经网络和前向反向计算、损失函数、自动训练的类

neural_net.py

  1 import numpy as np  2 import matplotlib.pyplot as plt  3   4 class TwoLayerNet(object):  5   """  6   两层的全连接网络。使用sotfmax损失函数和L2正则，非线性函数采用Relu函数。  7   网络结构：input - fully connected layer - ReLU - fully connected layer - softmax  8   """  9  10   def __init__(self, input_size, hidden_size, output_size, std=1e-4): 11     """ 12     初始化模型。 13     初始化权重矩阵W和偏置b。这里b置为零，但是Alexnet论文中说采用Relu函数激活时b置为1可以更快的收敛。 14     参数都保存在self.params字典中。 15     键为： 16     W1 (D, H) 17     b1 (H,) 18     W2 (H, C) 19     b2 (C,) 20     D,H,C分别表示输入数据的维度，隐藏层大小，输出类别的个数 21     """ 22     self.params = {} 23     self.params[‘W1‘] = std * np.random.randn(input_size, hidden_size) 24     self.params[‘b1‘] = np.zeros(hidden_size) 25     self.params[‘W2‘] = std * np.random.randn(hidden_size, output_size) 26     self.params[‘b2‘] = np.zeros(output_size) 27  28   def loss(self, X, y=None, reg=0.0): 29     """ 30     如果是在训练过程,计算损失和梯度，如果是在测试过程，返回最后一层的输入,即每个类的得分。 31  32     Inputs: 33     - X (N, D).  X[i] 为一个训练样本。 34     - y: 标签。如果为None则表示是在进行测试过程，否则是在进行训练过程。 35     - reg: Regularization strength. 36  37     Returns: 38     如果y=None，返回shape为(N, C)的矩阵，scores[i, c]表示输入i在c类上的得分。 39  40     如果y!=None, 返回一个tuple: 41     - loss: 包括数据损失和正则损失两部分。 42     - grads: 各个参数的梯度。 43     """ 44      45     W1, b1 = self.params[‘W1‘], self.params[‘b1‘] 46     W2, b2 = self.params[‘W2‘], self.params[‘b2‘] 47     N, D = X.shape 48     C=b2.shape[0] 49      50     #forward pass 51     h1=np.maximum(0,np.dot(X,W1)+b1) 52     h2=np.dot(h1,W2)+b2 53     scores=h2 54      55     if y is None: 56       return scores 57  58     # 计算loss 59     shift_scores=scores-np.max(scores,axis=1).reshape(-1,1) 60     exp_scores=np.exp(shift_scores) 61     softmax_out=exp_scores/np.sum(exp_scores,axis=1).reshape(-1,1) 62     loss=np.sum(-np.log(softmax_out[range(N),y]))/N+reg * (np.sum(W1 * W1) + np.sum(W2 * W2)) 63     print(np.sum(-np.log(softmax_out[range(N),y]))/N,reg * (np.sum(W1 * W1) + np.sum(W2 * W2))) 64  65     # Backward pass: 计算梯度，梯度的计算就是链式求导的过程 66     grads = {} 67  68     dscores = softmax_out.copy() 69     dscores[range(N),y]-=1 70     dscores /= N 71      72     grads[‘W2‘]=np.dot(h1.T,dscores)+2*reg*W2 73     grads[‘b2‘]=np.sum(dscores,axis=0) 74      75     dh=np.dot(dscores,W2.T) 76     d_max=(h1>0)*dh 77      78     grads[‘W1‘] = X.T.dot(d_max) + 2*reg * W1 79     grads[‘b1‘] = np.sum(d_max, axis = 0) 80  81     return loss, grads 82  83   def train(self, X, y, X_val, y_val, 84             learning_rate=1e-3, learning_rate_decay=0.95, 85             reg=5e-6, num_iters=100, 86             batch_size=200, verbose=False): 87     """ 88     自动化训练过程。采用SGD优化。 89  90     Inputs: 91     - X (N, D)：训练输入。 92     - y (N,) ：标签。 y[i] = c 表示X[i]的类别下标是c。 93     - X_val (N_val, D)：验证集输入。 94     - y_val (N_val,)： 验证集标签。 95     - learning_rate:  96     - learning_rate_decay: 学习率的损失因子。 97     - reg: regularization strength。 98     - num_iters: 迭代次数。 99     - batch_size: 每次迭代的数据批大小。.100     - verbose: 是否显示训练进度。101     """102     num_train = X.shape[0]103     iterations_per_epoch = max(num_train / batch_size, 1)104 105     loss_history = []106     train_acc_history = []107     val_acc_history = []108 109     for it in range(num_iters):110       #随机选择一批数据111       idx = np.random.choice(num_train, batch_size, replace=True)112       X_batch = X[idx]113       y_batch = y[idx]114       # 计算损失和梯度115       loss, grads = self.loss(X_batch, y=y_batch, reg=reg)116       loss_history.append(loss)117       #更新参数118       self.params[‘W2‘] += - learning_rate * grads[‘W2‘]119       self.params[‘b2‘] += - learning_rate * grads[‘b2‘]120       self.params[‘W1‘] += - learning_rate * grads[‘W1‘]121       self.params[‘b1‘] += - learning_rate * grads[‘b1‘]122       #可视化进度123       if verbose and it % 100 == 0:124         print(‘iteration %d / %d: loss %f‘ % (it, num_iters, loss))125 126       # 每个epoch保存一次数据记录127       if it % iterations_per_epoch == 0:128         train_acc = (self.predict(X_batch) == y_batch).mean()129         val_acc = (self.predict(X_val) == y_val).mean()130         train_acc_history.append(train_acc)131         val_acc_history.append(val_acc)132         #学习率衰减133         learning_rate *= learning_rate_decay134     return {135       ‘loss_history‘: loss_history,136       ‘train_acc_history‘: train_acc_history,137       ‘val_acc_history‘: val_acc_history,138     }139 140   def predict(self, X):141     """142     使用训练好的参数预测输入的标签。143 144     Inputs:145     - X (N, D)： 需要预测的输入。146 147     Returns:148     - y_pred (N,)：每个输入的预测分类下标。149     """150     151     h = np.maximum(0, X.dot(self.params[‘W1‘]) + self.params[‘b1‘])152     scores = h.dot(self.params[‘W2‘]) + self.params[‘b2‘]153     y_pred = np.argmax(scores, axis=1)154 155     return y_pred

View Code

载入CIFAR10数据的函数

data_utils.py

 1 from six.moves import cPickle as pickle 2 import numpy as np 3 import os 4 from scipy.misc import imread 5 import platform 6  7 def load_pickle(f): 8     version = platform.python_version_tuple() 9     if version[0] == ‘2‘:10         return  pickle.load(f)11     elif version[0] == ‘3‘:12         return  pickle.load(f, encoding=‘latin1‘)13     raise ValueError("invalid python version: {}".format(version))14 15 def load_CIFAR_batch(filename):16   """ CIRAR的数据是分批的，这个函数的功能是载入一批数据 """17   with open(filename, ‘rb‘) as f:18     datadict = load_pickle(f) #以二进制方式打开文件19     X = datadict[‘data‘]20     Y = datadict[‘labels‘]21     X = X.reshape(10000, 3, 32, 32).transpose(0,2,3,1).astype("float")22     Y = np.array(Y)23     return X, Y24 25 def load_CIFAR10(ROOT):26   """ load 所有的数据 """27   xs = []28   ys = []29   for b in range(1,6):30     f = os.path.join(ROOT, ‘data_batch_%d‘ % (b, ))31     X, Y = load_CIFAR_batch(f)32     xs.append(X)33     ys.append(Y)    34   Xtr = np.concatenate(xs)35   Ytr = np.concatenate(ys)36   del X, Y37   Xte, Yte = load_CIFAR_batch(os.path.join(ROOT, ‘test_batch‘))38   return Xtr, Ytr, Xte, Yte

View Code

可视化用到的函数

vis_utils.py

 1 from math import sqrt, ceil 2 import numpy as np 3  4 def visualize_grid(Xs, ubound=255.0, padding=1): 5   """ 6   #把4维的数据显示在平面图上，也就是把(N, H, W, C)N张3通道的图片同时显示出来 7  8   Inputs: 9   - Xs:(N, H, W, C)shape的数据10   - ubound: 像素会被放缩到【0，ubound】之间11   - padding: 方块之间的间隔填充12   """13   (N, H, W, C) = Xs.shape14   grid_size = int(ceil(sqrt(N)))15   grid_height = H * grid_size + padding * (grid_size - 1)16   grid_width = W * grid_size + padding * (grid_size - 1)17   grid = np.zeros((grid_height, grid_width, C))18   next_idx = 019   y0, y1 = 0, H20   for y in range(grid_size):21     x0, x1 = 0, W22     for x in range(grid_size):23       if next_idx < N:24         img = Xs[next_idx]25         low, high = np.min(img), np.max(img)26         grid[y0:y1, x0:x1] = ubound * (img - low) / (high - low)27         next_idx += 128       x0 += W + padding29       x1 += W + padding30     y0 += H + padding31     y1 += H + padding32   return grid

View Code

神经网络入门-用python实现一个两层神经网络并在CIFAR10数据集上调参