我正在使用众所周知的手写数字MNIST数据库开发第一个神经网络。我希望NN能够将给定图像的数字分类为0到9。
我的神经网络由三层组成：输入层（784个神经元，每个数字代表每个像素），隐藏层包含30个神经元（也可能是100或50个神经元，但是我不太担心超参数调整），输出层有10个神经元，每个神经元代表每个数字的激活。这给了我两个权重矩阵：一个是30x724，另一个是10x30。
我知道并理解反向传播，优化和其背后的数学公式的理论，这并不是问题。我可以优化第二个权重矩阵的权重，并且随着时间的推移，成本的确在降低。但是由于矩阵结构，我无法继续传播。
知道我已经找到了成本w.r.t.权重：
d(cost) / d(w) = d(cost) / d(f(z)) * d(f(z)) / d(z) * d(z) / d(w)
（是f激活函数，z点积加神经元的偏倚）
所以我在最右边的一层，有10个元素的输出数组。 d(cost) / d(f(z))是所观察到的预测值的减法。我可以将其乘以d(f(z)) / d(z)，它只是最右边一层的f'(z)，也是10个元素的一维向量，现在已经计算出d(cost) / d(z)。然后，d(z)/d(w)只是该层的输入，即上一层的输出，它是30个元素的向量。我认为可以对d(cost) / d(z)进行转置，以便T( d(cost) / d(z) ) * d(z) / d(w)给我一个（10，30）矩阵，这很有意义，因为它与最右边的权重矩阵的维数匹配。
但是后来我被卡住了。 d(cost) / d(f(z))的尺寸为（1，10），d(f(z)) / d(z)的尺寸为（1，30），而d(z) / d(w)的尺寸为（1，784）。我不知道该如何得出结果。
这是我到目前为止编写的代码。不完整的部分是_propagate_back方法。我还不在乎偏见，因为我只是执着于重量，首先我想弄清楚这一点。
import random
from typing import List, Tuple
import numpy as np
from matplotlib import pyplot as plt
import mnist_loader
np.random.seed(42)
NETWORK_LAYER_SIZES = [784, 30, 10]
LEARNING_RATE = 0.05
BATCH_SIZE = 20
NUMBER_OF_EPOCHS = 5000
def sigmoid(x):
return 1 / (1 + np.exp(-x))
def sigmoid_der(x):
return sigmoid(x) * (1 - sigmoid(x))
class Layer:
def __init__(self, input_size: int, output_size: int):
self.weights = np.random.uniform(-1, 1, [output_size, input_size])
self.biases = np.random.uniform(-1, 1, [output_size])
self.z = np.zeros(output_size)
self.a = np.zeros(output_size)
self.dz = np.zeros(output_size)
def feed_forward(self, input_data: np.ndarray):
input_data_t = np.atleast_2d(input_data).T
dot_product = self.weights.dot(input_data_t).T[0]
self.z = dot_product + self.biases
self.a = sigmoid(self.z)
self.dz = sigmoid_der(self.z)
class Network:
def __init__(self, layer_sizes: List[int], X_train: np.ndarray, y_train: np.ndarray):
self.layers = [
Layer(input_size, output_size)
for input_size, output_size
in zip(layer_sizes[0:], layer_sizes[1:])
]
self.X_train = X_train
self.y_train = y_train
@property
def predicted(self) -> np.ndarray:
return self.layers[-1].a
def _normalize_y(self, y: int) -> np.ndarray:
output_layer_size = len(self.predicted)
normalized_y = np.zeros(output_layer_size)
normalized_y[y] = 1.
return normalized_y
def _calculate_cost(self, y_observed: np.ndarray) -> int:
y_observed = self._normalize_y(y_observed)
y_predicted = self.layers[-1].a
squared_difference = (y_predicted - y_observed) ** 2
return np.sum(squared_difference)
def _get_training_batches(self, X_train: np.ndarray, y_train: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
train_batch_indexes = random.sample(range(len(X_train)), BATCH_SIZE)
return X_train[train_batch_indexes], y_train[train_batch_indexes]
def _feed_forward(self, input_data: np.ndarray):
for layer in self.layers:
layer.feed_forward(input_data)
input_data = layer.a
def _propagate_back(self, X: np.ndarray, y_observed: int):
"""
der(cost) / der(weight) = der(cost) / der(predicted) * der(predicted) / der(z) * der(z) / der(weight)
"""
y_observed = self._normalize_y(y_observed)
d_cost_d_pred = self.predicted - y_observed
hidden_layer = self.layers[0]
output_layer = self.layers[1]
# Output layer weights
d_pred_d_z = output_layer.dz
d_z_d_weight = hidden_layer.a # Input to the current layer, i.e. the output from the previous one
d_cost_d_z = d_cost_d_pred * d_pred_d_z
d_cost_d_weight = np.atleast_2d(d_cost_d_z).T * np.atleast_2d(d_z_d_weight)
output_layer.weights -= LEARNING_RATE * d_cost_d_weight
# Hidden layer weights
d_pred_d_z = hidden_layer.dz
d_z_d_weight = X
# ...
def train(self, X_train: np.ndarray, y_train: np.ndarray):
X_train_batch, y_train_batch = self._get_training_batches(X_train, y_train)
cost_over_epoch = []
for epoch_number in range(NUMBER_OF_EPOCHS):
X_train_batch, y_train_batch = self._get_training_batches(X_train, y_train)
cost = 0
for X_sample, y_observed in zip(X_train_batch, y_train_batch):
self._feed_forward(X_sample)
cost += self._calculate_cost(y_observed)
self._propagate_back(X_sample, y_observed)
cost_over_epoch.append(cost / BATCH_SIZE)
plt.plot(cost_over_epoch)
plt.ylabel('Cost')
plt.xlabel('Epoch')
plt.savefig('cost_over_epoch.png')
training_data, validation_data, test_data = mnist_loader.load_data()
X_train, y_train = training_data[0], training_data[1]
network = Network(NETWORK_LAYER_SIZES, training_data[0], training_data[1])
network.train(X_train, y_train)
这是mnist_loader的代码，以防有人想重现该示例：
import pickle
import gzip
def load_data():
f = gzip.open('data/mnist.pkl.gz', 'rb')
training_data, validation_data, test_data = pickle.load(f, encoding='latin-1')
f.close()
return training_data, validation_data, test_data
[一旦有了d(cost) / d(z)，我认为您实际上应该将其乘以权重矩阵：只有这样，您才能将误差d(cost) / d(z)移回新层（并获得有意义的矩阵形状）。
这是我如何更改后向传递功能：
def _propagate_back(self, X: np.ndarray, y_observed: int):
"""
der(cost) / der(weight) = der(cost) / der(predicted) * der(predicted) / der(z) * der(z) / der(weight)
"""
y_observed = self._normalize_y(y_observed)
d_cost_d_pred = self.predicted - y_observed
hidden_layer = self.layers[0]
output_layer = self.layers[1]
# Output layer weights
d_pred_d_z = output_layer.dz
d_z_d_weight = np.atleast_2d(hidden_layer.a) # Input to the current layer, i.e. the output from the previous one
d_cost_d_z = np.atleast_2d(d_cost_d_pred * d_pred_d_z)
d_cost_d_weight = np.dot(d_cost_d_z.T, d_z_d_weight)
output_layer.weights -= LEARNING_RATE * d_cost_d_weight
# Hidden layer weights
d_pred_d_z = hidden_layer.dz
d_z_d_weight = np.atleast_2d(X)
hidden_err = np.dot(d_cost_d_z, output_layer.weights)
d_cost_d_z = np.atleast_2d(hidden_err * d_pred_d_z)
d_cost_d_weight = np.dot(d_cost_d_z.T, d_z_d_weight)
hidden_layer.weights -= LEARNING_RATE * d_cost_d_weight
两个注释：
hidden_err = np.dot(d_cost_d_z, output_layer.weights)乘以权重矩阵的位置d(cost) / d(z)运算符（在Numpy中为Hadamard积，如果我是对的，则替换为*函数的应用（在Numpy中为矩阵乘法）的出现]我不是专家，所以我希望我不会犯一些可怕的错误……无论如何，我的回答主要基于Michael Nielsen的神经网络和深度学习的np.dot。