ニューラルネットワークを用いた教師あり学習のpythonプログラム

import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.optim as optim
import numpy as np

class Model(nn.Module):
    def __init__(self, input_dim, output_dim):
        super(Model, self).__init__()
        # NNの入出力
        self.fc1 = nn.Linear(input_dim, 100)
        self.fc2 = nn.Linear(100, 100)
        self.fc3 = nn.Linear(100, output_dim)
        # 学習率とパラメータの更新方法
        self.optimizer = optim.SGD(self.parameters(), lr=0.01)
        # loss関数
        self.criterion = nn.MSELoss()

    def forward(self, x):
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x

    def update(self,output, y):
        self.optimizer.zero_grad()
        loss = self.criterion(output, y)
        # 勾配の計算
        loss.backward()
        # パラメータの更新
        self.optimizer.step()
        return loss

import numpy as np

np.random.seed(0)


def sigmoid(z):
    return 1.0 / (1.0 + np.exp(-z))


def forward(X, W1, b1, W2, b2):
    Z1 = X @ W1 + b1
    A1 = sigmoid(Z1)
    Z2 = A1 @ W2 + b2
    return Z1, A1, Z2


def loss_fn(Z2, Y):
    N, d_out = Z2.shape
    return np.sum((Z2 - Y) ** 2) / (N * d_out)


def analytic_grad(X, Y, W1, b1, W2, b2):
    N, d_out = Y.shape
    Z1, A1, Z2 = forward(X, W1, b1, W2, b2)

    dZ2 = (2.0 / (N * d_out)) * (Z2 - Y)   # 式(5)のバッチ版
    dW2 = A1.T @ dZ2                        # 式(7)
    db2 = dZ2.sum(axis=0)                   # 式(8)

    dA1 = dZ2 @ W2.T                        # 式(9)
    dZ1 = dA1 * (A1 * (1 - A1))             # 式(11): sigmoid'(Z1) = A1*(1-A1)

    dW1 = X.T @ dZ1                         # 式(12)
    db1 = dZ1.sum(axis=0)                   # 式(12)

    return dW1, db1, dW2, db2


def numerical_grad(param, X, Y, W1, b1, W2, b2, eps=1e-5):
    grad = np.zeros_like(param)
    it = np.nditer(param, flags=["multi_index"])
    while not it.finished:
        idx = it.multi_index
        orig = param[idx]
        param[idx] = orig + eps
        Lp = loss_fn(forward(X, W1, b1, W2, b2)[2], Y)
        param[idx] = orig - eps
        Lm = loss_fn(forward(X, W1, b1, W2, b2)[2], Y)
        param[idx] = orig
        grad[idx] = (Lp - Lm) / (2 * eps)   # 式(16)
        it.iternext()
    return grad


N, d_in, d_h, d_out = 5, 3, 4, 2
X = np.random.randn(N, d_in)
Y = np.random.randn(N, d_out)
W1 = np.random.randn(d_in, d_h) * 0.5
b1 = np.random.randn(d_h) * 0.5
W2 = np.random.randn(d_h, d_out) * 0.5
b2 = np.random.randn(d_out) * 0.5

dW1, db1, dW2, db2 = analytic_grad(X, Y, W1, b1, W2, b2)
dW1_num = numerical_grad(W1, X, Y, W1, b1, W2, b2)
db1_num = numerical_grad(b1, X, Y, W1, b1, W2, b2)
dW2_num = numerical_grad(W2, X, Y, W1, b1, W2, b2)
db2_num = numerical_grad(b2, X, Y, W1, b1, W2, b2)

for name, ana, num in [("W1", dW1, dW1_num), ("b1", db1, db1_num),
                        ("W2", dW2, dW2_num), ("b2", db2, db2_num)]:
    abs_err = np.max(np.abs(ana - num))
    rel_err = np.max(np.abs(ana - num) / (np.abs(ana) + np.abs(num) + 1e-8))
    print(f"{name}: max_abs_err={abs_err:.3e}  max_rel_err={rel_err:.3e}")

W1: max_abs_err=6.087e-11  max_rel_err=3.444e-10
b1: max_abs_err=2.493e-11  max_rel_err=3.596e-11
W2: max_abs_err=6.114e-11  max_rel_err=1.208e-10
b2: max_abs_err=1.385e-11  max_rel_err=2.144e-11
overall relative error = 2.411e-11

import torch

Xt, Yt = torch.tensor(X), torch.tensor(Y)
W1t = torch.tensor(W1, requires_grad=True)
b1t = torch.tensor(b1, requires_grad=True)
W2t = torch.tensor(W2, requires_grad=True)
b2t = torch.tensor(b2, requires_grad=True)

Z1t = Xt @ W1t + b1t
A1t = torch.sigmoid(Z1t)
Z2t = A1t @ W2t + b2t
Lt = torch.mean((Z2t - Yt) ** 2)
Lt.backward()

for name, ana, tgrad in [("W1", dW1, W1t.grad), ("b1", db1, b1t.grad),
                          ("W2", dW2, W2t.grad), ("b2", db2, b2t.grad)]:
    err = (torch.tensor(ana) - tgrad).abs().max().item()
    print(f"{name} 最大誤差: {err:.3e}")

import torch
import torch.nn as nn

torch.manual_seed(0)

N_LAYERS, WIDTH, BATCH = 10, 64, 32


class DeepMLP(nn.Module):
    def __init__(self, n_layers, width, activation):
        super().__init__()
        self.layers = nn.ModuleList(nn.Linear(width, width) for _ in range(n_layers))
        self.out = nn.Linear(width, 1)
        self.activation = activation
        for layer in self.layers:
            nn.init.xavier_uniform_(layer.weight)
            nn.init.zeros_(layer.bias)
        nn.init.xavier_uniform_(self.out.weight)
        nn.init.zeros_(self.out.bias)

    def forward(self, x):
        for layer in self.layers:
            x = self.activation(layer(x))
        return self.out(x)


def run(activation):
    torch.manual_seed(0)
    model = DeepMLP(N_LAYERS, WIDTH, activation)
    x, y = torch.randn(BATCH, WIDTH), torch.randn(BATCH, 1)
    loss = nn.functional.mse_loss(model(x), y)
    loss.backward()
    return [layer.weight.grad.norm().item() for layer in model.layers]


sig_norms = run(torch.sigmoid)
relu_norms = run(torch.relu)

Sigmoid:  layer0=1.281e-06  layer1=6.133e-06  layer2=2.554e-05  ...  layer9=7.325e-01
          ratio (layer0 / layer9) = 1.749e-06

ReLU:     layer0=6.106e-02  layer1=5.718e-02  layer2=5.403e-02  ...  layer9=6.160e-02
          ratio (layer0 / layer9) = 9.912e-01

# テストデータの作成
def math(x):
    return x*x + 2*x + 1

def make():
    x = np.random.rand(1000)
    y = math(x)
    return x,y

train_x, train_y = make()
test_x, test_y = make()
model = Model(1,1)

# テストデータ
gosa = 0
for i in range(1000):
    output = model(torch.tensor([[float(test_x[i])]]))
    gosa += abs((output - test_y[i]))

print("学習前の誤差平均：", gosa/1000)

# 訓練データを学習
for i in range(1000):
    output = model(torch.tensor([[float(train_x[i])]]))
    loss = model.update(output, torch.tensor([[float(train_y[i])]]))

# テストデータ
gosa = 0
for i in range(1000):
    output = model(torch.tensor([[float(test_x[i])]]))
    gosa += abs((output - test_y[i])[0])

print("学習後の誤差平均：", gosa/1000)

学習前の誤差平均： tensor([[2.4819]], grad_fn=<DivBackward0>)
学習後の誤差平均： tensor([0.0326], grad_fn=<DivBackward0>)