31 KiB
31 KiB
In [1]:
import d2l
import torch
import d2l
import numpy
import torch.nn as nn
import torch.nn.functional as FIn [2]:
net = nn.Sequential(nn.Linear(20, 256), nn.ReLU(), nn.Linear(256, 10))
X = torch.rand(2, 20)
net(X)Out [2]:
tensor([[ 0.0362, 0.0737, -0.0211, 0.0666, -0.1115, 0.0158, -0.1162, 0.0884,
0.1486, -0.1063],
[ 0.1796, -0.0009, 0.1236, -0.0783, -0.0937, -0.0560, 0.0441, 0.0812,
0.2236, -0.0597]], grad_fn=<AddmmBackward0>)In [3]:
class MLP(nn.Module):
def __init__(self):
super().__init__()
self.hidden=nn.Linear(20,256)
self.out=nn.Linear(256,10)
def forward(self,X):
return self.out(F.relu(self.hidden(X)))
In [4]:
net=MLP()
net(X)Out [4]:
tensor([[ 0.0376, -0.2522, -0.0243, -0.0838, 0.1215, 0.0258, -0.2358, 0.0799,
0.0756, 0.0520],
[ 0.0098, -0.2070, 0.0638, 0.1173, 0.0275, 0.0116, -0.0448, -0.0448,
-0.0309, -0.0976]], grad_fn=<AddmmBackward0>)In [5]:
class FixedHiddenMLP(nn.Module):
def __init__(self):
super().__init__()
# 不计算梯度的随机权重参数。因此其在训练期间保持不变
self.rand_weight = torch.rand((20, 20), requires_grad=False)
self.linear = nn.Linear(20, 20)
def forward(self, X):
X = self.linear(X)
# 使用创建的常量参数以及relu和mm函数
X = F.relu(torch.mm(X, self.rand_weight) + 1)
# 复用全连接层。这相当于两个全连接层共享参数
X = self.linear(X)
# 控制流
while X.abs().sum() > 1:
X /= 2
return X.sum()In [6]:
net = FixedHiddenMLP()
net(X)Out [6]:
tensor(0.1704, grad_fn=<SumBackward0>)
In [7]:
class NestMLP(nn.Module):
def __init__(self):
super().__init__()
self.net = nn.Sequential(nn.Linear(20, 64), nn.ReLU(),
nn.Linear(64, 32), nn.ReLU())
self.linear = nn.Linear(32, 16)
def forward(self, X):
return self.linear(self.net(X))
chimera = nn.Sequential(NestMLP(), nn.Linear(16, 20), FixedHiddenMLP())
chimera(X)In [8]:
net = nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 1))
X = torch.rand(size=(2, 4))
net(X)Out [8]:
tensor([[-0.2445],
[-0.2901]], grad_fn=<AddmmBackward0>)In [9]:
print(net[2].state_dict())OrderedDict([('weight', tensor([[-0.2116, 0.3448, 0.0726, -0.0626, -0.2922, 0.3172, 0.3025, -0.3025]])), ('bias', tensor([-0.3315]))])
In [10]:
net[2].state_dict()Out [10]:
OrderedDict([('weight',
tensor([[-0.2116, 0.3448, 0.0726, -0.0626, -0.2922, 0.3172, 0.3025, -0.3025]])),
('bias', tensor([-0.3315]))])In [11]:
print(type(net[2].bias))<class 'torch.nn.parameter.Parameter'>
In [12]:
print(net[2].bias)
print(net[2].bias.data)
Parameter containing: tensor([-0.3315], requires_grad=True) tensor([-0.3315])
In [13]:
net[2].weight.grad==NoneOut [13]:
True
In [14]:
print(*[(name, param.shape) for name, param in net[0].named_parameters()])
print(*[(name, param.shape) for name, param in net.named_parameters()])('weight', torch.Size([8, 4])) ('bias', torch.Size([8]))
('0.weight', torch.Size([8, 4])) ('0.bias', torch.Size([8])) ('2.weight', torch.Size([1, 8])) ('2.bias', torch.Size([1]))
In [15]:
net.state_dict()['2.bias'].dataOut [15]:
tensor([-0.3315])
In [16]:
def block1():
return nn.Sequential(nn.Linear(4, 8), nn.ReLU(), nn.Linear(8, 4),nn.ReLU())
def block2():
net = nn.Sequential()
for i in range(4):
net.add_module(f'block{i}', block1())
return netIn [17]:
rgnet = nn.Sequential(block2(),nn.Linear(4,1))
rgnet(X)Out [17]:
tensor([[-0.3640],
[-0.3640]], grad_fn=<AddmmBackward0>)In [18]:
print(rgnet)Sequential(
(0): Sequential(
(block0): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block1): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block2): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
(block3): Sequential(
(0): Linear(in_features=4, out_features=8, bias=True)
(1): ReLU()
(2): Linear(in_features=8, out_features=4, bias=True)
(3): ReLU()
)
)
(1): Linear(in_features=4, out_features=1, bias=True)
)
In [19]:
rgnet[0][1][0].bias.dataOut [19]:
tensor([ 0.3672, -0.3124, -0.3113, -0.3251, -0.4771, -0.3622, 0.1464, -0.4632])
In [20]:
def init_normal(m):
if type(m) == nn.Linear:
nn.init.normal_(m.weight, mean=0, std=0.01)
nn.init.zeros_(m.bias)
net.apply(init_normal)
net[0].weight.data[0], net[0].bias.data[0]Out [20]:
(tensor([-0.0004, 0.0166, -0.0085, -0.0099]), tensor(0.))
In [21]:
def init_xavier(m):
if type(m) == nn.Linear:
nn.init.xavier_uniform_(m.weight)
def init_42(m):
if type(m) == nn.Linear:
nn.init.constant_(m.weight, 42)
net[0].apply(init_xavier)
net[2].apply(init_42)
print(net[0].weight.data[0])
print(net[2].weight.data)tensor([-0.3265, -0.5057, -0.5062, -0.2116]) tensor([[42., 42., 42., 42., 42., 42., 42., 42.]])
In [22]:
x = torch.arange(4)
torch.save(x, 'x-file')In [23]:
x2 = torch.load('x-file')
x2Out [23]:
tensor([0, 1, 2, 3])
In [24]:
class MLP(nn.Module):
def __init__(self):
super().__init__()
self.hidden = nn.Linear(20, 256)
self.output = nn.Linear(256, 10)
def forward(self, x):
return self.output(F.relu(self.hidden(x)))
net = MLP()
X = torch.randn(size=(2, 20))
Y = net(X)In [25]:
torch.save(net.state_dict(), 'mlp.params')In [26]:
clone = MLP()
clone.load_state_dict(torch.load('mlp.params'))
clone.eval()Out [26]:
MLP( (hidden): Linear(in_features=20, out_features=256, bias=True) (output): Linear(in_features=256, out_features=10, bias=True) )
In [27]:
Y_clone = clone(X)
Y_clone == YOut [27]:
tensor([[True, True, True, True, True, True, True, True, True, True],
[True, True, True, True, True, True, True, True, True, True]])In [28]:
def corr2d(X,K):
h,w=K.shape
Y=torch.ones((X.shape[0]-h+1,X.shape[1]-w+1))
for i in range(Y.shape[0]):
for j in range(Y.shape[1]):
Y[i,j]=(X[i:i+h,j:j+w]*K).sum()
return Y
In [29]:
X = torch.tensor([[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]])
K = torch.tensor([[0.0, 1.0], [2.0, 3.0]])
corr2d(X,K)Out [29]:
tensor([[19., 25.],
[37., 43.]])In [30]:
class Conv2D(nn.Module):
def __init__(self, kernel_size):
super().__init__()
self.weight = nn.Parameter(torch.rand(kernel_size))
self.bias = nn.Parameter(torch.zeros(1))
def forward(self, x):
return corr2d(x, self.weight) + self.bias
In [31]:
X = torch.ones((6, 8))
X[:, 2:6] = 0
XOut [31]:
tensor([[1., 1., 0., 0., 0., 0., 1., 1.],
[1., 1., 0., 0., 0., 0., 1., 1.],
[1., 1., 0., 0., 0., 0., 1., 1.],
[1., 1., 0., 0., 0., 0., 1., 1.],
[1., 1., 0., 0., 0., 0., 1., 1.],
[1., 1., 0., 0., 0., 0., 1., 1.]])In [32]:
K = torch.tensor([[1.0, -1.0]])
Y = corr2d(X, K)
YOut [32]:
tensor([[ 0., 1., 0., 0., 0., -1., 0.],
[ 0., 1., 0., 0., 0., -1., 0.],
[ 0., 1., 0., 0., 0., -1., 0.],
[ 0., 1., 0., 0., 0., -1., 0.],
[ 0., 1., 0., 0., 0., -1., 0.],
[ 0., 1., 0., 0., 0., -1., 0.]])In [33]:
corr2d(X.t(), K)Out [33]:
tensor([[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.]])In [34]:
conv2d = nn.Conv2d(1,1, kernel_size=(1, 2), bias=False)In [35]:
X = X.reshape((1, 1, 6, 8))
Y = Y.reshape((1, 1, 6, 7))
lr = 3e-2In [36]:
for i in range(100):
Y_hat = conv2d(X)
l = (Y_hat - Y) ** 2
conv2d.zero_grad()
l.sum().backward()
# 迭代卷积核
conv2d.weight.data[:] -= lr * conv2d.weight.grad
if (i + 1) % 20 == 0:
print(f'epoch {i+1}, loss {l.sum():.3f}')epoch 20, loss 0.000 epoch 40, loss 0.000 epoch 60, loss 0.000 epoch 80, loss 0.000 epoch 100, loss 0.000
In [37]:
conv2d.weight.data.reshape((1, 2))
Out [37]:
tensor([[ 1.0000, -1.0000]])
In [38]:
# 为了方便起见，我们定义了一个计算卷积层的函数。
# 此函数初始化卷积层权重，并对输入和输出提高和缩减相应的维数
def comp_conv2d(conv2d, X):
# 这里的（1，1）表示批量大小和通道数都是1
X = X.reshape((1, 1) + X.shape)
Y = conv2d(X)
# 省略前两个维度：批量大小和通道
return Y.reshape(Y.shape[2:])
# 请注意，这里每边都填充了1行或1列，因此总共添加了2行或2列
conv2d = nn.Conv2d(1, 1, kernel_size=3, padding=1)In [39]:
X = torch.rand(size=(8, 8))
comp_conv2d(conv2d, X).shapeOut [39]:
torch.Size([8, 8])
In [40]:
conv2d = nn.Conv2d(1, 1, kernel_size=(5, 3), padding=(2, 1))
comp_conv2d(conv2d, X).shapeOut [40]:
torch.Size([8, 8])
In [41]:
conv2d = nn.Conv2d(1, 1, kernel_size=3, padding=1, stride=2)
comp_conv2d(conv2d, X).shapeOut [41]:
torch.Size([4, 4])
In [42]:
conv2d = nn.Conv2d(1, 1, kernel_size=(3, 5), padding=(0, 1), stride=(3, 4))
comp_conv2d(conv2d, X).shapeOut [42]:
torch.Size([2, 2])
In [50]:
def corr2d_multi_in(X,K):
return sum(corr2d(x,k) for x,k in zip(X,K))
X = torch.tensor([[[0.0, 1.0, 2.0], [3.0, 4.0, 5.0], [6.0, 7.0, 8.0]],
[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]]])
K = torch.tensor([[[0.0, 1.0], [2.0, 3.0]], [[1.0, 2.0], [3.0, 4.0]]])
corr2d_multi_in(X, K)Out [50]:
tensor([[ 56., 72.],
[104., 120.]])In [47]:
def corr2d_multi_in_out(X,K) ->torch.Tensor :
return torch.stack([corr2d_multi_in(X,k) for k in K],0)
In [51]:
K = torch.stack((K, K + 1, K + 2), 0)
K.shapeOut [51]:
torch.Size([3, 2, 2, 2])
In [52]:
corr2d_multi_in_out(X, K)Out [52]:
tensor([[[ 56., 72.],
[104., 120.]],
[[ 76., 100.],
[148., 172.]],
[[ 96., 128.],
[192., 224.]]])In [56]:
def corr2d_multi_in_out_1x1(X, K):
h_i,h,w=X.shape
h_o=K.shape[0]
X=X.reshape((h_i,h*w))
print(X.shape)
K=K.reshape((h_o,h_i))
print(K.shape)
Y=torch.matmul(K,X)
return Y.reshape((h_o,h,w))In [57]:
X = torch.normal(0, 1, (3, 3, 3))
K = torch.normal(0, 1, (2, 3, 1, 1))In [58]:
Y1 = corr2d_multi_in_out_1x1(X, K)torch.Size([3, 9]) torch.Size([2, 3])
In [59]:
Y2 = corr2d_multi_in_out(X, K)
assert float(torch.abs(Y1 - Y2).sum()) < 1e-6In [ ]: