Notebook

Neural Networks in PyTorch¶

In [1]:

import torch
from torch.autograd import Variable
import torch.nn as nn
import torch.nn.functional as F

In [2]:

class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv1 = nn.Conv2d(1, 6, 5)
        self.conv2 = nn.Conv2d(6, 16, 5)
        self.fc1 = nn.Linear(16*5*5, 120)
        self.fc2 = nn.Linear(120, 84)
        self.fc3 = nn.Linear(84, 10)
        
    def forward(self, x):
        x = F.max_pool2d(F.relu(self.conv1(x)), (2, 2))
        x = F.max_pool2d(F.relu(self.conv2(x)), 2)
        x = x.view(-1, self.num_flat_features(x))
        x = F.relu(self.fc1(x))
        x = F.relu(self.fc2(x))
        x = self.fc3(x)
        return x
    def num_flat_features(self, x):
        size = x.size()[1:]
        num_features = 1
        for s in size:
            num_features *= s
        return num_features
    
net = Net()
print(net)

Net(
  (conv1): Conv2d (1, 6, kernel_size=(5, 5), stride=(1, 1))
  (conv2): Conv2d (6, 16, kernel_size=(5, 5), stride=(1, 1))
  (fc1): Linear(in_features=400, out_features=120)
  (fc2): Linear(in_features=120, out_features=84)
  (fc3): Linear(in_features=84, out_features=10)
)

In [3]:

params = list(net.parameters())
print(len(params))
print(params[0].size())

10
torch.Size([6, 1, 5, 5])

In [4]:

input = Variable(torch.randn(1, 1, 32, 32))
out = net(input)
print(out)

Variable containing:
 0.0520 -0.0491  0.0239  0.1516 -0.0555  0.0332  0.0873 -0.0206 -0.0700  0.0891
[torch.FloatTensor of size 1x10]

In [5]:

target = Variable(torch.arange(1, 11))
target = target.view(1, -1)
criterion = nn.MSELoss()

loss = criterion(out, target)
print(loss)

Variable containing:
 38.2532
[torch.FloatTensor of size 1]

In [6]:

print(loss.grad_fn)
print(loss.grad_fn.next_functions[0][0])
print(loss.grad_fn.next_functions[0][0].next_functions[0][0])
print(loss.grad_fn.next_functions[0][0].next_functions[0][0].next_functions[0][0])

<MseLossBackward object at 0x7f0d3d0da050>
<AddmmBackward object at 0x7f0d3d0da150>
<ExpandBackward object at 0x7f0d3d0da110>
<AccumulateGrad object at 0x7f0d3d0da190>

In [7]:

print('conv1.bias.grad before backward')
print(net.conv1.bias.grad)

loss.backward(retain_graph=True)

print('conv1.bias.grad after backward')
print(net.conv1.bias.grad)

conv1.bias.grad before backward
None
conv1.bias.grad after backward
Variable containing:
-0.0810
-0.0638
-0.0474
-0.1109
 0.0235
 0.0689
[torch.FloatTensor of size 6]

In [9]:

%%time
import torch.optim as optim

net.zero_grad()
optimizer = optim.SGD(net.parameters(), lr = 0.01)
optimizer.zero_grad()
for i in range(10):
    output = net(input)
    loss = criterion(output, target)
    loss.backward()
    optimizer.step()
    print(output, target)

(Variable containing:

Columns 0 to 7 
  4.7988  -1.8145   9.6046   5.1899   2.7905  11.5400  19.0878  12.0102

Columns 8 to 9 
 19.2533  18.5875
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0659  0.2023  0.1443  0.1505  0.2695  0.1457  0.1217  0.1000  0.2713  0.2817
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0519  0.2295  0.1683  0.2111  0.3358  0.2225  0.1956  0.2225  0.3733  0.4116
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0323  0.2573  0.1696  0.2499  0.3687  0.2751  0.2512  0.3136  0.4476  0.4794
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0463  0.2677  0.1846  0.2679  0.3976  0.2892  0.2572  0.3281  0.4646  0.5145
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0291  0.2927  0.2018  0.3052  0.4399  0.3470  0.3272  0.4194  0.5561  0.6071
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0247  0.3149  0.2164  0.3453  0.4927  0.3971  0.3772  0.4971  0.6338  0.6973
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0192  0.3426  0.2362  0.3930  0.5573  0.4562  0.4400  0.5915  0.7289  0.8067
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
-0.0102  0.3746  0.2632  0.4507  0.6344  0.5283  0.5213  0.7050  0.8480  0.9410
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
(Variable containing:
 0.0019  0.4116  0.2986  0.5196  0.7248  0.6148  0.6235  0.8407  0.9949  1.1035
[torch.FloatTensor of size 1x10]
, Variable containing:
    1     2     3     4     5     6     7     8     9    10
[torch.FloatTensor of size 1x10]
)
CPU times: user 208 ms, sys: 0 ns, total: 208 ms
Wall time: 50.1 ms