Pytorch Notes

可视化梯度数据

可视化需要用到的一些重要函数。

register_backward_hook

The hook will be called every time the gradients with respect to module inputs are computed. The hook should have the following signature:

hook(module, grad_input, grad_output) -> Tensor or None

he hook should not modify its arguments, but it can optionally return a new gradient with respect to input that will be used in place of grad_input in subsequent computations.

register_forward_hook

The hook will be called every time after forward() has computed an output. It should have the following signature:

hook(module, input, output) -> None or modified output

The hook can modify the output. It can modify the input inplace but it will not have effect on forward since this is called after forward() is called`

register_forward_pre_hook

The hook will be called every time before forward() is invoked. It should have the following signature:

hook(module, input) -> None or modified input

The hook can modify the input. User can either return a tuple or a single modified value in the hook.

state_dict

Returns a dictionary containing a whole state of the module. Both parameters and persistent buffers (e.g. running averages) are included. Keys are corresponding parameter and buffer names.

parameters

Returns an iterator over module parameters.

1
2


for param in model.parameters():
    print(type(param), param.size())

Examples

Example 1

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42


import torch
import torch.nn as nn

class myNet(nn.Module):
  def __init__(self):
    super().__init__()
    self.conv = nn.Conv2d(3,10,2, stride = 2)
    self.relu = nn.ReLU()
    self.flatten = lambda x: x.view(-1)
    self.fc1 = nn.Linear(160,5)


  def forward(self, x):
    x = self.relu(self.conv(x))
    return self.fc1(self.flatten(x))


net = myNet()

def hook_fn(m, i, o):
  print(m)
  print("------------Input Grad------------")

  for grad in i:
    try:
      print(grad.shape)
    except AttributeError:
      print ("None found for Gradient")

  print("------------Output Grad------------")
  for grad in o:
    try:
      print(grad.shape)
    except AttributeError:
      print ("None found for Gradient")
  print("\n")
net.conv.register_backward_hook(hook_fn)
net.fc1.register_backward_hook(hook_fn)
inp = torch.randn(1,3,8,8)
out = net(inp)

(1 - out.mean()).backward()

Example 2

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77


import torch
import torch.nn as nn
import numpy as np

class MyLayer(nn.Module):
    def forward(self, input):
        out = input
        input.register_hook(self.hook)
        return out
    def hook(self, grad):
        print(grad.size())
        array = torch.tensor([1, 1, 1, 2, 2, 3], dtype = torch.float)
        grad2 = array.reshape(3, 1, 1, 2)
        return grad2

class myNet(nn.Module):

    def __init__(self):
        super().__init__()
        self.fc1 = nn.Linear(4, 2)
        self.s = MyLayer()


    def forward(self, x):
        x = self.fc1(x);
        x = self.s(x)
        return x

def backward_hook(m, i, o):

    print("=============backward===========")
    print("------------Input Grad------------")
    print(i)
    print("------------Output Grad------------")
    print(o)
    print("\n")

def forward_hook(m, i, o):

    print("=========forward===========")
    for param in m.parameters():
        print("parameters:\n", param)

    print("------------Input ------------")
    print(i)

    print("------------Output ------------")
    print(o)
    print("\n")

def preforward_hook(m, i):

    # print("========preforward===========")
    i = 0
    for param in m.parameters():
        if i == 0:
            param.data = torch.as_tensor(np.array([[2, 2, 2, 2], [1, 1, 1, 1]], dtype=np.float32))
            i += 1
        else:
            param.data = torch.as_tensor(np.array([1, 1], dtype = np.float32))

if __name__ == "__main__":
    net = myNet()
    fc1 = net.fc1
    net.fc1.register_backward_hook(backward_hook)
    net.fc1.register_forward_pre_hook(preforward_hook)
    net.fc1.register_forward_hook(forward_hook)
    array = np.array([1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3], dtype = np.float32)
    array = array.reshape(3, 1, 1, 4)
    inp = torch.as_tensor(array)
    inp.requires_grad = True
    out = net(inp)
    (1 - out.mean()).backward()
    print("result:", out.data)
    print("W grad:\n", fc1.weight.grad.data)
    print("b grad:\n", fc1.bias.grad.data)
    print("input grad:\n", inp.grad.data)