Pytorch how and when to use Module Sequential ModuleList and ModuleDict
1.0.0
Pytorch 1.5 更新
您可以在这里找到代码
Pytorch 是一个开源深度学习框架，提供创建 ML 模型的智能方法。即使文档制作精良，我仍然发现大多数人都没有在 PyTorch 中编写良好且有组织的代码。
今天，我们将了解如何使用 PyTorch 的三个主要构建块： Module, Sequential and ModuleList 。我们将从一个示例开始，然后迭代地使其变得更好。
所有这四个类都包含在torch.nn中
import torch . nn as nn
# nn.Module
# nn.Sequential
# nn.Module 模块是主要构建块，它定义了所有神经网络的基类，您必须对其进行子类化。
让我们创建一个经典的 CNN 分类器作为示例：
import torch . nn . functional as F
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , n_classes ):
super (). __init__ ()
self . conv1 = nn . Conv2d ( in_c , 32 , kernel_size = 3 , stride = 1 , padding = 1 )
self . bn1 = nn . BatchNorm2d ( 32 )
self . conv2 = nn . Conv2d ( 32 , 64 , kernel_size = 3 , stride = 1 , padding = 1 )
self . bn2 = nn . BatchNorm2d ( 64 )
self . fc1 = nn . Linear ( 64 * 28 * 28 , 1024 )
self . fc2 = nn . Linear ( 1024 , n_classes )
def forward ( self , x ):
x = self . conv1 ( x )
x = self . bn1 ( x )
x = F . relu ( x )
x = self . conv2 ( x )
x = self . bn2 ( x )
x = F . relu ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . fc1 ( x )
x = F . sigmoid ( x )
x = self . fc2 ( x )
return x model = MyCNNClassifier ( 1 , 10 )
print ( model ) MyCNNClassifier(
(conv1): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(conv2): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(fc1): Linear(in_features=50176, out_features=1024, bias=True)
(fc2): Linear(in_features=1024, out_features=10, bias=True)
)
这是一个非常简单的分类器，其编码部分使用具有 3x3 卷积 + batchnorm + relu 的两个层，以及具有两个线性层的解码部分。如果您对 PyTorch 不陌生，您可能以前见过这种类型的编码，但有两个问题。
如果我们想添加一层，我们必须再次在__init__和forward函数中编写大量代码。另外，如果我们有一些公共模块想要在另一个模型中使用，例如 3x3 conv + batchnorm + relu，我们必须重新编写它。
Sequential 是一个可以堆叠在一起并同时运行的模块的容器。
您可以注意到我们必须将所有内容存储到self中。我们可以使用Sequential来改进我们的代码。
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , n_classes ):
super (). __init__ ()
self . conv_block1 = nn . Sequential (
nn . Conv2d ( in_c , 32 , kernel_size = 3 , stride = 1 , padding = 1 ),
nn . BatchNorm2d ( 32 ),
nn . ReLU ()
)
self . conv_block2 = nn . Sequential (
nn . Conv2d ( 32 , 64 , kernel_size = 3 , stride = 1 , padding = 1 ),
nn . BatchNorm2d ( 64 ),
nn . ReLU ()
)
self . decoder = nn . Sequential (
nn . Linear ( 64 * 28 * 28 , 1024 ),
nn . Sigmoid (),
nn . Linear ( 1024 , n_classes )
)
def forward ( self , x ):
x = self . conv_block1 ( x )
x = self . conv_block2 ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , 10 )
print ( model ) MyCNNClassifier(
(conv_block1): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(conv_block2): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(decoder): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
(2): Linear(in_features=1024, out_features=10, bias=True)
)
)
好多了呃？
您是否注意到conv_block1和conv_block2看起来几乎相同？我们可以创建一个返回nn.Sequential的函数来简化代码！
def conv_block ( in_f , out_f , * args , ** kwargs ):
return nn . Sequential (
nn . Conv2d ( in_f , out_f , * args , ** kwargs ),
nn . BatchNorm2d ( out_f ),
nn . ReLU ()
)然后我们就可以在我们的模块中调用这个函数
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , n_classes ):
super (). __init__ ()
self . conv_block1 = conv_block ( in_c , 32 , kernel_size = 3 , padding = 1 )
self . conv_block2 = conv_block ( 32 , 64 , kernel_size = 3 , padding = 1 )
self . decoder = nn . Sequential (
nn . Linear ( 64 * 28 * 28 , 1024 ),
nn . Sigmoid (),
nn . Linear ( 1024 , n_classes )
)
def forward ( self , x ):
x = self . conv_block1 ( x )
x = self . conv_block2 ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , 10 )
print ( model ) MyCNNClassifier(
(conv_block1): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(conv_block2): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(decoder): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
(2): Linear(in_features=1024, out_features=10, bias=True)
)
)
更干净！仍然conv_block1和conv_block2几乎相同！我们可以使用nn.Sequential合并它们
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , n_classes ):
super (). __init__ ()
self . encoder = nn . Sequential (
conv_block ( in_c , 32 , kernel_size = 3 , padding = 1 ),
conv_block ( 32 , 64 , kernel_size = 3 , padding = 1 )
)
self . decoder = nn . Sequential (
nn . Linear ( 64 * 28 * 28 , 1024 ),
nn . Sigmoid (),
nn . Linear ( 1024 , n_classes )
)
def forward ( self , x ):
x = self . encoder ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , 10 )
print ( model ) MyCNNClassifier(
(encoder): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
)
(decoder): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
(2): Linear(in_features=1024, out_features=10, bias=True)
)
)
self.encoder现在持有 Booth conv_block 。我们为模型解耦了逻辑，使其更易于阅读和重用。我们的conv_block函数可以导入并在另一个模型中使用。
如果我们可以在self.encoder中添加新层怎么办，对它们进行硬编码并不方便：
self . encoder = nn . Sequential (
conv_block ( in_c , 32 , kernel_size = 3 , padding = 1 ),
conv_block ( 32 , 64 , kernel_size = 3 , padding = 1 ),
conv_block ( 64 , 128 , kernel_size = 3 , padding = 1 ),
conv_block ( 128 , 256 , kernel_size = 3 , padding = 1 ),
)如果我们可以将大小定义为一个数组并自动创建所有层而不需要编写每一层，那该多好？幸运的是，我们可以创建一个数组并将其传递给Sequential
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , n_classes ):
super (). __init__ ()
self . enc_sizes = [ in_c , 32 , 64 ]
conv_blocks = [ conv_block ( in_f , out_f , kernel_size = 3 , padding = 1 )
for in_f , out_f in zip ( self . enc_sizes , self . enc_sizes [ 1 :])]
self . encoder = nn . Sequential ( * conv_blocks )
self . decoder = nn . Sequential (
nn . Linear ( 64 * 28 * 28 , 1024 ),
nn . Sigmoid (),
nn . Linear ( 1024 , n_classes )
)
def forward ( self , x ):
x = self . encoder ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , 10 )
print ( model ) MyCNNClassifier(
(encoder): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
)
(decoder): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
(2): Linear(in_features=1024, out_features=10, bias=True)
)
)
让我们来分解一下。我们创建了一个数组self.enc_sizes来保存编码器的大小。然后我们通过迭代大小来创建一个数组conv_blocks 。由于我们必须为每一层指定展位的 in 尺寸和 outsize，因此我们通过将 size' 数组移动一位来将其自身zip 。
为了清楚起见，请看以下示例：
sizes = [ 1 , 32 , 64 ]
for in_f , out_f in zip ( sizes , sizes [ 1 :]):
print ( in_f , out_f ) 1 32
32 64
然后，由于Sequential不接受列表，因此我们使用*运算符对其进行分解。
田田！现在，如果我们只想添加尺寸，我们可以轻松地将新数字添加到列表中。将尺寸作为参数是一种常见的做法。
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , enc_sizes , n_classes ):
super (). __init__ ()
self . enc_sizes = [ in_c , * enc_sizes ]
conv_blocks = [ conv_block ( in_f , out_f , kernel_size = 3 , padding = 1 )
for in_f , out_f in zip ( self . enc_sizes , self . enc_sizes [ 1 :])]
self . encoder = nn . Sequential ( * conv_blocks )
self . decoder = nn . Sequential (
nn . Linear ( 64 * 28 * 28 , 1024 ),
nn . Sigmoid (),
nn . Linear ( 1024 , n_classes )
)
def forward ( self , x ):
x = self . encoder ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , [ 32 , 64 , 128 ], 10 )
print ( model ) MyCNNClassifier(
(encoder): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(2): Sequential(
(0): Conv2d(64, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
)
(decoder): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
(2): Linear(in_features=1024, out_features=10, bias=True)
)
)
我们可以对解码器部分做同样的事情
def dec_block ( in_f , out_f ):
return nn . Sequential (
nn . Linear ( in_f , out_f ),
nn . Sigmoid ()
)
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , enc_sizes , dec_sizes , n_classes ):
super (). __init__ ()
self . enc_sizes = [ in_c , * enc_sizes ]
self . dec_sizes = [ 64 * 28 * 28 , * dec_sizes ]
conv_blocks = [ conv_block ( in_f , out_f , kernel_size = 3 , padding = 1 )
for in_f , out_f in zip ( self . enc_sizes , self . enc_sizes [ 1 :])]
self . encoder = nn . Sequential ( * conv_blocks )
dec_blocks = [ dec_block ( in_f , out_f )
for in_f , out_f in zip ( self . dec_sizes , self . dec_sizes [ 1 :])]
self . decoder = nn . Sequential ( * dec_blocks )
self . last = nn . Linear ( self . dec_sizes [ - 1 ], n_classes )
def forward ( self , x ):
x = self . encoder ( x )
x = x . view ( x . size ( 0 ), - 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , [ 32 , 64 ], [ 1024 , 512 ], 10 )
print ( model ) MyCNNClassifier(
(encoder): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
)
(decoder): Sequential(
(0): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
)
(1): Sequential(
(0): Linear(in_features=1024, out_features=512, bias=True)
(1): Sigmoid()
)
)
(last): Linear(in_features=512, out_features=10, bias=True)
)
我们遵循相同的模式，为解码部分创建一个新块，线性 + sigmoid，并传递一个具有大小的数组。我们必须添加self.last因为我们不想激活输出
现在，我们甚至可以将模型一分为二！编码器+解码器
class MyEncoder ( nn . Module ):
def __init__ ( self , enc_sizes ):
super (). __init__ ()
self . conv_blocks = nn . Sequential ( * [ conv_block ( in_f , out_f , kernel_size = 3 , padding = 1 )
for in_f , out_f in zip ( enc_sizes , enc_sizes [ 1 :])])
def forward ( self , x ):
return self . conv_blocks ( x )
class MyDecoder ( nn . Module ):
def __init__ ( self , dec_sizes , n_classes ):
super (). __init__ ()
self . dec_blocks = nn . Sequential ( * [ dec_block ( in_f , out_f )
for in_f , out_f in zip ( dec_sizes , dec_sizes [ 1 :])])
self . last = nn . Linear ( dec_sizes [ - 1 ], n_classes )
def forward ( self , x ):
return self . dec_blocks ()
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , enc_sizes , dec_sizes , n_classes ):
super (). __init__ ()
self . enc_sizes = [ in_c , * enc_sizes ]
self . dec_sizes = [ self . enc_sizes [ - 1 ] * 28 * 28 , * dec_sizes ]
self . encoder = MyEncoder ( self . enc_sizes )
self . decoder = MyDecoder ( self . dec_sizes , n_classes )
def forward ( self , x ):
x = self . encoder ( x )
x = x . flatten ( 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , [ 32 , 64 ], [ 1024 , 512 ], 10 )
print ( model ) MyCNNClassifier(
(encoder): MyEncoder(
(conv_blocks): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
)
)
(decoder): MyDecoder(
(dec_blocks): Sequential(
(0): Sequential(
(0): Linear(in_features=50176, out_features=1024, bias=True)
(1): Sigmoid()
)
(1): Sequential(
(0): Linear(in_features=1024, out_features=512, bias=True)
(1): Sigmoid()
)
)
(last): Linear(in_features=512, out_features=10, bias=True)
)
)
请注意， MyEncoder和MyDecoder也可以是返回nn.Sequential的函数。我更喜欢将第一个模式用于模型，将第二个模式用于构建块。
通过将我们的模块分成子模块，可以更轻松地共享代码、调试和测试它。
ModuleList允许您将Module存储为列表。当您需要遍历图层并存储/使用某些信息（例如在 U-net 中）时，它会很有用。
Sequential的主要区别在于ModuleList没有forward方法，因此内层没有连接。假设我们需要解码器中每一层的每个输出，我们可以通过以下方式存储它：
class MyModule ( nn . Module ):
def __init__ ( self , sizes ):
super (). __init__ ()
self . layers = nn . ModuleList ([ nn . Linear ( in_f , out_f ) for in_f , out_f in zip ( sizes , sizes [ 1 :])])
self . trace = []
def forward ( self , x ):
for layer in self . layers :
x = layer ( x )
self . trace . append ( x )
return x model = MyModule ([ 1 , 16 , 32 ])
import torch
model ( torch . rand (( 4 , 1 )))
[ print ( trace . shape ) for trace in model . trace ] torch.Size([4, 16])
torch.Size([4, 32])
[None, None]
如果我们想在conv_block中切换到LearkyRelu该怎么办？我们可以使用ModuleDict创建Module的字典，并在需要时动态切换Module
def conv_block ( in_f , out_f , activation = 'relu' , * args , ** kwargs ):
activations = nn . ModuleDict ([
[ 'lrelu' , nn . LeakyReLU ()],
[ 'relu' , nn . ReLU ()]
])
return nn . Sequential (
nn . Conv2d ( in_f , out_f , * args , ** kwargs ),
nn . BatchNorm2d ( out_f ),
activations [ activation ]
) print ( conv_block ( 1 , 32 , 'lrelu' , kernel_size = 3 , padding = 1 ))
print ( conv_block ( 1 , 32 , 'relu' , kernel_size = 3 , padding = 1 )) Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): LeakyReLU(negative_slope=0.01)
)
Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): ReLU()
)
让我们把一切都包起来吧！
def conv_block ( in_f , out_f , activation = 'relu' , * args , ** kwargs ):
activations = nn . ModuleDict ([
[ 'lrelu' , nn . LeakyReLU ()],
[ 'relu' , nn . ReLU ()]
])
return nn . Sequential (
nn . Conv2d ( in_f , out_f , * args , ** kwargs ),
nn . BatchNorm2d ( out_f ),
activations [ activation ]
)
def dec_block ( in_f , out_f ):
return nn . Sequential (
nn . Linear ( in_f , out_f ),
nn . Sigmoid ()
)
class MyEncoder ( nn . Module ):
def __init__ ( self , enc_sizes , * args , ** kwargs ):
super (). __init__ ()
self . conv_blocks = nn . Sequential ( * [ conv_block ( in_f , out_f , kernel_size = 3 , padding = 1 , * args , ** kwargs )
for in_f , out_f in zip ( enc_sizes , enc_sizes [ 1 :])])
def forward ( self , x ):
return self . conv_blocks ( x )
class MyDecoder ( nn . Module ):
def __init__ ( self , dec_sizes , n_classes ):
super (). __init__ ()
self . dec_blocks = nn . Sequential ( * [ dec_block ( in_f , out_f )
for in_f , out_f in zip ( dec_sizes , dec_sizes [ 1 :])])
self . last = nn . Linear ( dec_sizes [ - 1 ], n_classes )
def forward ( self , x ):
return self . dec_blocks ()
class MyCNNClassifier ( nn . Module ):
def __init__ ( self , in_c , enc_sizes , dec_sizes , n_classes , activation = 'relu' ):
super (). __init__ ()
self . enc_sizes = [ in_c , * enc_sizes ]
self . dec_sizes = [ 32 * 28 * 28 , * dec_sizes ]
self . encoder = MyEncoder ( self . enc_sizes , activation = activation )
self . decoder = MyDecoder ( dec_sizes , n_classes )
def forward ( self , x ):
x = self . encoder ( x )
x = x . flatten ( 1 ) # flat
x = self . decoder ( x )
return x model = MyCNNClassifier ( 1 , [ 32 , 64 ], [ 1024 , 512 ], 10 , activation = 'lrelu' )
print ( model ) MyCNNClassifier(
(encoder): MyEncoder(
(conv_blocks): Sequential(
(0): Sequential(
(0): Conv2d(1, 32, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(32, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): LeakyReLU(negative_slope=0.01)
)
(1): Sequential(
(0): Conv2d(32, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))
(1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)
(2): LeakyReLU(negative_slope=0.01)
)
)
)
(decoder): MyDecoder(
(dec_blocks): Sequential(
(0): Sequential(
(0): Linear(in_features=1024, out_features=512, bias=True)
(1): Sigmoid()
)
)
(last): Linear(in_features=512, out_features=10, bias=True)
)
)
所以，总而言之。
ModuleSequentialModuleListModuleDict这就是大家！
感谢您的阅读
