fpn.py 9.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import numpy as np
  15. import paddle
  16. import paddle.nn as nn
  17. import paddle.nn.functional as F
  18. from paddle import ParamAttr
  19. from paddle.nn.initializer import XavierUniform
  20. from paddle.regularizer import L2Decay
  21. from paddlex.ppdet.core.workspace import register, serializable
  22. from paddlex.ppdet.modeling.layers import ConvNormLayer
  23. from ..shape_spec import ShapeSpec
  24. __all__ = ['FPN']
  25. @register
  26. @serializable
  27. class FPN(nn.Layer):
  28. """
  29. Feature Pyramid Network, see https://arxiv.org/abs/1612.03144
  30. Args:
  31. in_channels (list[int]): input channels of each level which can be
  32. derived from the output shape of backbone by from_config
  33. out_channel (list[int]): output channel of each level
  34. spatial_scales (list[float]): the spatial scales between input feature
  35. maps and original input image which can be derived from the output
  36. shape of backbone by from_config
  37. has_extra_convs (bool): whether to add extra conv to the last level.
  38. default False
  39. extra_stage (int): the number of extra stages added to the last level.
  40. default 1
  41. use_c5 (bool): Whether to use c5 as the input of extra stage,
  42. otherwise p5 is used. default True
  43. norm_type (string|None): The normalization type in FPN module. If
  44. norm_type is None, norm will not be used after conv and if
  45. norm_type is string, bn, gn, sync_bn are available. default None
  46. norm_decay (float): weight decay for normalization layer weights.
  47. default 0.
  48. freeze_norm (bool): whether to freeze normalization layer.
  49. default False
  50. relu_before_extra_convs (bool): whether to add relu before extra convs.
  51. default False
  52. """
  53. def __init__(self,
  54. in_channels,
  55. out_channel,
  56. spatial_scales=[0.25, 0.125, 0.0625, 0.03125],
  57. has_extra_convs=False,
  58. extra_stage=1,
  59. use_c5=True,
  60. norm_type=None,
  61. norm_decay=0.,
  62. freeze_norm=False,
  63. relu_before_extra_convs=True):
  64. super(FPN, self).__init__()
  65. self.out_channel = out_channel
  66. for s in range(extra_stage):
  67. spatial_scales = spatial_scales + [spatial_scales[-1] / 2.]
  68. self.spatial_scales = spatial_scales
  69. self.has_extra_convs = has_extra_convs
  70. self.extra_stage = extra_stage
  71. self.use_c5 = use_c5
  72. self.relu_before_extra_convs = relu_before_extra_convs
  73. self.norm_type = norm_type
  74. self.norm_decay = norm_decay
  75. self.freeze_norm = freeze_norm
  76. self.lateral_convs = []
  77. self.fpn_convs = []
  78. fan = out_channel * 3 * 3
  79. # stage index 0,1,2,3 stands for res2,res3,res4,res5 on ResNet Backbone
  80. # 0 <= st_stage < ed_stage <= 3
  81. st_stage = 4 - len(in_channels)
  82. ed_stage = st_stage + len(in_channels) - 1
  83. for i in range(st_stage, ed_stage + 1):
  84. if i == 3:
  85. lateral_name = 'fpn_inner_res5_sum'
  86. else:
  87. lateral_name = 'fpn_inner_res{}_sum_lateral'.format(i + 2)
  88. in_c = in_channels[i - st_stage]
  89. if self.norm_type is not None:
  90. lateral = self.add_sublayer(
  91. lateral_name,
  92. ConvNormLayer(
  93. ch_in=in_c,
  94. ch_out=out_channel,
  95. filter_size=1,
  96. stride=1,
  97. norm_type=self.norm_type,
  98. norm_decay=self.norm_decay,
  99. freeze_norm=self.freeze_norm,
  100. initializer=XavierUniform(fan_out=in_c)))
  101. else:
  102. lateral = self.add_sublayer(
  103. lateral_name,
  104. nn.Conv2D(
  105. in_channels=in_c,
  106. out_channels=out_channel,
  107. kernel_size=1,
  108. weight_attr=ParamAttr(
  109. initializer=XavierUniform(fan_out=in_c))))
  110. self.lateral_convs.append(lateral)
  111. fpn_name = 'fpn_res{}_sum'.format(i + 2)
  112. if self.norm_type is not None:
  113. fpn_conv = self.add_sublayer(
  114. fpn_name,
  115. ConvNormLayer(
  116. ch_in=out_channel,
  117. ch_out=out_channel,
  118. filter_size=3,
  119. stride=1,
  120. norm_type=self.norm_type,
  121. norm_decay=self.norm_decay,
  122. freeze_norm=self.freeze_norm,
  123. initializer=XavierUniform(fan_out=fan)))
  124. else:
  125. fpn_conv = self.add_sublayer(
  126. fpn_name,
  127. nn.Conv2D(
  128. in_channels=out_channel,
  129. out_channels=out_channel,
  130. kernel_size=3,
  131. padding=1,
  132. weight_attr=ParamAttr(
  133. initializer=XavierUniform(fan_out=fan))))
  134. self.fpn_convs.append(fpn_conv)
  135. # add extra conv levels for RetinaNet(use_c5)/FCOS(use_p5)
  136. if self.has_extra_convs:
  137. for i in range(self.extra_stage):
  138. lvl = ed_stage + 1 + i
  139. if i == 0 and self.use_c5:
  140. in_c = in_channels[-1]
  141. else:
  142. in_c = out_channel
  143. extra_fpn_name = 'fpn_{}'.format(lvl + 2)
  144. if self.norm_type is not None:
  145. extra_fpn_conv = self.add_sublayer(
  146. extra_fpn_name,
  147. ConvNormLayer(
  148. ch_in=in_c,
  149. ch_out=out_channel,
  150. filter_size=3,
  151. stride=2,
  152. norm_type=self.norm_type,
  153. norm_decay=self.norm_decay,
  154. freeze_norm=self.freeze_norm,
  155. initializer=XavierUniform(fan_out=fan)))
  156. else:
  157. extra_fpn_conv = self.add_sublayer(
  158. extra_fpn_name,
  159. nn.Conv2D(
  160. in_channels=in_c,
  161. out_channels=out_channel,
  162. kernel_size=3,
  163. stride=2,
  164. padding=1,
  165. weight_attr=ParamAttr(
  166. initializer=XavierUniform(fan_out=fan))))
  167. self.fpn_convs.append(extra_fpn_conv)
  168. @classmethod
  169. def from_config(cls, cfg, input_shape):
  170. return {
  171. 'in_channels': [i.channels for i in input_shape],
  172. 'spatial_scales': [1.0 / i.stride for i in input_shape],
  173. }
  174. def forward(self, body_feats):
  175. laterals = []
  176. num_levels = len(body_feats)
  177. for i in range(num_levels):
  178. laterals.append(self.lateral_convs[i](body_feats[i]))
  179. for i in range(1, num_levels):
  180. lvl = num_levels - i
  181. upsample = F.interpolate(
  182. laterals[lvl],
  183. scale_factor=2.,
  184. mode='nearest', )
  185. laterals[lvl - 1] += upsample
  186. fpn_output = []
  187. for lvl in range(num_levels):
  188. fpn_output.append(self.fpn_convs[lvl](laterals[lvl]))
  189. if self.extra_stage > 0:
  190. # use max pool to get more levels on top of outputs (Faster R-CNN, Mask R-CNN)
  191. if not self.has_extra_convs:
  192. assert self.extra_stage == 1, 'extra_stage should be 1 if FPN has not extra convs'
  193. fpn_output.append(F.max_pool2d(fpn_output[-1], 1, stride=2))
  194. # add extra conv levels for RetinaNet(use_c5)/FCOS(use_p5)
  195. else:
  196. if self.use_c5:
  197. extra_source = body_feats[-1]
  198. else:
  199. extra_source = fpn_output[-1]
  200. fpn_output.append(self.fpn_convs[num_levels](extra_source))
  201. for i in range(1, self.extra_stage):
  202. if self.relu_before_extra_convs:
  203. fpn_output.append(self.fpn_convs[num_levels + i](F.relu(
  204. fpn_output[-1])))
  205. else:
  206. fpn_output.append(self.fpn_convs[num_levels + i](
  207. fpn_output[-1]))
  208. return fpn_output
  209. @property
  210. def out_shape(self):
  211. return [
  212. ShapeSpec(
  213. channels=self.out_channel, stride=1. / s)
  214. for s in self.spatial_scales
  215. ]