resnet.py 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487
  1. # copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. from __future__ import division
  16. from __future__ import print_function
  17. import math
  18. from collections import OrderedDict
  19. import paddle
  20. import paddle.fluid as fluid
  21. from paddle.fluid.param_attr import ParamAttr
  22. from paddle.fluid.framework import Variable
  23. from paddle.fluid.regularizer import L2Decay
  24. from paddle.fluid.initializer import Constant
  25. from numbers import Integral
  26. from .backbone_utils import NameAdapter
  27. __all__ = ['ResNet', 'ResNetC5']
  28. class ResNet(object):
  29. """
  30. Residual Network, see https://arxiv.org/abs/1512.03385
  31. Args:
  32. layers (int): ResNet layers, should be 18, 34, 50, 101, 152.
  33. freeze_at (int): freeze the backbone at which stage
  34. norm_type (str): normalization type, 'bn'/'sync_bn'/'affine_channel'
  35. freeze_norm (bool): freeze normalization layers
  36. norm_decay (float): weight decay for normalization layer weights
  37. variant (str): ResNet variant, supports 'a', 'b', 'c', 'd' currently
  38. feature_maps (list): index of stages whose feature maps are returned
  39. dcn_v2_stages (list): index of stages who select deformable conv v2
  40. nonlocal_stages (list): index of stages who select nonlocal networks
  41. gcb_stages (list): index of stages who select gc blocks
  42. gcb_params (dict): gc blocks config, includes ratio(default as 1.0/16),
  43. pooling_type(default as "att") and
  44. fusion_types(default as ['channel_add'])
  45. """
  46. def __init__(self,
  47. layers=50,
  48. freeze_at=0,
  49. norm_type='bn',
  50. freeze_norm=False,
  51. norm_decay=0.,
  52. variant='b',
  53. feature_maps=[2, 3, 4, 5],
  54. dcn_v2_stages=[],
  55. weight_prefix_name='',
  56. nonlocal_stages=[],
  57. gcb_stages=[],
  58. gcb_params=dict(),
  59. num_classes=None):
  60. super(ResNet, self).__init__()
  61. if isinstance(feature_maps, Integral):
  62. feature_maps = [feature_maps]
  63. assert layers in [18, 34, 50, 101, 152, 200], \
  64. "layers {} not in [18, 34, 50, 101, 152, 200]"
  65. assert variant in ['a', 'b', 'c', 'd'], "invalid ResNet variant"
  66. assert 0 <= freeze_at <= 5, "freeze_at should be 0, 1, 2, 3, 4 or 5"
  67. assert len(feature_maps) > 0, "need one or more feature maps"
  68. assert norm_type in ['bn', 'sync_bn', 'affine_channel']
  69. assert not (len(nonlocal_stages)>0 and layers<50), \
  70. "non-local is not supported for resnet18 or resnet34"
  71. self.layers = layers
  72. self.freeze_at = freeze_at
  73. self.norm_type = norm_type
  74. self.norm_decay = norm_decay
  75. self.freeze_norm = freeze_norm
  76. self.variant = variant
  77. self._model_type = 'ResNet'
  78. self.feature_maps = feature_maps
  79. self.dcn_v2_stages = dcn_v2_stages
  80. self.layers_cfg = {
  81. 18: ([2, 2, 2, 2], self.basicblock),
  82. 34: ([3, 4, 6, 3], self.basicblock),
  83. 50: ([3, 4, 6, 3], self.bottleneck),
  84. 101: ([3, 4, 23, 3], self.bottleneck),
  85. 152: ([3, 8, 36, 3], self.bottleneck),
  86. 200: ([3, 12, 48, 3], self.bottleneck),
  87. }
  88. self.stage_filters = [64, 128, 256, 512]
  89. self._c1_out_chan_num = 64
  90. self.na = NameAdapter(self)
  91. self.prefix_name = weight_prefix_name
  92. self.nonlocal_stages = nonlocal_stages
  93. self.nonlocal_mod_cfg = {
  94. 50: 2,
  95. 101: 5,
  96. 152: 8,
  97. 200: 12,
  98. }
  99. self.gcb_stages = gcb_stages
  100. self.gcb_params = gcb_params
  101. self.num_classes = num_classes
  102. def _conv_offset(self,
  103. input,
  104. filter_size,
  105. stride,
  106. padding,
  107. act=None,
  108. name=None):
  109. out_channel = filter_size * filter_size * 3
  110. out = fluid.layers.conv2d(
  111. input,
  112. num_filters=out_channel,
  113. filter_size=filter_size,
  114. stride=stride,
  115. padding=padding,
  116. param_attr=ParamAttr(
  117. initializer=Constant(0.0), name=name + ".w_0"),
  118. bias_attr=ParamAttr(initializer=Constant(0.0), name=name + ".b_0"),
  119. act=act,
  120. name=name)
  121. return out
  122. def _conv_norm(self,
  123. input,
  124. num_filters,
  125. filter_size,
  126. stride=1,
  127. groups=1,
  128. act=None,
  129. name=None,
  130. dcn_v2=False):
  131. _name = self.prefix_name + name if self.prefix_name != '' else name
  132. if not dcn_v2:
  133. conv = fluid.layers.conv2d(
  134. input=input,
  135. num_filters=num_filters,
  136. filter_size=filter_size,
  137. stride=stride,
  138. padding=(filter_size - 1) // 2,
  139. groups=groups,
  140. act=None,
  141. param_attr=ParamAttr(name=_name + "_weights"),
  142. bias_attr=False,
  143. name=_name + '.conv2d.output.1')
  144. else:
  145. # select deformable conv"
  146. offset_mask = self._conv_offset(
  147. input=input,
  148. filter_size=filter_size,
  149. stride=stride,
  150. padding=(filter_size - 1) // 2,
  151. act=None,
  152. name=_name + "_conv_offset")
  153. offset_channel = filter_size**2 * 2
  154. mask_channel = filter_size**2
  155. offset, mask = fluid.layers.split(
  156. input=offset_mask,
  157. num_or_sections=[offset_channel, mask_channel],
  158. dim=1)
  159. mask = fluid.layers.sigmoid(mask)
  160. conv = fluid.layers.deformable_conv(
  161. input=input,
  162. offset=offset,
  163. mask=mask,
  164. num_filters=num_filters,
  165. filter_size=filter_size,
  166. stride=stride,
  167. padding=(filter_size - 1) // 2,
  168. groups=groups,
  169. deformable_groups=1,
  170. im2col_step=1,
  171. param_attr=ParamAttr(name=_name + "_weights"),
  172. bias_attr=False,
  173. name=_name + ".conv2d.output.1")
  174. bn_name = self.na.fix_conv_norm_name(name)
  175. bn_name = self.prefix_name + bn_name if self.prefix_name != '' else bn_name
  176. norm_lr = 0. if self.freeze_norm else 1.
  177. norm_decay = self.norm_decay
  178. pattr = ParamAttr(
  179. name=bn_name + '_scale',
  180. learning_rate=norm_lr,
  181. regularizer=L2Decay(norm_decay))
  182. battr = ParamAttr(
  183. name=bn_name + '_offset',
  184. learning_rate=norm_lr,
  185. regularizer=L2Decay(norm_decay))
  186. if self.norm_type in ['bn', 'sync_bn']:
  187. global_stats = True if self.freeze_norm else False
  188. out = fluid.layers.batch_norm(
  189. input=conv,
  190. act=act,
  191. name=bn_name + '.output.1',
  192. param_attr=pattr,
  193. bias_attr=battr,
  194. moving_mean_name=bn_name + '_mean',
  195. moving_variance_name=bn_name + '_variance',
  196. use_global_stats=global_stats)
  197. scale = fluid.framework._get_var(pattr.name)
  198. bias = fluid.framework._get_var(battr.name)
  199. elif self.norm_type == 'affine_channel':
  200. scale = fluid.layers.create_parameter(
  201. shape=[conv.shape[1]],
  202. dtype=conv.dtype,
  203. attr=pattr,
  204. default_initializer=fluid.initializer.Constant(1.))
  205. bias = fluid.layers.create_parameter(
  206. shape=[conv.shape[1]],
  207. dtype=conv.dtype,
  208. attr=battr,
  209. default_initializer=fluid.initializer.Constant(0.))
  210. out = fluid.layers.affine_channel(
  211. x=conv, scale=scale, bias=bias, act=act)
  212. if self.freeze_norm:
  213. scale.stop_gradient = True
  214. bias.stop_gradient = True
  215. return out
  216. def _shortcut(self, input, ch_out, stride, is_first, name):
  217. max_pooling_in_short_cut = self.variant == 'd'
  218. ch_in = input.shape[1]
  219. # the naming rule is same as pretrained weight
  220. name = self.na.fix_shortcut_name(name)
  221. std_senet = getattr(self, 'std_senet', False)
  222. if ch_in != ch_out or stride != 1 or (self.layers < 50 and is_first):
  223. if std_senet:
  224. if is_first:
  225. return self._conv_norm(input, ch_out, 1, stride, name=name)
  226. else:
  227. return self._conv_norm(input, ch_out, 3, stride, name=name)
  228. if max_pooling_in_short_cut and not is_first:
  229. input = fluid.layers.pool2d(
  230. input=input,
  231. pool_size=2,
  232. pool_stride=2,
  233. pool_padding=0,
  234. ceil_mode=True,
  235. pool_type='avg')
  236. return self._conv_norm(input, ch_out, 1, 1, name=name)
  237. return self._conv_norm(input, ch_out, 1, stride, name=name)
  238. else:
  239. return input
  240. def bottleneck(self,
  241. input,
  242. num_filters,
  243. stride,
  244. is_first,
  245. name,
  246. dcn_v2=False,
  247. gcb=False,
  248. gcb_name=None):
  249. if self.variant == 'a':
  250. stride1, stride2 = stride, 1
  251. else:
  252. stride1, stride2 = 1, stride
  253. # ResNeXt
  254. groups = getattr(self, 'groups', 1)
  255. group_width = getattr(self, 'group_width', -1)
  256. if groups == 1:
  257. expand = 4
  258. elif (groups * group_width) == 256:
  259. expand = 1
  260. else: # FIXME hard code for now, handles 32x4d, 64x4d and 32x8d
  261. num_filters = num_filters // 2
  262. expand = 2
  263. conv_name1, conv_name2, conv_name3, \
  264. shortcut_name = self.na.fix_bottleneck_name(name)
  265. std_senet = getattr(self, 'std_senet', False)
  266. if std_senet:
  267. conv_def = [[
  268. int(num_filters / 2), 1, stride1, 'relu', 1, conv_name1
  269. ], [num_filters, 3, stride2, 'relu', groups, conv_name2],
  270. [num_filters * expand, 1, 1, None, 1, conv_name3]]
  271. else:
  272. conv_def = [[num_filters, 1, stride1, 'relu', 1, conv_name1],
  273. [num_filters, 3, stride2, 'relu', groups, conv_name2],
  274. [num_filters * expand, 1, 1, None, 1, conv_name3]]
  275. residual = input
  276. for i, (c, k, s, act, g, _name) in enumerate(conv_def):
  277. residual = self._conv_norm(
  278. input=residual,
  279. num_filters=c,
  280. filter_size=k,
  281. stride=s,
  282. act=act,
  283. groups=g,
  284. name=_name,
  285. dcn_v2=(i == 1 and dcn_v2))
  286. short = self._shortcut(
  287. input,
  288. num_filters * expand,
  289. stride,
  290. is_first=is_first,
  291. name=shortcut_name)
  292. # Squeeze-and-Excitation
  293. if callable(getattr(self, '_squeeze_excitation', None)):
  294. residual = self._squeeze_excitation(
  295. input=residual, num_channels=num_filters, name='fc' + name)
  296. if gcb:
  297. residual = add_gc_block(residual, name=gcb_name, **self.gcb_params)
  298. return fluid.layers.elementwise_add(
  299. x=short, y=residual, act='relu', name=name + ".add.output.5")
  300. def basicblock(self,
  301. input,
  302. num_filters,
  303. stride,
  304. is_first,
  305. name,
  306. dcn_v2=False,
  307. gcb=False,
  308. gcb_name=None):
  309. assert dcn_v2 is False, "Not implemented yet."
  310. assert gcb is False, "Not implemented yet."
  311. conv0 = self._conv_norm(
  312. input=input,
  313. num_filters=num_filters,
  314. filter_size=3,
  315. act='relu',
  316. stride=stride,
  317. name=name + "_branch2a")
  318. conv1 = self._conv_norm(
  319. input=conv0,
  320. num_filters=num_filters,
  321. filter_size=3,
  322. act=None,
  323. name=name + "_branch2b")
  324. short = self._shortcut(
  325. input, num_filters, stride, is_first, name=name + "_branch1")
  326. return fluid.layers.elementwise_add(x=short, y=conv1, act='relu')
  327. def layer_warp(self, input, stage_num):
  328. """
  329. Args:
  330. input (Variable): input variable.
  331. stage_num (int): the stage number, should be 2, 3, 4, 5
  332. Returns:
  333. The last variable in endpoint-th stage.
  334. """
  335. assert stage_num in [2, 3, 4, 5]
  336. stages, block_func = self.layers_cfg[self.layers]
  337. count = stages[stage_num - 2]
  338. ch_out = self.stage_filters[stage_num - 2]
  339. is_first = False if stage_num != 2 else True
  340. dcn_v2 = True if stage_num in self.dcn_v2_stages else False
  341. nonlocal_mod = 1000
  342. if stage_num in self.nonlocal_stages:
  343. nonlocal_mod = self.nonlocal_mod_cfg[
  344. self.layers] if stage_num == 4 else 2
  345. # Make the layer name and parameter name consistent
  346. # with ImageNet pre-trained model
  347. conv = input
  348. for i in range(count):
  349. conv_name = self.na.fix_layer_warp_name(stage_num, count, i)
  350. if self.layers < 50:
  351. is_first = True if i == 0 and stage_num == 2 else False
  352. gcb = stage_num in self.gcb_stages
  353. gcb_name = "gcb_res{}_b{}".format(stage_num, i)
  354. conv = block_func(
  355. input=conv,
  356. num_filters=ch_out,
  357. stride=2 if i == 0 and stage_num != 2 else 1,
  358. is_first=is_first,
  359. name=conv_name,
  360. dcn_v2=dcn_v2,
  361. gcb=gcb,
  362. gcb_name=gcb_name)
  363. # add non local model
  364. dim_in = conv.shape[1]
  365. nonlocal_name = "nonlocal_conv{}".format(stage_num)
  366. if i % nonlocal_mod == nonlocal_mod - 1:
  367. conv = add_space_nonlocal(conv, dim_in, dim_in,
  368. nonlocal_name + '_{}'.format(i),
  369. int(dim_in / 2))
  370. return conv
  371. def c1_stage(self, input):
  372. out_chan = self._c1_out_chan_num
  373. conv1_name = self.na.fix_c1_stage_name()
  374. if self.variant in ['c', 'd']:
  375. conv_def = [
  376. [out_chan // 2, 3, 2, "conv1_1"],
  377. [out_chan // 2, 3, 1, "conv1_2"],
  378. [out_chan, 3, 1, "conv1_3"],
  379. ]
  380. else:
  381. conv_def = [[out_chan, 7, 2, conv1_name]]
  382. for (c, k, s, _name) in conv_def:
  383. input = self._conv_norm(
  384. input=input,
  385. num_filters=c,
  386. filter_size=k,
  387. stride=s,
  388. act='relu',
  389. name=_name)
  390. output = fluid.layers.pool2d(
  391. input=input,
  392. pool_size=3,
  393. pool_stride=2,
  394. pool_padding=1,
  395. pool_type='max')
  396. return output
  397. def __call__(self, input):
  398. assert isinstance(input, Variable)
  399. assert not (set(self.feature_maps) - set([1, 2, 3, 4, 5])), \
  400. "feature maps {} not in [1, 2, 3, 4, 5]".format(self.feature_maps)
  401. res_endpoints = []
  402. res = input
  403. feature_maps = self.feature_maps
  404. severed_head = getattr(self, 'severed_head', False)
  405. if not severed_head:
  406. res = self.c1_stage(res)
  407. feature_maps = range(2, max(self.feature_maps) + 1)
  408. for i in feature_maps:
  409. res = self.layer_warp(res, i)
  410. if i in self.feature_maps:
  411. res_endpoints.append(res)
  412. if self.freeze_at >= i:
  413. res.stop_gradient = True
  414. if self.num_classes is not None:
  415. pool = fluid.layers.pool2d(
  416. input=res, pool_type='avg', global_pooling=True)
  417. stdv = 1.0 / math.sqrt(pool.shape[1] * 1.0)
  418. out = fluid.layers.fc(
  419. input=pool,
  420. size=self.num_classes,
  421. param_attr=fluid.param_attr.ParamAttr(
  422. initializer=fluid.initializer.Uniform(-stdv, stdv)))
  423. return out
  424. return OrderedDict([('res{}_sum'.format(self.feature_maps[idx]), feat)
  425. for idx, feat in enumerate(res_endpoints)])
  426. class ResNetC5(ResNet):
  427. __doc__ = ResNet.__doc__
  428. def __init__(self,
  429. layers=50,
  430. freeze_at=2,
  431. norm_type='affine_channel',
  432. freeze_norm=True,
  433. norm_decay=0.,
  434. variant='b',
  435. feature_maps=[5],
  436. weight_prefix_name=''):
  437. super(ResNetC5,
  438. self).__init__(layers, freeze_at, norm_type, freeze_norm,
  439. norm_decay, variant, feature_maps)
  440. self.severed_head = True