mobilenet_v3.py 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import paddle.fluid as fluid
  15. from paddle.fluid.param_attr import ParamAttr
  16. from paddle.fluid.regularizer import L2Decay
  17. import math
  18. class MobileNetV3():
  19. """
  20. MobileNet v3, see https://arxiv.org/abs/1905.02244
  21. Args:
  22. scale (float): scaling factor for convolution groups proportion of mobilenet_v3.
  23. model_name (str): There are two modes, small and large.
  24. norm_type (str): normalization type, 'bn' and 'sync_bn' are supported.
  25. norm_decay (float): weight decay for normalization layer weights.
  26. conv_decay (float): weight decay for convolution layer weights.
  27. with_extra_blocks (bool): if extra blocks should be added.
  28. extra_block_filters (list): number of filter for each extra block.
  29. """
  30. def __init__(self,
  31. scale=1.0,
  32. model_name='small',
  33. with_extra_blocks=False,
  34. conv_decay=0.0,
  35. norm_type='bn',
  36. norm_decay=0.0,
  37. extra_block_filters=[[256, 512], [128, 256], [128, 256],
  38. [64, 128]],
  39. num_classes=None,
  40. lr_mult_list=[1.0, 1.0, 1.0, 1.0, 1.0],
  41. for_seg=False,
  42. output_stride=None):
  43. assert len(lr_mult_list) == 5, \
  44. "lr_mult_list length in MobileNetV3 must be 5 but got {}!!".format(
  45. len(lr_mult_list))
  46. self.scale = scale
  47. self.with_extra_blocks = with_extra_blocks
  48. self.extra_block_filters = extra_block_filters
  49. self.conv_decay = conv_decay
  50. self.norm_decay = norm_decay
  51. self.inplanes = 16
  52. self.end_points = []
  53. self.block_stride = 1
  54. self.num_classes = num_classes
  55. self.lr_mult_list = lr_mult_list
  56. self.curr_stage = 0
  57. self.for_seg = for_seg
  58. self.decode_point = None
  59. if self.for_seg:
  60. if model_name == "large":
  61. self.cfg = [
  62. # k, exp, c, se, nl, s,
  63. [3, 16, 16, False, 'relu', 1],
  64. [3, 64, 24, False, 'relu', 2],
  65. [3, 72, 24, False, 'relu', 1],
  66. [5, 72, 40, True, 'relu', 2],
  67. [5, 120, 40, True, 'relu', 1],
  68. [5, 120, 40, True, 'relu', 1],
  69. [3, 240, 80, False, 'hard_swish', 2],
  70. [3, 200, 80, False, 'hard_swish', 1],
  71. [3, 184, 80, False, 'hard_swish', 1],
  72. [3, 184, 80, False, 'hard_swish', 1],
  73. [3, 480, 112, True, 'hard_swish', 1],
  74. [3, 672, 112, True, 'hard_swish', 1],
  75. # The number of channels in the last 4 stages is reduced by a
  76. # factor of 2 compared to the standard implementation.
  77. [5, 336, 80, True, 'hard_swish', 2],
  78. [5, 480, 80, True, 'hard_swish', 1],
  79. [5, 480, 80, True, 'hard_swish', 1],
  80. ]
  81. self.cls_ch_squeeze = 480
  82. self.cls_ch_expand = 1280
  83. self.lr_interval = 3
  84. elif model_name == "small":
  85. self.cfg = [
  86. # k, exp, c, se, nl, s,
  87. [3, 16, 16, True, 'relu', 2],
  88. [3, 72, 24, False, 'relu', 2],
  89. [3, 88, 24, False, 'relu', 1],
  90. [5, 96, 40, True, 'hard_swish', 2],
  91. [5, 240, 40, True, 'hard_swish', 1],
  92. [5, 240, 40, True, 'hard_swish', 1],
  93. [5, 120, 48, True, 'hard_swish', 1],
  94. [5, 144, 48, True, 'hard_swish', 1],
  95. # The number of channels in the last 4 stages is reduced by a
  96. # factor of 2 compared to the standard implementation.
  97. [5, 144, 48, True, 'hard_swish', 2],
  98. [5, 288, 48, True, 'hard_swish', 1],
  99. [5, 288, 48, True, 'hard_swish', 1],
  100. ]
  101. else:
  102. raise NotImplementedError
  103. else:
  104. if model_name == "large":
  105. self.cfg = [
  106. # kernel_size, expand, channel, se_block, act_mode, stride
  107. [3, 16, 16, False, 'relu', 1],
  108. [3, 64, 24, False, 'relu', 2],
  109. [3, 72, 24, False, 'relu', 1],
  110. [5, 72, 40, True, 'relu', 2],
  111. [5, 120, 40, True, 'relu', 1],
  112. [5, 120, 40, True, 'relu', 1],
  113. [3, 240, 80, False, 'hard_swish', 2],
  114. [3, 200, 80, False, 'hard_swish', 1],
  115. [3, 184, 80, False, 'hard_swish', 1],
  116. [3, 184, 80, False, 'hard_swish', 1],
  117. [3, 480, 112, True, 'hard_swish', 1],
  118. [3, 672, 112, True, 'hard_swish', 1],
  119. [5, 672, 160, True, 'hard_swish', 2],
  120. [5, 960, 160, True, 'hard_swish', 1],
  121. [5, 960, 160, True, 'hard_swish', 1],
  122. ]
  123. self.cls_ch_squeeze = 960
  124. self.cls_ch_expand = 1280
  125. self.lr_interval = 3
  126. elif model_name == "small":
  127. self.cfg = [
  128. # kernel_size, expand, channel, se_block, act_mode, stride
  129. [3, 16, 16, True, 'relu', 2],
  130. [3, 72, 24, False, 'relu', 2],
  131. [3, 88, 24, False, 'relu', 1],
  132. [5, 96, 40, True, 'hard_swish', 2],
  133. [5, 240, 40, True, 'hard_swish', 1],
  134. [5, 240, 40, True, 'hard_swish', 1],
  135. [5, 120, 48, True, 'hard_swish', 1],
  136. [5, 144, 48, True, 'hard_swish', 1],
  137. [5, 288, 96, True, 'hard_swish', 2],
  138. [5, 576, 96, True, 'hard_swish', 1],
  139. [5, 576, 96, True, 'hard_swish', 1],
  140. ]
  141. self.cls_ch_squeeze = 576
  142. self.cls_ch_expand = 1280
  143. self.lr_interval = 2
  144. else:
  145. raise NotImplementedError
  146. self.modify_bottle_params(output_stride)
  147. def modify_bottle_params(self, output_stride=None):
  148. if output_stride is not None and output_stride % 2 != 0:
  149. raise Exception("output stride must to be even number")
  150. if output_stride is None:
  151. return
  152. else:
  153. stride = 2
  154. for i, _cfg in enumerate(self.cfg):
  155. stride = stride * _cfg[-1]
  156. if stride > output_stride:
  157. s = 1
  158. self.cfg[i][-1] = s
  159. def _conv_bn_layer(self,
  160. input,
  161. filter_size,
  162. num_filters,
  163. stride,
  164. padding,
  165. num_groups=1,
  166. if_act=True,
  167. act=None,
  168. name=None,
  169. use_cudnn=True):
  170. lr_idx = self.curr_stage // self.lr_interval
  171. lr_idx = min(lr_idx, len(self.lr_mult_list) - 1)
  172. lr_mult = self.lr_mult_list[lr_idx]
  173. if self.num_classes:
  174. regularizer = None
  175. else:
  176. regularizer = L2Decay(self.conv_decay)
  177. conv_param_attr = ParamAttr(
  178. name=name + '_weights',
  179. learning_rate=lr_mult,
  180. regularizer=regularizer)
  181. conv = fluid.layers.conv2d(
  182. input=input,
  183. num_filters=num_filters,
  184. filter_size=filter_size,
  185. stride=stride,
  186. padding=padding,
  187. groups=num_groups,
  188. act=None,
  189. use_cudnn=use_cudnn,
  190. param_attr=conv_param_attr,
  191. bias_attr=False)
  192. bn_name = name + '_bn'
  193. bn_param_attr = ParamAttr(
  194. name=bn_name + "_scale", regularizer=L2Decay(self.norm_decay))
  195. bn_bias_attr = ParamAttr(
  196. name=bn_name + "_offset", regularizer=L2Decay(self.norm_decay))
  197. bn = fluid.layers.batch_norm(
  198. input=conv,
  199. param_attr=bn_param_attr,
  200. bias_attr=bn_bias_attr,
  201. moving_mean_name=bn_name + '_mean',
  202. moving_variance_name=bn_name + '_variance')
  203. if if_act:
  204. if act == 'relu':
  205. bn = fluid.layers.relu(bn)
  206. elif act == 'hard_swish':
  207. bn = self._hard_swish(bn)
  208. elif act == 'relu6':
  209. bn = fluid.layers.relu6(bn)
  210. return bn
  211. def make_divisible(self, v, divisor=8, min_value=None):
  212. if min_value is None:
  213. min_value = divisor
  214. new_v = max(min_value, int(v + divisor / 2) // divisor * divisor)
  215. if new_v < 0.9 * v:
  216. new_v += divisor
  217. return new_v
  218. def _hard_swish(self, x):
  219. return x * fluid.layers.relu6(x + 3) / 6.
  220. def _se_block(self, input, num_out_filter, ratio=4, name=None):
  221. lr_idx = self.curr_stage // self.lr_interval
  222. lr_idx = min(lr_idx, len(self.lr_mult_list) - 1)
  223. lr_mult = self.lr_mult_list[lr_idx]
  224. num_mid_filter = int(num_out_filter // ratio)
  225. pool = fluid.layers.pool2d(
  226. input=input, pool_type='avg', global_pooling=True, use_cudnn=False)
  227. conv1 = fluid.layers.conv2d(
  228. input=pool,
  229. filter_size=1,
  230. num_filters=num_mid_filter,
  231. act='relu',
  232. param_attr=ParamAttr(
  233. name=name + '_1_weights', learning_rate=lr_mult),
  234. bias_attr=ParamAttr(
  235. name=name + '_1_offset', learning_rate=lr_mult))
  236. conv2 = fluid.layers.conv2d(
  237. input=conv1,
  238. filter_size=1,
  239. num_filters=num_out_filter,
  240. act='hard_sigmoid',
  241. param_attr=ParamAttr(
  242. name=name + '_2_weights', learning_rate=lr_mult),
  243. bias_attr=ParamAttr(
  244. name=name + '_2_offset', learning_rate=lr_mult))
  245. scale = fluid.layers.elementwise_mul(x=input, y=conv2, axis=0)
  246. return scale
  247. def _residual_unit(self,
  248. input,
  249. num_in_filter,
  250. num_mid_filter,
  251. num_out_filter,
  252. stride,
  253. filter_size,
  254. act=None,
  255. use_se=False,
  256. name=None):
  257. input_data = input
  258. conv0 = self._conv_bn_layer(
  259. input=input,
  260. filter_size=1,
  261. num_filters=num_mid_filter,
  262. stride=1,
  263. padding=0,
  264. if_act=True,
  265. act=act,
  266. name=name + '_expand')
  267. if self.block_stride == 16 and stride == 2:
  268. self.end_points.append(conv0)
  269. conv1 = self._conv_bn_layer(
  270. input=conv0,
  271. filter_size=filter_size,
  272. num_filters=num_mid_filter,
  273. stride=stride,
  274. padding=int((filter_size - 1) // 2),
  275. if_act=True,
  276. act=act,
  277. num_groups=num_mid_filter,
  278. use_cudnn=False,
  279. name=name + '_depthwise')
  280. if self.curr_stage == 5:
  281. self.decode_point = conv1
  282. if use_se:
  283. conv1 = self._se_block(
  284. input=conv1, num_out_filter=num_mid_filter, name=name + '_se')
  285. conv2 = self._conv_bn_layer(
  286. input=conv1,
  287. filter_size=1,
  288. num_filters=num_out_filter,
  289. stride=1,
  290. padding=0,
  291. if_act=False,
  292. name=name + '_linear')
  293. if num_in_filter != num_out_filter or stride != 1:
  294. return conv2
  295. else:
  296. return fluid.layers.elementwise_add(
  297. x=input_data, y=conv2, act=None)
  298. def _extra_block_dw(self,
  299. input,
  300. num_filters1,
  301. num_filters2,
  302. stride,
  303. name=None):
  304. pointwise_conv = self._conv_bn_layer(
  305. input=input,
  306. filter_size=1,
  307. num_filters=int(num_filters1),
  308. stride=1,
  309. padding="SAME",
  310. act='relu6',
  311. name=name + "_extra1")
  312. depthwise_conv = self._conv_bn_layer(
  313. input=pointwise_conv,
  314. filter_size=3,
  315. num_filters=int(num_filters2),
  316. stride=stride,
  317. padding="SAME",
  318. num_groups=int(num_filters1),
  319. act='relu6',
  320. use_cudnn=False,
  321. name=name + "_extra2_dw")
  322. normal_conv = self._conv_bn_layer(
  323. input=depthwise_conv,
  324. filter_size=1,
  325. num_filters=int(num_filters2),
  326. stride=1,
  327. padding="SAME",
  328. act='relu6',
  329. name=name + "_extra2_sep")
  330. return normal_conv
  331. def __call__(self, input):
  332. scale = self.scale
  333. inplanes = self.inplanes
  334. cfg = self.cfg
  335. blocks = []
  336. #conv1
  337. conv = self._conv_bn_layer(
  338. input,
  339. filter_size=3,
  340. num_filters=self.make_divisible(inplanes * scale),
  341. stride=2,
  342. padding=1,
  343. num_groups=1,
  344. if_act=True,
  345. act='hard_swish',
  346. name='conv1')
  347. i = 0
  348. inplanes = self.make_divisible(inplanes * scale)
  349. for layer_cfg in cfg:
  350. self.block_stride *= layer_cfg[5]
  351. if layer_cfg[5] == 2:
  352. blocks.append(conv)
  353. conv = self._residual_unit(
  354. input=conv,
  355. num_in_filter=inplanes,
  356. num_mid_filter=self.make_divisible(scale * layer_cfg[1]),
  357. num_out_filter=self.make_divisible(scale * layer_cfg[2]),
  358. act=layer_cfg[4],
  359. stride=layer_cfg[5],
  360. filter_size=layer_cfg[0],
  361. use_se=layer_cfg[3],
  362. name='conv' + str(i + 2))
  363. inplanes = self.make_divisible(scale * layer_cfg[2])
  364. i += 1
  365. self.curr_stage = i
  366. blocks.append(conv)
  367. if self.for_seg:
  368. conv = self._conv_bn_layer(
  369. input=conv,
  370. filter_size=1,
  371. num_filters=self.make_divisible(scale * self.cls_ch_squeeze),
  372. stride=1,
  373. padding=0,
  374. num_groups=1,
  375. if_act=True,
  376. act='hard_swish',
  377. name='conv_last')
  378. return conv, self.decode_point
  379. if self.num_classes:
  380. conv = self._conv_bn_layer(
  381. input=conv,
  382. filter_size=1,
  383. num_filters=int(scale * self.cls_ch_squeeze),
  384. stride=1,
  385. padding=0,
  386. num_groups=1,
  387. if_act=True,
  388. act='hard_swish',
  389. name='conv_last')
  390. conv = fluid.layers.pool2d(
  391. input=conv,
  392. pool_type='avg',
  393. global_pooling=True,
  394. use_cudnn=False)
  395. conv = fluid.layers.conv2d(
  396. input=conv,
  397. num_filters=self.cls_ch_expand,
  398. filter_size=1,
  399. stride=1,
  400. padding=0,
  401. act=None,
  402. param_attr=ParamAttr(name='last_1x1_conv_weights'),
  403. bias_attr=False)
  404. conv = self._hard_swish(conv)
  405. drop = fluid.layers.dropout(x=conv, dropout_prob=0.2)
  406. out = fluid.layers.fc(input=drop,
  407. size=self.num_classes,
  408. param_attr=ParamAttr(name='fc_weights'),
  409. bias_attr=ParamAttr(name='fc_offset'))
  410. return out
  411. if not self.with_extra_blocks:
  412. return blocks
  413. # extra block
  414. conv_extra = self._conv_bn_layer(
  415. conv,
  416. filter_size=1,
  417. num_filters=int(scale * cfg[-1][1]),
  418. stride=1,
  419. padding="SAME",
  420. num_groups=1,
  421. if_act=True,
  422. act='hard_swish',
  423. name='conv' + str(i + 2))
  424. self.end_points.append(conv_extra)
  425. i += 1
  426. for block_filter in self.extra_block_filters:
  427. conv_extra = self._extra_block_dw(conv_extra, block_filter[0],
  428. block_filter[1], 2,
  429. 'conv' + str(i + 2))
  430. self.end_points.append(conv_extra)
  431. i += 1
  432. return self.end_points