yolo_fpn.py 35 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import paddle
  15. import paddle.nn as nn
  16. import paddle.nn.functional as F
  17. from paddle import ParamAttr
  18. from paddlex.ppdet.core.workspace import register, serializable
  19. from ..backbones.darknet import ConvBNLayer
  20. from ..shape_spec import ShapeSpec
  21. __all__ = ['YOLOv3FPN', 'PPYOLOFPN', 'PPYOLOTinyFPN', 'PPYOLOPAN']
  22. def add_coord(x, data_format):
  23. b = x.shape[0]
  24. if data_format == 'NCHW':
  25. h = x.shape[2]
  26. w = x.shape[3]
  27. else:
  28. h = x.shape[1]
  29. w = x.shape[2]
  30. gx = paddle.arange(w, dtype='float32') / (w - 1.) * 2.0 - 1.
  31. if data_format == 'NCHW':
  32. gx = gx.reshape([1, 1, 1, w]).expand([b, 1, h, w])
  33. else:
  34. gx = gx.reshape([1, 1, w, 1]).expand([b, h, w, 1])
  35. gx.stop_gradient = True
  36. gy = paddle.arange(h, dtype='float32') / (h - 1.) * 2.0 - 1.
  37. if data_format == 'NCHW':
  38. gy = gy.reshape([1, 1, h, 1]).expand([b, 1, h, w])
  39. else:
  40. gy = gy.reshape([1, h, 1, 1]).expand([b, h, w, 1])
  41. gy.stop_gradient = True
  42. return gx, gy
  43. class YoloDetBlock(nn.Layer):
  44. def __init__(self,
  45. ch_in,
  46. channel,
  47. norm_type,
  48. freeze_norm=False,
  49. name='',
  50. data_format='NCHW'):
  51. """
  52. YOLODetBlock layer for yolov3, see https://arxiv.org/abs/1804.02767
  53. Args:
  54. ch_in (int): input channel
  55. channel (int): base channel
  56. norm_type (str): batch norm type
  57. freeze_norm (bool): whether to freeze norm, default False
  58. name (str): layer name
  59. data_format (str): data format, NCHW or NHWC
  60. """
  61. super(YoloDetBlock, self).__init__()
  62. self.ch_in = ch_in
  63. self.channel = channel
  64. assert channel % 2 == 0, \
  65. "channel {} cannot be divided by 2".format(channel)
  66. conv_def = [
  67. ['conv0', ch_in, channel, 1, '.0.0'],
  68. ['conv1', channel, channel * 2, 3, '.0.1'],
  69. ['conv2', channel * 2, channel, 1, '.1.0'],
  70. ['conv3', channel, channel * 2, 3, '.1.1'],
  71. ['route', channel * 2, channel, 1, '.2'],
  72. ]
  73. self.conv_module = nn.Sequential()
  74. for idx, (conv_name, ch_in, ch_out, filter_size,
  75. post_name) in enumerate(conv_def):
  76. self.conv_module.add_sublayer(
  77. conv_name,
  78. ConvBNLayer(
  79. ch_in=ch_in,
  80. ch_out=ch_out,
  81. filter_size=filter_size,
  82. padding=(filter_size - 1) // 2,
  83. norm_type=norm_type,
  84. freeze_norm=freeze_norm,
  85. data_format=data_format,
  86. name=name + post_name))
  87. self.tip = ConvBNLayer(
  88. ch_in=channel,
  89. ch_out=channel * 2,
  90. filter_size=3,
  91. padding=1,
  92. norm_type=norm_type,
  93. freeze_norm=freeze_norm,
  94. data_format=data_format,
  95. name=name + '.tip')
  96. def forward(self, inputs):
  97. route = self.conv_module(inputs)
  98. tip = self.tip(route)
  99. return route, tip
  100. class SPP(nn.Layer):
  101. def __init__(self,
  102. ch_in,
  103. ch_out,
  104. k,
  105. pool_size,
  106. norm_type,
  107. freeze_norm=False,
  108. name='',
  109. act='leaky',
  110. data_format='NCHW'):
  111. """
  112. SPP layer, which consist of four pooling layer follwed by conv layer
  113. Args:
  114. ch_in (int): input channel of conv layer
  115. ch_out (int): output channel of conv layer
  116. k (int): kernel size of conv layer
  117. norm_type (str): batch norm type
  118. freeze_norm (bool): whether to freeze norm, default False
  119. name (str): layer name
  120. act (str): activation function
  121. data_format (str): data format, NCHW or NHWC
  122. """
  123. super(SPP, self).__init__()
  124. self.pool = []
  125. self.data_format = data_format
  126. for size in pool_size:
  127. pool = self.add_sublayer(
  128. '{}.pool1'.format(name),
  129. nn.MaxPool2D(
  130. kernel_size=size,
  131. stride=1,
  132. padding=size // 2,
  133. data_format=data_format,
  134. ceil_mode=False))
  135. self.pool.append(pool)
  136. self.conv = ConvBNLayer(
  137. ch_in,
  138. ch_out,
  139. k,
  140. padding=k // 2,
  141. norm_type=norm_type,
  142. freeze_norm=freeze_norm,
  143. name=name,
  144. act=act,
  145. data_format=data_format)
  146. def forward(self, x):
  147. outs = [x]
  148. for pool in self.pool:
  149. outs.append(pool(x))
  150. if self.data_format == "NCHW":
  151. y = paddle.concat(outs, axis=1)
  152. else:
  153. y = paddle.concat(outs, axis=-1)
  154. y = self.conv(y)
  155. return y
  156. class DropBlock(nn.Layer):
  157. def __init__(self, block_size, keep_prob, name, data_format='NCHW'):
  158. """
  159. DropBlock layer, see https://arxiv.org/abs/1810.12890
  160. Args:
  161. block_size (int): block size
  162. keep_prob (int): keep probability
  163. name (str): layer name
  164. data_format (str): data format, NCHW or NHWC
  165. """
  166. super(DropBlock, self).__init__()
  167. self.block_size = block_size
  168. self.keep_prob = keep_prob
  169. self.name = name
  170. self.data_format = data_format
  171. def forward(self, x):
  172. if not self.training or self.keep_prob == 1:
  173. return x
  174. else:
  175. gamma = (1. - self.keep_prob) / (self.block_size**2)
  176. if self.data_format == 'NCHW':
  177. shape = x.shape[2:]
  178. else:
  179. shape = x.shape[1:3]
  180. for s in shape:
  181. gamma *= s / (s - self.block_size + 1)
  182. matrix = paddle.cast(paddle.rand(x.shape, x.dtype) < gamma, x.dtype)
  183. mask_inv = F.max_pool2d(
  184. matrix,
  185. self.block_size,
  186. stride=1,
  187. padding=self.block_size // 2,
  188. data_format=self.data_format)
  189. mask = 1. - mask_inv
  190. y = x * mask * (mask.numel() / mask.sum())
  191. return y
  192. class CoordConv(nn.Layer):
  193. def __init__(self,
  194. ch_in,
  195. ch_out,
  196. filter_size,
  197. padding,
  198. norm_type,
  199. freeze_norm=False,
  200. name='',
  201. data_format='NCHW'):
  202. """
  203. CoordConv layer
  204. Args:
  205. ch_in (int): input channel
  206. ch_out (int): output channel
  207. filter_size (int): filter size, default 3
  208. padding (int): padding size, default 0
  209. norm_type (str): batch norm type, default bn
  210. name (str): layer name
  211. data_format (str): data format, NCHW or NHWC
  212. """
  213. super(CoordConv, self).__init__()
  214. self.conv = ConvBNLayer(
  215. ch_in + 2,
  216. ch_out,
  217. filter_size=filter_size,
  218. padding=padding,
  219. norm_type=norm_type,
  220. freeze_norm=freeze_norm,
  221. data_format=data_format,
  222. name=name)
  223. self.data_format = data_format
  224. def forward(self, x):
  225. gx, gy = add_coord(x, self.data_format)
  226. if self.data_format == 'NCHW':
  227. y = paddle.concat([x, gx, gy], axis=1)
  228. else:
  229. y = paddle.concat([x, gx, gy], axis=-1)
  230. y = self.conv(y)
  231. return y
  232. class PPYOLODetBlock(nn.Layer):
  233. def __init__(self, cfg, name, data_format='NCHW'):
  234. """
  235. PPYOLODetBlock layer
  236. Args:
  237. cfg (list): layer configs for this block
  238. name (str): block name
  239. data_format (str): data format, NCHW or NHWC
  240. """
  241. super(PPYOLODetBlock, self).__init__()
  242. self.conv_module = nn.Sequential()
  243. for idx, (conv_name, layer, args, kwargs) in enumerate(cfg[:-1]):
  244. kwargs.update(
  245. name='{}.{}'.format(name, conv_name), data_format=data_format)
  246. self.conv_module.add_sublayer(conv_name, layer(*args, **kwargs))
  247. conv_name, layer, args, kwargs = cfg[-1]
  248. kwargs.update(
  249. name='{}.{}'.format(name, conv_name), data_format=data_format)
  250. self.tip = layer(*args, **kwargs)
  251. def forward(self, inputs):
  252. route = self.conv_module(inputs)
  253. tip = self.tip(route)
  254. return route, tip
  255. class PPYOLOTinyDetBlock(nn.Layer):
  256. def __init__(self,
  257. ch_in,
  258. ch_out,
  259. name,
  260. drop_block=False,
  261. block_size=3,
  262. keep_prob=0.9,
  263. data_format='NCHW'):
  264. """
  265. PPYOLO Tiny DetBlock layer
  266. Args:
  267. ch_in (list): input channel number
  268. ch_out (list): output channel number
  269. name (str): block name
  270. drop_block: whether user DropBlock
  271. block_size: drop block size
  272. keep_prob: probability to keep block in DropBlock
  273. data_format (str): data format, NCHW or NHWC
  274. """
  275. super(PPYOLOTinyDetBlock, self).__init__()
  276. self.drop_block_ = drop_block
  277. self.conv_module = nn.Sequential()
  278. cfgs = [
  279. # name, in channels, out channels, filter_size,
  280. # stride, padding, groups
  281. ['.0', ch_in, ch_out, 1, 1, 0, 1],
  282. ['.1', ch_out, ch_out, 5, 1, 2, ch_out],
  283. ['.2', ch_out, ch_out, 1, 1, 0, 1],
  284. ['.route', ch_out, ch_out, 5, 1, 2, ch_out],
  285. ]
  286. for cfg in cfgs:
  287. conv_name, conv_ch_in, conv_ch_out, filter_size, stride, padding, \
  288. groups = cfg
  289. self.conv_module.add_sublayer(
  290. name + conv_name,
  291. ConvBNLayer(
  292. ch_in=conv_ch_in,
  293. ch_out=conv_ch_out,
  294. filter_size=filter_size,
  295. stride=stride,
  296. padding=padding,
  297. groups=groups,
  298. name=name + conv_name))
  299. self.tip = ConvBNLayer(
  300. ch_in=ch_out,
  301. ch_out=ch_out,
  302. filter_size=1,
  303. stride=1,
  304. padding=0,
  305. groups=1,
  306. name=name + conv_name)
  307. if self.drop_block_:
  308. self.drop_block = DropBlock(
  309. block_size=block_size,
  310. keep_prob=keep_prob,
  311. data_format=data_format,
  312. name=name + '.dropblock')
  313. def forward(self, inputs):
  314. if self.drop_block_:
  315. inputs = self.drop_block(inputs)
  316. route = self.conv_module(inputs)
  317. tip = self.tip(route)
  318. return route, tip
  319. class PPYOLODetBlockCSP(nn.Layer):
  320. def __init__(self,
  321. cfg,
  322. ch_in,
  323. ch_out,
  324. act,
  325. norm_type,
  326. name,
  327. data_format='NCHW'):
  328. """
  329. PPYOLODetBlockCSP layer
  330. Args:
  331. cfg (list): layer configs for this block
  332. ch_in (int): input channel
  333. ch_out (int): output channel
  334. act (str): default mish
  335. name (str): block name
  336. data_format (str): data format, NCHW or NHWC
  337. """
  338. super(PPYOLODetBlockCSP, self).__init__()
  339. self.data_format = data_format
  340. self.conv1 = ConvBNLayer(
  341. ch_in,
  342. ch_out,
  343. 1,
  344. padding=0,
  345. act=act,
  346. norm_type=norm_type,
  347. name=name + '.left',
  348. data_format=data_format)
  349. self.conv2 = ConvBNLayer(
  350. ch_in,
  351. ch_out,
  352. 1,
  353. padding=0,
  354. act=act,
  355. norm_type=norm_type,
  356. name=name + '.right',
  357. data_format=data_format)
  358. self.conv3 = ConvBNLayer(
  359. ch_out * 2,
  360. ch_out * 2,
  361. 1,
  362. padding=0,
  363. act=act,
  364. norm_type=norm_type,
  365. name=name,
  366. data_format=data_format)
  367. self.conv_module = nn.Sequential()
  368. for idx, (layer_name, layer, args, kwargs) in enumerate(cfg):
  369. kwargs.update(name=name + layer_name, data_format=data_format)
  370. self.conv_module.add_sublayer(layer_name, layer(*args, **kwargs))
  371. def forward(self, inputs):
  372. conv_left = self.conv1(inputs)
  373. conv_right = self.conv2(inputs)
  374. conv_left = self.conv_module(conv_left)
  375. if self.data_format == 'NCHW':
  376. conv = paddle.concat([conv_left, conv_right], axis=1)
  377. else:
  378. conv = paddle.concat([conv_left, conv_right], axis=-1)
  379. conv = self.conv3(conv)
  380. return conv, conv
  381. @register
  382. @serializable
  383. class YOLOv3FPN(nn.Layer):
  384. __shared__ = ['norm_type', 'data_format']
  385. def __init__(self,
  386. in_channels=[256, 512, 1024],
  387. norm_type='bn',
  388. freeze_norm=False,
  389. data_format='NCHW'):
  390. """
  391. YOLOv3FPN layer
  392. Args:
  393. in_channels (list): input channels for fpn
  394. norm_type (str): batch norm type, default bn
  395. data_format (str): data format, NCHW or NHWC
  396. """
  397. super(YOLOv3FPN, self).__init__()
  398. assert len(in_channels) > 0, "in_channels length should > 0"
  399. self.in_channels = in_channels
  400. self.num_blocks = len(in_channels)
  401. self._out_channels = []
  402. self.yolo_blocks = []
  403. self.routes = []
  404. self.data_format = data_format
  405. for i in range(self.num_blocks):
  406. name = 'yolo_block.{}'.format(i)
  407. in_channel = in_channels[-i - 1]
  408. if i > 0:
  409. in_channel += 512 // (2**i)
  410. yolo_block = self.add_sublayer(
  411. name,
  412. YoloDetBlock(
  413. in_channel,
  414. channel=512 // (2**i),
  415. norm_type=norm_type,
  416. freeze_norm=freeze_norm,
  417. data_format=data_format,
  418. name=name))
  419. self.yolo_blocks.append(yolo_block)
  420. # tip layer output channel doubled
  421. self._out_channels.append(1024 // (2**i))
  422. if i < self.num_blocks - 1:
  423. name = 'yolo_transition.{}'.format(i)
  424. route = self.add_sublayer(
  425. name,
  426. ConvBNLayer(
  427. ch_in=512 // (2**i),
  428. ch_out=256 // (2**i),
  429. filter_size=1,
  430. stride=1,
  431. padding=0,
  432. norm_type=norm_type,
  433. freeze_norm=freeze_norm,
  434. data_format=data_format,
  435. name=name))
  436. self.routes.append(route)
  437. def forward(self, blocks, for_mot=False):
  438. assert len(blocks) == self.num_blocks
  439. blocks = blocks[::-1]
  440. yolo_feats = []
  441. # add embedding features output for multi-object tracking model
  442. if for_mot:
  443. emb_feats = []
  444. for i, block in enumerate(blocks):
  445. if i > 0:
  446. if self.data_format == 'NCHW':
  447. block = paddle.concat([route, block], axis=1)
  448. else:
  449. block = paddle.concat([route, block], axis=-1)
  450. route, tip = self.yolo_blocks[i](block)
  451. yolo_feats.append(tip)
  452. if for_mot:
  453. # add embedding features output
  454. emb_feats.append(route)
  455. if i < self.num_blocks - 1:
  456. route = self.routes[i](route)
  457. route = F.interpolate(
  458. route, scale_factor=2., data_format=self.data_format)
  459. if for_mot:
  460. return {'yolo_feats': yolo_feats, 'emb_feats': emb_feats}
  461. else:
  462. return yolo_feats
  463. @classmethod
  464. def from_config(cls, cfg, input_shape):
  465. return {'in_channels': [i.channels for i in input_shape], }
  466. @property
  467. def out_shape(self):
  468. return [ShapeSpec(channels=c) for c in self._out_channels]
  469. @register
  470. @serializable
  471. class PPYOLOFPN(nn.Layer):
  472. __shared__ = ['norm_type', 'data_format']
  473. def __init__(self,
  474. in_channels=[512, 1024, 2048],
  475. norm_type='bn',
  476. freeze_norm=False,
  477. data_format='NCHW',
  478. coord_conv=False,
  479. conv_block_num=2,
  480. drop_block=False,
  481. block_size=3,
  482. keep_prob=0.9,
  483. spp=False):
  484. """
  485. PPYOLOFPN layer
  486. Args:
  487. in_channels (list): input channels for fpn
  488. norm_type (str): batch norm type, default bn
  489. data_format (str): data format, NCHW or NHWC
  490. coord_conv (bool): whether use CoordConv or not
  491. conv_block_num (int): conv block num of each pan block
  492. drop_block (bool): whether use DropBlock or not
  493. block_size (int): block size of DropBlock
  494. keep_prob (float): keep probability of DropBlock
  495. spp (bool): whether use spp or not
  496. """
  497. super(PPYOLOFPN, self).__init__()
  498. assert len(in_channels) > 0, "in_channels length should > 0"
  499. self.in_channels = in_channels
  500. self.num_blocks = len(in_channels)
  501. # parse kwargs
  502. self.coord_conv = coord_conv
  503. self.drop_block = drop_block
  504. self.block_size = block_size
  505. self.keep_prob = keep_prob
  506. self.spp = spp
  507. self.conv_block_num = conv_block_num
  508. self.data_format = data_format
  509. if self.coord_conv:
  510. ConvLayer = CoordConv
  511. else:
  512. ConvLayer = ConvBNLayer
  513. if self.drop_block:
  514. dropblock_cfg = [[
  515. 'dropblock', DropBlock, [self.block_size, self.keep_prob],
  516. dict()
  517. ]]
  518. else:
  519. dropblock_cfg = []
  520. self._out_channels = []
  521. self.yolo_blocks = []
  522. self.routes = []
  523. for i, ch_in in enumerate(self.in_channels[::-1]):
  524. if i > 0:
  525. ch_in += 512 // (2**i)
  526. channel = 64 * (2**self.num_blocks) // (2**i)
  527. base_cfg = []
  528. c_in, c_out = ch_in, channel
  529. for j in range(self.conv_block_num):
  530. base_cfg += [
  531. [
  532. 'conv{}'.format(2 * j), ConvLayer, [c_in, c_out, 1],
  533. dict(
  534. padding=0,
  535. norm_type=norm_type,
  536. freeze_norm=freeze_norm)
  537. ],
  538. [
  539. 'conv{}'.format(2 * j + 1), ConvBNLayer,
  540. [c_out, c_out * 2, 3], dict(
  541. padding=1,
  542. norm_type=norm_type,
  543. freeze_norm=freeze_norm)
  544. ],
  545. ]
  546. c_in, c_out = c_out * 2, c_out
  547. base_cfg += [[
  548. 'route', ConvLayer, [c_in, c_out, 1], dict(
  549. padding=0, norm_type=norm_type, freeze_norm=freeze_norm)
  550. ], [
  551. 'tip', ConvLayer, [c_out, c_out * 2, 3], dict(
  552. padding=1, norm_type=norm_type, freeze_norm=freeze_norm)
  553. ]]
  554. if self.conv_block_num == 2:
  555. if i == 0:
  556. if self.spp:
  557. spp_cfg = [[
  558. 'spp', SPP, [channel * 4, channel, 1], dict(
  559. pool_size=[5, 9, 13],
  560. norm_type=norm_type,
  561. freeze_norm=freeze_norm)
  562. ]]
  563. else:
  564. spp_cfg = []
  565. cfg = base_cfg[0:3] + spp_cfg + base_cfg[
  566. 3:4] + dropblock_cfg + base_cfg[4:6]
  567. else:
  568. cfg = base_cfg[0:2] + dropblock_cfg + base_cfg[2:6]
  569. elif self.conv_block_num == 0:
  570. if self.spp and i == 0:
  571. spp_cfg = [[
  572. 'spp', SPP, [c_in * 4, c_in, 1], dict(
  573. pool_size=[5, 9, 13],
  574. norm_type=norm_type,
  575. freeze_norm=freeze_norm)
  576. ]]
  577. else:
  578. spp_cfg = []
  579. cfg = spp_cfg + dropblock_cfg + base_cfg
  580. name = 'yolo_block.{}'.format(i)
  581. yolo_block = self.add_sublayer(name, PPYOLODetBlock(cfg, name))
  582. self.yolo_blocks.append(yolo_block)
  583. self._out_channels.append(channel * 2)
  584. if i < self.num_blocks - 1:
  585. name = 'yolo_transition.{}'.format(i)
  586. route = self.add_sublayer(
  587. name,
  588. ConvBNLayer(
  589. ch_in=channel,
  590. ch_out=256 // (2**i),
  591. filter_size=1,
  592. stride=1,
  593. padding=0,
  594. norm_type=norm_type,
  595. freeze_norm=freeze_norm,
  596. data_format=data_format,
  597. name=name))
  598. self.routes.append(route)
  599. def forward(self, blocks, for_mot=False):
  600. assert len(blocks) == self.num_blocks
  601. blocks = blocks[::-1]
  602. yolo_feats = []
  603. # add embedding features output for multi-object tracking model
  604. if for_mot:
  605. emb_feats = []
  606. for i, block in enumerate(blocks):
  607. if i > 0:
  608. if self.data_format == 'NCHW':
  609. block = paddle.concat([route, block], axis=1)
  610. else:
  611. block = paddle.concat([route, block], axis=-1)
  612. route, tip = self.yolo_blocks[i](block)
  613. yolo_feats.append(tip)
  614. if for_mot:
  615. # add embedding features output
  616. emb_feats.append(route)
  617. if i < self.num_blocks - 1:
  618. route = self.routes[i](route)
  619. route = F.interpolate(
  620. route, scale_factor=2., data_format=self.data_format)
  621. if for_mot:
  622. return {'yolo_feats': yolo_feats, 'emb_feats': emb_feats}
  623. else:
  624. return yolo_feats
  625. @classmethod
  626. def from_config(cls, cfg, input_shape):
  627. return {'in_channels': [i.channels for i in input_shape], }
  628. @property
  629. def out_shape(self):
  630. return [ShapeSpec(channels=c) for c in self._out_channels]
  631. @register
  632. @serializable
  633. class PPYOLOTinyFPN(nn.Layer):
  634. __shared__ = ['norm_type', 'data_format']
  635. def __init__(self,
  636. in_channels=[80, 56, 34],
  637. detection_block_channels=[160, 128, 96],
  638. norm_type='bn',
  639. data_format='NCHW',
  640. **kwargs):
  641. """
  642. PPYOLO Tiny FPN layer
  643. Args:
  644. in_channels (list): input channels for fpn
  645. detection_block_channels (list): channels in fpn
  646. norm_type (str): batch norm type, default bn
  647. data_format (str): data format, NCHW or NHWC
  648. kwargs: extra key-value pairs, such as parameter of DropBlock and spp
  649. """
  650. super(PPYOLOTinyFPN, self).__init__()
  651. assert len(in_channels) > 0, "in_channels length should > 0"
  652. self.in_channels = in_channels[::-1]
  653. assert len(detection_block_channels
  654. ) > 0, "detection_block_channelslength should > 0"
  655. self.detection_block_channels = detection_block_channels
  656. self.data_format = data_format
  657. self.num_blocks = len(in_channels)
  658. # parse kwargs
  659. self.drop_block = kwargs.get('drop_block', False)
  660. self.block_size = kwargs.get('block_size', 3)
  661. self.keep_prob = kwargs.get('keep_prob', 0.9)
  662. self.spp_ = kwargs.get('spp', False)
  663. if self.spp_:
  664. self.spp = SPP(self.in_channels[0] * 4,
  665. self.in_channels[0],
  666. k=1,
  667. pool_size=[5, 9, 13],
  668. norm_type=norm_type,
  669. name='spp')
  670. self._out_channels = []
  671. self.yolo_blocks = []
  672. self.routes = []
  673. for i, (
  674. ch_in, ch_out
  675. ) in enumerate(zip(self.in_channels, self.detection_block_channels)):
  676. name = 'yolo_block.{}'.format(i)
  677. if i > 0:
  678. ch_in += self.detection_block_channels[i - 1]
  679. yolo_block = self.add_sublayer(
  680. name,
  681. PPYOLOTinyDetBlock(
  682. ch_in,
  683. ch_out,
  684. name,
  685. drop_block=self.drop_block,
  686. block_size=self.block_size,
  687. keep_prob=self.keep_prob))
  688. self.yolo_blocks.append(yolo_block)
  689. self._out_channels.append(ch_out)
  690. if i < self.num_blocks - 1:
  691. name = 'yolo_transition.{}'.format(i)
  692. route = self.add_sublayer(
  693. name,
  694. ConvBNLayer(
  695. ch_in=ch_out,
  696. ch_out=ch_out,
  697. filter_size=1,
  698. stride=1,
  699. padding=0,
  700. norm_type=norm_type,
  701. data_format=data_format,
  702. name=name))
  703. self.routes.append(route)
  704. def forward(self, blocks, for_mot=False):
  705. assert len(blocks) == self.num_blocks
  706. blocks = blocks[::-1]
  707. yolo_feats = []
  708. # add embedding features output for multi-object tracking model
  709. if for_mot:
  710. emb_feats = []
  711. for i, block in enumerate(blocks):
  712. if i == 0 and self.spp_:
  713. block = self.spp(block)
  714. if i > 0:
  715. if self.data_format == 'NCHW':
  716. block = paddle.concat([route, block], axis=1)
  717. else:
  718. block = paddle.concat([route, block], axis=-1)
  719. route, tip = self.yolo_blocks[i](block)
  720. yolo_feats.append(tip)
  721. if for_mot:
  722. # add embedding features output
  723. emb_feats.append(route)
  724. if i < self.num_blocks - 1:
  725. route = self.routes[i](route)
  726. route = F.interpolate(
  727. route, scale_factor=2., data_format=self.data_format)
  728. if for_mot:
  729. return {'yolo_feats': yolo_feats, 'emb_feats': emb_feats}
  730. else:
  731. return yolo_feats
  732. @classmethod
  733. def from_config(cls, cfg, input_shape):
  734. return {'in_channels': [i.channels for i in input_shape], }
  735. @property
  736. def out_shape(self):
  737. return [ShapeSpec(channels=c) for c in self._out_channels]
  738. @register
  739. @serializable
  740. class PPYOLOPAN(nn.Layer):
  741. __shared__ = ['norm_type', 'data_format']
  742. def __init__(self,
  743. in_channels=[512, 1024, 2048],
  744. norm_type='bn',
  745. data_format='NCHW',
  746. act='mish',
  747. conv_block_num=3,
  748. drop_block=False,
  749. block_size=3,
  750. keep_prob=0.9,
  751. spp=False):
  752. """
  753. PPYOLOPAN layer with SPP, DropBlock and CSP connection.
  754. Args:
  755. in_channels (list): input channels for fpn
  756. norm_type (str): batch norm type, default bn
  757. data_format (str): data format, NCHW or NHWC
  758. act (str): activation function, default mish
  759. conv_block_num (int): conv block num of each pan block
  760. drop_block (bool): whether use DropBlock or not
  761. block_size (int): block size of DropBlock
  762. keep_prob (float): keep probability of DropBlock
  763. spp (bool): whether use spp or not
  764. """
  765. super(PPYOLOPAN, self).__init__()
  766. assert len(in_channels) > 0, "in_channels length should > 0"
  767. self.in_channels = in_channels
  768. self.num_blocks = len(in_channels)
  769. # parse kwargs
  770. self.drop_block = drop_block
  771. self.block_size = block_size
  772. self.keep_prob = keep_prob
  773. self.spp = spp
  774. self.conv_block_num = conv_block_num
  775. self.data_format = data_format
  776. if self.drop_block:
  777. dropblock_cfg = [[
  778. 'dropblock', DropBlock, [self.block_size, self.keep_prob],
  779. dict()
  780. ]]
  781. else:
  782. dropblock_cfg = []
  783. # fpn
  784. self.fpn_blocks = []
  785. self.fpn_routes = []
  786. fpn_channels = []
  787. for i, ch_in in enumerate(self.in_channels[::-1]):
  788. if i > 0:
  789. ch_in += 512 // (2**(i - 1))
  790. channel = 512 // (2**i)
  791. base_cfg = []
  792. for j in range(self.conv_block_num):
  793. base_cfg += [
  794. # name, layer, args
  795. [
  796. '{}.0'.format(j), ConvBNLayer, [channel, channel, 1],
  797. dict(
  798. padding=0, act=act, norm_type=norm_type)
  799. ],
  800. [
  801. '{}.1'.format(j), ConvBNLayer, [channel, channel, 3],
  802. dict(
  803. padding=1, act=act, norm_type=norm_type)
  804. ]
  805. ]
  806. if i == 0 and self.spp:
  807. base_cfg[3] = [
  808. 'spp', SPP, [channel * 4, channel, 1], dict(
  809. pool_size=[5, 9, 13], act=act, norm_type=norm_type)
  810. ]
  811. cfg = base_cfg[:4] + dropblock_cfg + base_cfg[4:]
  812. name = 'fpn.{}'.format(i)
  813. fpn_block = self.add_sublayer(
  814. name,
  815. PPYOLODetBlockCSP(cfg, ch_in, channel, act, norm_type, name,
  816. data_format))
  817. self.fpn_blocks.append(fpn_block)
  818. fpn_channels.append(channel * 2)
  819. if i < self.num_blocks - 1:
  820. name = 'fpn_transition.{}'.format(i)
  821. route = self.add_sublayer(
  822. name,
  823. ConvBNLayer(
  824. ch_in=channel * 2,
  825. ch_out=channel,
  826. filter_size=1,
  827. stride=1,
  828. padding=0,
  829. act=act,
  830. norm_type=norm_type,
  831. data_format=data_format,
  832. name=name))
  833. self.fpn_routes.append(route)
  834. # pan
  835. self.pan_blocks = []
  836. self.pan_routes = []
  837. self._out_channels = [512 // (2**(self.num_blocks - 2)), ]
  838. for i in reversed(range(self.num_blocks - 1)):
  839. name = 'pan_transition.{}'.format(i)
  840. route = self.add_sublayer(
  841. name,
  842. ConvBNLayer(
  843. ch_in=fpn_channels[i + 1],
  844. ch_out=fpn_channels[i + 1],
  845. filter_size=3,
  846. stride=2,
  847. padding=1,
  848. act=act,
  849. norm_type=norm_type,
  850. data_format=data_format,
  851. name=name))
  852. self.pan_routes = [route, ] + self.pan_routes
  853. base_cfg = []
  854. ch_in = fpn_channels[i] + fpn_channels[i + 1]
  855. channel = 512 // (2**i)
  856. for j in range(self.conv_block_num):
  857. base_cfg += [
  858. # name, layer, args
  859. [
  860. '{}.0'.format(j), ConvBNLayer, [channel, channel, 1],
  861. dict(
  862. padding=0, act=act, norm_type=norm_type)
  863. ],
  864. [
  865. '{}.1'.format(j), ConvBNLayer, [channel, channel, 3],
  866. dict(
  867. padding=1, act=act, norm_type=norm_type)
  868. ]
  869. ]
  870. cfg = base_cfg[:4] + dropblock_cfg + base_cfg[4:]
  871. name = 'pan.{}'.format(i)
  872. pan_block = self.add_sublayer(
  873. name,
  874. PPYOLODetBlockCSP(cfg, ch_in, channel, act, norm_type, name,
  875. data_format))
  876. self.pan_blocks = [pan_block, ] + self.pan_blocks
  877. self._out_channels.append(channel * 2)
  878. self._out_channels = self._out_channels[::-1]
  879. def forward(self, blocks, for_mot=False):
  880. assert len(blocks) == self.num_blocks
  881. blocks = blocks[::-1]
  882. fpn_feats = []
  883. # add embedding features output for multi-object tracking model
  884. if for_mot:
  885. emb_feats = []
  886. for i, block in enumerate(blocks):
  887. if i > 0:
  888. if self.data_format == 'NCHW':
  889. block = paddle.concat([route, block], axis=1)
  890. else:
  891. block = paddle.concat([route, block], axis=-1)
  892. route, tip = self.fpn_blocks[i](block)
  893. fpn_feats.append(tip)
  894. if for_mot:
  895. # add embedding features output
  896. emb_feats.append(route)
  897. if i < self.num_blocks - 1:
  898. route = self.fpn_routes[i](route)
  899. route = F.interpolate(
  900. route, scale_factor=2., data_format=self.data_format)
  901. pan_feats = [fpn_feats[-1], ]
  902. route = fpn_feats[self.num_blocks - 1]
  903. for i in reversed(range(self.num_blocks - 1)):
  904. block = fpn_feats[i]
  905. route = self.pan_routes[i](route)
  906. if self.data_format == 'NCHW':
  907. block = paddle.concat([route, block], axis=1)
  908. else:
  909. block = paddle.concat([route, block], axis=-1)
  910. route, tip = self.pan_blocks[i](block)
  911. pan_feats.append(tip)
  912. if for_mot:
  913. return {'yolo_feats': pan_feats[::-1], 'emb_feats': emb_feats}
  914. else:
  915. return pan_feats[::-1]
  916. @classmethod
  917. def from_config(cls, cfg, input_shape):
  918. return {'in_channels': [i.channels for i in input_shape], }
  919. @property
  920. def out_shape(self):
  921. return [ShapeSpec(channels=c) for c in self._out_channels]