detector.py 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. import collections
  16. import copy
  17. import os
  18. import os.path as osp
  19. import six
  20. import numpy as np
  21. import paddle
  22. from paddle.static import InputSpec
  23. import ppdet
  24. from ppdet.modeling.proposal_generator.target_layer import BBoxAssigner, MaskAssigner
  25. import paddlex
  26. import paddlex.utils.logging as logging
  27. from paddlex.cv.transforms.operators import _NormalizeBox, _PadBox, _BboxXYXY2XYWH
  28. from paddlex.cv.transforms.batch_operators import BatchCompose, BatchRandomResize, BatchRandomResizeByShort, _BatchPadding, _Gt2YoloTarget
  29. from paddlex.cv.transforms import arrange_transforms
  30. from .base import BaseModel
  31. from .utils.det_metrics import VOCMetric, COCOMetric
  32. from .utils.ema import ExponentialMovingAverage
  33. from paddlex.utils.checkpoint import det_pretrain_weights_dict
  34. __all__ = [
  35. "YOLOv3", "FasterRCNN", "PPYOLO", "PPYOLOTiny", "PPYOLOv2", "MaskRCNN"
  36. ]
  37. class BaseDetector(BaseModel):
  38. def __init__(self, model_name, num_classes=80, **params):
  39. self.init_params.update(locals())
  40. del self.init_params['params']
  41. super(BaseDetector, self).__init__('detector')
  42. if not hasattr(ppdet.modeling, model_name):
  43. raise Exception("ERROR: There's no model named {}.".format(
  44. model_name))
  45. self.model_name = model_name
  46. self.num_classes = num_classes
  47. self.labels = None
  48. self.net = self.build_net(**params)
  49. def build_net(self, **params):
  50. with paddle.utils.unique_name.guard():
  51. net = ppdet.modeling.__dict__[self.model_name](**params)
  52. return net
  53. def get_test_inputs(self, image_shape):
  54. input_spec = [{
  55. "image": InputSpec(
  56. shape=[None, 3] + image_shape, name='image', dtype='float32'),
  57. "im_shape": InputSpec(
  58. shape=[None, 2], name='im_shape', dtype='float32'),
  59. "scale_factor": InputSpec(
  60. shape=[None, 2], name='scale_factor', dtype='float32')
  61. }]
  62. return input_spec
  63. def _get_backbone(self, backbone_name, **params):
  64. backbone = getattr(ppdet.modeling, backbone_name)(**params)
  65. return backbone
  66. def run(self, net, inputs, mode):
  67. net_out = net(inputs)
  68. if mode in ['train', 'eval']:
  69. outputs = net_out
  70. else:
  71. for key in ['im_shape', 'scale_factor']:
  72. net_out[key] = inputs[key]
  73. outputs = dict()
  74. for key in net_out:
  75. outputs[key] = net_out[key].numpy()
  76. return outputs
  77. def default_optimizer(self, parameters, learning_rate, warmup_steps,
  78. warmup_start_lr, lr_decay_epochs, lr_decay_gamma,
  79. num_steps_each_epoch):
  80. boundaries = [b * num_steps_each_epoch for b in lr_decay_epochs]
  81. values = [(lr_decay_gamma**i) * learning_rate
  82. for i in range(len(lr_decay_epochs) + 1)]
  83. scheduler = paddle.optimizer.lr.PiecewiseDecay(
  84. boundaries=boundaries, values=values)
  85. if warmup_steps > 0:
  86. if warmup_steps > lr_decay_epochs[0] * num_steps_each_epoch:
  87. logging.error(
  88. "In function train(), parameters should satisfy: "
  89. "warmup_steps <= lr_decay_epochs[0]*num_samples_in_train_dataset",
  90. exit=False)
  91. logging.error(
  92. "See this doc for more information: "
  93. "https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/appendix/parameters.md#notice",
  94. exit=False)
  95. scheduler = paddle.optimizer.lr.LinearWarmup(
  96. learning_rate=scheduler,
  97. warmup_steps=warmup_steps,
  98. start_lr=warmup_start_lr,
  99. end_lr=learning_rate)
  100. optimizer = paddle.optimizer.Momentum(
  101. scheduler,
  102. momentum=.9,
  103. weight_decay=paddle.regularizer.L2Decay(coeff=1e-04),
  104. parameters=parameters)
  105. return optimizer
  106. def train(self,
  107. num_epochs,
  108. train_dataset,
  109. train_batch_size=64,
  110. eval_dataset=None,
  111. optimizer=None,
  112. save_interval_epochs=1,
  113. log_interval_steps=10,
  114. save_dir='output',
  115. pretrain_weights='IMAGENET',
  116. learning_rate=.001,
  117. warmup_steps=0,
  118. warmup_start_lr=0.0,
  119. lr_decay_epochs=(216, 243),
  120. lr_decay_gamma=0.1,
  121. metric=None,
  122. use_ema=False,
  123. early_stop=False,
  124. early_stop_patience=5,
  125. use_vdl=True):
  126. """
  127. Train the model.
  128. Args:
  129. num_epochs(int): The number of epochs.
  130. train_dataset(paddlex.dataset): Training dataset.
  131. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  132. eval_dataset(paddlex.dataset, optional):
  133. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  134. optimizer(paddle.optimizer.Optimizer or None, optional):
  135. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  136. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  137. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  138. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  139. pretrain_weights(str or None, optional):
  140. None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'IMAGENET'.
  141. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  142. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  143. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  144. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  145. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  146. metric({'VOC', 'COCO', None}, optional):
  147. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  148. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  149. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  150. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  151. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  152. """
  153. if train_dataset.__class__.__name__ == 'VOCDetection':
  154. train_dataset.data_fields = {
  155. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  156. 'difficult'
  157. }
  158. elif train_dataset.__class__.__name__ == 'CocoDetection':
  159. if self.__class__.__name__ == 'MaskRCNN':
  160. train_dataset.data_fields = {
  161. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  162. 'gt_poly', 'is_crowd'
  163. }
  164. else:
  165. train_dataset.data_fields = {
  166. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  167. 'is_crowd'
  168. }
  169. if metric is None:
  170. if eval_dataset.__class__.__name__ == 'VOCDetection':
  171. self.metric = 'voc'
  172. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  173. self.metric = 'coco'
  174. else:
  175. assert metric.lower() in ['coco', 'voc'], \
  176. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  177. self.metric = metric.lower()
  178. train_dataset.batch_transforms = self._compose_batch_transform(
  179. train_dataset.transforms, mode='train')
  180. self.labels = train_dataset.labels
  181. # build optimizer if not defined
  182. if optimizer is None:
  183. num_steps_each_epoch = len(train_dataset) // train_batch_size
  184. self.optimizer = self.default_optimizer(
  185. parameters=self.net.parameters(),
  186. learning_rate=learning_rate,
  187. warmup_steps=warmup_steps,
  188. warmup_start_lr=warmup_start_lr,
  189. lr_decay_epochs=lr_decay_epochs,
  190. lr_decay_gamma=lr_decay_gamma,
  191. num_steps_each_epoch=num_steps_each_epoch)
  192. else:
  193. self.optimizer = optimizer
  194. # initiate weights
  195. if pretrain_weights is not None and not osp.exists(pretrain_weights):
  196. if pretrain_weights not in det_pretrain_weights_dict['_'.join(
  197. [self.model_name, self.backbone_name])]:
  198. logging.warning(
  199. "Path of pretrain_weights('{}') does not exist!".format(
  200. pretrain_weights))
  201. pretrain_weights = det_pretrain_weights_dict['_'.join(
  202. [self.model_name, self.backbone_name])][0]
  203. logging.warning("Pretrain_weights is forcibly set to '{}'. "
  204. "If you don't want to use pretrain weights, "
  205. "set pretrain_weights to be None.".format(
  206. pretrain_weights))
  207. pretrained_dir = osp.join(save_dir, 'pretrain')
  208. self.net_initialize(
  209. pretrain_weights=pretrain_weights, save_dir=pretrained_dir)
  210. if use_ema:
  211. ema = ExponentialMovingAverage(
  212. decay=.9998, model=self.net, use_thres_step=True)
  213. else:
  214. ema = None
  215. # start train loop
  216. self.train_loop(
  217. num_epochs=num_epochs,
  218. train_dataset=train_dataset,
  219. train_batch_size=train_batch_size,
  220. eval_dataset=eval_dataset,
  221. save_interval_epochs=save_interval_epochs,
  222. log_interval_steps=log_interval_steps,
  223. save_dir=save_dir,
  224. ema=ema,
  225. early_stop=early_stop,
  226. early_stop_patience=early_stop_patience,
  227. use_vdl=use_vdl)
  228. def quant_aware_train(self,
  229. num_epochs,
  230. train_dataset,
  231. train_batch_size=64,
  232. eval_dataset=None,
  233. optimizer=None,
  234. save_interval_epochs=1,
  235. log_interval_steps=10,
  236. save_dir='output',
  237. learning_rate=.00001,
  238. warmup_steps=0,
  239. warmup_start_lr=0.0,
  240. lr_decay_epochs=(216, 243),
  241. lr_decay_gamma=0.1,
  242. metric=None,
  243. use_ema=False,
  244. early_stop=False,
  245. early_stop_patience=5,
  246. use_vdl=True,
  247. quant_config=None):
  248. """
  249. Quantization-aware training.
  250. Args:
  251. num_epochs(int): The number of epochs.
  252. train_dataset(paddlex.dataset): Training dataset.
  253. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  254. eval_dataset(paddlex.dataset, optional):
  255. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  256. optimizer(paddle.optimizer.Optimizer or None, optional):
  257. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  258. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  259. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  260. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  261. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  262. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  263. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  264. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  265. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  266. metric({'VOC', 'COCO', None}, optional):
  267. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  268. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  269. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  270. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  271. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  272. quant_config(dict or None, optional): Quantization configuration. If None, a default rule of thumb
  273. configuration will be used. Defaults to None.
  274. """
  275. self._prepare_qat(quant_config)
  276. self.train(
  277. num_epochs=num_epochs,
  278. train_dataset=train_dataset,
  279. train_batch_size=train_batch_size,
  280. eval_dataset=eval_dataset,
  281. optimizer=optimizer,
  282. save_interval_epochs=save_interval_epochs,
  283. log_interval_steps=log_interval_steps,
  284. save_dir=save_dir,
  285. pretrain_weights=None,
  286. learning_rate=learning_rate,
  287. warmup_steps=warmup_steps,
  288. warmup_start_lr=warmup_start_lr,
  289. lr_decay_epochs=lr_decay_epochs,
  290. lr_decay_gamma=lr_decay_gamma,
  291. metric=metric,
  292. use_ema=use_ema,
  293. early_stop=early_stop,
  294. early_stop_patience=early_stop_patience,
  295. use_vdl=use_vdl)
  296. def evaluate(self,
  297. eval_dataset,
  298. batch_size=1,
  299. metric=None,
  300. return_details=False):
  301. """
  302. Evaluate the model.
  303. Args:
  304. eval_dataset(paddlex.dataset): Evaluation dataset.
  305. batch_size(int, optional): Total batch size among all cards used for evaluation. Defaults to 1.
  306. metric({'VOC', 'COCO', None}, optional):
  307. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  308. return_details(bool, optional): Whether to return evaluation details. Defaults to False.
  309. Returns:
  310. collections.OrderedDict with key-value pairs: {"mAP(0.50, 11point)":`mean average precision`}.
  311. """
  312. if metric is None:
  313. if not hasattr(self, 'metric'):
  314. if eval_dataset.__class__.__name__ == 'VOCDetection':
  315. self.metric = 'voc'
  316. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  317. self.metric = 'coco'
  318. else:
  319. assert metric.lower() in ['coco', 'voc'], \
  320. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  321. self.metric = metric.lower()
  322. if self.metric == 'voc':
  323. eval_dataset.data_fields = {
  324. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  325. 'difficult'
  326. }
  327. elif self.metric == 'coco':
  328. if self.__class__.__name__ == 'MaskRCNN':
  329. eval_dataset.data_fields = {
  330. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  331. 'gt_poly', 'is_crowd'
  332. }
  333. else:
  334. eval_dataset.data_fields = {
  335. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  336. 'is_crowd'
  337. }
  338. eval_dataset.batch_transforms = self._compose_batch_transform(
  339. eval_dataset.transforms, mode='eval')
  340. arrange_transforms(
  341. model_type=self.model_type,
  342. transforms=eval_dataset.transforms,
  343. mode='eval')
  344. self.net.eval()
  345. nranks = paddle.distributed.get_world_size()
  346. local_rank = paddle.distributed.get_rank()
  347. if nranks > 1:
  348. # Initialize parallel environment if not done.
  349. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(
  350. ):
  351. paddle.distributed.init_parallel_env()
  352. if batch_size > 1:
  353. logging.warning(
  354. "Detector only supports single card evaluation with batch_size=1 "
  355. "during evaluation, so batch_size is forcibly set to 1.")
  356. batch_size = 1
  357. if nranks < 2 or local_rank == 0:
  358. self.eval_data_loader = self.build_data_loader(
  359. eval_dataset, batch_size=batch_size, mode='eval')
  360. is_bbox_normalized = False
  361. if eval_dataset.batch_transforms is not None:
  362. is_bbox_normalized = any(
  363. isinstance(t, _NormalizeBox)
  364. for t in eval_dataset.batch_transforms.batch_transforms)
  365. if self.metric == 'voc':
  366. eval_metric = VOCMetric(
  367. labels=eval_dataset.labels,
  368. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  369. is_bbox_normalized=is_bbox_normalized,
  370. classwise=False)
  371. else:
  372. eval_metric = COCOMetric(
  373. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  374. classwise=False)
  375. scores = collections.OrderedDict()
  376. logging.info(
  377. "Start to evaluate(total_samples={}, total_steps={})...".
  378. format(eval_dataset.num_samples, eval_dataset.num_samples))
  379. with paddle.no_grad():
  380. for step, data in enumerate(self.eval_data_loader):
  381. outputs = self.run(self.net, data, 'eval')
  382. eval_metric.update(data, outputs)
  383. eval_metric.accumulate()
  384. self.eval_details = eval_metric.details
  385. scores.update(eval_metric.get())
  386. eval_metric.reset()
  387. if return_details:
  388. return scores, self.eval_details
  389. return scores
  390. def predict(self, img_file, transforms=None):
  391. """
  392. Do inference.
  393. Args:
  394. img_file(List[np.ndarray or str], str or np.ndarray): img_file(list or str or np.array):
  395. Image path or decoded image data in a BGR format, which also could constitute a list,
  396. meaning all images to be predicted as a mini-batch.
  397. transforms(paddlex.transforms.Compose or None, optional):
  398. Transforms for inputs. If None, the transforms for evaluation process will be used. Defaults to None.
  399. Returns:
  400. If img_file is a string or np.array, the result is a list of dict with key-value pairs:
  401. {"category_id": `category_id`, "category": `category`, "bbox": `[x, y, w, h]`, "score": `score`}.
  402. If img_file is a list, the result is a list composed of dicts with the corresponding fields:
  403. category_id(int): the predicted category ID
  404. category(str): category name
  405. bbox(list): bounding box in [x, y, w, h] format
  406. score(str): confidence
  407. """
  408. if transforms is None and not hasattr(self, 'test_transforms'):
  409. raise Exception("transforms need to be defined, now is None.")
  410. if transforms is None:
  411. transforms = self.test_transforms
  412. if isinstance(img_file, (str, np.ndarray)):
  413. images = [img_file]
  414. else:
  415. images = img_file
  416. batch_samples = self._preprocess(images, transforms)
  417. self.net.eval()
  418. outputs = self.run(self.net, batch_samples, 'test')
  419. prediction = self._postprocess(outputs)
  420. if isinstance(img_file, (str, np.ndarray)):
  421. prediction = prediction[0]
  422. return prediction
  423. def _preprocess(self, images, transforms):
  424. arrange_transforms(
  425. model_type=self.model_type, transforms=transforms, mode='test')
  426. batch_samples = list()
  427. for im in images:
  428. sample = {'image': im}
  429. batch_samples.append(transforms(sample))
  430. batch_transforms = self._compose_batch_transform(transforms, 'test')
  431. batch_samples = batch_transforms(batch_samples)
  432. for k, v in batch_samples.items():
  433. batch_samples[k] = paddle.to_tensor(v)
  434. return batch_samples
  435. def _postprocess(self, batch_pred):
  436. infer_result = {}
  437. if 'bbox' in batch_pred:
  438. bboxes = batch_pred['bbox']
  439. bbox_nums = batch_pred['bbox_num']
  440. det_res = []
  441. k = 0
  442. for i in range(len(bbox_nums)):
  443. det_nums = bbox_nums[i]
  444. for j in range(det_nums):
  445. dt = bboxes[k]
  446. k = k + 1
  447. num_id, score, xmin, ymin, xmax, ymax = dt.tolist()
  448. if int(num_id) < 0:
  449. continue
  450. category = self.labels[int(num_id)]
  451. w = xmax - xmin
  452. h = ymax - ymin
  453. bbox = [xmin, ymin, w, h]
  454. dt_res = {
  455. 'category_id': int(num_id),
  456. 'category': category,
  457. 'bbox': bbox,
  458. 'score': score
  459. }
  460. det_res.append(dt_res)
  461. infer_result['bbox'] = det_res
  462. if 'mask' in batch_pred:
  463. masks = batch_pred['mask']
  464. bboxes = batch_pred['bbox']
  465. mask_nums = batch_pred['bbox_num']
  466. seg_res = []
  467. k = 0
  468. for i in range(len(mask_nums)):
  469. det_nums = mask_nums[i]
  470. for j in range(det_nums):
  471. mask = masks[k].astype(np.uint8)
  472. score = float(bboxes[k][1])
  473. label = int(bboxes[k][0])
  474. k = k + 1
  475. if label == -1:
  476. continue
  477. category = self.labels[int(label)]
  478. import pycocotools.mask as mask_util
  479. rle = mask_util.encode(
  480. np.array(
  481. mask[:, :, None], order="F", dtype="uint8"))[0]
  482. if six.PY3:
  483. if 'counts' in rle:
  484. rle['counts'] = rle['counts'].decode("utf8")
  485. sg_res = {
  486. 'category': category,
  487. 'segmentation': rle,
  488. 'score': score
  489. }
  490. seg_res.append(sg_res)
  491. infer_result['mask'] = seg_res
  492. bbox_num = batch_pred['bbox_num']
  493. results = []
  494. start = 0
  495. for num in bbox_num:
  496. end = start + num
  497. curr_res = infer_result['bbox'][start:end]
  498. if 'mask' in infer_result:
  499. mask_res = infer_result['mask'][start:end]
  500. for box, mask in zip(curr_res, mask_res):
  501. box.update(mask)
  502. results.append(curr_res)
  503. start = end
  504. return results
  505. class YOLOv3(BaseDetector):
  506. def __init__(self,
  507. num_classes=80,
  508. backbone='MobileNetV1',
  509. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  510. [59, 119], [116, 90], [156, 198], [373, 326]],
  511. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  512. ignore_threshold=0.7,
  513. nms_score_threshold=0.01,
  514. nms_topk=1000,
  515. nms_keep_topk=100,
  516. nms_iou_threshold=0.45,
  517. label_smooth=False):
  518. self.init_params = locals()
  519. if backbone not in [
  520. 'MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3',
  521. 'MobileNetV3_ssld', 'DarkNet53', 'ResNet50_vd_dcn', 'ResNet34'
  522. ]:
  523. raise ValueError(
  524. "backbone: {} is not supported. Please choose one of "
  525. "('MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3', 'MobileNetV3_ssld', 'DarkNet53', 'ResNet50_vd_dcn', 'ResNet34')".
  526. format(backbone))
  527. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  528. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  529. norm_type = 'sync_bn'
  530. else:
  531. norm_type = 'bn'
  532. self.backbone_name = backbone
  533. if 'MobileNetV1' in backbone:
  534. norm_type = 'bn'
  535. backbone = self._get_backbone('MobileNet', norm_type=norm_type)
  536. elif 'MobileNetV3' in backbone:
  537. backbone = self._get_backbone(
  538. 'MobileNetV3', norm_type=norm_type, feature_maps=[7, 13, 16])
  539. elif backbone == 'ResNet50_vd_dcn':
  540. backbone = self._get_backbone(
  541. 'ResNet',
  542. norm_type=norm_type,
  543. variant='d',
  544. return_idx=[1, 2, 3],
  545. dcn_v2_stages=[3],
  546. freeze_at=-1,
  547. freeze_norm=False)
  548. elif backbone == 'ResNet34':
  549. backbone = self._get_backbone(
  550. 'ResNet',
  551. depth=34,
  552. norm_type=norm_type,
  553. return_idx=[1, 2, 3],
  554. freeze_at=-1,
  555. freeze_norm=False,
  556. norm_decay=0.)
  557. else:
  558. backbone = self._get_backbone('DarkNet', norm_type=norm_type)
  559. neck = ppdet.modeling.YOLOv3FPN(
  560. norm_type=norm_type,
  561. in_channels=[i.channels for i in backbone.out_shape])
  562. loss = ppdet.modeling.YOLOv3Loss(
  563. num_classes=num_classes,
  564. ignore_thresh=ignore_threshold,
  565. label_smooth=label_smooth)
  566. yolo_head = ppdet.modeling.YOLOv3Head(
  567. in_channels=[i.channels for i in neck.out_shape],
  568. anchors=anchors,
  569. anchor_masks=anchor_masks,
  570. num_classes=num_classes,
  571. loss=loss)
  572. post_process = ppdet.modeling.BBoxPostProcess(
  573. decode=ppdet.modeling.YOLOBox(num_classes=num_classes),
  574. nms=ppdet.modeling.MultiClassNMS(
  575. score_threshold=nms_score_threshold,
  576. nms_top_k=nms_topk,
  577. keep_top_k=nms_keep_topk,
  578. nms_threshold=nms_iou_threshold))
  579. params = {
  580. 'backbone': backbone,
  581. 'neck': neck,
  582. 'yolo_head': yolo_head,
  583. 'post_process': post_process
  584. }
  585. super(YOLOv3, self).__init__(
  586. model_name='YOLOv3', num_classes=num_classes, **params)
  587. self.anchors = anchors
  588. self.anchor_masks = anchor_masks
  589. def _compose_batch_transform(self, transforms, mode='train'):
  590. if mode == 'train':
  591. default_batch_transforms = [
  592. _BatchPadding(pad_to_stride=-1), _NormalizeBox(),
  593. _PadBox(getattr(self, 'num_max_boxes', 50)), _BboxXYXY2XYWH(),
  594. _Gt2YoloTarget(
  595. anchor_masks=self.anchor_masks,
  596. anchors=self.anchors,
  597. downsample_ratios=getattr(self, 'downsample_ratios',
  598. [32, 16, 8]),
  599. num_classes=self.num_classes)
  600. ]
  601. else:
  602. default_batch_transforms = [_BatchPadding(pad_to_stride=-1)]
  603. if mode == 'eval' and self.metric == 'voc':
  604. collate_batch = False
  605. else:
  606. collate_batch = True
  607. custom_batch_transforms = []
  608. for i, op in enumerate(transforms.transforms):
  609. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  610. if mode != 'train':
  611. raise Exception(
  612. "{} cannot be present in the {} transforms. ".format(
  613. op.__class__.__name__, mode) +
  614. "Please check the {} transforms.".format(mode))
  615. custom_batch_transforms.insert(0, copy.deepcopy(op))
  616. batch_transforms = BatchCompose(
  617. custom_batch_transforms + default_batch_transforms,
  618. collate_batch=collate_batch)
  619. return batch_transforms
  620. class FasterRCNN(BaseDetector):
  621. def __init__(self,
  622. num_classes=80,
  623. backbone='ResNet50',
  624. with_fpn=True,
  625. aspect_ratios=[0.5, 1.0, 2.0],
  626. anchor_sizes=[[32], [64], [128], [256], [512]],
  627. keep_top_k=100,
  628. nms_threshold=0.5,
  629. score_threshold=0.05,
  630. fpn_num_channels=256,
  631. rpn_batch_size_per_im=256,
  632. rpn_fg_fraction=0.5,
  633. test_pre_nms_top_n=None,
  634. test_post_nms_top_n=1000):
  635. self.init_params = locals()
  636. if backbone not in [
  637. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34',
  638. 'ResNet34_vd', 'ResNet101', 'ResNet101_vd', 'HRNet_W18'
  639. ]:
  640. raise ValueError(
  641. "backbone: {} is not supported. Please choose one of "
  642. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34', 'ResNet34_vd', "
  643. "'ResNet101', 'ResNet101_vd', 'HRNet_W18')".format(backbone))
  644. self.backbone_name = backbone
  645. if backbone == 'HRNet_W18':
  646. if not with_fpn:
  647. logging.warning(
  648. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  649. format(backbone))
  650. with_fpn = True
  651. backbone = self._get_backbone(
  652. 'HRNet', width=18, freeze_at=0, return_idx=[0, 1, 2, 3])
  653. elif backbone == 'ResNet50_vd_ssld':
  654. if not with_fpn:
  655. logging.warning(
  656. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  657. format(backbone))
  658. with_fpn = True
  659. backbone = self._get_backbone(
  660. 'ResNet',
  661. variant='d',
  662. norm_type='bn',
  663. freeze_at=0,
  664. return_idx=[0, 1, 2, 3],
  665. num_stages=4,
  666. lr_mult_list=[0.05, 0.05, 0.1, 0.15])
  667. elif 'ResNet50' in backbone:
  668. if with_fpn:
  669. backbone = self._get_backbone(
  670. 'ResNet',
  671. variant='d' if '_vd' in backbone else 'b',
  672. norm_type='bn',
  673. freeze_at=0,
  674. return_idx=[0, 1, 2, 3],
  675. num_stages=4)
  676. else:
  677. backbone = self._get_backbone(
  678. 'ResNet',
  679. variant='d' if '_vd' in backbone else 'b',
  680. norm_type='bn',
  681. freeze_at=0,
  682. return_idx=[2],
  683. num_stages=3)
  684. elif 'ResNet34' in backbone:
  685. if not with_fpn:
  686. logging.warning(
  687. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  688. format(backbone))
  689. with_fpn = True
  690. backbone = self._get_backbone(
  691. 'ResNet',
  692. depth=34,
  693. variant='d' if 'vd' in backbone else 'b',
  694. norm_type='bn',
  695. freeze_at=0,
  696. return_idx=[0, 1, 2, 3],
  697. num_stages=4)
  698. else:
  699. if not with_fpn:
  700. logging.warning(
  701. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  702. format(backbone))
  703. with_fpn = True
  704. backbone = self._get_backbone(
  705. 'ResNet',
  706. depth=101,
  707. variant='d' if 'vd' in backbone else 'b',
  708. norm_type='bn',
  709. freeze_at=0,
  710. return_idx=[0, 1, 2, 3],
  711. num_stages=4)
  712. rpn_in_channel = backbone.out_shape[0].channels
  713. if with_fpn:
  714. self.backbone_name = self.backbone_name + '_fpn'
  715. if 'HRNet' in self.backbone_name:
  716. neck = ppdet.modeling.HRFPN(
  717. in_channels=[i.channels for i in backbone.out_shape],
  718. out_channel=fpn_num_channels,
  719. spatial_scales=[
  720. 1.0 / i.stride for i in backbone.out_shape
  721. ],
  722. share_conv=False)
  723. else:
  724. neck = ppdet.modeling.FPN(
  725. in_channels=[i.channels for i in backbone.out_shape],
  726. out_channel=fpn_num_channels,
  727. spatial_scales=[
  728. 1.0 / i.stride for i in backbone.out_shape
  729. ])
  730. rpn_in_channel = neck.out_shape[0].channels
  731. anchor_generator_cfg = {
  732. 'aspect_ratios': aspect_ratios,
  733. 'anchor_sizes': anchor_sizes,
  734. 'strides': [4, 8, 16, 32, 64]
  735. }
  736. train_proposal_cfg = {
  737. 'min_size': 0.0,
  738. 'nms_thresh': .7,
  739. 'pre_nms_top_n': 2000,
  740. 'post_nms_top_n': 1000,
  741. 'topk_after_collect': True
  742. }
  743. test_proposal_cfg = {
  744. 'min_size': 0.0,
  745. 'nms_thresh': .7,
  746. 'pre_nms_top_n': 1000
  747. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  748. 'post_nms_top_n': test_post_nms_top_n
  749. }
  750. head = ppdet.modeling.TwoFCHead(out_channel=1024)
  751. roi_extractor_cfg = {
  752. 'resolution': 7,
  753. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  754. 'sampling_ratio': 0,
  755. 'aligned': True
  756. }
  757. with_pool = False
  758. else:
  759. neck = None
  760. anchor_generator_cfg = {
  761. 'aspect_ratios': aspect_ratios,
  762. 'anchor_sizes': anchor_sizes,
  763. 'strides': [16]
  764. }
  765. train_proposal_cfg = {
  766. 'min_size': 0.0,
  767. 'nms_thresh': .7,
  768. 'pre_nms_top_n': 12000,
  769. 'post_nms_top_n': 2000,
  770. 'topk_after_collect': False
  771. }
  772. test_proposal_cfg = {
  773. 'min_size': 0.0,
  774. 'nms_thresh': .7,
  775. 'pre_nms_top_n': 6000
  776. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  777. 'post_nms_top_n': test_post_nms_top_n
  778. }
  779. head = ppdet.modeling.Res5Head()
  780. roi_extractor_cfg = {
  781. 'resolution': 14,
  782. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  783. 'sampling_ratio': 0,
  784. 'aligned': True
  785. }
  786. with_pool = True
  787. rpn_target_assign_cfg = {
  788. 'batch_size_per_im': rpn_batch_size_per_im,
  789. 'fg_fraction': rpn_fg_fraction,
  790. 'negative_overlap': .3,
  791. 'positive_overlap': .7,
  792. 'use_random': True
  793. }
  794. rpn_head = ppdet.modeling.RPNHead(
  795. anchor_generator=anchor_generator_cfg,
  796. rpn_target_assign=rpn_target_assign_cfg,
  797. train_proposal=train_proposal_cfg,
  798. test_proposal=test_proposal_cfg,
  799. in_channel=rpn_in_channel)
  800. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  801. bbox_head = ppdet.modeling.BBoxHead(
  802. head=head,
  803. in_channel=head.out_shape[0].channels,
  804. roi_extractor=roi_extractor_cfg,
  805. with_pool=with_pool,
  806. bbox_assigner=bbox_assigner,
  807. num_classes=num_classes)
  808. bbox_post_process = ppdet.modeling.BBoxPostProcess(
  809. num_classes=num_classes,
  810. decode=ppdet.modeling.RCNNBox(num_classes=num_classes),
  811. nms=ppdet.modeling.MultiClassNMS(
  812. score_threshold=score_threshold,
  813. keep_top_k=keep_top_k,
  814. nms_threshold=nms_threshold))
  815. params = {
  816. 'backbone': backbone,
  817. 'neck': neck,
  818. 'rpn_head': rpn_head,
  819. 'bbox_head': bbox_head,
  820. 'bbox_post_process': bbox_post_process
  821. }
  822. self.with_fpn = with_fpn
  823. super(FasterRCNN, self).__init__(
  824. model_name='FasterRCNN', num_classes=num_classes, **params)
  825. def _compose_batch_transform(self, transforms, mode='train'):
  826. if mode == 'train':
  827. default_batch_transforms = [
  828. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  829. ]
  830. collate_batch = False
  831. else:
  832. default_batch_transforms = [
  833. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  834. ]
  835. collate_batch = True
  836. custom_batch_transforms = []
  837. for i, op in enumerate(transforms.transforms):
  838. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  839. if mode != 'train':
  840. raise Exception(
  841. "{} cannot be present in the {} transforms. ".format(
  842. op.__class__.__name__, mode) +
  843. "Please check the {} transforms.".format(mode))
  844. custom_batch_transforms.insert(0, copy.deepcopy(op))
  845. batch_transforms = BatchCompose(
  846. custom_batch_transforms + default_batch_transforms,
  847. collate_batch=collate_batch)
  848. return batch_transforms
  849. class PPYOLO(YOLOv3):
  850. def __init__(self,
  851. num_classes=80,
  852. backbone='ResNet50_vd_dcn',
  853. anchors=None,
  854. anchor_masks=None,
  855. use_coord_conv=True,
  856. use_iou_aware=True,
  857. use_spp=True,
  858. use_drop_block=True,
  859. scale_x_y=1.05,
  860. ignore_threshold=0.7,
  861. label_smooth=False,
  862. use_iou_loss=True,
  863. use_matrix_nms=True,
  864. nms_score_threshold=0.01,
  865. nms_topk=-1,
  866. nms_keep_topk=100,
  867. nms_iou_threshold=0.45):
  868. self.init_params = locals()
  869. if backbone not in [
  870. 'ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large',
  871. 'MobileNetV3_small'
  872. ]:
  873. raise ValueError(
  874. "backbone: {} is not supported. Please choose one of "
  875. "('ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large', 'MobileNetV3_small')".
  876. format(backbone))
  877. self.backbone_name = backbone
  878. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  879. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  880. norm_type = 'sync_bn'
  881. else:
  882. norm_type = 'bn'
  883. if anchors is None and anchor_masks is None:
  884. if 'MobileNetV3' in backbone:
  885. anchors = [[11, 18], [34, 47], [51, 126], [115, 71],
  886. [120, 195], [254, 235]]
  887. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  888. elif backbone == 'ResNet50_vd_dcn':
  889. anchors = [[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  890. [59, 119], [116, 90], [156, 198], [373, 326]]
  891. anchor_masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
  892. else:
  893. anchors = [[10, 14], [23, 27], [37, 58], [81, 82], [135, 169],
  894. [344, 319]]
  895. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  896. elif anchors is None or anchor_masks is None:
  897. raise ValueError("Please define both anchors and anchor_masks.")
  898. if backbone == 'ResNet50_vd_dcn':
  899. backbone = self._get_backbone(
  900. 'ResNet',
  901. variant='d',
  902. norm_type=norm_type,
  903. return_idx=[1, 2, 3],
  904. dcn_v2_stages=[3],
  905. freeze_at=-1,
  906. freeze_norm=False,
  907. norm_decay=0.)
  908. downsample_ratios = [32, 16, 8]
  909. elif backbone == 'ResNet18_vd':
  910. backbone = self._get_backbone(
  911. 'ResNet',
  912. depth=18,
  913. variant='d',
  914. norm_type=norm_type,
  915. return_idx=[2, 3],
  916. freeze_at=-1,
  917. freeze_norm=False,
  918. norm_decay=0.)
  919. downsample_ratios = [32, 16, 8]
  920. elif backbone == 'MobileNetV3_large':
  921. backbone = self._get_backbone(
  922. 'MobileNetV3',
  923. model_name='large',
  924. norm_type=norm_type,
  925. scale=1,
  926. with_extra_blocks=False,
  927. extra_block_filters=[],
  928. feature_maps=[13, 16])
  929. downsample_ratios = [32, 16]
  930. elif backbone == 'MobileNetV3_small':
  931. backbone = self._get_backbone(
  932. 'MobileNetV3',
  933. model_name='small',
  934. norm_type=norm_type,
  935. scale=1,
  936. with_extra_blocks=False,
  937. extra_block_filters=[],
  938. feature_maps=[9, 12])
  939. downsample_ratios = [32, 16]
  940. neck = ppdet.modeling.PPYOLOFPN(
  941. norm_type=norm_type,
  942. in_channels=[i.channels for i in backbone.out_shape],
  943. coord_conv=use_coord_conv,
  944. drop_block=use_drop_block,
  945. spp=use_spp,
  946. conv_block_num=0 if ('MobileNetV3' in self.backbone_name or
  947. self.backbone_name == 'ResNet18_vd') else 2)
  948. loss = ppdet.modeling.YOLOv3Loss(
  949. num_classes=num_classes,
  950. ignore_thresh=ignore_threshold,
  951. downsample=downsample_ratios,
  952. label_smooth=label_smooth,
  953. scale_x_y=scale_x_y,
  954. iou_loss=ppdet.modeling.IouLoss(
  955. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  956. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  957. if use_iou_aware else None)
  958. yolo_head = ppdet.modeling.YOLOv3Head(
  959. in_channels=[i.channels for i in neck.out_shape],
  960. anchors=anchors,
  961. anchor_masks=anchor_masks,
  962. num_classes=num_classes,
  963. loss=loss,
  964. iou_aware=use_iou_aware)
  965. if use_matrix_nms:
  966. nms = ppdet.modeling.MatrixNMS(
  967. keep_top_k=nms_keep_topk,
  968. score_threshold=nms_score_threshold,
  969. post_threshold=.05
  970. if 'MobileNetV3' in self.backbone_name else .01,
  971. nms_top_k=nms_topk,
  972. background_label=-1)
  973. else:
  974. nms = ppdet.modeling.MultiClassNMS(
  975. score_threshold=nms_score_threshold,
  976. nms_top_k=nms_topk,
  977. keep_top_k=nms_keep_topk,
  978. nms_threshold=nms_iou_threshold)
  979. post_process = ppdet.modeling.BBoxPostProcess(
  980. decode=ppdet.modeling.YOLOBox(
  981. num_classes=num_classes,
  982. conf_thresh=.005
  983. if 'MobileNetV3' in self.backbone_name else .01,
  984. scale_x_y=scale_x_y),
  985. nms=nms)
  986. params = {
  987. 'backbone': backbone,
  988. 'neck': neck,
  989. 'yolo_head': yolo_head,
  990. 'post_process': post_process
  991. }
  992. super(YOLOv3, self).__init__(
  993. model_name='YOLOv3', num_classes=num_classes, **params)
  994. self.anchors = anchors
  995. self.anchor_masks = anchor_masks
  996. self.downsample_ratios = downsample_ratios
  997. self.model_name = 'PPYOLO'
  998. class PPYOLOTiny(YOLOv3):
  999. def __init__(self,
  1000. num_classes=80,
  1001. backbone='MobileNetV3',
  1002. anchors=[[10, 15], [24, 36], [72, 42], [35, 87], [102, 96],
  1003. [60, 170], [220, 125], [128, 222], [264, 266]],
  1004. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  1005. use_iou_aware=False,
  1006. use_spp=True,
  1007. use_drop_block=True,
  1008. scale_x_y=1.05,
  1009. ignore_threshold=0.5,
  1010. label_smooth=False,
  1011. use_iou_loss=True,
  1012. use_matrix_nms=False,
  1013. nms_score_threshold=0.005,
  1014. nms_topk=1000,
  1015. nms_keep_topk=100,
  1016. nms_iou_threshold=0.45):
  1017. self.init_params = locals()
  1018. if backbone != 'MobileNetV3':
  1019. logging.warning(
  1020. "PPYOLOTiny only supports MobileNetV3 as backbone. "
  1021. "Backbone is forcibly set to MobileNetV3.")
  1022. self.backbone_name = 'MobileNetV3'
  1023. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1024. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1025. norm_type = 'sync_bn'
  1026. else:
  1027. norm_type = 'bn'
  1028. backbone = self._get_backbone(
  1029. 'MobileNetV3',
  1030. model_name='large',
  1031. norm_type=norm_type,
  1032. scale=.5,
  1033. with_extra_blocks=False,
  1034. extra_block_filters=[],
  1035. feature_maps=[7, 13, 16])
  1036. downsample_ratios = [32, 16, 8]
  1037. neck = ppdet.modeling.PPYOLOTinyFPN(
  1038. detection_block_channels=[160, 128, 96],
  1039. in_channels=[i.channels for i in backbone.out_shape],
  1040. spp=use_spp,
  1041. drop_block=use_drop_block)
  1042. loss = ppdet.modeling.YOLOv3Loss(
  1043. num_classes=num_classes,
  1044. ignore_thresh=ignore_threshold,
  1045. downsample=downsample_ratios,
  1046. label_smooth=label_smooth,
  1047. scale_x_y=scale_x_y,
  1048. iou_loss=ppdet.modeling.IouLoss(
  1049. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  1050. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  1051. if use_iou_aware else None)
  1052. yolo_head = ppdet.modeling.YOLOv3Head(
  1053. in_channels=[i.channels for i in neck.out_shape],
  1054. anchors=anchors,
  1055. anchor_masks=anchor_masks,
  1056. num_classes=num_classes,
  1057. loss=loss,
  1058. iou_aware=use_iou_aware)
  1059. if use_matrix_nms:
  1060. nms = ppdet.modeling.MatrixNMS(
  1061. keep_top_k=nms_keep_topk,
  1062. score_threshold=nms_score_threshold,
  1063. post_threshold=.05,
  1064. nms_top_k=nms_topk,
  1065. background_label=-1)
  1066. else:
  1067. nms = ppdet.modeling.MultiClassNMS(
  1068. score_threshold=nms_score_threshold,
  1069. nms_top_k=nms_topk,
  1070. keep_top_k=nms_keep_topk,
  1071. nms_threshold=nms_iou_threshold)
  1072. post_process = ppdet.modeling.BBoxPostProcess(
  1073. decode=ppdet.modeling.YOLOBox(
  1074. num_classes=num_classes,
  1075. conf_thresh=.005,
  1076. downsample_ratio=32,
  1077. clip_bbox=True,
  1078. scale_x_y=scale_x_y),
  1079. nms=nms)
  1080. params = {
  1081. 'backbone': backbone,
  1082. 'neck': neck,
  1083. 'yolo_head': yolo_head,
  1084. 'post_process': post_process
  1085. }
  1086. super(YOLOv3, self).__init__(
  1087. model_name='YOLOv3', num_classes=num_classes, **params)
  1088. self.anchors = anchors
  1089. self.anchor_masks = anchor_masks
  1090. self.downsample_ratios = downsample_ratios
  1091. self.num_max_boxes = 100
  1092. self.model_name = 'PPYOLOTiny'
  1093. class PPYOLOv2(YOLOv3):
  1094. def __init__(self,
  1095. num_classes=80,
  1096. backbone='ResNet50_vd_dcn',
  1097. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  1098. [59, 119], [116, 90], [156, 198], [373, 326]],
  1099. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  1100. use_iou_aware=True,
  1101. use_spp=True,
  1102. use_drop_block=True,
  1103. scale_x_y=1.05,
  1104. ignore_threshold=0.7,
  1105. label_smooth=False,
  1106. use_iou_loss=True,
  1107. use_matrix_nms=True,
  1108. nms_score_threshold=0.01,
  1109. nms_topk=-1,
  1110. nms_keep_topk=100,
  1111. nms_iou_threshold=0.45):
  1112. self.init_params = locals()
  1113. if backbone not in ['ResNet50_vd_dcn', 'ResNet101_vd_dcn']:
  1114. raise ValueError(
  1115. "backbone: {} is not supported. Please choose one of "
  1116. "('ResNet50_vd_dcn', 'ResNet18_vd')".format(backbone))
  1117. self.backbone_name = backbone
  1118. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1119. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1120. norm_type = 'sync_bn'
  1121. else:
  1122. norm_type = 'bn'
  1123. if backbone == 'ResNet50_vd_dcn':
  1124. backbone = self._get_backbone(
  1125. 'ResNet',
  1126. variant='d',
  1127. norm_type=norm_type,
  1128. return_idx=[1, 2, 3],
  1129. dcn_v2_stages=[3],
  1130. freeze_at=-1,
  1131. freeze_norm=False,
  1132. norm_decay=0.)
  1133. downsample_ratios = [32, 16, 8]
  1134. elif backbone == 'ResNet101_vd_dcn':
  1135. backbone = self._get_backbone(
  1136. 'ResNet',
  1137. depth=101,
  1138. variant='d',
  1139. norm_type=norm_type,
  1140. return_idx=[1, 2, 3],
  1141. dcn_v2_stages=[3],
  1142. freeze_at=-1,
  1143. freeze_norm=False,
  1144. norm_decay=0.)
  1145. downsample_ratios = [32, 16, 8]
  1146. neck = ppdet.modeling.PPYOLOPAN(
  1147. norm_type=norm_type,
  1148. in_channels=[i.channels for i in backbone.out_shape],
  1149. drop_block=use_drop_block,
  1150. block_size=3,
  1151. keep_prob=.9,
  1152. spp=use_spp)
  1153. loss = ppdet.modeling.YOLOv3Loss(
  1154. num_classes=num_classes,
  1155. ignore_thresh=ignore_threshold,
  1156. downsample=downsample_ratios,
  1157. label_smooth=label_smooth,
  1158. scale_x_y=scale_x_y,
  1159. iou_loss=ppdet.modeling.IouLoss(
  1160. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  1161. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  1162. if use_iou_aware else None)
  1163. yolo_head = ppdet.modeling.YOLOv3Head(
  1164. in_channels=[i.channels for i in neck.out_shape],
  1165. anchors=anchors,
  1166. anchor_masks=anchor_masks,
  1167. num_classes=num_classes,
  1168. loss=loss,
  1169. iou_aware=use_iou_aware,
  1170. iou_aware_factor=.5)
  1171. if use_matrix_nms:
  1172. nms = ppdet.modeling.MatrixNMS(
  1173. keep_top_k=nms_keep_topk,
  1174. score_threshold=nms_score_threshold,
  1175. post_threshold=.01,
  1176. nms_top_k=nms_topk,
  1177. background_label=-1)
  1178. else:
  1179. nms = ppdet.modeling.MultiClassNMS(
  1180. score_threshold=nms_score_threshold,
  1181. nms_top_k=nms_topk,
  1182. keep_top_k=nms_keep_topk,
  1183. nms_threshold=nms_iou_threshold)
  1184. post_process = ppdet.modeling.BBoxPostProcess(
  1185. decode=ppdet.modeling.YOLOBox(
  1186. num_classes=num_classes,
  1187. conf_thresh=.01,
  1188. downsample_ratio=32,
  1189. clip_bbox=True,
  1190. scale_x_y=scale_x_y),
  1191. nms=nms)
  1192. params = {
  1193. 'backbone': backbone,
  1194. 'neck': neck,
  1195. 'yolo_head': yolo_head,
  1196. 'post_process': post_process
  1197. }
  1198. super(YOLOv3, self).__init__(
  1199. model_name='YOLOv3', num_classes=num_classes, **params)
  1200. self.anchors = anchors
  1201. self.anchor_masks = anchor_masks
  1202. self.downsample_ratios = downsample_ratios
  1203. self.num_max_boxes = 100
  1204. self.model_name = 'PPYOLOv2'
  1205. class MaskRCNN(BaseDetector):
  1206. def __init__(self,
  1207. num_classes=80,
  1208. backbone='ResNet50_vd',
  1209. with_fpn=True,
  1210. aspect_ratios=[0.5, 1.0, 2.0],
  1211. anchor_sizes=[[32], [64], [128], [256], [512]],
  1212. keep_top_k=100,
  1213. nms_threshold=0.5,
  1214. score_threshold=0.05,
  1215. fpn_num_channels=256,
  1216. rpn_batch_size_per_im=256,
  1217. rpn_fg_fraction=0.5,
  1218. test_pre_nms_top_n=None,
  1219. test_post_nms_top_n=1000):
  1220. self.init_params = locals()
  1221. if backbone not in [
  1222. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101',
  1223. 'ResNet101_vd'
  1224. ]:
  1225. raise ValueError(
  1226. "backbone: {} is not supported. Please choose one of "
  1227. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101', 'ResNet101_vd')".
  1228. format(backbone))
  1229. self.backbone_name = backbone + '_fpn' if with_fpn else backbone
  1230. if backbone == 'ResNet50':
  1231. if with_fpn:
  1232. backbone = self._get_backbone(
  1233. 'ResNet',
  1234. norm_type='bn',
  1235. freeze_at=0,
  1236. return_idx=[0, 1, 2, 3],
  1237. num_stages=4)
  1238. else:
  1239. backbone = self._get_backbone(
  1240. 'ResNet',
  1241. norm_type='bn',
  1242. freeze_at=0,
  1243. return_idx=[2],
  1244. num_stages=3)
  1245. elif 'ResNet50_vd' in backbone:
  1246. if not with_fpn:
  1247. logging.warning(
  1248. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1249. format(backbone))
  1250. with_fpn = True
  1251. backbone = self._get_backbone(
  1252. 'ResNet',
  1253. variant='d',
  1254. norm_type='bn',
  1255. freeze_at=0,
  1256. return_idx=[0, 1, 2, 3],
  1257. num_stages=4,
  1258. lr_mult_list=[0.05, 0.05, 0.1, 0.15]
  1259. if '_ssld' in backbone else [1.0, 1.0, 1.0, 1.0])
  1260. else:
  1261. if not with_fpn:
  1262. logging.warning(
  1263. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1264. format(backbone))
  1265. with_fpn = True
  1266. backbone = self._get_backbone(
  1267. 'ResNet',
  1268. variant='d' if '_vd' in backbone else 'b',
  1269. depth=101,
  1270. norm_type='bn',
  1271. freeze_at=0,
  1272. return_idx=[0, 1, 2, 3],
  1273. num_stages=4)
  1274. rpn_in_channel = backbone.out_shape[0].channels
  1275. if with_fpn:
  1276. neck = ppdet.modeling.FPN(
  1277. in_channels=[i.channels for i in backbone.out_shape],
  1278. out_channel=fpn_num_channels,
  1279. spatial_scales=[1.0 / i.stride for i in backbone.out_shape])
  1280. rpn_in_channel = neck.out_shape[0].channels
  1281. anchor_generator_cfg = {
  1282. 'aspect_ratios': aspect_ratios,
  1283. 'anchor_sizes': anchor_sizes,
  1284. 'strides': [4, 8, 16, 32, 64]
  1285. }
  1286. train_proposal_cfg = {
  1287. 'min_size': 0.0,
  1288. 'nms_thresh': .7,
  1289. 'pre_nms_top_n': 2000,
  1290. 'post_nms_top_n': 1000,
  1291. 'topk_after_collect': True
  1292. }
  1293. test_proposal_cfg = {
  1294. 'min_size': 0.0,
  1295. 'nms_thresh': .7,
  1296. 'pre_nms_top_n': 1000
  1297. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1298. 'post_nms_top_n': test_post_nms_top_n
  1299. }
  1300. bb_head = ppdet.modeling.TwoFCHead(
  1301. in_channel=neck.out_shape[0].channels, out_channel=1024)
  1302. bb_roi_extractor_cfg = {
  1303. 'resolution': 7,
  1304. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1305. 'sampling_ratio': 0,
  1306. 'aligned': True
  1307. }
  1308. with_pool = False
  1309. m_head = ppdet.modeling.MaskFeat(
  1310. in_channel=neck.out_shape[0].channels,
  1311. out_channel=256,
  1312. num_convs=4)
  1313. m_roi_extractor_cfg = {
  1314. 'resolution': 14,
  1315. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1316. 'sampling_ratio': 0,
  1317. 'aligned': True
  1318. }
  1319. mask_assigner = MaskAssigner(
  1320. num_classes=num_classes, mask_resolution=28)
  1321. share_bbox_feat = False
  1322. else:
  1323. neck = None
  1324. anchor_generator_cfg = {
  1325. 'aspect_ratios': aspect_ratios,
  1326. 'anchor_sizes': anchor_sizes,
  1327. 'strides': [16]
  1328. }
  1329. train_proposal_cfg = {
  1330. 'min_size': 0.0,
  1331. 'nms_thresh': .7,
  1332. 'pre_nms_top_n': 12000,
  1333. 'post_nms_top_n': 2000,
  1334. 'topk_after_collect': False
  1335. }
  1336. test_proposal_cfg = {
  1337. 'min_size': 0.0,
  1338. 'nms_thresh': .7,
  1339. 'pre_nms_top_n': 6000
  1340. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1341. 'post_nms_top_n': test_post_nms_top_n
  1342. }
  1343. bb_head = ppdet.modeling.Res5Head()
  1344. bb_roi_extractor_cfg = {
  1345. 'resolution': 14,
  1346. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  1347. 'sampling_ratio': 0,
  1348. 'aligned': True
  1349. }
  1350. with_pool = True
  1351. m_head = ppdet.modeling.MaskFeat(
  1352. in_channel=bb_head.out_shape[0].channels,
  1353. out_channel=256,
  1354. num_convs=0)
  1355. m_roi_extractor_cfg = {
  1356. 'resolution': 14,
  1357. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  1358. 'sampling_ratio': 0,
  1359. 'aligned': True
  1360. }
  1361. mask_assigner = MaskAssigner(
  1362. num_classes=num_classes, mask_resolution=14)
  1363. share_bbox_feat = True
  1364. rpn_target_assign_cfg = {
  1365. 'batch_size_per_im': rpn_batch_size_per_im,
  1366. 'fg_fraction': rpn_fg_fraction,
  1367. 'negative_overlap': .3,
  1368. 'positive_overlap': .7,
  1369. 'use_random': True
  1370. }
  1371. rpn_head = ppdet.modeling.RPNHead(
  1372. anchor_generator=anchor_generator_cfg,
  1373. rpn_target_assign=rpn_target_assign_cfg,
  1374. train_proposal=train_proposal_cfg,
  1375. test_proposal=test_proposal_cfg,
  1376. in_channel=rpn_in_channel)
  1377. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  1378. bbox_head = ppdet.modeling.BBoxHead(
  1379. head=bb_head,
  1380. in_channel=bb_head.out_shape[0].channels,
  1381. roi_extractor=bb_roi_extractor_cfg,
  1382. with_pool=with_pool,
  1383. bbox_assigner=bbox_assigner,
  1384. num_classes=num_classes)
  1385. mask_head = ppdet.modeling.MaskHead(
  1386. head=m_head,
  1387. roi_extractor=m_roi_extractor_cfg,
  1388. mask_assigner=mask_assigner,
  1389. share_bbox_feat=share_bbox_feat,
  1390. num_classes=num_classes)
  1391. bbox_post_process = ppdet.modeling.BBoxPostProcess(
  1392. num_classes=num_classes,
  1393. decode=ppdet.modeling.RCNNBox(num_classes=num_classes),
  1394. nms=ppdet.modeling.MultiClassNMS(
  1395. score_threshold=score_threshold,
  1396. keep_top_k=keep_top_k,
  1397. nms_threshold=nms_threshold))
  1398. mask_post_process = ppdet.modeling.MaskPostProcess(binary_thresh=.5)
  1399. params = {
  1400. 'backbone': backbone,
  1401. 'neck': neck,
  1402. 'rpn_head': rpn_head,
  1403. 'bbox_head': bbox_head,
  1404. 'mask_head': mask_head,
  1405. 'bbox_post_process': bbox_post_process,
  1406. 'mask_post_process': mask_post_process
  1407. }
  1408. self.with_fpn = with_fpn
  1409. super(MaskRCNN, self).__init__(
  1410. model_name='MaskRCNN', num_classes=num_classes, **params)
  1411. def _compose_batch_transform(self, transforms, mode='train'):
  1412. if mode == 'train':
  1413. default_batch_transforms = [
  1414. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1415. ]
  1416. collate_batch = False
  1417. else:
  1418. default_batch_transforms = [
  1419. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1420. ]
  1421. collate_batch = True
  1422. custom_batch_transforms = []
  1423. for i, op in enumerate(transforms.transforms):
  1424. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  1425. if mode != 'train':
  1426. raise Exception(
  1427. "{} cannot be present in the {} transforms. ".format(
  1428. op.__class__.__name__, mode) +
  1429. "Please check the {} transforms.".format(mode))
  1430. custom_batch_transforms.insert(0, copy.deepcopy(op))
  1431. batch_transforms = BatchCompose(
  1432. custom_batch_transforms + default_batch_transforms,
  1433. collate_batch=collate_batch)
  1434. return batch_transforms