detector.py 57 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. import collections
  16. import copy
  17. import os
  18. import os.path as osp
  19. from paddle.static import InputSpec
  20. import paddlex
  21. import paddlex.utils.logging as logging
  22. from paddlex.cv.nets.ppdet.modeling.proposal_generator.target_layer import BBoxAssigner, MaskAssigner
  23. from paddlex.cv.nets.ppdet.modeling import *
  24. from paddlex.cv.nets.ppdet.modeling.post_process import *
  25. from paddlex.cv.nets.ppdet.modeling.layers import YOLOBox, MultiClassNMS, RCNNBox
  26. from paddlex.cv.transforms.operators import _NormalizeBox, _PadBox, _BboxXYXY2XYWH
  27. from paddlex.cv.transforms.batch_operators import BatchCompose, BatchRandomResize, BatchRandomResizeByShort, _BatchPadding, _Gt2YoloTarget
  28. from paddlex.cv.transforms import arrange_transforms
  29. from .base import BaseModel
  30. from .utils.det_metrics import VOCMetric, COCOMetric
  31. from paddlex.utils.checkpoint import det_pretrain_weights_dict
  32. __all__ = [
  33. "YOLOv3", "FasterRCNN", "PPYOLO", "PPYOLOTiny", "PPYOLOv2", "MaskRCNN"
  34. ]
  35. class BaseDetector(BaseModel):
  36. def __init__(self, model_name, num_classes=80, **params):
  37. self.init_params.update(locals())
  38. del self.init_params['params']
  39. super(BaseDetector, self).__init__('detector')
  40. if not hasattr(architectures, model_name):
  41. raise Exception("ERROR: There's no model named {}.".format(
  42. model_name))
  43. self.model_name = model_name
  44. self.num_classes = num_classes
  45. self.labels = None
  46. self.net = self.build_net(**params)
  47. def build_net(self, **params):
  48. with paddle.utils.unique_name.guard():
  49. net = architectures.__dict__[self.model_name](**params)
  50. return net
  51. def get_test_inputs(self, image_shape):
  52. input_spec = [{
  53. "image": InputSpec(
  54. shape=[None, 3] + image_shape, name='image', dtype='float32'),
  55. "im_shape": InputSpec(
  56. shape=[None, 2], name='im_shape', dtype='float32'),
  57. "scale_factor": InputSpec(
  58. shape=[None, 2], name='scale_factor', dtype='float32')
  59. }]
  60. return input_spec
  61. def _get_backbone(self, backbone_name, **params):
  62. backbone = backbones.__dict__[backbone_name](**params)
  63. return backbone
  64. def run(self, net, inputs, mode):
  65. net_out = net(inputs)
  66. if mode in ['train', 'eval']:
  67. outputs = net_out
  68. else:
  69. for key in ['im_shape', 'scale_factor']:
  70. net_out[key] = inputs[key]
  71. outputs = dict()
  72. for key in net_out:
  73. outputs[key] = net_out[key].numpy()
  74. return outputs
  75. def default_optimizer(self, parameters, learning_rate, warmup_steps,
  76. warmup_start_lr, lr_decay_epochs, lr_decay_gamma,
  77. num_steps_each_epoch):
  78. boundaries = [b * num_steps_each_epoch for b in lr_decay_epochs]
  79. values = [(lr_decay_gamma**i) * learning_rate
  80. for i in range(len(lr_decay_epochs) + 1)]
  81. scheduler = paddle.optimizer.lr.PiecewiseDecay(
  82. boundaries=boundaries, values=values)
  83. if warmup_steps > 0:
  84. if warmup_steps > lr_decay_epochs[0] * num_steps_each_epoch:
  85. logging.error(
  86. "In function train(), parameters should satisfy: "
  87. "warmup_steps <= lr_decay_epochs[0]*num_samples_in_train_dataset",
  88. exit=False)
  89. logging.error(
  90. "See this doc for more information: "
  91. "https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/appendix/parameters.md#notice",
  92. exit=False)
  93. scheduler = paddle.optimizer.lr.LinearWarmup(
  94. learning_rate=scheduler,
  95. warmup_steps=warmup_steps,
  96. start_lr=warmup_start_lr,
  97. end_lr=learning_rate)
  98. optimizer = paddle.optimizer.Momentum(
  99. scheduler,
  100. momentum=.9,
  101. weight_decay=paddle.regularizer.L2Decay(coeff=1e-04),
  102. parameters=parameters)
  103. return optimizer
  104. def train(self,
  105. num_epochs,
  106. train_dataset,
  107. train_batch_size=64,
  108. eval_dataset=None,
  109. optimizer=None,
  110. save_interval_epochs=1,
  111. log_interval_steps=10,
  112. save_dir='output',
  113. pretrain_weights='IMAGENET',
  114. learning_rate=.001,
  115. warmup_steps=0,
  116. warmup_start_lr=0.0,
  117. lr_decay_epochs=(216, 243),
  118. lr_decay_gamma=0.1,
  119. metric=None,
  120. early_stop=False,
  121. early_stop_patience=5,
  122. use_vdl=True):
  123. """
  124. Train the model.
  125. Args:
  126. num_epochs(int): The number of epochs.
  127. train_dataset(paddlex.dataset): Training dataset.
  128. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  129. eval_dataset(paddlex.dataset, optional):
  130. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  131. optimizer(paddle.optimizer.Optimizer or None, optional):
  132. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  133. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  134. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  135. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  136. pretrain_weights(str or None, optional):
  137. None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'IMAGENET'.
  138. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  139. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  140. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  141. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  142. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  143. metric({'VOC', 'COCO', None}, optional):
  144. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  145. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  146. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  147. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  148. """
  149. if train_dataset.__class__.__name__ == 'VOCDetection':
  150. train_dataset.data_fields = {
  151. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  152. 'difficult'
  153. }
  154. elif train_dataset.__class__.__name__ == 'CocoDetection':
  155. if self.__class__.__name__ == 'MaskRCNN':
  156. train_dataset.data_fields = {
  157. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  158. 'gt_poly', 'is_crowd'
  159. }
  160. else:
  161. train_dataset.data_fields = {
  162. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  163. 'is_crowd'
  164. }
  165. if metric is None:
  166. if eval_dataset.__class__.__name__ == 'VOCDetection':
  167. self.metric = 'voc'
  168. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  169. self.metric = 'coco'
  170. else:
  171. assert metric.lower() in ['coco', 'voc'], \
  172. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  173. self.metric = metric.lower()
  174. train_dataset.batch_transforms = self._compose_batch_transform(
  175. train_dataset.transforms, mode='train')
  176. self.labels = train_dataset.labels
  177. # build optimizer if not defined
  178. if optimizer is None:
  179. num_steps_each_epoch = len(train_dataset) // train_batch_size
  180. self.optimizer = self.default_optimizer(
  181. parameters=self.net.parameters(),
  182. learning_rate=learning_rate,
  183. warmup_steps=warmup_steps,
  184. warmup_start_lr=warmup_start_lr,
  185. lr_decay_epochs=lr_decay_epochs,
  186. lr_decay_gamma=lr_decay_gamma,
  187. num_steps_each_epoch=num_steps_each_epoch)
  188. else:
  189. self.optimizer = optimizer
  190. # initiate weights
  191. if pretrain_weights is not None and not osp.exists(pretrain_weights):
  192. if pretrain_weights not in det_pretrain_weights_dict['_'.join(
  193. [self.model_name, self.backbone_name])]:
  194. logging.warning(
  195. "Path of pretrain_weights('{}') does not exist!".format(
  196. pretrain_weights))
  197. pretrain_weights = det_pretrain_weights_dict['_'.join(
  198. [self.model_name, self.backbone_name])][0]
  199. logging.warning("Pretrain_weights is forcibly set to '{}'. "
  200. "If don't want to use pretrain weights, "
  201. "set pretrain_weights to be None.".format(
  202. pretrain_weights))
  203. pretrained_dir = osp.join(save_dir, 'pretrain')
  204. self.net_initialize(
  205. pretrain_weights=pretrain_weights, save_dir=pretrained_dir)
  206. # start train loop
  207. self.train_loop(
  208. num_epochs=num_epochs,
  209. train_dataset=train_dataset,
  210. train_batch_size=train_batch_size,
  211. eval_dataset=eval_dataset,
  212. save_interval_epochs=save_interval_epochs,
  213. log_interval_steps=log_interval_steps,
  214. save_dir=save_dir,
  215. early_stop=early_stop,
  216. early_stop_patience=early_stop_patience,
  217. use_vdl=use_vdl)
  218. def evaluate(self,
  219. eval_dataset,
  220. batch_size=1,
  221. metric=None,
  222. return_details=False):
  223. """
  224. Evaluate the model.
  225. Args:
  226. eval_dataset(paddlex.dataset): Evaluation dataset.
  227. batch_size(int, optional): Total batch size among all cards used for evaluation. Defaults to 1.
  228. metric({'VOC', 'COCO', None}, optional):
  229. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  230. return_details(bool, optional): Whether to return evaluation details. Defaults to False.
  231. Returns:
  232. collections.OrderedDict with key-value pairs: {"mAP(0.50, 11point)":`mean average precision`}.
  233. """
  234. if eval_dataset.__class__.__name__ == 'VOCDetection':
  235. eval_dataset.data_fields = {
  236. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  237. 'difficult'
  238. }
  239. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  240. if self.__class__.__name__ == 'MaskRCNN':
  241. eval_dataset.data_fields = {
  242. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  243. 'gt_poly', 'is_crowd'
  244. }
  245. else:
  246. eval_dataset.data_fields = {
  247. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  248. 'is_crowd'
  249. }
  250. eval_dataset.batch_transforms = self._compose_batch_transform(
  251. eval_dataset.transforms, mode='eval')
  252. arrange_transforms(
  253. model_type=self.model_type,
  254. transforms=eval_dataset.transforms,
  255. mode='eval')
  256. self.net.eval()
  257. nranks = paddle.distributed.get_world_size()
  258. local_rank = paddle.distributed.get_rank()
  259. if nranks > 1:
  260. # Initialize parallel environment if not done.
  261. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(
  262. ):
  263. paddle.distributed.init_parallel_env()
  264. if batch_size > 1:
  265. logging.warning(
  266. "Detector only supports single card evaluation with batch_size=1 "
  267. "during evaluation, so batch_size is forcibly set to 1.")
  268. batch_size = 1
  269. if nranks < 2 or local_rank == 0:
  270. self.eval_data_loader = self.build_data_loader(
  271. eval_dataset, batch_size=batch_size, mode='eval')
  272. is_bbox_normalized = False
  273. if eval_dataset.batch_transforms is not None:
  274. is_bbox_normalized = any(
  275. isinstance(t, _NormalizeBox)
  276. for t in eval_dataset.batch_transforms.batch_transforms)
  277. if metric is None:
  278. if getattr(self, 'metric', None) is not None:
  279. if self.metric == 'voc':
  280. eval_metric = VOCMetric(
  281. labels=eval_dataset.labels,
  282. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  283. is_bbox_normalized=is_bbox_normalized,
  284. classwise=False)
  285. else:
  286. eval_metric = COCOMetric(
  287. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  288. classwise=False)
  289. else:
  290. if eval_dataset.__class__.__name__ == 'VOCDetection':
  291. eval_metric = VOCMetric(
  292. labels=eval_dataset.labels,
  293. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  294. is_bbox_normalized=is_bbox_normalized,
  295. classwise=False)
  296. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  297. eval_metric = COCOMetric(
  298. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  299. classwise=False)
  300. else:
  301. assert metric.lower() in ['coco', 'voc'], \
  302. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  303. if metric.lower() == 'coco':
  304. eval_metric = COCOMetric(
  305. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  306. classwise=False)
  307. else:
  308. eval_metric = VOCMetric(
  309. labels=eval_dataset.labels,
  310. is_bbox_normalized=is_bbox_normalized,
  311. classwise=False)
  312. scores = collections.OrderedDict()
  313. logging.info(
  314. "Start to evaluate(total_samples={}, total_steps={})...".
  315. format(eval_dataset.num_samples, eval_dataset.num_samples))
  316. with paddle.no_grad():
  317. for step, data in enumerate(self.eval_data_loader):
  318. outputs = self.run(self.net, data, 'eval')
  319. eval_metric.update(data, outputs)
  320. eval_metric.accumulate()
  321. self.eval_details = eval_metric.details
  322. scores.update(eval_metric.get())
  323. eval_metric.reset()
  324. if return_details:
  325. return scores, self.eval_details
  326. return scores
  327. def predict(self, img_file, transforms=None):
  328. """
  329. Do inference.
  330. Args:
  331. img_file(List[np.ndarray or str], str or np.ndarray): img_file(list or str or np.array):
  332. Image path or decoded image data in a BGR format, which also could constitute a list,
  333. meaning all images to be predicted as a mini-batch.
  334. transforms(paddlex.transforms.Compose or None, optional):
  335. Transforms for inputs. If None, the transforms for evaluation process will be used. Defaults to None.
  336. Returns:
  337. If img_file is a string or np.array, the result is a list of dict with key-value pairs:
  338. {"category_id": `category_id`, "category": `category`, "bbox": `[x, y, w, h]`, "score": `score`}.
  339. If img_file is a list, the result is a list composed of dicts with the corresponding fields:
  340. category_id(int): the predicted category ID
  341. category(str): category name
  342. bbox(list): bounding box in [x, y, w, h] format
  343. score(str): confidence
  344. """
  345. if transforms is None and not hasattr(self, 'test_transforms'):
  346. raise Exception("transforms need to be defined, now is None.")
  347. if transforms is None:
  348. transforms = self.test_transforms
  349. if isinstance(img_file, (str, np.ndarray)):
  350. images = [img_file]
  351. else:
  352. images = img_file
  353. batch_samples = self._preprocess(images, transforms)
  354. self.net.eval()
  355. outputs = self.run(self.net, batch_samples, 'test')
  356. prediction = self._postprocess(outputs)
  357. if isinstance(img_file, (str, np.ndarray)):
  358. prediction = prediction[0]
  359. return prediction
  360. def _preprocess(self, images, transforms):
  361. arrange_transforms(
  362. model_type=self.model_type, transforms=transforms, mode='test')
  363. batch_samples = list()
  364. for im in images:
  365. sample = {'image': im}
  366. batch_samples.append(transforms(sample))
  367. batch_transforms = self._compose_batch_transform(transforms, 'test')
  368. batch_samples = batch_transforms(batch_samples)
  369. for k, v in batch_samples.items():
  370. batch_samples[k] = paddle.to_tensor(v)
  371. return batch_samples
  372. def _postprocess(self, batch_pred):
  373. infer_result = {}
  374. if 'bbox' in batch_pred:
  375. bboxes = batch_pred['bbox']
  376. bbox_nums = batch_pred['bbox_num']
  377. det_res = []
  378. k = 0
  379. for i in range(len(bbox_nums)):
  380. det_nums = bbox_nums[i]
  381. for j in range(det_nums):
  382. dt = bboxes[k]
  383. k = k + 1
  384. num_id, score, xmin, ymin, xmax, ymax = dt.tolist()
  385. if int(num_id) < 0:
  386. continue
  387. category = self.labels[int(num_id)]
  388. w = xmax - xmin
  389. h = ymax - ymin
  390. bbox = [xmin, ymin, w, h]
  391. dt_res = {
  392. 'category_id': int(num_id),
  393. 'category': category,
  394. 'bbox': bbox,
  395. 'score': score
  396. }
  397. det_res.append(dt_res)
  398. infer_result['bbox'] = det_res
  399. if 'mask' in batch_pred:
  400. masks = batch_pred['mask']
  401. bboxes = batch_pred['bbox']
  402. mask_nums = batch_pred['bbox_num']
  403. seg_res = []
  404. k = 0
  405. for i in range(len(mask_nums)):
  406. det_nums = mask_nums[i]
  407. for j in range(det_nums):
  408. mask = masks[k].astype(np.uint8)
  409. score = float(bboxes[k][1])
  410. label = int(bboxes[k][0])
  411. k = k + 1
  412. if label == -1:
  413. continue
  414. category = self.labels[int(label)]
  415. import pycocotools.mask as mask_util
  416. rle = mask_util.encode(
  417. np.array(
  418. mask[:, :, None], order="F", dtype="uint8"))[0]
  419. if six.PY3:
  420. if 'counts' in rle:
  421. rle['counts'] = rle['counts'].decode("utf8")
  422. sg_res = {
  423. 'category': category,
  424. 'segmentation': rle,
  425. 'score': score
  426. }
  427. seg_res.append(sg_res)
  428. infer_result['mask'] = seg_res
  429. bbox_num = batch_pred['bbox_num']
  430. results = []
  431. start = 0
  432. for num in bbox_num:
  433. end = start + num
  434. curr_res = infer_result['bbox'][start:end]
  435. if 'mask' in infer_result:
  436. mask_res = infer_result['mask'][start:end]
  437. for box, mask in zip(curr_res, mask_res):
  438. box.update(mask)
  439. results.append(curr_res)
  440. start = end
  441. return results
  442. class YOLOv3(BaseDetector):
  443. def __init__(self,
  444. num_classes=80,
  445. backbone='MobileNetV1',
  446. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  447. [59, 119], [116, 90], [156, 198], [373, 326]],
  448. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  449. ignore_threshold=0.7,
  450. nms_score_threshold=0.01,
  451. nms_topk=1000,
  452. nms_keep_topk=100,
  453. nms_iou_threshold=0.45,
  454. label_smooth=False):
  455. self.init_params = locals()
  456. if backbone not in [
  457. 'MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3',
  458. 'MobileNetV3_ssld', 'DarkNet53', 'ResNet50_vd_dcn', 'ResNet34'
  459. ]:
  460. raise ValueError(
  461. "backbone: {} is not supported. Please choose one of "
  462. "('MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3', 'MobileNetV3_ssld', 'DarkNet53', 'ResNet50_vd_dcn', 'ResNet34')".
  463. format(backbone))
  464. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  465. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  466. norm_type = 'sync_bn'
  467. else:
  468. norm_type = 'bn'
  469. self.backbone_name = backbone
  470. if 'MobileNetV1' in backbone:
  471. norm_type = 'bn'
  472. backbone = self._get_backbone('MobileNet', norm_type=norm_type)
  473. elif 'MobileNetV3' in backbone:
  474. backbone = self._get_backbone(
  475. 'MobileNetV3', norm_type=norm_type, feature_maps=[7, 13, 16])
  476. elif backbone == 'ResNet50_vd_dcn':
  477. backbone = self._get_backbone(
  478. 'ResNet',
  479. norm_type=norm_type,
  480. variant='d',
  481. return_idx=[1, 2, 3],
  482. dcn_v2_stages=[3],
  483. freeze_at=-1,
  484. freeze_norm=False)
  485. elif backbone == 'ResNet34':
  486. backbone = self._get_backbone(
  487. 'ResNet',
  488. depth=34,
  489. norm_type=norm_type,
  490. return_idx=[1, 2, 3],
  491. freeze_at=-1,
  492. freeze_norm=False,
  493. norm_decay=0.)
  494. else:
  495. backbone = self._get_backbone('DarkNet', norm_type=norm_type)
  496. neck = necks.YOLOv3FPN(
  497. norm_type=norm_type,
  498. in_channels=[i.channels for i in backbone.out_shape])
  499. loss = losses.YOLOv3Loss(
  500. num_classes=num_classes,
  501. ignore_thresh=ignore_threshold,
  502. label_smooth=label_smooth)
  503. yolo_head = heads.YOLOv3Head(
  504. in_channels=[i.channels for i in neck.out_shape],
  505. anchors=anchors,
  506. anchor_masks=anchor_masks,
  507. num_classes=num_classes,
  508. loss=loss)
  509. post_process = BBoxPostProcess(
  510. decode=YOLOBox(num_classes=num_classes),
  511. nms=MultiClassNMS(
  512. score_threshold=nms_score_threshold,
  513. nms_top_k=nms_topk,
  514. keep_top_k=nms_keep_topk,
  515. nms_threshold=nms_iou_threshold))
  516. params = {
  517. 'backbone': backbone,
  518. 'neck': neck,
  519. 'yolo_head': yolo_head,
  520. 'post_process': post_process
  521. }
  522. super(YOLOv3, self).__init__(
  523. model_name='YOLOv3', num_classes=num_classes, **params)
  524. self.anchors = anchors
  525. self.anchor_masks = anchor_masks
  526. def _compose_batch_transform(self, transforms, mode='train'):
  527. if mode == 'train':
  528. default_batch_transforms = [
  529. _BatchPadding(
  530. pad_to_stride=-1, pad_gt=False), _NormalizeBox(),
  531. _PadBox(getattr(self, 'num_max_boxes', 50)), _BboxXYXY2XYWH(),
  532. _Gt2YoloTarget(
  533. anchor_masks=self.anchor_masks,
  534. anchors=self.anchors,
  535. downsample_ratios=getattr(self, 'downsample_ratios',
  536. [32, 16, 8]),
  537. num_classes=self.num_classes)
  538. ]
  539. else:
  540. default_batch_transforms = [
  541. _BatchPadding(
  542. pad_to_stride=-1, pad_gt=False)
  543. ]
  544. custom_batch_transforms = []
  545. for i, op in enumerate(transforms.transforms):
  546. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  547. if mode != 'train':
  548. raise Exception(
  549. "{} cannot be present in the {} transforms. ".format(
  550. op.__class__.__name__, mode) +
  551. "Please check the {} transforms.".format(mode))
  552. custom_batch_transforms.insert(0, copy.deepcopy(op))
  553. batch_transforms = BatchCompose(custom_batch_transforms +
  554. default_batch_transforms)
  555. return batch_transforms
  556. class FasterRCNN(BaseDetector):
  557. def __init__(self,
  558. num_classes=80,
  559. backbone='ResNet50',
  560. with_fpn=True,
  561. aspect_ratios=[0.5, 1.0, 2.0],
  562. anchor_sizes=[[32], [64], [128], [256], [512]],
  563. keep_top_k=100,
  564. nms_threshold=0.5,
  565. score_threshold=0.05,
  566. fpn_num_channels=256,
  567. rpn_batch_size_per_im=256,
  568. rpn_fg_fraction=0.5,
  569. test_pre_nms_top_n=None,
  570. test_post_nms_top_n=1000):
  571. self.init_params = locals()
  572. if backbone not in [
  573. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34',
  574. 'ResNet34_vd', 'ResNet101', 'ResNet101_vd'
  575. ]:
  576. raise ValueError(
  577. "backbone: {} is not supported. Please choose one of "
  578. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34', 'ResNet34_vd', "
  579. "'ResNet101', 'ResNet101_vd')".format(backbone))
  580. self.backbone_name = backbone + '_fpn' if with_fpn else backbone
  581. if backbone == 'ResNet50_vd_ssld':
  582. if not with_fpn:
  583. logging.warning(
  584. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  585. format(backbone))
  586. with_fpn = True
  587. backbone = self._get_backbone(
  588. 'ResNet',
  589. variant='d',
  590. norm_type='bn',
  591. freeze_at=0,
  592. return_idx=[0, 1, 2, 3],
  593. num_stages=4,
  594. lr_mult_list=[0.05, 0.05, 0.1, 0.15])
  595. elif 'ResNet50' in backbone:
  596. if with_fpn:
  597. backbone = self._get_backbone(
  598. 'ResNet',
  599. variant='d' if '_vd' in backbone else 'b',
  600. norm_type='bn',
  601. freeze_at=0,
  602. return_idx=[0, 1, 2, 3],
  603. num_stages=4)
  604. else:
  605. backbone = self._get_backbone(
  606. 'ResNet',
  607. variant='d' if '_vd' in backbone else 'b',
  608. norm_type='bn',
  609. freeze_at=0,
  610. return_idx=[2],
  611. num_stages=3)
  612. elif 'ResNet34' in backbone:
  613. if not with_fpn:
  614. logging.warning(
  615. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  616. format(backbone))
  617. with_fpn = True
  618. backbone = self._get_backbone(
  619. 'ResNet',
  620. depth=34,
  621. variant='d' if 'vd' in backbone else 'b',
  622. norm_type='bn',
  623. freeze_at=0,
  624. return_idx=[0, 1, 2, 3],
  625. num_stages=4)
  626. else:
  627. if not with_fpn:
  628. logging.warning(
  629. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  630. format(backbone))
  631. with_fpn = True
  632. backbone = self._get_backbone(
  633. 'ResNet',
  634. depth=101,
  635. variant='d' if 'vd' in backbone else 'b',
  636. norm_type='bn',
  637. freeze_at=0,
  638. return_idx=[0, 1, 2, 3],
  639. num_stages=4)
  640. rpn_in_channel = backbone.out_shape[0].channels
  641. if with_fpn:
  642. neck = necks.FPN(
  643. in_channels=[i.channels for i in backbone.out_shape],
  644. out_channel=fpn_num_channels,
  645. spatial_scales=[1.0 / i.stride for i in backbone.out_shape])
  646. rpn_in_channel = neck.out_shape[0].channels
  647. anchor_generator_cfg = {
  648. 'aspect_ratios': aspect_ratios,
  649. 'anchor_sizes': anchor_sizes,
  650. 'strides': [4, 8, 16, 32, 64]
  651. }
  652. train_proposal_cfg = {
  653. 'min_size': 0.0,
  654. 'nms_thresh': .7,
  655. 'pre_nms_top_n': 2000,
  656. 'post_nms_top_n': 1000,
  657. 'topk_after_collect': True
  658. }
  659. test_proposal_cfg = {
  660. 'min_size': 0.0,
  661. 'nms_thresh': .7,
  662. 'pre_nms_top_n': 1000
  663. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  664. 'post_nms_top_n': test_post_nms_top_n
  665. }
  666. head = heads.TwoFCHead(out_channel=1024)
  667. roi_extractor_cfg = {
  668. 'resolution': 7,
  669. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  670. 'sampling_ratio': 0,
  671. 'aligned': True
  672. }
  673. with_pool = False
  674. else:
  675. neck = None
  676. anchor_generator_cfg = {
  677. 'aspect_ratios': aspect_ratios,
  678. 'anchor_sizes': anchor_sizes,
  679. 'strides': [16]
  680. }
  681. train_proposal_cfg = {
  682. 'min_size': 0.0,
  683. 'nms_thresh': .7,
  684. 'pre_nms_top_n': 12000,
  685. 'post_nms_top_n': 2000,
  686. 'topk_after_collect': False
  687. }
  688. test_proposal_cfg = {
  689. 'min_size': 0.0,
  690. 'nms_thresh': .7,
  691. 'pre_nms_top_n': 6000
  692. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  693. 'post_nms_top_n': test_post_nms_top_n
  694. }
  695. head = backbones.Res5Head()
  696. roi_extractor_cfg = {
  697. 'resolution': 14,
  698. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  699. 'sampling_ratio': 0,
  700. 'aligned': True
  701. }
  702. with_pool = True
  703. rpn_target_assign_cfg = {
  704. 'batch_size_per_im': rpn_batch_size_per_im,
  705. 'fg_fraction': rpn_fg_fraction,
  706. 'negative_overlap': .3,
  707. 'positive_overlap': .7,
  708. 'use_random': True
  709. }
  710. rpn_head = RPNHead(
  711. anchor_generator=anchor_generator_cfg,
  712. rpn_target_assign=rpn_target_assign_cfg,
  713. train_proposal=train_proposal_cfg,
  714. test_proposal=test_proposal_cfg,
  715. in_channel=rpn_in_channel)
  716. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  717. bbox_head = heads.BBoxHead(
  718. head=head,
  719. in_channel=head.out_shape[0].channels,
  720. roi_extractor=roi_extractor_cfg,
  721. with_pool=with_pool,
  722. bbox_assigner=bbox_assigner,
  723. num_classes=num_classes)
  724. bbox_post_process = BBoxPostProcess(
  725. num_classes=num_classes,
  726. decode=RCNNBox(num_classes=num_classes),
  727. nms=MultiClassNMS(
  728. score_threshold=score_threshold,
  729. keep_top_k=keep_top_k,
  730. nms_threshold=nms_threshold))
  731. params = {
  732. 'backbone': backbone,
  733. 'neck': neck,
  734. 'rpn_head': rpn_head,
  735. 'bbox_head': bbox_head,
  736. 'bbox_post_process': bbox_post_process
  737. }
  738. self.with_fpn = with_fpn
  739. super(FasterRCNN, self).__init__(
  740. model_name='FasterRCNN', num_classes=num_classes, **params)
  741. def _compose_batch_transform(self, transforms, mode='train'):
  742. if mode == 'train':
  743. default_batch_transforms = [
  744. _BatchPadding(
  745. pad_to_stride=32 if self.with_fpn else -1, pad_gt=True)
  746. ]
  747. else:
  748. default_batch_transforms = [
  749. _BatchPadding(
  750. pad_to_stride=32 if self.with_fpn else -1, pad_gt=False)
  751. ]
  752. custom_batch_transforms = []
  753. for i, op in enumerate(transforms.transforms):
  754. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  755. if mode != 'train':
  756. raise Exception(
  757. "{} cannot be present in the {} transforms. ".format(
  758. op.__class__.__name__, mode) +
  759. "Please check the {} transforms.".format(mode))
  760. custom_batch_transforms.insert(0, copy.deepcopy(op))
  761. batch_transforms = BatchCompose(custom_batch_transforms +
  762. default_batch_transforms)
  763. return batch_transforms
  764. class PPYOLO(YOLOv3):
  765. def __init__(self,
  766. num_classes=80,
  767. backbone='ResNet50_vd_dcn',
  768. anchors=None,
  769. anchor_masks=None,
  770. use_coord_conv=True,
  771. use_iou_aware=True,
  772. use_spp=True,
  773. use_drop_block=True,
  774. scale_x_y=1.05,
  775. ignore_threshold=0.7,
  776. label_smooth=False,
  777. use_iou_loss=True,
  778. use_matrix_nms=True,
  779. nms_score_threshold=0.01,
  780. nms_topk=-1,
  781. nms_keep_topk=100,
  782. nms_iou_threshold=0.45):
  783. self.init_params = locals()
  784. if backbone not in [
  785. 'ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large',
  786. 'MobileNetV3_small'
  787. ]:
  788. raise ValueError(
  789. "backbone: {} is not supported. Please choose one of "
  790. "('ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large', 'MobileNetV3_small')".
  791. format(backbone))
  792. self.backbone_name = backbone
  793. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  794. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  795. norm_type = 'sync_bn'
  796. else:
  797. norm_type = 'bn'
  798. if anchors is None and anchor_masks is None:
  799. if 'MobileNetV3' in backbone:
  800. anchors = [[11, 18], [34, 47], [51, 126], [115, 71],
  801. [120, 195], [254, 235]]
  802. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  803. elif backbone == 'ResNet50_vd_dcn':
  804. anchors = [[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  805. [59, 119], [116, 90], [156, 198], [373, 326]]
  806. anchor_masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
  807. else:
  808. anchors = [[10, 14], [23, 27], [37, 58], [81, 82], [135, 169],
  809. [344, 319]]
  810. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  811. elif anchors is None or anchor_masks is None:
  812. raise ValueError("Please define both anchors and anchor_masks.")
  813. if backbone == 'ResNet50_vd_dcn':
  814. backbone = self._get_backbone(
  815. 'ResNet',
  816. variant='d',
  817. norm_type=norm_type,
  818. return_idx=[1, 2, 3],
  819. dcn_v2_stages=[3],
  820. freeze_at=-1,
  821. freeze_norm=False,
  822. norm_decay=0.)
  823. downsample_ratios = [32, 16, 8]
  824. elif backbone == 'ResNet18_vd':
  825. backbone = self._get_backbone(
  826. 'ResNet',
  827. depth=18,
  828. variant='d',
  829. norm_type=norm_type,
  830. return_idx=[2, 3],
  831. freeze_at=-1,
  832. freeze_norm=False,
  833. norm_decay=0.)
  834. downsample_ratios = [32, 16, 8]
  835. elif backbone == 'MobileNetV3_large':
  836. backbone = self._get_backbone(
  837. 'MobileNetV3',
  838. model_name='large',
  839. norm_type=norm_type,
  840. scale=1,
  841. with_extra_blocks=False,
  842. extra_block_filters=[],
  843. feature_maps=[13, 16])
  844. downsample_ratios = [32, 16]
  845. elif backbone == 'MobileNetV3_small':
  846. backbone = self._get_backbone(
  847. 'MobileNetV3',
  848. model_name='small',
  849. norm_type=norm_type,
  850. scale=1,
  851. with_extra_blocks=False,
  852. extra_block_filters=[],
  853. feature_maps=[9, 12])
  854. downsample_ratios = [32, 16]
  855. neck = necks.PPYOLOFPN(
  856. norm_type=norm_type,
  857. in_channels=[i.channels for i in backbone.out_shape],
  858. coord_conv=use_coord_conv,
  859. drop_block=use_drop_block,
  860. spp=use_spp,
  861. conv_block_num=0 if ('MobileNetV3' in self.backbone_name or
  862. self.backbone_name == 'ResNet18_vd') else 2)
  863. loss = losses.YOLOv3Loss(
  864. num_classes=num_classes,
  865. ignore_thresh=ignore_threshold,
  866. downsample=downsample_ratios,
  867. label_smooth=label_smooth,
  868. scale_x_y=scale_x_y,
  869. iou_loss=losses.IouLoss(
  870. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  871. iou_aware_loss=losses.IouAwareLoss(loss_weight=1.0)
  872. if use_iou_aware else None)
  873. yolo_head = heads.YOLOv3Head(
  874. in_channels=[i.channels for i in neck.out_shape],
  875. anchors=anchors,
  876. anchor_masks=anchor_masks,
  877. num_classes=num_classes,
  878. loss=loss,
  879. iou_aware=use_iou_aware)
  880. if use_matrix_nms:
  881. nms = MatrixNMS(
  882. keep_top_k=nms_keep_topk,
  883. score_threshold=nms_score_threshold,
  884. post_threshold=.05
  885. if 'MobileNetV3' in self.backbone_name else .01,
  886. nms_top_k=nms_topk,
  887. background_label=-1)
  888. else:
  889. nms = MultiClassNMS(
  890. score_threshold=nms_score_threshold,
  891. nms_top_k=nms_topk,
  892. keep_top_k=nms_keep_topk,
  893. nms_threshold=nms_iou_threshold)
  894. post_process = BBoxPostProcess(
  895. decode=YOLOBox(
  896. num_classes=num_classes,
  897. conf_thresh=.005
  898. if 'MobileNetV3' in self.backbone_name else .01,
  899. scale_x_y=scale_x_y),
  900. nms=nms)
  901. params = {
  902. 'backbone': backbone,
  903. 'neck': neck,
  904. 'yolo_head': yolo_head,
  905. 'post_process': post_process
  906. }
  907. super(YOLOv3, self).__init__(
  908. model_name='YOLOv3', num_classes=num_classes, **params)
  909. self.anchors = anchors
  910. self.anchor_masks = anchor_masks
  911. self.downsample_ratios = downsample_ratios
  912. self.model_name = 'PPYOLO'
  913. class PPYOLOTiny(YOLOv3):
  914. def __init__(self,
  915. num_classes=80,
  916. backbone='MobileNetV3',
  917. anchors=[[10, 15], [24, 36], [72, 42], [35, 87], [102, 96],
  918. [60, 170], [220, 125], [128, 222], [264, 266]],
  919. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  920. use_iou_aware=False,
  921. use_spp=True,
  922. use_drop_block=True,
  923. scale_x_y=1.05,
  924. ignore_threshold=0.5,
  925. label_smooth=False,
  926. use_iou_loss=True,
  927. use_matrix_nms=False,
  928. nms_score_threshold=0.005,
  929. nms_topk=1000,
  930. nms_keep_topk=100,
  931. nms_iou_threshold=0.45):
  932. self.init_params = locals()
  933. if backbone != 'MobileNetV3':
  934. logging.warning(
  935. "PPYOLOTiny only supports MobileNetV3 as backbone. "
  936. "Backbone is forcibly set to MobileNetV3.")
  937. self.backbone_name = 'MobileNetV3'
  938. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  939. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  940. norm_type = 'sync_bn'
  941. else:
  942. norm_type = 'bn'
  943. backbone = self._get_backbone(
  944. 'MobileNetV3',
  945. model_name='large',
  946. norm_type=norm_type,
  947. scale=.5,
  948. with_extra_blocks=False,
  949. extra_block_filters=[],
  950. feature_maps=[7, 13, 16])
  951. downsample_ratios = [32, 16, 8]
  952. neck = necks.PPYOLOTinyFPN(
  953. detection_block_channels=[160, 128, 96],
  954. in_channels=[i.channels for i in backbone.out_shape],
  955. spp=use_spp,
  956. drop_block=use_drop_block)
  957. loss = losses.YOLOv3Loss(
  958. num_classes=num_classes,
  959. ignore_thresh=ignore_threshold,
  960. downsample=downsample_ratios,
  961. label_smooth=label_smooth,
  962. scale_x_y=scale_x_y,
  963. iou_loss=losses.IouLoss(
  964. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  965. iou_aware_loss=losses.IouAwareLoss(loss_weight=1.0)
  966. if use_iou_aware else None)
  967. yolo_head = heads.YOLOv3Head(
  968. in_channels=[i.channels for i in neck.out_shape],
  969. anchors=anchors,
  970. anchor_masks=anchor_masks,
  971. num_classes=num_classes,
  972. loss=loss,
  973. iou_aware=use_iou_aware)
  974. if use_matrix_nms:
  975. nms = MatrixNMS(
  976. keep_top_k=nms_keep_topk,
  977. score_threshold=nms_score_threshold,
  978. post_threshold=.05,
  979. nms_top_k=nms_topk,
  980. background_label=-1)
  981. else:
  982. nms = MultiClassNMS(
  983. score_threshold=nms_score_threshold,
  984. nms_top_k=nms_topk,
  985. keep_top_k=nms_keep_topk,
  986. nms_threshold=nms_iou_threshold)
  987. post_process = BBoxPostProcess(
  988. decode=YOLOBox(
  989. num_classes=num_classes,
  990. conf_thresh=.005,
  991. downsample_ratio=32,
  992. clip_bbox=True,
  993. scale_x_y=scale_x_y),
  994. nms=nms)
  995. params = {
  996. 'backbone': backbone,
  997. 'neck': neck,
  998. 'yolo_head': yolo_head,
  999. 'post_process': post_process
  1000. }
  1001. super(YOLOv3, self).__init__(
  1002. model_name='YOLOv3', num_classes=num_classes, **params)
  1003. self.anchors = anchors
  1004. self.anchor_masks = anchor_masks
  1005. self.downsample_ratios = downsample_ratios
  1006. self.num_max_boxes = 100
  1007. self.model_name = 'PPYOLOTiny'
  1008. class PPYOLOv2(YOLOv3):
  1009. def __init__(self,
  1010. num_classes=80,
  1011. backbone='ResNet50_vd_dcn',
  1012. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  1013. [59, 119], [116, 90], [156, 198], [373, 326]],
  1014. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  1015. use_iou_aware=True,
  1016. use_spp=True,
  1017. use_drop_block=True,
  1018. scale_x_y=1.05,
  1019. ignore_threshold=0.7,
  1020. label_smooth=False,
  1021. use_iou_loss=True,
  1022. use_matrix_nms=True,
  1023. nms_score_threshold=0.01,
  1024. nms_topk=-1,
  1025. nms_keep_topk=100,
  1026. nms_iou_threshold=0.45):
  1027. self.init_params = locals()
  1028. if backbone not in ['ResNet50_vd_dcn', 'ResNet101_vd_dcn']:
  1029. raise ValueError(
  1030. "backbone: {} is not supported. Please choose one of "
  1031. "('ResNet50_vd_dcn', 'ResNet18_vd')".format(backbone))
  1032. self.backbone_name = backbone
  1033. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1034. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1035. norm_type = 'sync_bn'
  1036. else:
  1037. norm_type = 'bn'
  1038. if backbone == 'ResNet50_vd_dcn':
  1039. backbone = self._get_backbone(
  1040. 'ResNet',
  1041. variant='d',
  1042. norm_type=norm_type,
  1043. return_idx=[1, 2, 3],
  1044. dcn_v2_stages=[3],
  1045. freeze_at=-1,
  1046. freeze_norm=False,
  1047. norm_decay=0.)
  1048. downsample_ratios = [32, 16, 8]
  1049. elif backbone == 'ResNet101_vd_dcn':
  1050. backbone = self._get_backbone(
  1051. 'ResNet',
  1052. depth=101,
  1053. variant='d',
  1054. norm_type=norm_type,
  1055. return_idx=[1, 2, 3],
  1056. dcn_v2_stages=[3],
  1057. freeze_at=-1,
  1058. freeze_norm=False,
  1059. norm_decay=0.)
  1060. downsample_ratios = [32, 16, 8]
  1061. neck = necks.PPYOLOPAN(
  1062. norm_type=norm_type,
  1063. in_channels=[i.channels for i in backbone.out_shape],
  1064. drop_block=use_drop_block,
  1065. block_size=3,
  1066. keep_prob=.9,
  1067. spp=use_spp)
  1068. loss = losses.YOLOv3Loss(
  1069. num_classes=num_classes,
  1070. ignore_thresh=ignore_threshold,
  1071. downsample=downsample_ratios,
  1072. label_smooth=label_smooth,
  1073. scale_x_y=scale_x_y,
  1074. iou_loss=losses.IouLoss(
  1075. loss_weight=2.5, loss_square=True) if use_iou_loss else None,
  1076. iou_aware_loss=losses.IouAwareLoss(loss_weight=1.0)
  1077. if use_iou_aware else None)
  1078. yolo_head = heads.YOLOv3Head(
  1079. in_channels=[i.channels for i in neck.out_shape],
  1080. anchors=anchors,
  1081. anchor_masks=anchor_masks,
  1082. num_classes=num_classes,
  1083. loss=loss,
  1084. iou_aware=use_iou_aware,
  1085. iou_aware_factor=.5)
  1086. if use_matrix_nms:
  1087. nms = MatrixNMS(
  1088. keep_top_k=nms_keep_topk,
  1089. score_threshold=nms_score_threshold,
  1090. post_threshold=.01,
  1091. nms_top_k=nms_topk,
  1092. background_label=-1)
  1093. else:
  1094. nms = MultiClassNMS(
  1095. score_threshold=nms_score_threshold,
  1096. nms_top_k=nms_topk,
  1097. keep_top_k=nms_keep_topk,
  1098. nms_threshold=nms_iou_threshold)
  1099. post_process = BBoxPostProcess(
  1100. decode=YOLOBox(
  1101. num_classes=num_classes,
  1102. conf_thresh=.01,
  1103. downsample_ratio=32,
  1104. clip_bbox=True,
  1105. scale_x_y=scale_x_y),
  1106. nms=nms)
  1107. params = {
  1108. 'backbone': backbone,
  1109. 'neck': neck,
  1110. 'yolo_head': yolo_head,
  1111. 'post_process': post_process
  1112. }
  1113. super(YOLOv3, self).__init__(
  1114. model_name='YOLOv3', num_classes=num_classes, **params)
  1115. self.anchors = anchors
  1116. self.anchor_masks = anchor_masks
  1117. self.downsample_ratios = downsample_ratios
  1118. self.num_max_boxes = 100
  1119. self.model_name = 'PPYOLOv2'
  1120. class MaskRCNN(BaseDetector):
  1121. def __init__(self,
  1122. num_classes=80,
  1123. backbone='ResNet50_vd',
  1124. with_fpn=True,
  1125. aspect_ratios=[0.5, 1.0, 2.0],
  1126. anchor_sizes=[[32], [64], [128], [256], [512]],
  1127. keep_top_k=100,
  1128. nms_threshold=0.5,
  1129. score_threshold=0.05,
  1130. fpn_num_channels=256,
  1131. rpn_batch_size_per_im=256,
  1132. rpn_fg_fraction=0.5,
  1133. test_pre_nms_top_n=None,
  1134. test_post_nms_top_n=1000):
  1135. self.init_params = locals()
  1136. if backbone not in [
  1137. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101',
  1138. 'ResNet101_vd'
  1139. ]:
  1140. raise ValueError(
  1141. "backbone: {} is not supported. Please choose one of "
  1142. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101', 'ResNet101_vd')".
  1143. format(backbone))
  1144. self.backbone_name = backbone + '_fpn' if with_fpn else backbone
  1145. if backbone == 'ResNet50':
  1146. if with_fpn:
  1147. backbone = self._get_backbone(
  1148. 'ResNet',
  1149. norm_type='bn',
  1150. freeze_at=0,
  1151. return_idx=[0, 1, 2, 3],
  1152. num_stages=4)
  1153. else:
  1154. backbone = self._get_backbone(
  1155. 'ResNet',
  1156. norm_type='bn',
  1157. freeze_at=0,
  1158. return_idx=[2],
  1159. num_stages=3)
  1160. elif 'ResNet50_vd' in backbone:
  1161. if not with_fpn:
  1162. logging.warning(
  1163. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1164. format(backbone))
  1165. with_fpn = True
  1166. backbone = self._get_backbone(
  1167. 'ResNet',
  1168. variant='d',
  1169. norm_type='bn',
  1170. freeze_at=0,
  1171. return_idx=[0, 1, 2, 3],
  1172. num_stages=4,
  1173. lr_mult_list=[0.05, 0.05, 0.1, 0.15]
  1174. if '_ssld' in backbone else [1.0, 1.0, 1.0, 1.0])
  1175. else:
  1176. if not with_fpn:
  1177. logging.warning(
  1178. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1179. format(backbone))
  1180. with_fpn = True
  1181. backbone = self._get_backbone(
  1182. 'ResNet',
  1183. variant='d' if '_vd' in backbone else 'b',
  1184. depth=101,
  1185. norm_type='bn',
  1186. freeze_at=0,
  1187. return_idx=[0, 1, 2, 3],
  1188. num_stages=4)
  1189. rpn_in_channel = backbone.out_shape[0].channels
  1190. if with_fpn:
  1191. neck = necks.FPN(
  1192. in_channels=[i.channels for i in backbone.out_shape],
  1193. out_channel=fpn_num_channels,
  1194. spatial_scales=[1.0 / i.stride for i in backbone.out_shape])
  1195. rpn_in_channel = neck.out_shape[0].channels
  1196. anchor_generator_cfg = {
  1197. 'aspect_ratios': aspect_ratios,
  1198. 'anchor_sizes': anchor_sizes,
  1199. 'strides': [4, 8, 16, 32, 64]
  1200. }
  1201. train_proposal_cfg = {
  1202. 'min_size': 0.0,
  1203. 'nms_thresh': .7,
  1204. 'pre_nms_top_n': 2000,
  1205. 'post_nms_top_n': 1000,
  1206. 'topk_after_collect': True
  1207. }
  1208. test_proposal_cfg = {
  1209. 'min_size': 0.0,
  1210. 'nms_thresh': .7,
  1211. 'pre_nms_top_n': 1000
  1212. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1213. 'post_nms_top_n': test_post_nms_top_n
  1214. }
  1215. bb_head = heads.TwoFCHead(
  1216. in_channel=neck.out_shape[0].channels, out_channel=1024)
  1217. bb_roi_extractor_cfg = {
  1218. 'resolution': 7,
  1219. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1220. 'sampling_ratio': 0,
  1221. 'aligned': True
  1222. }
  1223. with_pool = False
  1224. m_head = heads.MaskFeat(
  1225. in_channel=neck.out_shape[0].channels,
  1226. out_channel=256,
  1227. num_convs=4)
  1228. m_roi_extractor_cfg = {
  1229. 'resolution': 14,
  1230. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1231. 'sampling_ratio': 0,
  1232. 'aligned': True
  1233. }
  1234. mask_assigner = MaskAssigner(
  1235. num_classes=num_classes, mask_resolution=28)
  1236. share_bbox_feat = False
  1237. else:
  1238. neck = None
  1239. anchor_generator_cfg = {
  1240. 'aspect_ratios': aspect_ratios,
  1241. 'anchor_sizes': anchor_sizes,
  1242. 'strides': [16]
  1243. }
  1244. train_proposal_cfg = {
  1245. 'min_size': 0.0,
  1246. 'nms_thresh': .7,
  1247. 'pre_nms_top_n': 12000,
  1248. 'post_nms_top_n': 2000,
  1249. 'topk_after_collect': False
  1250. }
  1251. test_proposal_cfg = {
  1252. 'min_size': 0.0,
  1253. 'nms_thresh': .7,
  1254. 'pre_nms_top_n': 6000
  1255. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1256. 'post_nms_top_n': test_post_nms_top_n
  1257. }
  1258. bb_head = backbones.Res5Head()
  1259. bb_roi_extractor_cfg = {
  1260. 'resolution': 14,
  1261. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  1262. 'sampling_ratio': 0,
  1263. 'aligned': True
  1264. }
  1265. with_pool = True
  1266. m_head = heads.MaskFeat(
  1267. in_channel=bb_head.out_shape[0].channels,
  1268. out_channel=256,
  1269. num_convs=0)
  1270. m_roi_extractor_cfg = {
  1271. 'resolution': 14,
  1272. 'spatial_scale': [1. / i.stride for i in backbone.out_shape],
  1273. 'sampling_ratio': 0,
  1274. 'aligned': True
  1275. }
  1276. mask_assigner = MaskAssigner(
  1277. num_classes=num_classes, mask_resolution=14)
  1278. share_bbox_feat = True
  1279. rpn_target_assign_cfg = {
  1280. 'batch_size_per_im': rpn_batch_size_per_im,
  1281. 'fg_fraction': rpn_fg_fraction,
  1282. 'negative_overlap': .3,
  1283. 'positive_overlap': .7,
  1284. 'use_random': True
  1285. }
  1286. rpn_head = RPNHead(
  1287. anchor_generator=anchor_generator_cfg,
  1288. rpn_target_assign=rpn_target_assign_cfg,
  1289. train_proposal=train_proposal_cfg,
  1290. test_proposal=test_proposal_cfg,
  1291. in_channel=rpn_in_channel)
  1292. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  1293. bbox_head = heads.BBoxHead(
  1294. head=bb_head,
  1295. in_channel=bb_head.out_shape[0].channels,
  1296. roi_extractor=bb_roi_extractor_cfg,
  1297. with_pool=with_pool,
  1298. bbox_assigner=bbox_assigner,
  1299. num_classes=num_classes)
  1300. mask_head = heads.MaskHead(
  1301. head=m_head,
  1302. roi_extractor=m_roi_extractor_cfg,
  1303. mask_assigner=mask_assigner,
  1304. share_bbox_feat=share_bbox_feat,
  1305. num_classes=num_classes)
  1306. bbox_post_process = BBoxPostProcess(
  1307. num_classes=num_classes,
  1308. decode=RCNNBox(num_classes=num_classes),
  1309. nms=MultiClassNMS(
  1310. score_threshold=score_threshold,
  1311. keep_top_k=keep_top_k,
  1312. nms_threshold=nms_threshold))
  1313. mask_post_process = MaskPostProcess(binary_thresh=.5)
  1314. params = {
  1315. 'backbone': backbone,
  1316. 'neck': neck,
  1317. 'rpn_head': rpn_head,
  1318. 'bbox_head': bbox_head,
  1319. 'mask_head': mask_head,
  1320. 'bbox_post_process': bbox_post_process,
  1321. 'mask_post_process': mask_post_process
  1322. }
  1323. self.with_fpn = with_fpn
  1324. super(MaskRCNN, self).__init__(
  1325. model_name='MaskRCNN', num_classes=num_classes, **params)
  1326. def _compose_batch_transform(self, transforms, mode='train'):
  1327. if mode == 'train':
  1328. default_batch_transforms = [
  1329. _BatchPadding(
  1330. pad_to_stride=32 if self.with_fpn else -1, pad_gt=True)
  1331. ]
  1332. else:
  1333. default_batch_transforms = [
  1334. _BatchPadding(
  1335. pad_to_stride=32 if self.with_fpn else -1, pad_gt=False)
  1336. ]
  1337. custom_batch_transforms = []
  1338. for i, op in enumerate(transforms.transforms):
  1339. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  1340. if mode != 'train':
  1341. raise Exception(
  1342. "{} cannot be present in the {} transforms. ".format(
  1343. op.__class__.__name__, mode) +
  1344. "Please check the {} transforms.".format(mode))
  1345. custom_batch_transforms.insert(0, copy.deepcopy(op))
  1346. batch_transforms = BatchCompose(custom_batch_transforms +
  1347. default_batch_transforms)
  1348. return batch_transforms