detector.py 92 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. import collections
  16. import copy
  17. import os
  18. import os.path as osp
  19. import numpy as np
  20. import paddle
  21. from paddle.static import InputSpec
  22. import paddlex.ppdet as ppdet
  23. from paddlex.ppdet.modeling.proposal_generator.target_layer import BBoxAssigner, MaskAssigner
  24. import paddlex
  25. import paddlex.utils.logging as logging
  26. from paddlex.cv.transforms.operators import _NormalizeBox, _PadBox, _BboxXYXY2XYWH, Resize, Padding
  27. from paddlex.cv.transforms.batch_operators import BatchCompose, BatchRandomResize, BatchRandomResizeByShort, \
  28. _BatchPadding, _Gt2YoloTarget
  29. from paddlex.cv.transforms import arrange_transforms
  30. from .base import BaseModel
  31. from .utils.det_metrics import VOCMetric, COCOMetric
  32. from .utils.ema import ExponentialMovingAverage
  33. from paddlex.utils.checkpoint import det_pretrain_weights_dict
  34. __all__ = [
  35. "YOLOv3", "FasterRCNN", "PPYOLO", "PPYOLOTiny", "PPYOLOv2", "MaskRCNN",
  36. "PicoDet"
  37. ]
  38. class BaseDetector(BaseModel):
  39. def __init__(self, model_name, num_classes=80, **params):
  40. self.init_params.update(locals())
  41. if 'with_net' in self.init_params:
  42. del self.init_params['with_net']
  43. super(BaseDetector, self).__init__('detector')
  44. if not hasattr(ppdet.modeling, model_name):
  45. raise Exception("ERROR: There's no model named {}.".format(
  46. model_name))
  47. self.model_name = model_name
  48. self.num_classes = num_classes
  49. self.labels = None
  50. if params.get('with_net', True):
  51. params.pop('with_net', None)
  52. self.net = self.build_net(**params)
  53. def build_net(self, **params):
  54. with paddle.utils.unique_name.guard():
  55. net = ppdet.modeling.__dict__[self.model_name](**params)
  56. return net
  57. def _fix_transforms_shape(self, image_shape):
  58. raise NotImplementedError("_fix_transforms_shape: not implemented!")
  59. def _define_input_spec(self, image_shape):
  60. input_spec = [{
  61. "image": InputSpec(
  62. shape=image_shape, name='image', dtype='float32'),
  63. "im_shape": InputSpec(
  64. shape=[image_shape[0], 2], name='im_shape', dtype='float32'),
  65. "scale_factor": InputSpec(
  66. shape=[image_shape[0], 2],
  67. name='scale_factor',
  68. dtype='float32')
  69. }]
  70. return input_spec
  71. def _check_image_shape(self, image_shape):
  72. if len(image_shape) == 2:
  73. image_shape = [1, 3] + image_shape
  74. if image_shape[-2] % 32 > 0 or image_shape[-1] % 32 > 0:
  75. raise Exception(
  76. "Height and width in fixed_input_shape must be a multiple of 32, but received {}.".
  77. format(image_shape[-2:]))
  78. return image_shape
  79. def _get_test_inputs(self, image_shape):
  80. if image_shape is not None:
  81. image_shape = self._check_image_shape(image_shape)
  82. self._fix_transforms_shape(image_shape[-2:])
  83. else:
  84. image_shape = [None, 3, -1, -1]
  85. self.fixed_input_shape = image_shape
  86. return self._define_input_spec(image_shape)
  87. def _get_backbone(self, backbone_name, **params):
  88. backbone = getattr(ppdet.modeling, backbone_name)(**params)
  89. return backbone
  90. def run(self, net, inputs, mode):
  91. net_out = net(inputs)
  92. if mode in ['train', 'eval']:
  93. outputs = net_out
  94. else:
  95. outputs = dict()
  96. for key in net_out:
  97. outputs[key] = net_out[key].numpy()
  98. return outputs
  99. def default_optimizer(self, parameters, learning_rate, warmup_steps,
  100. warmup_start_lr, lr_decay_epochs, lr_decay_gamma,
  101. num_steps_each_epoch):
  102. boundaries = [b * num_steps_each_epoch for b in lr_decay_epochs]
  103. values = [(lr_decay_gamma**i) * learning_rate
  104. for i in range(len(lr_decay_epochs) + 1)]
  105. scheduler = paddle.optimizer.lr.PiecewiseDecay(
  106. boundaries=boundaries, values=values)
  107. if warmup_steps > 0:
  108. if warmup_steps > lr_decay_epochs[0] * num_steps_each_epoch:
  109. logging.error(
  110. "In function train(), parameters should satisfy: "
  111. "warmup_steps <= lr_decay_epochs[0]*num_samples_in_train_dataset",
  112. exit=False)
  113. logging.error(
  114. "See this doc for more information: "
  115. "https://github.com/PaddlePaddle/PaddleX/blob/develop/docs/parameters.md",
  116. exit=False)
  117. scheduler = paddle.optimizer.lr.LinearWarmup(
  118. learning_rate=scheduler,
  119. warmup_steps=warmup_steps,
  120. start_lr=warmup_start_lr,
  121. end_lr=learning_rate)
  122. optimizer = paddle.optimizer.Momentum(
  123. scheduler,
  124. momentum=.9,
  125. weight_decay=paddle.regularizer.L2Decay(coeff=1e-04),
  126. parameters=parameters)
  127. return optimizer
  128. def train(self,
  129. num_epochs,
  130. train_dataset,
  131. train_batch_size=64,
  132. eval_dataset=None,
  133. optimizer=None,
  134. save_interval_epochs=1,
  135. log_interval_steps=10,
  136. save_dir='output',
  137. pretrain_weights='IMAGENET',
  138. learning_rate=.001,
  139. warmup_steps=0,
  140. warmup_start_lr=0.0,
  141. lr_decay_epochs=(216, 243),
  142. lr_decay_gamma=0.1,
  143. metric=None,
  144. use_ema=False,
  145. early_stop=False,
  146. early_stop_patience=5,
  147. use_vdl=True,
  148. resume_checkpoint=None):
  149. """
  150. Train the model.
  151. Args:
  152. num_epochs(int): The number of epochs.
  153. train_dataset(paddlex.dataset): Training dataset.
  154. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  155. eval_dataset(paddlex.dataset, optional):
  156. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  157. optimizer(paddle.optimizer.Optimizer or None, optional):
  158. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  159. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  160. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  161. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  162. pretrain_weights(str or None, optional):
  163. None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'IMAGENET'.
  164. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  165. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  166. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  167. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  168. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  169. metric({'VOC', 'COCO', None}, optional):
  170. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  171. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  172. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  173. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  174. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  175. resume_checkpoint(str or None, optional): The path of the checkpoint to resume training from.
  176. If None, no training checkpoint will be resumed. At most one of `resume_checkpoint` and
  177. `pretrain_weights` can be set simultaneously. Defaults to None.
  178. """
  179. if self.status == 'Infer':
  180. logging.error(
  181. "Exported inference model does not support training.",
  182. exit=True)
  183. if pretrain_weights is not None and resume_checkpoint is not None:
  184. logging.error(
  185. "pretrain_weights and resume_checkpoint cannot be set simultaneously.",
  186. exit=True)
  187. if train_dataset.__class__.__name__ == 'VOCDetection':
  188. train_dataset.data_fields = {
  189. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  190. 'difficult'
  191. }
  192. elif train_dataset.__class__.__name__ == 'CocoDetection':
  193. if self.__class__.__name__ == 'MaskRCNN':
  194. train_dataset.data_fields = {
  195. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  196. 'gt_poly', 'is_crowd'
  197. }
  198. else:
  199. train_dataset.data_fields = {
  200. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  201. 'is_crowd'
  202. }
  203. if metric is None:
  204. if eval_dataset.__class__.__name__ == 'VOCDetection':
  205. self.metric = 'voc'
  206. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  207. self.metric = 'coco'
  208. else:
  209. assert metric.lower() in ['coco', 'voc'], \
  210. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  211. self.metric = metric.lower()
  212. self.labels = train_dataset.labels
  213. self.num_max_boxes = train_dataset.num_max_boxes
  214. train_dataset.batch_transforms = self._compose_batch_transform(
  215. train_dataset.transforms, mode='train')
  216. # build optimizer if not defined
  217. if optimizer is None:
  218. num_steps_each_epoch = len(train_dataset) // train_batch_size
  219. self.optimizer = self.default_optimizer(
  220. parameters=self.net.parameters(),
  221. learning_rate=learning_rate,
  222. warmup_steps=warmup_steps,
  223. warmup_start_lr=warmup_start_lr,
  224. lr_decay_epochs=lr_decay_epochs,
  225. lr_decay_gamma=lr_decay_gamma,
  226. num_steps_each_epoch=num_steps_each_epoch)
  227. else:
  228. self.optimizer = optimizer
  229. # initiate weights
  230. if pretrain_weights is not None and not osp.exists(pretrain_weights):
  231. if pretrain_weights not in det_pretrain_weights_dict['_'.join(
  232. [self.model_name, self.backbone_name])]:
  233. logging.warning(
  234. "Path of pretrain_weights('{}') does not exist!".format(
  235. pretrain_weights))
  236. pretrain_weights = det_pretrain_weights_dict['_'.join(
  237. [self.model_name, self.backbone_name])][0]
  238. logging.warning("Pretrain_weights is forcibly set to '{}'. "
  239. "If you don't want to use pretrain weights, "
  240. "set pretrain_weights to be None.".format(
  241. pretrain_weights))
  242. elif pretrain_weights is not None and osp.exists(pretrain_weights):
  243. if osp.splitext(pretrain_weights)[-1] != '.pdparams':
  244. logging.error(
  245. "Invalid pretrain weights. Please specify a '.pdparams' file.",
  246. exit=True)
  247. pretrained_dir = osp.join(save_dir, 'pretrain')
  248. self.net_initialize(
  249. pretrain_weights=pretrain_weights,
  250. save_dir=pretrained_dir,
  251. resume_checkpoint=resume_checkpoint,
  252. is_backbone_weights=(pretrain_weights == 'IMAGENET' and
  253. 'ESNet_' in self.backbone_name))
  254. if use_ema:
  255. ema = ExponentialMovingAverage(
  256. decay=.9998, model=self.net, use_thres_step=True)
  257. else:
  258. ema = None
  259. # start train loop
  260. self.train_loop(
  261. num_epochs=num_epochs,
  262. train_dataset=train_dataset,
  263. train_batch_size=train_batch_size,
  264. eval_dataset=eval_dataset,
  265. save_interval_epochs=save_interval_epochs,
  266. log_interval_steps=log_interval_steps,
  267. save_dir=save_dir,
  268. ema=ema,
  269. early_stop=early_stop,
  270. early_stop_patience=early_stop_patience,
  271. use_vdl=use_vdl)
  272. def quant_aware_train(self,
  273. num_epochs,
  274. train_dataset,
  275. train_batch_size=64,
  276. eval_dataset=None,
  277. optimizer=None,
  278. save_interval_epochs=1,
  279. log_interval_steps=10,
  280. save_dir='output',
  281. learning_rate=.00001,
  282. warmup_steps=0,
  283. warmup_start_lr=0.0,
  284. lr_decay_epochs=(216, 243),
  285. lr_decay_gamma=0.1,
  286. metric=None,
  287. use_ema=False,
  288. early_stop=False,
  289. early_stop_patience=5,
  290. use_vdl=True,
  291. resume_checkpoint=None,
  292. quant_config=None):
  293. """
  294. Quantization-aware training.
  295. Args:
  296. num_epochs(int): The number of epochs.
  297. train_dataset(paddlex.dataset): Training dataset.
  298. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  299. eval_dataset(paddlex.dataset, optional):
  300. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  301. optimizer(paddle.optimizer.Optimizer or None, optional):
  302. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  303. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  304. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  305. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  306. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  307. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  308. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  309. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  310. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  311. metric({'VOC', 'COCO', None}, optional):
  312. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  313. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  314. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  315. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  316. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  317. quant_config(dict or None, optional): Quantization configuration. If None, a default rule of thumb
  318. configuration will be used. Defaults to None.
  319. resume_checkpoint(str or None, optional): The path of the checkpoint to resume quantization-aware training
  320. from. If None, no training checkpoint will be resumed. Defaults to None.
  321. """
  322. self._prepare_qat(quant_config)
  323. self.train(
  324. num_epochs=num_epochs,
  325. train_dataset=train_dataset,
  326. train_batch_size=train_batch_size,
  327. eval_dataset=eval_dataset,
  328. optimizer=optimizer,
  329. save_interval_epochs=save_interval_epochs,
  330. log_interval_steps=log_interval_steps,
  331. save_dir=save_dir,
  332. pretrain_weights=None,
  333. learning_rate=learning_rate,
  334. warmup_steps=warmup_steps,
  335. warmup_start_lr=warmup_start_lr,
  336. lr_decay_epochs=lr_decay_epochs,
  337. lr_decay_gamma=lr_decay_gamma,
  338. metric=metric,
  339. use_ema=use_ema,
  340. early_stop=early_stop,
  341. early_stop_patience=early_stop_patience,
  342. use_vdl=use_vdl,
  343. resume_checkpoint=resume_checkpoint)
  344. def evaluate(self,
  345. eval_dataset,
  346. batch_size=1,
  347. metric=None,
  348. return_details=False):
  349. """
  350. Evaluate the model.
  351. Args:
  352. eval_dataset(paddlex.dataset): Evaluation dataset.
  353. batch_size(int, optional): Total batch size among all cards used for evaluation. Defaults to 1.
  354. metric({'VOC', 'COCO', None}, optional):
  355. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  356. return_details(bool, optional): Whether to return evaluation details. Defaults to False.
  357. Returns:
  358. collections.OrderedDict with key-value pairs: {"mAP(0.50, 11point)":`mean average precision`}.
  359. """
  360. if metric is None:
  361. if not hasattr(self, 'metric'):
  362. if eval_dataset.__class__.__name__ == 'VOCDetection':
  363. self.metric = 'voc'
  364. elif eval_dataset.__class__.__name__ == 'CocoDetection':
  365. self.metric = 'coco'
  366. else:
  367. assert metric.lower() in ['coco', 'voc'], \
  368. "Evaluation metric {} is not supported, please choose form 'COCO' and 'VOC'"
  369. self.metric = metric.lower()
  370. if self.metric == 'voc':
  371. eval_dataset.data_fields = {
  372. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  373. 'difficult'
  374. }
  375. elif self.metric == 'coco':
  376. if self.__class__.__name__ == 'MaskRCNN':
  377. eval_dataset.data_fields = {
  378. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  379. 'gt_poly', 'is_crowd'
  380. }
  381. else:
  382. eval_dataset.data_fields = {
  383. 'im_id', 'image_shape', 'image', 'gt_bbox', 'gt_class',
  384. 'is_crowd'
  385. }
  386. eval_dataset.batch_transforms = self._compose_batch_transform(
  387. eval_dataset.transforms, mode='eval')
  388. arrange_transforms(
  389. model_type=self.model_type,
  390. transforms=eval_dataset.transforms,
  391. mode='eval')
  392. self.net.eval()
  393. nranks = paddle.distributed.get_world_size()
  394. local_rank = paddle.distributed.get_rank()
  395. if nranks > 1:
  396. # Initialize parallel environment if not done.
  397. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(
  398. ):
  399. paddle.distributed.init_parallel_env()
  400. if batch_size > 1:
  401. logging.warning(
  402. "Detector only supports single card evaluation with batch_size=1 "
  403. "during evaluation, so batch_size is forcibly set to 1.")
  404. batch_size = 1
  405. if nranks < 2 or local_rank == 0:
  406. self.eval_data_loader = self.build_data_loader(
  407. eval_dataset, batch_size=batch_size, mode='eval')
  408. is_bbox_normalized = False
  409. if eval_dataset.batch_transforms is not None:
  410. is_bbox_normalized = any(
  411. isinstance(t, _NormalizeBox)
  412. for t in eval_dataset.batch_transforms.batch_transforms)
  413. if self.metric == 'voc':
  414. eval_metric = VOCMetric(
  415. labels=eval_dataset.labels,
  416. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  417. is_bbox_normalized=is_bbox_normalized,
  418. classwise=False)
  419. else:
  420. eval_metric = COCOMetric(
  421. coco_gt=copy.deepcopy(eval_dataset.coco_gt),
  422. classwise=False)
  423. scores = collections.OrderedDict()
  424. logging.info(
  425. "Start to evaluate(total_samples={}, total_steps={})...".
  426. format(eval_dataset.num_samples, eval_dataset.num_samples))
  427. with paddle.no_grad():
  428. for step, data in enumerate(self.eval_data_loader):
  429. outputs = self.run(self.net, data, 'eval')
  430. eval_metric.update(data, outputs)
  431. eval_metric.accumulate()
  432. self.eval_details = eval_metric.details
  433. scores.update(eval_metric.get())
  434. eval_metric.reset()
  435. if return_details:
  436. return scores, self.eval_details
  437. return scores
  438. def predict(self, img_file, transforms=None):
  439. """
  440. Do inference.
  441. Args:
  442. img_file(List[np.ndarray or str], str or np.ndarray):
  443. Image path or decoded image data in a BGR format, which also could constitute a list,
  444. meaning all images to be predicted as a mini-batch.
  445. transforms(paddlex.transforms.Compose or None, optional):
  446. Transforms for inputs. If None, the transforms for evaluation process will be used. Defaults to None.
  447. Returns:
  448. If img_file is a string or np.array, the result is a list of dict with key-value pairs:
  449. {"category_id": `category_id`, "category": `category`, "bbox": `[x, y, w, h]`, "score": `score`}.
  450. If img_file is a list, the result is a list composed of dicts with the corresponding fields:
  451. category_id(int): the predicted category ID. 0 represents the first category in the dataset, and so on.
  452. category(str): category name
  453. bbox(list): bounding box in [x, y, w, h] format
  454. score(str): confidence
  455. mask(dict): Only for instance segmentation task. Mask of the object in RLE format
  456. """
  457. if transforms is None and not hasattr(self, 'test_transforms'):
  458. raise Exception("transforms need to be defined, now is None.")
  459. if transforms is None:
  460. transforms = self.test_transforms
  461. if isinstance(img_file, (str, np.ndarray)):
  462. images = [img_file]
  463. else:
  464. images = img_file
  465. batch_samples = self._preprocess(images, transforms)
  466. self.net.eval()
  467. outputs = self.run(self.net, batch_samples, 'test')
  468. prediction = self._postprocess(outputs)
  469. if isinstance(img_file, (str, np.ndarray)):
  470. prediction = prediction[0]
  471. return prediction
  472. def _preprocess(self, images, transforms, to_tensor=True):
  473. arrange_transforms(
  474. model_type=self.model_type, transforms=transforms, mode='test')
  475. batch_samples = list()
  476. for im in images:
  477. sample = {'image': im}
  478. batch_samples.append(transforms(sample))
  479. batch_transforms = self._compose_batch_transform(transforms, 'test')
  480. batch_samples = batch_transforms(batch_samples)
  481. if to_tensor:
  482. for k in batch_samples:
  483. batch_samples[k] = paddle.to_tensor(batch_samples[k])
  484. return batch_samples
  485. def _postprocess(self, batch_pred):
  486. infer_result = {}
  487. if 'bbox' in batch_pred:
  488. bboxes = batch_pred['bbox']
  489. bbox_nums = batch_pred['bbox_num']
  490. det_res = []
  491. k = 0
  492. for i in range(len(bbox_nums)):
  493. det_nums = bbox_nums[i]
  494. for j in range(det_nums):
  495. dt = bboxes[k]
  496. k = k + 1
  497. num_id, score, xmin, ymin, xmax, ymax = dt.tolist()
  498. if int(num_id) < 0:
  499. continue
  500. category = self.labels[int(num_id)]
  501. w = xmax - xmin
  502. h = ymax - ymin
  503. bbox = [xmin, ymin, w, h]
  504. dt_res = {
  505. 'category_id': int(num_id),
  506. 'category': category,
  507. 'bbox': bbox,
  508. 'score': score
  509. }
  510. det_res.append(dt_res)
  511. infer_result['bbox'] = det_res
  512. if 'mask' in batch_pred:
  513. masks = batch_pred['mask']
  514. bboxes = batch_pred['bbox']
  515. mask_nums = batch_pred['bbox_num']
  516. seg_res = []
  517. k = 0
  518. for i in range(len(mask_nums)):
  519. det_nums = mask_nums[i]
  520. for j in range(det_nums):
  521. mask = masks[k].astype(np.uint8)
  522. score = float(bboxes[k][1])
  523. label = int(bboxes[k][0])
  524. k = k + 1
  525. if label == -1:
  526. continue
  527. category = self.labels[int(label)]
  528. sg_res = {
  529. 'category_id': int(label),
  530. 'category': category,
  531. 'mask': mask.astype('uint8'),
  532. 'score': score
  533. }
  534. seg_res.append(sg_res)
  535. infer_result['mask'] = seg_res
  536. bbox_num = batch_pred['bbox_num']
  537. results = []
  538. start = 0
  539. for num in bbox_num:
  540. end = start + num
  541. curr_res = infer_result['bbox'][start:end]
  542. if 'mask' in infer_result:
  543. mask_res = infer_result['mask'][start:end]
  544. for box, mask in zip(curr_res, mask_res):
  545. box.update(mask)
  546. results.append(curr_res)
  547. start = end
  548. return results
  549. class PicoDet(BaseDetector):
  550. def __init__(self,
  551. num_classes=80,
  552. backbone='ESNet_m',
  553. nms_score_threshold=.025,
  554. nms_top_k=1000,
  555. nms_keep_top_k=100,
  556. nms_iou_threshold=.6,
  557. **params):
  558. self.init_params = locals()
  559. if backbone not in {
  560. 'ESNet_s', 'ESNet_m', 'ESNet_l', 'LCNet', 'MobileNetV3',
  561. 'ResNet18_vd'
  562. }:
  563. raise ValueError(
  564. "backbone: {} is not supported. Please choose one of "
  565. "('ESNet_s', 'ESNet_m', 'ESNet_l', 'LCNet', 'MobileNetV3', 'ResNet18_vd')".
  566. format(backbone))
  567. self.backbone_name = backbone
  568. if params.get('with_net', True):
  569. if backbone == 'ESNet_s':
  570. backbone = self._get_backbone(
  571. 'ESNet',
  572. scale=.75,
  573. feature_maps=[4, 11, 14],
  574. act="hard_swish",
  575. channel_ratio=[
  576. 0.875, 0.5, 0.5, 0.5, 0.625, 0.5, 0.625, 0.5, 0.5, 0.5,
  577. 0.5, 0.5, 0.5
  578. ])
  579. neck_out_channels = 96
  580. head_num_convs = 2
  581. elif backbone == 'ESNet_m':
  582. backbone = self._get_backbone(
  583. 'ESNet',
  584. scale=1.0,
  585. feature_maps=[4, 11, 14],
  586. act="hard_swish",
  587. channel_ratio=[
  588. 0.875, 0.5, 1.0, 0.625, 0.5, 0.75, 0.625, 0.625, 0.5,
  589. 0.625, 1.0, 0.625, 0.75
  590. ])
  591. neck_out_channels = 128
  592. head_num_convs = 4
  593. elif backbone == 'ESNet_l':
  594. backbone = self._get_backbone(
  595. 'ESNet',
  596. scale=1.25,
  597. feature_maps=[4, 11, 14],
  598. act="hard_swish",
  599. channel_ratio=[
  600. 0.875, 0.5, 1.0, 0.625, 0.5, 0.75, 0.625, 0.625, 0.5,
  601. 0.625, 1.0, 0.625, 0.75
  602. ])
  603. neck_out_channels = 160
  604. head_num_convs = 4
  605. elif backbone == 'LCNet':
  606. backbone = self._get_backbone(
  607. 'LCNet', scale=1.5, feature_maps=[3, 4, 5])
  608. neck_out_channels = 128
  609. head_num_convs = 4
  610. elif backbone == 'MobileNetV3':
  611. backbone = self._get_backbone(
  612. 'MobileNetV3',
  613. scale=1.0,
  614. with_extra_blocks=False,
  615. extra_block_filters=[],
  616. feature_maps=[7, 13, 16])
  617. neck_out_channels = 128
  618. head_num_convs = 4
  619. else:
  620. backbone = self._get_backbone(
  621. 'ResNet',
  622. depth=18,
  623. variant='d',
  624. return_idx=[1, 2, 3],
  625. freeze_at=-1,
  626. freeze_norm=False,
  627. norm_decay=0.)
  628. neck_out_channels = 128
  629. head_num_convs = 4
  630. neck = ppdet.modeling.CSPPAN(
  631. in_channels=[i.channels for i in backbone.out_shape],
  632. out_channels=neck_out_channels,
  633. num_features=4,
  634. num_csp_blocks=1,
  635. use_depthwise=True)
  636. head_conv_feat = ppdet.modeling.PicoFeat(
  637. feat_in=neck_out_channels,
  638. feat_out=neck_out_channels,
  639. num_fpn_stride=4,
  640. num_convs=head_num_convs,
  641. norm_type='bn',
  642. share_cls_reg=True, )
  643. loss_class = ppdet.modeling.VarifocalLoss(
  644. use_sigmoid=True, iou_weighted=True, loss_weight=1.0)
  645. loss_dfl = ppdet.modeling.DistributionFocalLoss(loss_weight=.25)
  646. loss_bbox = ppdet.modeling.GIoULoss(loss_weight=2.0)
  647. assigner = ppdet.modeling.SimOTAAssigner(
  648. candidate_topk=10, iou_weight=6, num_classes=num_classes)
  649. nms = ppdet.modeling.MultiClassNMS(
  650. nms_top_k=nms_top_k,
  651. keep_top_k=nms_keep_top_k,
  652. score_threshold=nms_score_threshold,
  653. nms_threshold=nms_iou_threshold)
  654. head = ppdet.modeling.PicoHead(
  655. conv_feat=head_conv_feat,
  656. num_classes=num_classes,
  657. fpn_stride=[8, 16, 32, 64],
  658. prior_prob=0.01,
  659. reg_max=7,
  660. cell_offset=.5,
  661. loss_class=loss_class,
  662. loss_dfl=loss_dfl,
  663. loss_bbox=loss_bbox,
  664. assigner=assigner,
  665. feat_in_chan=neck_out_channels,
  666. nms=nms)
  667. params.update({
  668. 'backbone': backbone,
  669. 'neck': neck,
  670. 'head': head,
  671. })
  672. super(PicoDet, self).__init__(
  673. model_name='PicoDet', num_classes=num_classes, **params)
  674. def _compose_batch_transform(self, transforms, mode='train'):
  675. default_batch_transforms = [_BatchPadding(pad_to_stride=32)]
  676. if mode == 'eval':
  677. collate_batch = True
  678. else:
  679. collate_batch = False
  680. custom_batch_transforms = []
  681. for i, op in enumerate(transforms.transforms):
  682. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  683. if mode != 'train':
  684. raise Exception(
  685. "{} cannot be present in the {} transforms. ".format(
  686. op.__class__.__name__, mode) +
  687. "Please check the {} transforms.".format(mode))
  688. custom_batch_transforms.insert(0, copy.deepcopy(op))
  689. batch_transforms = BatchCompose(
  690. custom_batch_transforms + default_batch_transforms,
  691. collate_batch=collate_batch)
  692. return batch_transforms
  693. class YOLOv3(BaseDetector):
  694. def __init__(self,
  695. num_classes=80,
  696. backbone='MobileNetV1',
  697. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  698. [59, 119], [116, 90], [156, 198], [373, 326]],
  699. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  700. ignore_threshold=0.7,
  701. nms_score_threshold=0.01,
  702. nms_topk=1000,
  703. nms_keep_topk=100,
  704. nms_iou_threshold=0.45,
  705. label_smooth=False,
  706. **params):
  707. self.init_params = locals()
  708. if backbone not in {
  709. 'MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3',
  710. 'MobileNetV3_ssld', 'DarkNet53', 'ResNet50_vd_dcn', 'ResNet34'
  711. }:
  712. raise ValueError(
  713. "backbone: {} is not supported. Please choose one of "
  714. "('MobileNetV1', 'MobileNetV1_ssld', 'MobileNetV3', 'MobileNetV3_ssld', 'DarkNet53', "
  715. "'ResNet50_vd_dcn', 'ResNet34')".format(backbone))
  716. self.backbone_name = backbone
  717. if params.get('with_net', True):
  718. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  719. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  720. norm_type = 'sync_bn'
  721. else:
  722. norm_type = 'bn'
  723. if 'MobileNetV1' in backbone:
  724. norm_type = 'bn'
  725. backbone = self._get_backbone('MobileNet', norm_type=norm_type)
  726. elif 'MobileNetV3' in backbone:
  727. backbone = self._get_backbone(
  728. 'MobileNetV3',
  729. norm_type=norm_type,
  730. feature_maps=[7, 13, 16])
  731. elif backbone == 'ResNet50_vd_dcn':
  732. backbone = self._get_backbone(
  733. 'ResNet',
  734. norm_type=norm_type,
  735. variant='d',
  736. return_idx=[1, 2, 3],
  737. dcn_v2_stages=[3],
  738. freeze_at=-1,
  739. freeze_norm=False)
  740. elif backbone == 'ResNet34':
  741. backbone = self._get_backbone(
  742. 'ResNet',
  743. depth=34,
  744. norm_type=norm_type,
  745. return_idx=[1, 2, 3],
  746. freeze_at=-1,
  747. freeze_norm=False,
  748. norm_decay=0.)
  749. else:
  750. backbone = self._get_backbone('DarkNet', norm_type=norm_type)
  751. neck = ppdet.modeling.YOLOv3FPN(
  752. norm_type=norm_type,
  753. in_channels=[i.channels for i in backbone.out_shape])
  754. loss = ppdet.modeling.YOLOv3Loss(
  755. num_classes=num_classes,
  756. ignore_thresh=ignore_threshold,
  757. label_smooth=label_smooth)
  758. yolo_head = ppdet.modeling.YOLOv3Head(
  759. in_channels=[i.channels for i in neck.out_shape],
  760. anchors=anchors,
  761. anchor_masks=anchor_masks,
  762. num_classes=num_classes,
  763. loss=loss)
  764. post_process = ppdet.modeling.BBoxPostProcess(
  765. decode=ppdet.modeling.YOLOBox(num_classes=num_classes),
  766. nms=ppdet.modeling.MultiClassNMS(
  767. score_threshold=nms_score_threshold,
  768. nms_top_k=nms_topk,
  769. keep_top_k=nms_keep_topk,
  770. nms_threshold=nms_iou_threshold))
  771. params.update({
  772. 'backbone': backbone,
  773. 'neck': neck,
  774. 'yolo_head': yolo_head,
  775. 'post_process': post_process
  776. })
  777. super(YOLOv3, self).__init__(
  778. model_name='YOLOv3', num_classes=num_classes, **params)
  779. self.anchors = anchors
  780. self.anchor_masks = anchor_masks
  781. def _compose_batch_transform(self, transforms, mode='train'):
  782. if mode == 'train':
  783. default_batch_transforms = [
  784. _BatchPadding(pad_to_stride=-1), _NormalizeBox(),
  785. _PadBox(getattr(self, 'num_max_boxes', 50)), _BboxXYXY2XYWH(),
  786. _Gt2YoloTarget(
  787. anchor_masks=self.anchor_masks,
  788. anchors=self.anchors,
  789. downsample_ratios=getattr(self, 'downsample_ratios',
  790. [32, 16, 8]),
  791. num_classes=self.num_classes)
  792. ]
  793. else:
  794. default_batch_transforms = [_BatchPadding(pad_to_stride=-1)]
  795. if mode == 'eval' and self.metric == 'voc':
  796. collate_batch = False
  797. else:
  798. collate_batch = True
  799. custom_batch_transforms = []
  800. for i, op in enumerate(transforms.transforms):
  801. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  802. if mode != 'train':
  803. raise Exception(
  804. "{} cannot be present in the {} transforms. ".format(
  805. op.__class__.__name__, mode) +
  806. "Please check the {} transforms.".format(mode))
  807. custom_batch_transforms.insert(0, copy.deepcopy(op))
  808. batch_transforms = BatchCompose(
  809. custom_batch_transforms + default_batch_transforms,
  810. collate_batch=collate_batch)
  811. return batch_transforms
  812. def _fix_transforms_shape(self, image_shape):
  813. if hasattr(self, 'test_transforms'):
  814. if self.test_transforms is not None:
  815. has_resize_op = False
  816. resize_op_idx = -1
  817. normalize_op_idx = len(self.test_transforms.transforms)
  818. for idx, op in enumerate(self.test_transforms.transforms):
  819. name = op.__class__.__name__
  820. if name == 'Resize':
  821. has_resize_op = True
  822. resize_op_idx = idx
  823. if name == 'Normalize':
  824. normalize_op_idx = idx
  825. if not has_resize_op:
  826. self.test_transforms.transforms.insert(
  827. normalize_op_idx,
  828. Resize(
  829. target_size=image_shape, interp='CUBIC'))
  830. else:
  831. self.test_transforms.transforms[
  832. resize_op_idx].target_size = image_shape
  833. class FasterRCNN(BaseDetector):
  834. def __init__(self,
  835. num_classes=80,
  836. backbone='ResNet50',
  837. with_fpn=True,
  838. with_dcn=False,
  839. aspect_ratios=[0.5, 1.0, 2.0],
  840. anchor_sizes=[[32], [64], [128], [256], [512]],
  841. keep_top_k=100,
  842. nms_threshold=0.5,
  843. score_threshold=0.05,
  844. fpn_num_channels=256,
  845. rpn_batch_size_per_im=256,
  846. rpn_fg_fraction=0.5,
  847. test_pre_nms_top_n=None,
  848. test_post_nms_top_n=1000,
  849. **params):
  850. self.init_params = locals()
  851. if backbone not in {
  852. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34',
  853. 'ResNet34_vd', 'ResNet101', 'ResNet101_vd', 'HRNet_W18'
  854. }:
  855. raise ValueError(
  856. "backbone: {} is not supported. Please choose one of "
  857. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet34', 'ResNet34_vd', "
  858. "'ResNet101', 'ResNet101_vd', 'HRNet_W18')".format(backbone))
  859. self.backbone_name = backbone
  860. if params.get('with_net', True):
  861. dcn_v2_stages = [1, 2, 3] if with_dcn else [-1]
  862. if backbone == 'HRNet_W18':
  863. if not with_fpn:
  864. logging.warning(
  865. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  866. format(backbone))
  867. with_fpn = True
  868. if with_dcn:
  869. logging.warning(
  870. "Backbone {} should be used along with dcn disabled, 'with_dcn' is forcibly set to False".
  871. format(backbone))
  872. backbone = self._get_backbone(
  873. 'HRNet', width=18, freeze_at=0, return_idx=[0, 1, 2, 3])
  874. elif backbone == 'ResNet50_vd_ssld':
  875. if not with_fpn:
  876. logging.warning(
  877. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  878. format(backbone))
  879. with_fpn = True
  880. backbone = self._get_backbone(
  881. 'ResNet',
  882. variant='d',
  883. norm_type='bn',
  884. freeze_at=0,
  885. return_idx=[0, 1, 2, 3],
  886. num_stages=4,
  887. lr_mult_list=[0.05, 0.05, 0.1, 0.15],
  888. dcn_v2_stages=dcn_v2_stages)
  889. elif 'ResNet50' in backbone:
  890. if with_fpn:
  891. backbone = self._get_backbone(
  892. 'ResNet',
  893. variant='d' if '_vd' in backbone else 'b',
  894. norm_type='bn',
  895. freeze_at=0,
  896. return_idx=[0, 1, 2, 3],
  897. num_stages=4,
  898. dcn_v2_stages=dcn_v2_stages)
  899. else:
  900. if with_dcn:
  901. logging.warning(
  902. "Backbone {} without fpn should be used along with dcn disabled, 'with_dcn' is forcibly set to False".
  903. format(backbone))
  904. backbone = self._get_backbone(
  905. 'ResNet',
  906. variant='d' if '_vd' in backbone else 'b',
  907. norm_type='bn',
  908. freeze_at=0,
  909. return_idx=[2],
  910. num_stages=3)
  911. elif 'ResNet34' in backbone:
  912. if not with_fpn:
  913. logging.warning(
  914. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  915. format(backbone))
  916. with_fpn = True
  917. backbone = self._get_backbone(
  918. 'ResNet',
  919. depth=34,
  920. variant='d' if 'vd' in backbone else 'b',
  921. norm_type='bn',
  922. freeze_at=0,
  923. return_idx=[0, 1, 2, 3],
  924. num_stages=4,
  925. dcn_v2_stages=dcn_v2_stages)
  926. else:
  927. if not with_fpn:
  928. logging.warning(
  929. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  930. format(backbone))
  931. with_fpn = True
  932. backbone = self._get_backbone(
  933. 'ResNet',
  934. depth=101,
  935. variant='d' if 'vd' in backbone else 'b',
  936. norm_type='bn',
  937. freeze_at=0,
  938. return_idx=[0, 1, 2, 3],
  939. num_stages=4,
  940. dcn_v2_stages=dcn_v2_stages)
  941. rpn_in_channel = backbone.out_shape[0].channels
  942. if with_fpn:
  943. self.backbone_name = self.backbone_name + '_fpn'
  944. if 'HRNet' in self.backbone_name:
  945. neck = ppdet.modeling.HRFPN(
  946. in_channels=[i.channels for i in backbone.out_shape],
  947. out_channel=fpn_num_channels,
  948. spatial_scales=[
  949. 1.0 / i.stride for i in backbone.out_shape
  950. ],
  951. share_conv=False)
  952. else:
  953. neck = ppdet.modeling.FPN(
  954. in_channels=[i.channels for i in backbone.out_shape],
  955. out_channel=fpn_num_channels,
  956. spatial_scales=[
  957. 1.0 / i.stride for i in backbone.out_shape
  958. ])
  959. rpn_in_channel = neck.out_shape[0].channels
  960. anchor_generator_cfg = {
  961. 'aspect_ratios': aspect_ratios,
  962. 'anchor_sizes': anchor_sizes,
  963. 'strides': [4, 8, 16, 32, 64]
  964. }
  965. train_proposal_cfg = {
  966. 'min_size': 0.0,
  967. 'nms_thresh': .7,
  968. 'pre_nms_top_n': 2000,
  969. 'post_nms_top_n': 1000,
  970. 'topk_after_collect': True
  971. }
  972. test_proposal_cfg = {
  973. 'min_size': 0.0,
  974. 'nms_thresh': .7,
  975. 'pre_nms_top_n': 1000
  976. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  977. 'post_nms_top_n': test_post_nms_top_n
  978. }
  979. head = ppdet.modeling.TwoFCHead(
  980. in_channel=neck.out_shape[0].channels, out_channel=1024)
  981. roi_extractor_cfg = {
  982. 'resolution': 7,
  983. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  984. 'sampling_ratio': 0,
  985. 'aligned': True
  986. }
  987. with_pool = False
  988. else:
  989. neck = None
  990. anchor_generator_cfg = {
  991. 'aspect_ratios': aspect_ratios,
  992. 'anchor_sizes': anchor_sizes,
  993. 'strides': [16]
  994. }
  995. train_proposal_cfg = {
  996. 'min_size': 0.0,
  997. 'nms_thresh': .7,
  998. 'pre_nms_top_n': 12000,
  999. 'post_nms_top_n': 2000,
  1000. 'topk_after_collect': False
  1001. }
  1002. test_proposal_cfg = {
  1003. 'min_size': 0.0,
  1004. 'nms_thresh': .7,
  1005. 'pre_nms_top_n': 6000
  1006. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1007. 'post_nms_top_n': test_post_nms_top_n
  1008. }
  1009. head = ppdet.modeling.Res5Head()
  1010. roi_extractor_cfg = {
  1011. 'resolution': 14,
  1012. 'spatial_scale':
  1013. [1. / i.stride for i in backbone.out_shape],
  1014. 'sampling_ratio': 0,
  1015. 'aligned': True
  1016. }
  1017. with_pool = True
  1018. rpn_target_assign_cfg = {
  1019. 'batch_size_per_im': rpn_batch_size_per_im,
  1020. 'fg_fraction': rpn_fg_fraction,
  1021. 'negative_overlap': .3,
  1022. 'positive_overlap': .7,
  1023. 'use_random': True
  1024. }
  1025. rpn_head = ppdet.modeling.RPNHead(
  1026. anchor_generator=anchor_generator_cfg,
  1027. rpn_target_assign=rpn_target_assign_cfg,
  1028. train_proposal=train_proposal_cfg,
  1029. test_proposal=test_proposal_cfg,
  1030. in_channel=rpn_in_channel)
  1031. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  1032. bbox_head = ppdet.modeling.BBoxHead(
  1033. head=head,
  1034. in_channel=head.out_shape[0].channels,
  1035. roi_extractor=roi_extractor_cfg,
  1036. with_pool=with_pool,
  1037. bbox_assigner=bbox_assigner,
  1038. num_classes=num_classes)
  1039. bbox_post_process = ppdet.modeling.BBoxPostProcess(
  1040. num_classes=num_classes,
  1041. decode=ppdet.modeling.RCNNBox(num_classes=num_classes),
  1042. nms=ppdet.modeling.MultiClassNMS(
  1043. score_threshold=score_threshold,
  1044. keep_top_k=keep_top_k,
  1045. nms_threshold=nms_threshold))
  1046. params.update({
  1047. 'backbone': backbone,
  1048. 'neck': neck,
  1049. 'rpn_head': rpn_head,
  1050. 'bbox_head': bbox_head,
  1051. 'bbox_post_process': bbox_post_process
  1052. })
  1053. else:
  1054. if backbone not in {'ResNet50', 'ResNet50_vd'}:
  1055. with_fpn = True
  1056. self.with_fpn = with_fpn
  1057. super(FasterRCNN, self).__init__(
  1058. model_name='FasterRCNN', num_classes=num_classes, **params)
  1059. def train(self,
  1060. num_epochs,
  1061. train_dataset,
  1062. train_batch_size=64,
  1063. eval_dataset=None,
  1064. optimizer=None,
  1065. save_interval_epochs=1,
  1066. log_interval_steps=10,
  1067. save_dir='output',
  1068. pretrain_weights='IMAGENET',
  1069. learning_rate=.001,
  1070. warmup_steps=0,
  1071. warmup_start_lr=0.0,
  1072. lr_decay_epochs=(216, 243),
  1073. lr_decay_gamma=0.1,
  1074. metric=None,
  1075. use_ema=False,
  1076. early_stop=False,
  1077. early_stop_patience=5,
  1078. use_vdl=True,
  1079. resume_checkpoint=None):
  1080. """
  1081. Train the model.
  1082. Args:
  1083. num_epochs(int): The number of epochs.
  1084. train_dataset(paddlex.dataset): Training dataset.
  1085. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  1086. eval_dataset(paddlex.dataset, optional):
  1087. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  1088. optimizer(paddle.optimizer.Optimizer or None, optional):
  1089. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  1090. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  1091. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  1092. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  1093. pretrain_weights(str or None, optional):
  1094. None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'IMAGENET'.
  1095. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  1096. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  1097. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  1098. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  1099. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  1100. metric({'VOC', 'COCO', None}, optional):
  1101. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  1102. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  1103. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  1104. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  1105. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  1106. resume_checkpoint(str or None, optional): The path of the checkpoint to resume training from.
  1107. If None, no training checkpoint will be resumed. At most one of `resume_checkpoint` and
  1108. `pretrain_weights` can be set simultaneously. Defaults to None.
  1109. """
  1110. if train_dataset.pos_num < len(train_dataset.file_list):
  1111. train_dataset.num_workers = 0
  1112. if train_batch_size != 1:
  1113. train_batch_size = 1
  1114. logging.warning(
  1115. "Training RCNN models with negative samples only support batch size equals to 1 "
  1116. "on a single gpu/cpu card, `train_batch_size` is forcibly set to 1."
  1117. )
  1118. nranks = paddle.distributed.get_world_size()
  1119. local_rank = paddle.distributed.get_rank()
  1120. # single card training
  1121. if nranks < 2 or local_rank == 0:
  1122. super(FasterRCNN, self).train(
  1123. num_epochs, train_dataset, train_batch_size, eval_dataset,
  1124. optimizer, save_interval_epochs, log_interval_steps,
  1125. save_dir, pretrain_weights, learning_rate, warmup_steps,
  1126. warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric,
  1127. use_ema, early_stop, early_stop_patience, use_vdl,
  1128. resume_checkpoint)
  1129. else:
  1130. super(FasterRCNN, self).train(
  1131. num_epochs, train_dataset, train_batch_size, eval_dataset,
  1132. optimizer, save_interval_epochs, log_interval_steps, save_dir,
  1133. pretrain_weights, learning_rate, warmup_steps, warmup_start_lr,
  1134. lr_decay_epochs, lr_decay_gamma, metric, use_ema, early_stop,
  1135. early_stop_patience, use_vdl, resume_checkpoint)
  1136. def _compose_batch_transform(self, transforms, mode='train'):
  1137. if mode == 'train':
  1138. default_batch_transforms = [
  1139. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1140. ]
  1141. collate_batch = False
  1142. else:
  1143. default_batch_transforms = [
  1144. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1145. ]
  1146. collate_batch = True
  1147. custom_batch_transforms = []
  1148. for i, op in enumerate(transforms.transforms):
  1149. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  1150. if mode != 'train':
  1151. raise Exception(
  1152. "{} cannot be present in the {} transforms. ".format(
  1153. op.__class__.__name__, mode) +
  1154. "Please check the {} transforms.".format(mode))
  1155. custom_batch_transforms.insert(0, copy.deepcopy(op))
  1156. batch_transforms = BatchCompose(
  1157. custom_batch_transforms + default_batch_transforms,
  1158. collate_batch=collate_batch)
  1159. return batch_transforms
  1160. def _fix_transforms_shape(self, image_shape):
  1161. if hasattr(self, 'test_transforms'):
  1162. if self.test_transforms is not None:
  1163. has_resize_op = False
  1164. resize_op_idx = -1
  1165. normalize_op_idx = len(self.test_transforms.transforms)
  1166. for idx, op in enumerate(self.test_transforms.transforms):
  1167. name = op.__class__.__name__
  1168. if name == 'ResizeByShort':
  1169. has_resize_op = True
  1170. resize_op_idx = idx
  1171. if name == 'Normalize':
  1172. normalize_op_idx = idx
  1173. if not has_resize_op:
  1174. self.test_transforms.transforms.insert(
  1175. normalize_op_idx,
  1176. Resize(
  1177. target_size=image_shape,
  1178. keep_ratio=True,
  1179. interp='CUBIC'))
  1180. else:
  1181. self.test_transforms.transforms[resize_op_idx] = Resize(
  1182. target_size=image_shape,
  1183. keep_ratio=True,
  1184. interp='CUBIC')
  1185. self.test_transforms.transforms.append(
  1186. Padding(im_padding_value=[0., 0., 0.]))
  1187. def _get_test_inputs(self, image_shape):
  1188. if image_shape is not None:
  1189. image_shape = self._check_image_shape(image_shape)
  1190. self._fix_transforms_shape(image_shape[-2:])
  1191. else:
  1192. image_shape = [None, 3, -1, -1]
  1193. if self.with_fpn:
  1194. self.test_transforms.transforms.append(
  1195. Padding(im_padding_value=[0., 0., 0.]))
  1196. self.fixed_input_shape = image_shape
  1197. return self._define_input_spec(image_shape)
  1198. class PPYOLO(YOLOv3):
  1199. def __init__(self,
  1200. num_classes=80,
  1201. backbone='ResNet50_vd_dcn',
  1202. anchors=None,
  1203. anchor_masks=None,
  1204. use_coord_conv=True,
  1205. use_iou_aware=True,
  1206. use_spp=True,
  1207. use_drop_block=True,
  1208. scale_x_y=1.05,
  1209. ignore_threshold=0.7,
  1210. label_smooth=False,
  1211. use_iou_loss=True,
  1212. use_matrix_nms=True,
  1213. nms_score_threshold=0.01,
  1214. nms_topk=-1,
  1215. nms_keep_topk=100,
  1216. nms_iou_threshold=0.45,
  1217. **params):
  1218. self.init_params = locals()
  1219. if backbone not in {
  1220. 'ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large',
  1221. 'MobileNetV3_small'
  1222. }:
  1223. raise ValueError(
  1224. "backbone: {} is not supported. Please choose one of "
  1225. "('ResNet50_vd_dcn', 'ResNet18_vd', 'MobileNetV3_large', 'MobileNetV3_small')".
  1226. format(backbone))
  1227. self.backbone_name = backbone
  1228. self.downsample_ratios = [
  1229. 32, 16, 8
  1230. ] if backbone == 'ResNet50_vd_dcn' else [32, 16]
  1231. if params.get('with_net', True):
  1232. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1233. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1234. norm_type = 'sync_bn'
  1235. else:
  1236. norm_type = 'bn'
  1237. if anchors is None and anchor_masks is None:
  1238. if 'MobileNetV3' in backbone:
  1239. anchors = [[11, 18], [34, 47], [51, 126], [115, 71],
  1240. [120, 195], [254, 235]]
  1241. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  1242. elif backbone == 'ResNet50_vd_dcn':
  1243. anchors = [[10, 13], [16, 30], [33, 23], [30, 61],
  1244. [62, 45], [59, 119], [116, 90], [156, 198],
  1245. [373, 326]]
  1246. anchor_masks = [[6, 7, 8], [3, 4, 5], [0, 1, 2]]
  1247. else:
  1248. anchors = [[10, 14], [23, 27], [37, 58], [81, 82],
  1249. [135, 169], [344, 319]]
  1250. anchor_masks = [[3, 4, 5], [0, 1, 2]]
  1251. elif anchors is None or anchor_masks is None:
  1252. raise ValueError(
  1253. "Please define both anchors and anchor_masks.")
  1254. if backbone == 'ResNet50_vd_dcn':
  1255. backbone = self._get_backbone(
  1256. 'ResNet',
  1257. variant='d',
  1258. norm_type=norm_type,
  1259. return_idx=[1, 2, 3],
  1260. dcn_v2_stages=[3],
  1261. freeze_at=-1,
  1262. freeze_norm=False,
  1263. norm_decay=0.)
  1264. elif backbone == 'ResNet18_vd':
  1265. backbone = self._get_backbone(
  1266. 'ResNet',
  1267. depth=18,
  1268. variant='d',
  1269. norm_type=norm_type,
  1270. return_idx=[2, 3],
  1271. freeze_at=-1,
  1272. freeze_norm=False,
  1273. norm_decay=0.)
  1274. elif backbone == 'MobileNetV3_large':
  1275. backbone = self._get_backbone(
  1276. 'MobileNetV3',
  1277. model_name='large',
  1278. norm_type=norm_type,
  1279. scale=1,
  1280. with_extra_blocks=False,
  1281. extra_block_filters=[],
  1282. feature_maps=[13, 16])
  1283. elif backbone == 'MobileNetV3_small':
  1284. backbone = self._get_backbone(
  1285. 'MobileNetV3',
  1286. model_name='small',
  1287. norm_type=norm_type,
  1288. scale=1,
  1289. with_extra_blocks=False,
  1290. extra_block_filters=[],
  1291. feature_maps=[9, 12])
  1292. neck = ppdet.modeling.PPYOLOFPN(
  1293. norm_type=norm_type,
  1294. in_channels=[i.channels for i in backbone.out_shape],
  1295. coord_conv=use_coord_conv,
  1296. drop_block=use_drop_block,
  1297. spp=use_spp,
  1298. conv_block_num=0
  1299. if ('MobileNetV3' in self.backbone_name or
  1300. self.backbone_name == 'ResNet18_vd') else 2)
  1301. loss = ppdet.modeling.YOLOv3Loss(
  1302. num_classes=num_classes,
  1303. ignore_thresh=ignore_threshold,
  1304. downsample=self.downsample_ratios,
  1305. label_smooth=label_smooth,
  1306. scale_x_y=scale_x_y,
  1307. iou_loss=ppdet.modeling.IouLoss(
  1308. loss_weight=2.5, loss_square=True)
  1309. if use_iou_loss else None,
  1310. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  1311. if use_iou_aware else None)
  1312. yolo_head = ppdet.modeling.YOLOv3Head(
  1313. in_channels=[i.channels for i in neck.out_shape],
  1314. anchors=anchors,
  1315. anchor_masks=anchor_masks,
  1316. num_classes=num_classes,
  1317. loss=loss,
  1318. iou_aware=use_iou_aware)
  1319. if use_matrix_nms:
  1320. nms = ppdet.modeling.MatrixNMS(
  1321. keep_top_k=nms_keep_topk,
  1322. score_threshold=nms_score_threshold,
  1323. post_threshold=.05
  1324. if 'MobileNetV3' in self.backbone_name else .01,
  1325. nms_top_k=nms_topk,
  1326. background_label=-1)
  1327. else:
  1328. nms = ppdet.modeling.MultiClassNMS(
  1329. score_threshold=nms_score_threshold,
  1330. nms_top_k=nms_topk,
  1331. keep_top_k=nms_keep_topk,
  1332. nms_threshold=nms_iou_threshold)
  1333. post_process = ppdet.modeling.BBoxPostProcess(
  1334. decode=ppdet.modeling.YOLOBox(
  1335. num_classes=num_classes,
  1336. conf_thresh=.005
  1337. if 'MobileNetV3' in self.backbone_name else .01,
  1338. scale_x_y=scale_x_y),
  1339. nms=nms)
  1340. params.update({
  1341. 'backbone': backbone,
  1342. 'neck': neck,
  1343. 'yolo_head': yolo_head,
  1344. 'post_process': post_process
  1345. })
  1346. super(YOLOv3, self).__init__(
  1347. model_name='YOLOv3', num_classes=num_classes, **params)
  1348. self.anchors = anchors
  1349. self.anchor_masks = anchor_masks
  1350. self.model_name = 'PPYOLO'
  1351. def _get_test_inputs(self, image_shape):
  1352. if image_shape is not None:
  1353. image_shape = self._check_image_shape(image_shape)
  1354. self._fix_transforms_shape(image_shape[-2:])
  1355. else:
  1356. image_shape = [None, 3, 608, 608]
  1357. if hasattr(self, 'test_transforms'):
  1358. if self.test_transforms is not None:
  1359. for idx, op in enumerate(self.test_transforms.transforms):
  1360. name = op.__class__.__name__
  1361. if name == 'Resize':
  1362. image_shape = [None, 3] + list(
  1363. self.test_transforms.transforms[
  1364. idx].target_size)
  1365. logging.warning(
  1366. '[Important!!!] When exporting inference model for {},'.format(
  1367. self.__class__.__name__) +
  1368. ' if fixed_input_shape is not set, it will be forcibly set to {}. '.
  1369. format(image_shape) +
  1370. 'Please check image shape after transforms is {}, if not, fixed_input_shape '.
  1371. format(image_shape[1:]) + 'should be specified manually.')
  1372. self.fixed_input_shape = image_shape
  1373. return self._define_input_spec(image_shape)
  1374. class PPYOLOTiny(YOLOv3):
  1375. def __init__(self,
  1376. num_classes=80,
  1377. backbone='MobileNetV3',
  1378. anchors=[[10, 15], [24, 36], [72, 42], [35, 87], [102, 96],
  1379. [60, 170], [220, 125], [128, 222], [264, 266]],
  1380. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  1381. use_iou_aware=False,
  1382. use_spp=True,
  1383. use_drop_block=True,
  1384. scale_x_y=1.05,
  1385. ignore_threshold=0.5,
  1386. label_smooth=False,
  1387. use_iou_loss=True,
  1388. use_matrix_nms=False,
  1389. nms_score_threshold=0.005,
  1390. nms_topk=1000,
  1391. nms_keep_topk=100,
  1392. nms_iou_threshold=0.45,
  1393. **params):
  1394. self.init_params = locals()
  1395. if backbone != 'MobileNetV3':
  1396. logging.warning(
  1397. "PPYOLOTiny only supports MobileNetV3 as backbone. "
  1398. "Backbone is forcibly set to MobileNetV3.")
  1399. self.backbone_name = 'MobileNetV3'
  1400. self.downsample_ratios = [32, 16, 8]
  1401. if params.get('with_net', True):
  1402. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1403. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1404. norm_type = 'sync_bn'
  1405. else:
  1406. norm_type = 'bn'
  1407. backbone = self._get_backbone(
  1408. 'MobileNetV3',
  1409. model_name='large',
  1410. norm_type=norm_type,
  1411. scale=.5,
  1412. with_extra_blocks=False,
  1413. extra_block_filters=[],
  1414. feature_maps=[7, 13, 16])
  1415. neck = ppdet.modeling.PPYOLOTinyFPN(
  1416. detection_block_channels=[160, 128, 96],
  1417. in_channels=[i.channels for i in backbone.out_shape],
  1418. spp=use_spp,
  1419. drop_block=use_drop_block)
  1420. loss = ppdet.modeling.YOLOv3Loss(
  1421. num_classes=num_classes,
  1422. ignore_thresh=ignore_threshold,
  1423. downsample=self.downsample_ratios,
  1424. label_smooth=label_smooth,
  1425. scale_x_y=scale_x_y,
  1426. iou_loss=ppdet.modeling.IouLoss(
  1427. loss_weight=2.5, loss_square=True)
  1428. if use_iou_loss else None,
  1429. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  1430. if use_iou_aware else None)
  1431. yolo_head = ppdet.modeling.YOLOv3Head(
  1432. in_channels=[i.channels for i in neck.out_shape],
  1433. anchors=anchors,
  1434. anchor_masks=anchor_masks,
  1435. num_classes=num_classes,
  1436. loss=loss,
  1437. iou_aware=use_iou_aware)
  1438. if use_matrix_nms:
  1439. nms = ppdet.modeling.MatrixNMS(
  1440. keep_top_k=nms_keep_topk,
  1441. score_threshold=nms_score_threshold,
  1442. post_threshold=.05,
  1443. nms_top_k=nms_topk,
  1444. background_label=-1)
  1445. else:
  1446. nms = ppdet.modeling.MultiClassNMS(
  1447. score_threshold=nms_score_threshold,
  1448. nms_top_k=nms_topk,
  1449. keep_top_k=nms_keep_topk,
  1450. nms_threshold=nms_iou_threshold)
  1451. post_process = ppdet.modeling.BBoxPostProcess(
  1452. decode=ppdet.modeling.YOLOBox(
  1453. num_classes=num_classes,
  1454. conf_thresh=.005,
  1455. downsample_ratio=32,
  1456. clip_bbox=True,
  1457. scale_x_y=scale_x_y),
  1458. nms=nms)
  1459. params.update({
  1460. 'backbone': backbone,
  1461. 'neck': neck,
  1462. 'yolo_head': yolo_head,
  1463. 'post_process': post_process
  1464. })
  1465. super(YOLOv3, self).__init__(
  1466. model_name='YOLOv3', num_classes=num_classes, **params)
  1467. self.anchors = anchors
  1468. self.anchor_masks = anchor_masks
  1469. self.model_name = 'PPYOLOTiny'
  1470. def _get_test_inputs(self, image_shape):
  1471. if image_shape is not None:
  1472. image_shape = self._check_image_shape(image_shape)
  1473. self._fix_transforms_shape(image_shape[-2:])
  1474. else:
  1475. image_shape = [None, 3, 320, 320]
  1476. if hasattr(self, 'test_transforms'):
  1477. if self.test_transforms is not None:
  1478. for idx, op in enumerate(self.test_transforms.transforms):
  1479. name = op.__class__.__name__
  1480. if name == 'Resize':
  1481. image_shape = [None, 3] + list(
  1482. self.test_transforms.transforms[
  1483. idx].target_size)
  1484. logging.warning(
  1485. '[Important!!!] When exporting inference model for {},'.format(
  1486. self.__class__.__name__) +
  1487. ' if fixed_input_shape is not set, it will be forcibly set to {}. '.
  1488. format(image_shape) +
  1489. 'Please check image shape after transforms is {}, if not, fixed_input_shape '.
  1490. format(image_shape[1:]) + 'should be specified manually.')
  1491. self.fixed_input_shape = image_shape
  1492. return self._define_input_spec(image_shape)
  1493. class PPYOLOv2(YOLOv3):
  1494. def __init__(self,
  1495. num_classes=80,
  1496. backbone='ResNet50_vd_dcn',
  1497. anchors=[[10, 13], [16, 30], [33, 23], [30, 61], [62, 45],
  1498. [59, 119], [116, 90], [156, 198], [373, 326]],
  1499. anchor_masks=[[6, 7, 8], [3, 4, 5], [0, 1, 2]],
  1500. use_iou_aware=True,
  1501. use_spp=True,
  1502. use_drop_block=True,
  1503. scale_x_y=1.05,
  1504. ignore_threshold=0.7,
  1505. label_smooth=False,
  1506. use_iou_loss=True,
  1507. use_matrix_nms=True,
  1508. nms_score_threshold=0.01,
  1509. nms_topk=-1,
  1510. nms_keep_topk=100,
  1511. nms_iou_threshold=0.45,
  1512. **params):
  1513. self.init_params = locals()
  1514. if backbone not in {'ResNet50_vd_dcn', 'ResNet101_vd_dcn'}:
  1515. raise ValueError(
  1516. "backbone: {} is not supported. Please choose one of "
  1517. "('ResNet50_vd_dcn', 'ResNet101_vd_dcn')".format(backbone))
  1518. self.backbone_name = backbone
  1519. self.downsample_ratios = [32, 16, 8]
  1520. if params.get('with_net', True):
  1521. if paddlex.env_info['place'] == 'gpu' and paddlex.env_info[
  1522. 'num'] > 1 and not os.environ.get('PADDLEX_EXPORT_STAGE'):
  1523. norm_type = 'sync_bn'
  1524. else:
  1525. norm_type = 'bn'
  1526. if backbone == 'ResNet50_vd_dcn':
  1527. backbone = self._get_backbone(
  1528. 'ResNet',
  1529. variant='d',
  1530. norm_type=norm_type,
  1531. return_idx=[1, 2, 3],
  1532. dcn_v2_stages=[3],
  1533. freeze_at=-1,
  1534. freeze_norm=False,
  1535. norm_decay=0.)
  1536. elif backbone == 'ResNet101_vd_dcn':
  1537. backbone = self._get_backbone(
  1538. 'ResNet',
  1539. depth=101,
  1540. variant='d',
  1541. norm_type=norm_type,
  1542. return_idx=[1, 2, 3],
  1543. dcn_v2_stages=[3],
  1544. freeze_at=-1,
  1545. freeze_norm=False,
  1546. norm_decay=0.)
  1547. neck = ppdet.modeling.PPYOLOPAN(
  1548. norm_type=norm_type,
  1549. in_channels=[i.channels for i in backbone.out_shape],
  1550. drop_block=use_drop_block,
  1551. block_size=3,
  1552. keep_prob=.9,
  1553. spp=use_spp)
  1554. loss = ppdet.modeling.YOLOv3Loss(
  1555. num_classes=num_classes,
  1556. ignore_thresh=ignore_threshold,
  1557. downsample=self.downsample_ratios,
  1558. label_smooth=label_smooth,
  1559. scale_x_y=scale_x_y,
  1560. iou_loss=ppdet.modeling.IouLoss(
  1561. loss_weight=2.5, loss_square=True)
  1562. if use_iou_loss else None,
  1563. iou_aware_loss=ppdet.modeling.IouAwareLoss(loss_weight=1.0)
  1564. if use_iou_aware else None)
  1565. yolo_head = ppdet.modeling.YOLOv3Head(
  1566. in_channels=[i.channels for i in neck.out_shape],
  1567. anchors=anchors,
  1568. anchor_masks=anchor_masks,
  1569. num_classes=num_classes,
  1570. loss=loss,
  1571. iou_aware=use_iou_aware,
  1572. iou_aware_factor=.5)
  1573. if use_matrix_nms:
  1574. nms = ppdet.modeling.MatrixNMS(
  1575. keep_top_k=nms_keep_topk,
  1576. score_threshold=nms_score_threshold,
  1577. post_threshold=.01,
  1578. nms_top_k=nms_topk,
  1579. background_label=-1)
  1580. else:
  1581. nms = ppdet.modeling.MultiClassNMS(
  1582. score_threshold=nms_score_threshold,
  1583. nms_top_k=nms_topk,
  1584. keep_top_k=nms_keep_topk,
  1585. nms_threshold=nms_iou_threshold)
  1586. post_process = ppdet.modeling.BBoxPostProcess(
  1587. decode=ppdet.modeling.YOLOBox(
  1588. num_classes=num_classes,
  1589. conf_thresh=.01,
  1590. downsample_ratio=32,
  1591. clip_bbox=True,
  1592. scale_x_y=scale_x_y),
  1593. nms=nms)
  1594. params.update({
  1595. 'backbone': backbone,
  1596. 'neck': neck,
  1597. 'yolo_head': yolo_head,
  1598. 'post_process': post_process
  1599. })
  1600. super(YOLOv3, self).__init__(
  1601. model_name='YOLOv3', num_classes=num_classes, **params)
  1602. self.anchors = anchors
  1603. self.anchor_masks = anchor_masks
  1604. self.model_name = 'PPYOLOv2'
  1605. def _get_test_inputs(self, image_shape):
  1606. if image_shape is not None:
  1607. image_shape = self._check_image_shape(image_shape)
  1608. self._fix_transforms_shape(image_shape[-2:])
  1609. else:
  1610. image_shape = [None, 3, 640, 640]
  1611. if hasattr(self, 'test_transforms'):
  1612. if self.test_transforms is not None:
  1613. for idx, op in enumerate(self.test_transforms.transforms):
  1614. name = op.__class__.__name__
  1615. if name == 'Resize':
  1616. image_shape = [None, 3] + list(
  1617. self.test_transforms.transforms[
  1618. idx].target_size)
  1619. logging.warning(
  1620. '[Important!!!] When exporting inference model for {},'.format(
  1621. self.__class__.__name__) +
  1622. ' if fixed_input_shape is not set, it will be forcibly set to {}. '.
  1623. format(image_shape) +
  1624. 'Please check image shape after transforms is {}, if not, fixed_input_shape '.
  1625. format(image_shape[1:]) + 'should be specified manually.')
  1626. self.fixed_input_shape = image_shape
  1627. return self._define_input_spec(image_shape)
  1628. class MaskRCNN(BaseDetector):
  1629. def __init__(self,
  1630. num_classes=80,
  1631. backbone='ResNet50_vd',
  1632. with_fpn=True,
  1633. with_dcn=False,
  1634. aspect_ratios=[0.5, 1.0, 2.0],
  1635. anchor_sizes=[[32], [64], [128], [256], [512]],
  1636. keep_top_k=100,
  1637. nms_threshold=0.5,
  1638. score_threshold=0.05,
  1639. fpn_num_channels=256,
  1640. rpn_batch_size_per_im=256,
  1641. rpn_fg_fraction=0.5,
  1642. test_pre_nms_top_n=None,
  1643. test_post_nms_top_n=1000,
  1644. **params):
  1645. self.init_params = locals()
  1646. if backbone not in {
  1647. 'ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101',
  1648. 'ResNet101_vd'
  1649. }:
  1650. raise ValueError(
  1651. "backbone: {} is not supported. Please choose one of "
  1652. "('ResNet50', 'ResNet50_vd', 'ResNet50_vd_ssld', 'ResNet101', 'ResNet101_vd')".
  1653. format(backbone))
  1654. self.backbone_name = backbone + '_fpn' if with_fpn else backbone
  1655. dcn_v2_stages = [1, 2, 3] if with_dcn else [-1]
  1656. if params.get('with_net', True):
  1657. if backbone == 'ResNet50':
  1658. if with_fpn:
  1659. backbone = self._get_backbone(
  1660. 'ResNet',
  1661. norm_type='bn',
  1662. freeze_at=0,
  1663. return_idx=[0, 1, 2, 3],
  1664. num_stages=4,
  1665. dcn_v2_stages=dcn_v2_stages)
  1666. else:
  1667. if with_dcn:
  1668. logging.warning(
  1669. "Backbone {} should be used along with dcn disabled, 'with_dcn' is forcibly set to False".
  1670. format(backbone))
  1671. backbone = self._get_backbone(
  1672. 'ResNet',
  1673. norm_type='bn',
  1674. freeze_at=0,
  1675. return_idx=[2],
  1676. num_stages=3)
  1677. elif 'ResNet50_vd' in backbone:
  1678. if not with_fpn:
  1679. logging.warning(
  1680. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1681. format(backbone))
  1682. with_fpn = True
  1683. backbone = self._get_backbone(
  1684. 'ResNet',
  1685. variant='d',
  1686. norm_type='bn',
  1687. freeze_at=0,
  1688. return_idx=[0, 1, 2, 3],
  1689. num_stages=4,
  1690. lr_mult_list=[0.05, 0.05, 0.1, 0.15]
  1691. if '_ssld' in backbone else [1.0, 1.0, 1.0, 1.0],
  1692. dcn_v2_stages=dcn_v2_stages)
  1693. else:
  1694. if not with_fpn:
  1695. logging.warning(
  1696. "Backbone {} should be used along with fpn enabled, 'with_fpn' is forcibly set to True".
  1697. format(backbone))
  1698. with_fpn = True
  1699. backbone = self._get_backbone(
  1700. 'ResNet',
  1701. variant='d' if '_vd' in backbone else 'b',
  1702. depth=101,
  1703. norm_type='bn',
  1704. freeze_at=0,
  1705. return_idx=[0, 1, 2, 3],
  1706. num_stages=4,
  1707. dcn_v2_stages=dcn_v2_stages)
  1708. rpn_in_channel = backbone.out_shape[0].channels
  1709. if with_fpn:
  1710. neck = ppdet.modeling.FPN(
  1711. in_channels=[i.channels for i in backbone.out_shape],
  1712. out_channel=fpn_num_channels,
  1713. spatial_scales=[
  1714. 1.0 / i.stride for i in backbone.out_shape
  1715. ])
  1716. rpn_in_channel = neck.out_shape[0].channels
  1717. anchor_generator_cfg = {
  1718. 'aspect_ratios': aspect_ratios,
  1719. 'anchor_sizes': anchor_sizes,
  1720. 'strides': [4, 8, 16, 32, 64]
  1721. }
  1722. train_proposal_cfg = {
  1723. 'min_size': 0.0,
  1724. 'nms_thresh': .7,
  1725. 'pre_nms_top_n': 2000,
  1726. 'post_nms_top_n': 1000,
  1727. 'topk_after_collect': True
  1728. }
  1729. test_proposal_cfg = {
  1730. 'min_size': 0.0,
  1731. 'nms_thresh': .7,
  1732. 'pre_nms_top_n': 1000
  1733. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1734. 'post_nms_top_n': test_post_nms_top_n
  1735. }
  1736. bb_head = ppdet.modeling.TwoFCHead(
  1737. in_channel=neck.out_shape[0].channels, out_channel=1024)
  1738. bb_roi_extractor_cfg = {
  1739. 'resolution': 7,
  1740. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1741. 'sampling_ratio': 0,
  1742. 'aligned': True
  1743. }
  1744. with_pool = False
  1745. m_head = ppdet.modeling.MaskFeat(
  1746. in_channel=neck.out_shape[0].channels,
  1747. out_channel=256,
  1748. num_convs=4)
  1749. m_roi_extractor_cfg = {
  1750. 'resolution': 14,
  1751. 'spatial_scale': [1. / i.stride for i in neck.out_shape],
  1752. 'sampling_ratio': 0,
  1753. 'aligned': True
  1754. }
  1755. mask_assigner = MaskAssigner(
  1756. num_classes=num_classes, mask_resolution=28)
  1757. share_bbox_feat = False
  1758. else:
  1759. neck = None
  1760. anchor_generator_cfg = {
  1761. 'aspect_ratios': aspect_ratios,
  1762. 'anchor_sizes': anchor_sizes,
  1763. 'strides': [16]
  1764. }
  1765. train_proposal_cfg = {
  1766. 'min_size': 0.0,
  1767. 'nms_thresh': .7,
  1768. 'pre_nms_top_n': 12000,
  1769. 'post_nms_top_n': 2000,
  1770. 'topk_after_collect': False
  1771. }
  1772. test_proposal_cfg = {
  1773. 'min_size': 0.0,
  1774. 'nms_thresh': .7,
  1775. 'pre_nms_top_n': 6000
  1776. if test_pre_nms_top_n is None else test_pre_nms_top_n,
  1777. 'post_nms_top_n': test_post_nms_top_n
  1778. }
  1779. bb_head = ppdet.modeling.Res5Head()
  1780. bb_roi_extractor_cfg = {
  1781. 'resolution': 14,
  1782. 'spatial_scale':
  1783. [1. / i.stride for i in backbone.out_shape],
  1784. 'sampling_ratio': 0,
  1785. 'aligned': True
  1786. }
  1787. with_pool = True
  1788. m_head = ppdet.modeling.MaskFeat(
  1789. in_channel=bb_head.out_shape[0].channels,
  1790. out_channel=256,
  1791. num_convs=0)
  1792. m_roi_extractor_cfg = {
  1793. 'resolution': 14,
  1794. 'spatial_scale':
  1795. [1. / i.stride for i in backbone.out_shape],
  1796. 'sampling_ratio': 0,
  1797. 'aligned': True
  1798. }
  1799. mask_assigner = MaskAssigner(
  1800. num_classes=num_classes, mask_resolution=14)
  1801. share_bbox_feat = True
  1802. rpn_target_assign_cfg = {
  1803. 'batch_size_per_im': rpn_batch_size_per_im,
  1804. 'fg_fraction': rpn_fg_fraction,
  1805. 'negative_overlap': .3,
  1806. 'positive_overlap': .7,
  1807. 'use_random': True
  1808. }
  1809. rpn_head = ppdet.modeling.RPNHead(
  1810. anchor_generator=anchor_generator_cfg,
  1811. rpn_target_assign=rpn_target_assign_cfg,
  1812. train_proposal=train_proposal_cfg,
  1813. test_proposal=test_proposal_cfg,
  1814. in_channel=rpn_in_channel)
  1815. bbox_assigner = BBoxAssigner(num_classes=num_classes)
  1816. bbox_head = ppdet.modeling.BBoxHead(
  1817. head=bb_head,
  1818. in_channel=bb_head.out_shape[0].channels,
  1819. roi_extractor=bb_roi_extractor_cfg,
  1820. with_pool=with_pool,
  1821. bbox_assigner=bbox_assigner,
  1822. num_classes=num_classes)
  1823. mask_head = ppdet.modeling.MaskHead(
  1824. head=m_head,
  1825. roi_extractor=m_roi_extractor_cfg,
  1826. mask_assigner=mask_assigner,
  1827. share_bbox_feat=share_bbox_feat,
  1828. num_classes=num_classes)
  1829. bbox_post_process = ppdet.modeling.BBoxPostProcess(
  1830. num_classes=num_classes,
  1831. decode=ppdet.modeling.RCNNBox(num_classes=num_classes),
  1832. nms=ppdet.modeling.MultiClassNMS(
  1833. score_threshold=score_threshold,
  1834. keep_top_k=keep_top_k,
  1835. nms_threshold=nms_threshold))
  1836. mask_post_process = ppdet.modeling.MaskPostProcess(
  1837. binary_thresh=.5)
  1838. params.update({
  1839. 'backbone': backbone,
  1840. 'neck': neck,
  1841. 'rpn_head': rpn_head,
  1842. 'bbox_head': bbox_head,
  1843. 'mask_head': mask_head,
  1844. 'bbox_post_process': bbox_post_process,
  1845. 'mask_post_process': mask_post_process
  1846. })
  1847. self.with_fpn = with_fpn
  1848. super(MaskRCNN, self).__init__(
  1849. model_name='MaskRCNN', num_classes=num_classes, **params)
  1850. def train(self,
  1851. num_epochs,
  1852. train_dataset,
  1853. train_batch_size=64,
  1854. eval_dataset=None,
  1855. optimizer=None,
  1856. save_interval_epochs=1,
  1857. log_interval_steps=10,
  1858. save_dir='output',
  1859. pretrain_weights='IMAGENET',
  1860. learning_rate=.001,
  1861. warmup_steps=0,
  1862. warmup_start_lr=0.0,
  1863. lr_decay_epochs=(216, 243),
  1864. lr_decay_gamma=0.1,
  1865. metric=None,
  1866. use_ema=False,
  1867. early_stop=False,
  1868. early_stop_patience=5,
  1869. use_vdl=True,
  1870. resume_checkpoint=None):
  1871. """
  1872. Train the model.
  1873. Args:
  1874. num_epochs(int): The number of epochs.
  1875. train_dataset(paddlex.dataset): Training dataset.
  1876. train_batch_size(int, optional): Total batch size among all cards used in training. Defaults to 64.
  1877. eval_dataset(paddlex.dataset, optional):
  1878. Evaluation dataset. If None, the model will not be evaluated during training process. Defaults to None.
  1879. optimizer(paddle.optimizer.Optimizer or None, optional):
  1880. Optimizer used for training. If None, a default optimizer is used. Defaults to None.
  1881. save_interval_epochs(int, optional): Epoch interval for saving the model. Defaults to 1.
  1882. log_interval_steps(int, optional): Step interval for printing training information. Defaults to 10.
  1883. save_dir(str, optional): Directory to save the model. Defaults to 'output'.
  1884. pretrain_weights(str or None, optional):
  1885. None or name/path of pretrained weights. If None, no pretrained weights will be loaded. Defaults to 'IMAGENET'.
  1886. learning_rate(float, optional): Learning rate for training. Defaults to .001.
  1887. warmup_steps(int, optional): The number of steps of warm-up training. Defaults to 0.
  1888. warmup_start_lr(float, optional): Start learning rate of warm-up training. Defaults to 0..
  1889. lr_decay_epochs(list or tuple, optional): Epoch milestones for learning rate decay. Defaults to (216, 243).
  1890. lr_decay_gamma(float, optional): Gamma coefficient of learning rate decay. Defaults to .1.
  1891. metric({'VOC', 'COCO', None}, optional):
  1892. Evaluation metric. If None, determine the metric according to the dataset format. Defaults to None.
  1893. use_ema(bool, optional): Whether to use exponential moving average strategy. Defaults to False.
  1894. early_stop(bool, optional): Whether to adopt early stop strategy. Defaults to False.
  1895. early_stop_patience(int, optional): Early stop patience. Defaults to 5.
  1896. use_vdl(bool, optional): Whether to use VisualDL to monitor the training process. Defaults to True.
  1897. resume_checkpoint(str or None, optional): The path of the checkpoint to resume training from.
  1898. If None, no training checkpoint will be resumed. At most one of `resume_checkpoint` and
  1899. `pretrain_weights` can be set simultaneously. Defaults to None.
  1900. """
  1901. if train_dataset.pos_num < len(train_dataset.file_list):
  1902. train_dataset.num_workers = 0
  1903. if train_batch_size != 1:
  1904. train_batch_size = 1
  1905. logging.warning(
  1906. "Training RCNN models with negative samples only support batch size equals to 1 "
  1907. "on a single gpu/cpu card, `train_batch_size` is forcibly set to 1."
  1908. )
  1909. nranks = paddle.distributed.get_world_size()
  1910. local_rank = paddle.distributed.get_rank()
  1911. # single card training
  1912. if nranks < 2 or local_rank == 0:
  1913. super(MaskRCNN, self).train(
  1914. num_epochs, train_dataset, train_batch_size, eval_dataset,
  1915. optimizer, save_interval_epochs, log_interval_steps,
  1916. save_dir, pretrain_weights, learning_rate, warmup_steps,
  1917. warmup_start_lr, lr_decay_epochs, lr_decay_gamma, metric,
  1918. use_ema, early_stop, early_stop_patience, use_vdl,
  1919. resume_checkpoint)
  1920. else:
  1921. super(MaskRCNN, self).train(
  1922. num_epochs, train_dataset, train_batch_size, eval_dataset,
  1923. optimizer, save_interval_epochs, log_interval_steps, save_dir,
  1924. pretrain_weights, learning_rate, warmup_steps, warmup_start_lr,
  1925. lr_decay_epochs, lr_decay_gamma, metric, use_ema, early_stop,
  1926. early_stop_patience, use_vdl, resume_checkpoint)
  1927. def _compose_batch_transform(self, transforms, mode='train'):
  1928. if mode == 'train':
  1929. default_batch_transforms = [
  1930. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1931. ]
  1932. collate_batch = False
  1933. else:
  1934. default_batch_transforms = [
  1935. _BatchPadding(pad_to_stride=32 if self.with_fpn else -1)
  1936. ]
  1937. collate_batch = True
  1938. custom_batch_transforms = []
  1939. for i, op in enumerate(transforms.transforms):
  1940. if isinstance(op, (BatchRandomResize, BatchRandomResizeByShort)):
  1941. if mode != 'train':
  1942. raise Exception(
  1943. "{} cannot be present in the {} transforms. ".format(
  1944. op.__class__.__name__, mode) +
  1945. "Please check the {} transforms.".format(mode))
  1946. custom_batch_transforms.insert(0, copy.deepcopy(op))
  1947. batch_transforms = BatchCompose(
  1948. custom_batch_transforms + default_batch_transforms,
  1949. collate_batch=collate_batch)
  1950. return batch_transforms
  1951. def _fix_transforms_shape(self, image_shape):
  1952. if hasattr(self, 'test_transforms'):
  1953. if self.test_transforms is not None:
  1954. has_resize_op = False
  1955. resize_op_idx = -1
  1956. normalize_op_idx = len(self.test_transforms.transforms)
  1957. for idx, op in enumerate(self.test_transforms.transforms):
  1958. name = op.__class__.__name__
  1959. if name == 'ResizeByShort':
  1960. has_resize_op = True
  1961. resize_op_idx = idx
  1962. if name == 'Normalize':
  1963. normalize_op_idx = idx
  1964. if not has_resize_op:
  1965. self.test_transforms.transforms.insert(
  1966. normalize_op_idx,
  1967. Resize(
  1968. target_size=image_shape,
  1969. keep_ratio=True,
  1970. interp='CUBIC'))
  1971. else:
  1972. self.test_transforms.transforms[resize_op_idx] = Resize(
  1973. target_size=image_shape,
  1974. keep_ratio=True,
  1975. interp='CUBIC')
  1976. self.test_transforms.transforms.append(
  1977. Padding(im_padding_value=[0., 0., 0.]))
  1978. def _get_test_inputs(self, image_shape):
  1979. if image_shape is not None:
  1980. image_shape = self._check_image_shape(image_shape)
  1981. self._fix_transforms_shape(image_shape[-2:])
  1982. else:
  1983. image_shape = [None, 3, -1, -1]
  1984. if self.with_fpn:
  1985. self.test_transforms.transforms.append(
  1986. Padding(im_padding_value=[0., 0., 0.]))
  1987. self.fixed_input_shape = image_shape
  1988. return self._define_input_spec(image_shape)