metrics.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. from __future__ import division
  16. from __future__ import print_function
  17. import os
  18. import sys
  19. import json
  20. import paddle
  21. import numpy as np
  22. from .map_utils import prune_zero_padding, DetectionMAP
  23. from .coco_utils import get_infer_results, cocoapi_eval
  24. from .widerface_utils import face_eval_run
  25. from paddlex.ppdet.data.source.category import get_categories
  26. from paddlex.ppdet.utils.logger import setup_logger
  27. logger = setup_logger(__name__)
  28. __all__ = [
  29. 'Metric',
  30. 'COCOMetric',
  31. 'VOCMetric',
  32. 'WiderFaceMetric',
  33. 'get_infer_results',
  34. 'RBoxMetric',
  35. ]
  36. COCO_SIGMAS = np.array([
  37. .26, .25, .25, .35, .35, .79, .79, .72, .72, .62, .62, 1.07, 1.07, .87,
  38. .87, .89, .89
  39. ]) / 10.0
  40. CROWD_SIGMAS = np.array(
  41. [.79, .79, .72, .72, .62, .62, 1.07, 1.07, .87, .87, .89, .89, .79,
  42. .79]) / 10.0
  43. class Metric(paddle.metric.Metric):
  44. def name(self):
  45. return self.__class__.__name__
  46. def reset(self):
  47. pass
  48. def accumulate(self):
  49. pass
  50. # paddle.metric.Metric defined :metch:`update`, :meth:`accumulate`
  51. # :metch:`reset`, in ppdet, we also need following 2 methods:
  52. # abstract method for logging metric results
  53. def log(self):
  54. pass
  55. # abstract method for getting metric results
  56. def get_results(self):
  57. pass
  58. class COCOMetric(Metric):
  59. def __init__(self, anno_file, **kwargs):
  60. assert os.path.isfile(anno_file), \
  61. "anno_file {} not a file".format(anno_file)
  62. self.anno_file = anno_file
  63. self.clsid2catid = kwargs.get('clsid2catid', None)
  64. if self.clsid2catid is None:
  65. self.clsid2catid, _ = get_categories('COCO', anno_file)
  66. self.classwise = kwargs.get('classwise', False)
  67. self.output_eval = kwargs.get('output_eval', None)
  68. # TODO: bias should be unified
  69. self.bias = kwargs.get('bias', 0)
  70. self.save_prediction_only = kwargs.get('save_prediction_only', False)
  71. self.iou_type = kwargs.get('IouType', 'bbox')
  72. self.reset()
  73. def reset(self):
  74. # only bbox and mask evaluation support currently
  75. self.results = {'bbox': [], 'mask': [], 'segm': [], 'keypoint': []}
  76. self.eval_results = {}
  77. def update(self, inputs, outputs):
  78. outs = {}
  79. # outputs Tensor -> numpy.ndarray
  80. for k, v in outputs.items():
  81. outs[k] = v.numpy() if isinstance(v, paddle.Tensor) else v
  82. im_id = inputs['im_id']
  83. outs['im_id'] = im_id.numpy() if isinstance(im_id,
  84. paddle.Tensor) else im_id
  85. infer_results = get_infer_results(
  86. outs, self.clsid2catid, bias=self.bias)
  87. self.results['bbox'] += infer_results[
  88. 'bbox'] if 'bbox' in infer_results else []
  89. self.results['mask'] += infer_results[
  90. 'mask'] if 'mask' in infer_results else []
  91. self.results['segm'] += infer_results[
  92. 'segm'] if 'segm' in infer_results else []
  93. self.results['keypoint'] += infer_results[
  94. 'keypoint'] if 'keypoint' in infer_results else []
  95. def accumulate(self):
  96. if len(self.results['bbox']) > 0:
  97. output = "bbox.json"
  98. if self.output_eval:
  99. output = os.path.join(self.output_eval, output)
  100. with open(output, 'w') as f:
  101. json.dump(self.results['bbox'], f)
  102. logger.info('The bbox result is saved to bbox.json.')
  103. if self.save_prediction_only:
  104. logger.info('The bbox result is saved to {} and do not '
  105. 'evaluate the mAP.'.format(output))
  106. else:
  107. bbox_stats = cocoapi_eval(
  108. output,
  109. 'bbox',
  110. anno_file=self.anno_file,
  111. classwise=self.classwise)
  112. self.eval_results['bbox'] = bbox_stats
  113. sys.stdout.flush()
  114. if len(self.results['mask']) > 0:
  115. output = "mask.json"
  116. if self.output_eval:
  117. output = os.path.join(self.output_eval, output)
  118. with open(output, 'w') as f:
  119. json.dump(self.results['mask'], f)
  120. logger.info('The mask result is saved to mask.json.')
  121. if self.save_prediction_only:
  122. logger.info('The mask result is saved to {} and do not '
  123. 'evaluate the mAP.'.format(output))
  124. else:
  125. seg_stats = cocoapi_eval(
  126. output,
  127. 'segm',
  128. anno_file=self.anno_file,
  129. classwise=self.classwise)
  130. self.eval_results['mask'] = seg_stats
  131. sys.stdout.flush()
  132. if len(self.results['segm']) > 0:
  133. output = "segm.json"
  134. if self.output_eval:
  135. output = os.path.join(self.output_eval, output)
  136. with open(output, 'w') as f:
  137. json.dump(self.results['segm'], f)
  138. logger.info('The segm result is saved to segm.json.')
  139. if self.save_prediction_only:
  140. logger.info('The segm result is saved to {} and do not '
  141. 'evaluate the mAP.'.format(output))
  142. else:
  143. seg_stats = cocoapi_eval(
  144. output,
  145. 'segm',
  146. anno_file=self.anno_file,
  147. classwise=self.classwise)
  148. self.eval_results['mask'] = seg_stats
  149. sys.stdout.flush()
  150. if len(self.results['keypoint']) > 0:
  151. output = "keypoint.json"
  152. if self.output_eval:
  153. output = os.path.join(self.output_eval, output)
  154. with open(output, 'w') as f:
  155. json.dump(self.results['keypoint'], f)
  156. logger.info('The keypoint result is saved to keypoint.json.')
  157. if self.save_prediction_only:
  158. logger.info('The keypoint result is saved to {} and do not '
  159. 'evaluate the mAP.'.format(output))
  160. else:
  161. style = 'keypoints'
  162. use_area = True
  163. sigmas = COCO_SIGMAS
  164. if self.iou_type == 'keypoints_crowd':
  165. style = 'keypoints_crowd'
  166. use_area = False
  167. sigmas = CROWD_SIGMAS
  168. keypoint_stats = cocoapi_eval(
  169. output,
  170. style,
  171. anno_file=self.anno_file,
  172. classwise=self.classwise,
  173. sigmas=sigmas,
  174. use_area=use_area)
  175. self.eval_results['keypoint'] = keypoint_stats
  176. sys.stdout.flush()
  177. def log(self):
  178. pass
  179. def get_results(self):
  180. return self.eval_results
  181. class VOCMetric(Metric):
  182. def __init__(self,
  183. label_list,
  184. class_num=20,
  185. overlap_thresh=0.5,
  186. map_type='11point',
  187. is_bbox_normalized=False,
  188. evaluate_difficult=False,
  189. classwise=False):
  190. assert os.path.isfile(label_list), \
  191. "label_list {} not a file".format(label_list)
  192. self.clsid2catid, self.catid2name = get_categories('VOC', label_list)
  193. self.overlap_thresh = overlap_thresh
  194. self.map_type = map_type
  195. self.evaluate_difficult = evaluate_difficult
  196. self.detection_map = DetectionMAP(
  197. class_num=class_num,
  198. overlap_thresh=overlap_thresh,
  199. map_type=map_type,
  200. is_bbox_normalized=is_bbox_normalized,
  201. evaluate_difficult=evaluate_difficult,
  202. catid2name=self.catid2name,
  203. classwise=classwise)
  204. self.reset()
  205. def reset(self):
  206. self.detection_map.reset()
  207. def update(self, inputs, outputs):
  208. bbox_np = outputs['bbox'].numpy()
  209. bboxes = bbox_np[:, 2:]
  210. scores = bbox_np[:, 1]
  211. labels = bbox_np[:, 0]
  212. bbox_lengths = outputs['bbox_num'].numpy()
  213. if bboxes.shape == (1, 1) or bboxes is None:
  214. return
  215. gt_boxes = inputs['gt_bbox']
  216. gt_labels = inputs['gt_class']
  217. difficults = inputs['difficult'] if not self.evaluate_difficult \
  218. else None
  219. scale_factor = inputs['scale_factor'].numpy(
  220. ) if 'scale_factor' in inputs else np.ones(
  221. (gt_boxes.shape[0], 2)).astype('float32')
  222. bbox_idx = 0
  223. for i in range(len(gt_boxes)):
  224. gt_box = gt_boxes[i].numpy()
  225. h, w = scale_factor[i]
  226. gt_box = gt_box / np.array([w, h, w, h])
  227. gt_label = gt_labels[i].numpy()
  228. difficult = None if difficults is None \
  229. else difficults[i].numpy()
  230. bbox_num = bbox_lengths[i]
  231. bbox = bboxes[bbox_idx:bbox_idx + bbox_num]
  232. score = scores[bbox_idx:bbox_idx + bbox_num]
  233. label = labels[bbox_idx:bbox_idx + bbox_num]
  234. gt_box, gt_label, difficult = prune_zero_padding(gt_box, gt_label,
  235. difficult)
  236. self.detection_map.update(bbox, score, label, gt_box, gt_label,
  237. difficult)
  238. bbox_idx += bbox_num
  239. def accumulate(self):
  240. logger.info("Accumulating evaluatation results...")
  241. self.detection_map.accumulate()
  242. def log(self):
  243. map_stat = 100. * self.detection_map.get_map()
  244. logger.info("mAP({:.2f}, {}) = {:.2f}%".format(
  245. self.overlap_thresh, self.map_type, map_stat))
  246. def get_results(self):
  247. return {'bbox': [self.detection_map.get_map()]}
  248. class WiderFaceMetric(Metric):
  249. def __init__(self, image_dir, anno_file, multi_scale=True):
  250. self.image_dir = image_dir
  251. self.anno_file = anno_file
  252. self.multi_scale = multi_scale
  253. self.clsid2catid, self.catid2name = get_categories('widerface')
  254. def update(self, model):
  255. face_eval_run(
  256. model,
  257. self.image_dir,
  258. self.anno_file,
  259. pred_dir='output/pred',
  260. eval_mode='widerface',
  261. multi_scale=self.multi_scale)
  262. class RBoxMetric(Metric):
  263. def __init__(self, anno_file, **kwargs):
  264. assert os.path.isfile(anno_file), \
  265. "anno_file {} not a file".format(anno_file)
  266. assert os.path.exists(anno_file), "anno_file {} not exists".format(
  267. anno_file)
  268. self.anno_file = anno_file
  269. self.gt_anno = json.load(open(self.anno_file))
  270. cats = self.gt_anno['categories']
  271. self.clsid2catid = {i: cat['id'] for i, cat in enumerate(cats)}
  272. self.catid2clsid = {cat['id']: i for i, cat in enumerate(cats)}
  273. self.catid2name = {cat['id']: cat['name'] for cat in cats}
  274. self.classwise = kwargs.get('classwise', False)
  275. self.output_eval = kwargs.get('output_eval', None)
  276. # TODO: bias should be unified
  277. self.bias = kwargs.get('bias', 0)
  278. self.save_prediction_only = kwargs.get('save_prediction_only', False)
  279. self.iou_type = kwargs.get('IouType', 'bbox')
  280. self.overlap_thresh = kwargs.get('overlap_thresh', 0.5)
  281. self.map_type = kwargs.get('map_type', '11point')
  282. self.evaluate_difficult = kwargs.get('evaluate_difficult', False)
  283. class_num = len(self.catid2name)
  284. self.detection_map = DetectionMAP(
  285. class_num=class_num,
  286. overlap_thresh=self.overlap_thresh,
  287. map_type=self.map_type,
  288. is_bbox_normalized=False,
  289. evaluate_difficult=self.evaluate_difficult,
  290. catid2name=self.catid2name,
  291. classwise=self.classwise)
  292. self.reset()
  293. def reset(self):
  294. self.result_bbox = []
  295. self.detection_map.reset()
  296. def update(self, inputs, outputs):
  297. outs = {}
  298. # outputs Tensor -> numpy.ndarray
  299. for k, v in outputs.items():
  300. outs[k] = v.numpy() if isinstance(v, paddle.Tensor) else v
  301. im_id = inputs['im_id']
  302. outs['im_id'] = im_id.numpy() if isinstance(im_id,
  303. paddle.Tensor) else im_id
  304. infer_results = get_infer_results(
  305. outs, self.clsid2catid, bias=self.bias)
  306. self.result_bbox += infer_results[
  307. 'bbox'] if 'bbox' in infer_results else []
  308. bbox = [b['bbox'] for b in self.result_bbox]
  309. score = [b['score'] for b in self.result_bbox]
  310. label = [b['category_id'] for b in self.result_bbox]
  311. label = [self.catid2clsid[e] for e in label]
  312. gt_box = [
  313. e['bbox'] for e in self.gt_anno['annotations']
  314. if e['image_id'] == outs['im_id']
  315. ]
  316. gt_label = [
  317. e['category_id'] for e in self.gt_anno['annotations']
  318. if e['image_id'] == outs['im_id']
  319. ]
  320. gt_label = [self.catid2clsid[e] for e in gt_label]
  321. self.detection_map.update(bbox, score, label, gt_box, gt_label)
  322. def accumulate(self):
  323. if len(self.result_bbox) > 0:
  324. output = "bbox.json"
  325. if self.output_eval:
  326. output = os.path.join(self.output_eval, output)
  327. with open(output, 'w') as f:
  328. json.dump(self.result_bbox, f)
  329. logger.info('The bbox result is saved to bbox.json.')
  330. if self.save_prediction_only:
  331. logger.info('The bbox result is saved to {} and do not '
  332. 'evaluate the mAP.'.format(output))
  333. else:
  334. logger.info("Accumulating evaluatation results...")
  335. self.detection_map.accumulate()
  336. def log(self):
  337. map_stat = 100. * self.detection_map.get_map()
  338. logger.info("mAP({:.2f}, {}) = {:.2f}%".format(
  339. self.overlap_thresh, self.map_type, map_stat))
  340. def get_results(self):
  341. return {'bbox': [self.detection_map.get_map()]}