mcmot_metrics.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. from __future__ import division
  16. from __future__ import print_function
  17. import os
  18. import copy
  19. import sys
  20. import math
  21. from collections import defaultdict
  22. from motmetrics.math_util import quiet_divide
  23. import numpy as np
  24. import pandas as pd
  25. from .metrics import Metric
  26. import motmetrics as mm
  27. import openpyxl
  28. metrics = mm.metrics.motchallenge_metrics
  29. mh = mm.metrics.create()
  30. from paddlex.ppdet.utils.logger import setup_logger
  31. logger = setup_logger(__name__)
  32. __all__ = ['MCMOTEvaluator', 'MCMOTMetric']
  33. METRICS_LIST = [
  34. 'num_frames', 'num_matches', 'num_switches', 'num_transfer', 'num_ascend',
  35. 'num_migrate', 'num_false_positives', 'num_misses', 'num_detections',
  36. 'num_objects', 'num_predictions', 'num_unique_objects', 'mostly_tracked',
  37. 'partially_tracked', 'mostly_lost', 'num_fragmentations', 'motp', 'mota',
  38. 'precision', 'recall', 'idfp', 'idfn', 'idtp', 'idp', 'idr', 'idf1'
  39. ]
  40. NAME_MAP = {
  41. 'num_frames': 'num_frames',
  42. 'num_matches': 'num_matches',
  43. 'num_switches': 'IDs',
  44. 'num_transfer': 'IDt',
  45. 'num_ascend': 'IDa',
  46. 'num_migrate': 'IDm',
  47. 'num_false_positives': 'FP',
  48. 'num_misses': 'FN',
  49. 'num_detections': 'num_detections',
  50. 'num_objects': 'num_objects',
  51. 'num_predictions': 'num_predictions',
  52. 'num_unique_objects': 'GT',
  53. 'mostly_tracked': 'MT',
  54. 'partially_tracked': 'partially_tracked',
  55. 'mostly_lost': 'ML',
  56. 'num_fragmentations': 'FM',
  57. 'motp': 'MOTP',
  58. 'mota': 'MOTA',
  59. 'precision': 'Prcn',
  60. 'recall': 'Rcll',
  61. 'idfp': 'idfp',
  62. 'idfn': 'idfn',
  63. 'idtp': 'idtp',
  64. 'idp': 'IDP',
  65. 'idr': 'IDR',
  66. 'idf1': 'IDF1'
  67. }
  68. def parse_accs_metrics(seq_acc, index_name, verbose=False):
  69. """
  70. Parse the evaluation indicators of multiple MOTAccumulator
  71. """
  72. mh = mm.metrics.create()
  73. summary = MCMOTEvaluator.get_summary(seq_acc, index_name, METRICS_LIST)
  74. summary.loc['OVERALL', 'motp'] = (summary['motp'] * summary['num_detections']).sum() / \
  75. summary.loc['OVERALL', 'num_detections']
  76. if verbose:
  77. strsummary = mm.io.render_summary(
  78. summary, formatters=mh.formatters, namemap=NAME_MAP)
  79. print(strsummary)
  80. return summary
  81. def seqs_overall_metrics(summary_df, verbose=False):
  82. """
  83. Calculate overall metrics for multiple sequences
  84. """
  85. add_col = [
  86. 'num_frames', 'num_matches', 'num_switches', 'num_transfer',
  87. 'num_ascend', 'num_migrate', 'num_false_positives', 'num_misses',
  88. 'num_detections', 'num_objects', 'num_predictions',
  89. 'num_unique_objects', 'mostly_tracked', 'partially_tracked',
  90. 'mostly_lost', 'num_fragmentations', 'idfp', 'idfn', 'idtp'
  91. ]
  92. calc_col = ['motp', 'mota', 'precision', 'recall', 'idp', 'idr', 'idf1']
  93. calc_df = summary_df.copy()
  94. overall_dic = {}
  95. for col in add_col:
  96. overall_dic[col] = calc_df[col].sum()
  97. for col in calc_col:
  98. overall_dic[col] = getattr(MCMOTMetricOverall, col + '_overall')(
  99. calc_df, overall_dic)
  100. overall_df = pd.DataFrame(overall_dic, index=['overall_calc'])
  101. calc_df = pd.concat([calc_df, overall_df])
  102. if verbose:
  103. mh = mm.metrics.create()
  104. str_calc_df = mm.io.render_summary(
  105. calc_df, formatters=mh.formatters, namemap=NAME_MAP)
  106. print(str_calc_df)
  107. return calc_df
  108. class MCMOTMetricOverall(object):
  109. def motp_overall(summary_df, overall_dic):
  110. motp = quiet_divide((summary_df['motp'] *
  111. summary_df['num_detections']).sum(),
  112. overall_dic['num_detections'])
  113. return motp
  114. def mota_overall(summary_df, overall_dic):
  115. del summary_df
  116. mota = 1. - quiet_divide(
  117. (overall_dic['num_misses'] + overall_dic['num_switches'] +
  118. overall_dic['num_false_positives']), overall_dic['num_objects'])
  119. return mota
  120. def precision_overall(summary_df, overall_dic):
  121. del summary_df
  122. precision = quiet_divide(overall_dic['num_detections'], (
  123. overall_dic['num_false_positives'] + overall_dic['num_detections']
  124. ))
  125. return precision
  126. def recall_overall(summary_df, overall_dic):
  127. del summary_df
  128. recall = quiet_divide(overall_dic['num_detections'],
  129. overall_dic['num_objects'])
  130. return recall
  131. def idp_overall(summary_df, overall_dic):
  132. del summary_df
  133. idp = quiet_divide(overall_dic['idtp'],
  134. (overall_dic['idtp'] + overall_dic['idfp']))
  135. return idp
  136. def idr_overall(summary_df, overall_dic):
  137. del summary_df
  138. idr = quiet_divide(overall_dic['idtp'],
  139. (overall_dic['idtp'] + overall_dic['idfn']))
  140. return idr
  141. def idf1_overall(summary_df, overall_dic):
  142. del summary_df
  143. idf1 = quiet_divide(2. * overall_dic['idtp'], (
  144. overall_dic['num_objects'] + overall_dic['num_predictions']))
  145. return idf1
  146. def read_mcmot_results_union(filename, is_gt, is_ignore):
  147. results_dict = dict()
  148. if os.path.isfile(filename):
  149. all_result = np.loadtxt(filename, delimiter=',')
  150. if all_result.shape[0] == 0 or all_result.shape[1] < 7:
  151. return results_dict
  152. if is_ignore:
  153. return results_dict
  154. if is_gt:
  155. # only for test use
  156. all_result = all_result[all_result[:, 7] != 0]
  157. all_result[:, 7] = all_result[:, 7] - 1
  158. if all_result.shape[0] == 0:
  159. return results_dict
  160. class_unique = np.unique(all_result[:, 7])
  161. last_max_id = 0
  162. result_cls_list = []
  163. for cls in class_unique:
  164. result_cls_split = all_result[all_result[:, 7] == cls]
  165. result_cls_split[:, 1] = result_cls_split[:, 1] + last_max_id
  166. # make sure track id different between every category
  167. last_max_id = max(np.unique(result_cls_split[:, 1])) + 1
  168. result_cls_list.append(result_cls_split)
  169. results_con = np.concatenate(result_cls_list)
  170. for line in range(len(results_con)):
  171. linelist = results_con[line]
  172. fid = int(linelist[0])
  173. if fid < 1:
  174. continue
  175. results_dict.setdefault(fid, list())
  176. if is_gt:
  177. score = 1
  178. else:
  179. score = float(linelist[6])
  180. tlwh = tuple(map(float, linelist[2:6]))
  181. target_id = int(linelist[1])
  182. cls = int(linelist[7])
  183. results_dict[fid].append((tlwh, target_id, cls, score))
  184. return results_dict
  185. def read_mcmot_results(filename, is_gt, is_ignore):
  186. results_dict = dict()
  187. if os.path.isfile(filename):
  188. with open(filename, 'r') as f:
  189. for line in f.readlines():
  190. linelist = line.strip().split(',')
  191. if len(linelist) < 7:
  192. continue
  193. fid = int(linelist[0])
  194. if fid < 1:
  195. continue
  196. cid = int(linelist[7])
  197. if is_gt:
  198. score = 1
  199. # only for test use
  200. cid -= 1
  201. else:
  202. score = float(linelist[6])
  203. cls_result_dict = results_dict.setdefault(cid, dict())
  204. cls_result_dict.setdefault(fid, list())
  205. tlwh = tuple(map(float, linelist[2:6]))
  206. target_id = int(linelist[1])
  207. cls_result_dict[fid].append((tlwh, target_id, score))
  208. return results_dict
  209. def read_results(filename,
  210. data_type,
  211. is_gt=False,
  212. is_ignore=False,
  213. multi_class=False,
  214. union=False):
  215. if data_type in ['mcmot', 'lab']:
  216. if multi_class:
  217. if union:
  218. # The results are evaluated by union all the categories.
  219. # Track IDs between different categories cannot be duplicate.
  220. read_fun = read_mcmot_results_union
  221. else:
  222. # The results are evaluated separately by category.
  223. read_fun = read_mcmot_results
  224. else:
  225. raise ValueError('multi_class: {}, MCMOT should have cls_id.'.
  226. format(multi_class))
  227. else:
  228. raise ValueError('Unknown data type: {}'.format(data_type))
  229. return read_fun(filename, is_gt, is_ignore)
  230. def unzip_objs(objs):
  231. if len(objs) > 0:
  232. tlwhs, ids, scores = zip(*objs)
  233. else:
  234. tlwhs, ids, scores = [], [], []
  235. tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)
  236. return tlwhs, ids, scores
  237. def unzip_objs_cls(objs):
  238. if len(objs) > 0:
  239. tlwhs, ids, cls, scores = zip(*objs)
  240. else:
  241. tlwhs, ids, cls, scores = [], [], [], []
  242. tlwhs = np.asarray(tlwhs, dtype=float).reshape(-1, 4)
  243. ids = np.array(ids)
  244. cls = np.array(cls)
  245. scores = np.array(scores)
  246. return tlwhs, ids, cls, scores
  247. class MCMOTEvaluator(object):
  248. def __init__(self, data_root, seq_name, data_type, num_classes):
  249. self.data_root = data_root
  250. self.seq_name = seq_name
  251. self.data_type = data_type
  252. self.num_classes = num_classes
  253. self.load_annotations()
  254. self.reset_accumulator()
  255. self.class_accs = []
  256. def load_annotations(self):
  257. assert self.data_type == 'mcmot'
  258. self.gt_filename = os.path.join(self.data_root, '../', 'sequences',
  259. '{}.txt'.format(self.seq_name))
  260. if not os.path.exists(self.gt_filename):
  261. logger.warning(
  262. "gt_filename '{}' of MCMOTEvaluator is not exist, so the MOTA will be -INF."
  263. )
  264. def reset_accumulator(self):
  265. import motmetrics as mm
  266. mm.lap.default_solver = 'lap'
  267. self.acc = mm.MOTAccumulator(auto_id=True)
  268. def eval_frame_dict(self, trk_objs, gt_objs, rtn_events=False,
  269. union=False):
  270. import motmetrics as mm
  271. mm.lap.default_solver = 'lap'
  272. if union:
  273. trk_tlwhs, trk_ids, trk_cls = unzip_objs_cls(trk_objs)[:3]
  274. gt_tlwhs, gt_ids, gt_cls = unzip_objs_cls(gt_objs)[:3]
  275. # get distance matrix
  276. iou_distance = mm.distances.iou_matrix(
  277. gt_tlwhs, trk_tlwhs, max_iou=0.5)
  278. # Set the distance between objects of different categories to nan
  279. gt_cls_len = len(gt_cls)
  280. trk_cls_len = len(trk_cls)
  281. # When the number of GT or Trk is 0, iou_distance dimension is (0,0)
  282. if gt_cls_len != 0 and trk_cls_len != 0:
  283. gt_cls = gt_cls.reshape(gt_cls_len, 1)
  284. gt_cls = np.repeat(gt_cls, trk_cls_len, axis=1)
  285. trk_cls = trk_cls.reshape(1, trk_cls_len)
  286. trk_cls = np.repeat(trk_cls, gt_cls_len, axis=0)
  287. iou_distance = np.where(gt_cls == trk_cls, iou_distance,
  288. np.nan)
  289. else:
  290. trk_tlwhs, trk_ids = unzip_objs(trk_objs)[:2]
  291. gt_tlwhs, gt_ids = unzip_objs(gt_objs)[:2]
  292. # get distance matrix
  293. iou_distance = mm.distances.iou_matrix(
  294. gt_tlwhs, trk_tlwhs, max_iou=0.5)
  295. self.acc.update(gt_ids, trk_ids, iou_distance)
  296. if rtn_events and iou_distance.size > 0 and hasattr(self.acc,
  297. 'mot_events'):
  298. events = self.acc.mot_events # only supported by https://github.com/longcw/py-motmetrics
  299. else:
  300. events = None
  301. return events
  302. def eval_file(self, result_filename):
  303. # evaluation of each category
  304. gt_frame_dict = read_results(
  305. self.gt_filename,
  306. self.data_type,
  307. is_gt=True,
  308. multi_class=True,
  309. union=False)
  310. result_frame_dict = read_results(
  311. result_filename,
  312. self.data_type,
  313. is_gt=False,
  314. multi_class=True,
  315. union=False)
  316. for cid in range(self.num_classes):
  317. self.reset_accumulator()
  318. cls_result_frame_dict = result_frame_dict.setdefault(cid, dict())
  319. cls_gt_frame_dict = gt_frame_dict.setdefault(cid, dict())
  320. # only labeled frames will be evaluated
  321. frames = sorted(list(set(cls_gt_frame_dict.keys())))
  322. for frame_id in frames:
  323. trk_objs = cls_result_frame_dict.get(frame_id, [])
  324. gt_objs = cls_gt_frame_dict.get(frame_id, [])
  325. self.eval_frame_dict(trk_objs, gt_objs, rtn_events=False)
  326. self.class_accs.append(self.acc)
  327. return self.class_accs
  328. @staticmethod
  329. def get_summary(accs,
  330. names,
  331. metrics=('mota', 'num_switches', 'idp', 'idr', 'idf1',
  332. 'precision', 'recall')):
  333. import motmetrics as mm
  334. mm.lap.default_solver = 'lap'
  335. names = copy.deepcopy(names)
  336. if metrics is None:
  337. metrics = mm.metrics.motchallenge_metrics
  338. metrics = copy.deepcopy(metrics)
  339. mh = mm.metrics.create()
  340. summary = mh.compute_many(
  341. accs, metrics=metrics, names=names, generate_overall=True)
  342. return summary
  343. @staticmethod
  344. def save_summary(summary, filename):
  345. import pandas as pd
  346. writer = pd.ExcelWriter(filename)
  347. summary.to_excel(writer)
  348. writer.save()
  349. class MCMOTMetric(Metric):
  350. def __init__(self, num_classes, save_summary=False):
  351. self.num_classes = num_classes
  352. self.save_summary = save_summary
  353. self.MCMOTEvaluator = MCMOTEvaluator
  354. self.result_root = None
  355. self.reset()
  356. self.seqs_overall = defaultdict(list)
  357. def reset(self):
  358. self.accs = []
  359. self.seqs = []
  360. def update(self, data_root, seq, data_type, result_root, result_filename):
  361. evaluator = self.MCMOTEvaluator(data_root, seq, data_type,
  362. self.num_classes)
  363. seq_acc = evaluator.eval_file(result_filename)
  364. self.accs.append(seq_acc)
  365. self.seqs.append(seq)
  366. self.result_root = result_root
  367. cls_index_name = [
  368. '{}_{}'.format(seq, i) for i in range(self.num_classes)
  369. ]
  370. summary = parse_accs_metrics(seq_acc, cls_index_name)
  371. summary.rename(
  372. index={'OVERALL': '{}_OVERALL'.format(seq)}, inplace=True)
  373. for row in range(len(summary)):
  374. self.seqs_overall[row].append(summary.iloc[row:row + 1])
  375. def accumulate(self):
  376. self.cls_summary_list = []
  377. for row in range(self.num_classes):
  378. seqs_cls_df = pd.concat(self.seqs_overall[row])
  379. seqs_cls_summary = seqs_overall_metrics(seqs_cls_df)
  380. cls_summary_overall = seqs_cls_summary.iloc[-1:].copy()
  381. cls_summary_overall.rename(
  382. index={'overall_calc': 'overall_calc_{}'.format(row)},
  383. inplace=True)
  384. self.cls_summary_list.append(cls_summary_overall)
  385. def log(self):
  386. seqs_summary = seqs_overall_metrics(
  387. pd.concat(self.seqs_overall[self.num_classes]), verbose=True)
  388. class_summary = seqs_overall_metrics(
  389. pd.concat(self.cls_summary_list), verbose=True)
  390. def get_results(self):
  391. return 1