coco_utils.py 8.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import, division, print_function
  15. import copy
  16. import sys
  17. import numpy as np
  18. from . import fd_logging as logging
  19. from .json_results import (
  20. get_det_poly_res,
  21. get_det_res,
  22. get_seg_res,
  23. get_solov2_segm_res,
  24. )
  25. from .map_utils import draw_pr_curve
  26. def loadRes(coco_obj, anns):
  27. """
  28. Load result file and return a result api object.
  29. :param resFile (str) : file name of result file
  30. :return: res (obj) : result api object
  31. """
  32. # This function has the same functionality as pycocotools.COCO.loadRes,
  33. # except that the input anns is list of results rather than a json file.
  34. # Refer to
  35. # https://github.com/cocodataset/cocoapi/blob/8c9bcc3cf640524c4c20a9c40e89cb6a2f2fa0e9/PythonAPI/pycocotools/coco.py#L305,
  36. # matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
  37. # or matplotlib.backends is imported for the first time
  38. # pycocotools import matplotlib
  39. import matplotlib
  40. matplotlib.use("Agg")
  41. import time
  42. import pycocotools.mask as maskUtils
  43. from pycocotools.coco import COCO
  44. res = COCO()
  45. res.dataset["images"] = [img for img in coco_obj.dataset["images"]]
  46. tic = time.time()
  47. assert isinstance(anns) == list, "results in not an array of objects"
  48. annsImgIds = [ann["image_id"] for ann in anns]
  49. assert set(annsImgIds) == (
  50. set(annsImgIds) & set(coco_obj.getImgIds())
  51. ), "Results do not correspond to current coco set"
  52. if "caption" in anns[0]:
  53. imgIds = set([img["id"] for img in res.dataset["images"]]) & set(
  54. [ann["image_id"] for ann in anns]
  55. )
  56. res.dataset["images"] = [
  57. img for img in res.dataset["images"] if img["id"] in imgIds
  58. ]
  59. for id, ann in enumerate(anns):
  60. ann["id"] = id + 1
  61. elif "bbox" in anns[0] and not anns[0]["bbox"] == []:
  62. res.dataset["categories"] = copy.deepcopy(coco_obj.dataset["categories"])
  63. for id, ann in enumerate(anns):
  64. bb = ann["bbox"]
  65. x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]]
  66. if not "segmentation" in ann:
  67. ann["segmentation"] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
  68. ann["area"] = bb[2] * bb[3]
  69. ann["id"] = id + 1
  70. ann["iscrowd"] = 0
  71. elif "segmentation" in anns[0]:
  72. res.dataset["categories"] = copy.deepcopy(coco_obj.dataset["categories"])
  73. for id, ann in enumerate(anns):
  74. # now only support compressed RLE format as segmentation results
  75. ann["area"] = maskUtils.area(ann["segmentation"])
  76. if not "bbox" in ann:
  77. ann["bbox"] = maskUtils.toBbox(ann["segmentation"])
  78. ann["id"] = id + 1
  79. ann["iscrowd"] = 0
  80. elif "keypoints" in anns[0]:
  81. res.dataset["categories"] = copy.deepcopy(coco_obj.dataset["categories"])
  82. for id, ann in enumerate(anns):
  83. s = ann["keypoints"]
  84. x = s[0::3]
  85. y = s[1::3]
  86. x0, x1, y0, y1 = np.min(x), np.max(x), np.min(y), np.max(y)
  87. ann["area"] = (x1 - x0) * (y1 - y0)
  88. ann["id"] = id + 1
  89. ann["bbox"] = [x0, y0, x1 - x0, y1 - y0]
  90. res.dataset["annotations"] = anns
  91. res.createIndex()
  92. return res
  93. def get_infer_results(outs, catid, bias=0):
  94. """
  95. Get result at the stage of inference.
  96. The output format is dictionary containing bbox or mask result.
  97. For example, bbox result is a list and each element contains
  98. image_id, category_id, bbox and score.
  99. """
  100. if outs is None or len(outs) == 0:
  101. raise ValueError(
  102. "The number of valid detection result if zero. Please use reasonable model and check input data."
  103. )
  104. im_id = outs["im_id"]
  105. infer_res = {}
  106. if "bbox" in outs:
  107. if len(outs["bbox"]) > 0 and len(outs["bbox"][0]) > 6:
  108. infer_res["bbox"] = get_det_poly_res(
  109. outs["bbox"], outs["bbox_num"], im_id, catid, bias=bias
  110. )
  111. else:
  112. infer_res["bbox"] = get_det_res(
  113. outs["bbox"], outs["bbox_num"], im_id, catid, bias=bias
  114. )
  115. if "mask" in outs:
  116. # mask post process
  117. infer_res["mask"] = get_seg_res(
  118. outs["mask"], outs["bbox"], outs["bbox_num"], im_id, catid
  119. )
  120. if "segm" in outs:
  121. infer_res["segm"] = get_solov2_segm_res(outs, im_id, catid)
  122. return infer_res
  123. def cocoapi_eval(
  124. anns,
  125. style,
  126. coco_gt=None,
  127. anno_file=None,
  128. max_dets=(100, 300, 1000),
  129. classwise=False,
  130. ):
  131. """
  132. Args:
  133. anns: Evaluation result.
  134. style (str): COCOeval style, can be `bbox` , `segm` and `proposal`.
  135. coco_gt (str): Whether to load COCOAPI through anno_file,
  136. eg: coco_gt = COCO(anno_file)
  137. anno_file (str): COCO annotations file.
  138. max_dets (tuple): COCO evaluation maxDets.
  139. classwise (bool): Whether per-category AP and draw P-R Curve or not.
  140. """
  141. assert coco_gt is not None or anno_file is not None
  142. from pycocotools.coco import COCO
  143. from pycocotools.cocoeval import COCOeval
  144. if coco_gt is None:
  145. coco_gt = COCO(anno_file)
  146. logging.info("Start evaluate...")
  147. coco_dt = loadRes(coco_gt, anns)
  148. if style == "proposal":
  149. coco_eval = COCOeval(coco_gt, coco_dt, "bbox")
  150. coco_eval.params.useCats = 0
  151. coco_eval.params.maxDets = list(max_dets)
  152. else:
  153. coco_eval = COCOeval(coco_gt, coco_dt, style)
  154. coco_eval.evaluate()
  155. coco_eval.accumulate()
  156. coco_eval.summarize()
  157. if classwise:
  158. # Compute per-category AP and PR curve
  159. try:
  160. from terminaltables import AsciiTable
  161. except Exception as e:
  162. logging.error(
  163. "terminaltables not found, plaese install terminaltables. "
  164. "for example: `pip install terminaltables`."
  165. )
  166. raise e
  167. precisions = coco_eval.eval["precision"]
  168. cat_ids = coco_gt.getCatIds()
  169. # precision: (iou, recall, cls, area range, max dets)
  170. assert len(cat_ids) == precisions.shape[2]
  171. results_per_category = []
  172. for idx, catId in enumerate(cat_ids):
  173. # area range index 0: all area ranges
  174. # max dets index -1: typically 100 per image
  175. nm = coco_gt.loadCats(catId)[0]
  176. precision = precisions[:, :, idx, 0, -1]
  177. precision = precision[precision > -1]
  178. if precision.size:
  179. ap = np.mean(precision)
  180. else:
  181. ap = float("nan")
  182. results_per_category.append((str(nm["name"]), "{:0.3f}".format(float(ap))))
  183. pr_array = precisions[0, :, idx, 0, 2]
  184. recall_array = np.arange(0.0, 1.01, 0.01)
  185. draw_pr_curve(
  186. pr_array,
  187. recall_array,
  188. out_dir=style + "_pr_curve",
  189. file_name="{}_precision_recall_curve.jpg".format(nm["name"]),
  190. )
  191. num_columns = min(6, len(results_per_category) * 2)
  192. import itertools
  193. results_flatten = list(itertools.chain(*results_per_category))
  194. headers = ["category", "AP"] * (num_columns // 2)
  195. results_2d = itertools.zip_longest(
  196. *[results_flatten[i::num_columns] for i in range(num_columns)]
  197. )
  198. table_data = [headers]
  199. table_data += [result for result in results_2d]
  200. table = AsciiTable(table_data)
  201. logging.info("Per-category of {} AP: \n{}".format(style, table.table))
  202. logging.info(
  203. "per-category PR curve has output to {} folder.".format(style + "_pr_curve")
  204. )
  205. # flush coco evaluation result
  206. sys.stdout.flush()
  207. return coco_eval.stats