topk_eval.py 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118
  1. # copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. import json
  16. import argparse
  17. from paddle import nn
  18. import paddle
  19. from ....utils import logging
  20. def parse_args():
  21. """Parse all arguments"""
  22. parser = argparse.ArgumentParser()
  23. parser.add_argument("--prediction_json_path", type=str, default="./pre_res.json")
  24. parser.add_argument("--gt_val_path", type=str, default="./val.txt")
  25. parser.add_argument("--image_dir", type=str)
  26. parser.add_argument("--num_classes", type=int)
  27. args = parser.parse_args()
  28. return args
  29. class AvgMetrics(nn.Layer):
  30. """Average metrics"""
  31. def __init__(self):
  32. super().__init__()
  33. self.avg_meters = {}
  34. @property
  35. def avg(self):
  36. """Return average value of each metric"""
  37. if self.avg_meters:
  38. for metric_key in self.avg_meters:
  39. return self.avg_meters[metric_key].avg
  40. @property
  41. def avg_info(self):
  42. """Return a formatted string of average values and names"""
  43. return ", ".join([self.avg_meters[key].avg_info for key in self.avg_meters])
  44. class TopkAcc(AvgMetrics):
  45. """Top-k accuracy metric"""
  46. def __init__(self, topk=(1, 5)):
  47. super().__init__()
  48. assert isinstance(topk, (int, list, tuple))
  49. if isinstance(topk, int):
  50. topk = [topk]
  51. self.topk = topk
  52. self.warned = False
  53. def forward(self, x, label):
  54. """forward function"""
  55. if isinstance(x, dict):
  56. x = x["logits"]
  57. output_dims = x.shape[-1]
  58. metric_dict = dict()
  59. for idx, k in enumerate(self.topk):
  60. if output_dims < k:
  61. if not self.warned:
  62. msg = f"The output dims({output_dims}) is less than k({k}), so the Top-{k} metric is meaningless."
  63. logging.info(msg)
  64. self.warned = True
  65. metric_dict[f"top{k}"] = 1
  66. else:
  67. metric_dict[f"top{k}"] = paddle.metric.accuracy(x, label, k=k).item()
  68. return metric_dict
  69. def prase_pt_info(pt_info, num_classes):
  70. """Parse prediction information to probability vector"""
  71. pre_list = [0.0] * num_classes
  72. for idx, val in zip(pt_info["class_ids"], pt_info["scores"]):
  73. pre_list[idx] = val
  74. return pre_list
  75. def main(args):
  76. """main function"""
  77. with open(args.prediction_json_path, "r") as fp:
  78. predication_result = json.load(fp)
  79. gt_info = {}
  80. pred = []
  81. label = []
  82. for line in open(args.gt_val_path):
  83. img_file, gt_label = line.strip().split(" ")
  84. img_file = img_file.split("/")[-1]
  85. gt_info[img_file] = int(gt_label)
  86. for pt_info in predication_result:
  87. img_file = os.path.relpath(pt_info["file_name"], args.image_dir)
  88. pred.append(prase_pt_info(pt_info, args.num_classes))
  89. label.append([gt_info[img_file]])
  90. metric_dict = TopkAcc()(paddle.to_tensor(pred), paddle.to_tensor(label))
  91. logging.info(metric_dict)
  92. if __name__ == "__main__":
  93. args = parse_args()
  94. main(args)