metrics.py 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308
  1. # copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import numpy as np
  15. import paddle
  16. import paddle.nn as nn
  17. import paddle.nn.functional as F
  18. from sklearn.metrics import hamming_loss
  19. from sklearn.metrics import accuracy_score as accuracy_metric
  20. from sklearn.metrics import multilabel_confusion_matrix
  21. from sklearn.preprocessing import binarize
  22. class TopkAcc(nn.Layer):
  23. def __init__(self, topk=(1, 5)):
  24. super().__init__()
  25. assert isinstance(topk, (int, list, tuple))
  26. if isinstance(topk, int):
  27. topk = [topk]
  28. self.topk = topk
  29. def forward(self, x, label):
  30. if isinstance(x, dict):
  31. x = x["logits"]
  32. metric_dict = dict()
  33. for k in self.topk:
  34. metric_dict["top{}".format(k)] = paddle.metric.accuracy(
  35. x, label, k=k)
  36. return metric_dict
  37. class mAP(nn.Layer):
  38. def __init__(self):
  39. super().__init__()
  40. def forward(self, similarities_matrix, query_img_id, gallery_img_id,
  41. keep_mask):
  42. metric_dict = dict()
  43. choosen_indices = paddle.argsort(
  44. similarities_matrix, axis=1, descending=True)
  45. gallery_labels_transpose = paddle.transpose(gallery_img_id, [1, 0])
  46. gallery_labels_transpose = paddle.broadcast_to(
  47. gallery_labels_transpose,
  48. shape=[
  49. choosen_indices.shape[0], gallery_labels_transpose.shape[1]
  50. ])
  51. choosen_label = paddle.index_sample(gallery_labels_transpose,
  52. choosen_indices)
  53. equal_flag = paddle.equal(choosen_label, query_img_id)
  54. if keep_mask is not None:
  55. keep_mask = paddle.index_sample(
  56. keep_mask.astype('float32'), choosen_indices)
  57. equal_flag = paddle.logical_and(equal_flag,
  58. keep_mask.astype('bool'))
  59. equal_flag = paddle.cast(equal_flag, 'float32')
  60. num_rel = paddle.sum(equal_flag, axis=1)
  61. num_rel = paddle.greater_than(num_rel, paddle.to_tensor(0.))
  62. num_rel_index = paddle.nonzero(num_rel.astype("int"))
  63. num_rel_index = paddle.reshape(num_rel_index, [num_rel_index.shape[0]])
  64. equal_flag = paddle.index_select(equal_flag, num_rel_index, axis=0)
  65. acc_sum = paddle.cumsum(equal_flag, axis=1)
  66. div = paddle.arange(acc_sum.shape[1]).astype("float32") + 1
  67. precision = paddle.divide(acc_sum, div)
  68. #calc map
  69. precision_mask = paddle.multiply(equal_flag, precision)
  70. ap = paddle.sum(precision_mask, axis=1) / paddle.sum(equal_flag,
  71. axis=1)
  72. metric_dict["mAP"] = paddle.mean(ap).numpy()[0]
  73. return metric_dict
  74. class mINP(nn.Layer):
  75. def __init__(self):
  76. super().__init__()
  77. def forward(self, similarities_matrix, query_img_id, gallery_img_id,
  78. keep_mask):
  79. metric_dict = dict()
  80. choosen_indices = paddle.argsort(
  81. similarities_matrix, axis=1, descending=True)
  82. gallery_labels_transpose = paddle.transpose(gallery_img_id, [1, 0])
  83. gallery_labels_transpose = paddle.broadcast_to(
  84. gallery_labels_transpose,
  85. shape=[
  86. choosen_indices.shape[0], gallery_labels_transpose.shape[1]
  87. ])
  88. choosen_label = paddle.index_sample(gallery_labels_transpose,
  89. choosen_indices)
  90. equal_flag = paddle.equal(choosen_label, query_img_id)
  91. if keep_mask is not None:
  92. keep_mask = paddle.index_sample(
  93. keep_mask.astype('float32'), choosen_indices)
  94. equal_flag = paddle.logical_and(equal_flag,
  95. keep_mask.astype('bool'))
  96. equal_flag = paddle.cast(equal_flag, 'float32')
  97. num_rel = paddle.sum(equal_flag, axis=1)
  98. num_rel = paddle.greater_than(num_rel, paddle.to_tensor(0.))
  99. num_rel_index = paddle.nonzero(num_rel.astype("int"))
  100. num_rel_index = paddle.reshape(num_rel_index, [num_rel_index.shape[0]])
  101. equal_flag = paddle.index_select(equal_flag, num_rel_index, axis=0)
  102. #do accumulative sum
  103. div = paddle.arange(equal_flag.shape[1]).astype("float32") + 2
  104. minus = paddle.divide(equal_flag, div)
  105. auxilary = paddle.subtract(equal_flag, minus)
  106. hard_index = paddle.argmax(auxilary, axis=1).astype("float32")
  107. all_INP = paddle.divide(paddle.sum(equal_flag, axis=1), hard_index)
  108. mINP = paddle.mean(all_INP)
  109. metric_dict["mINP"] = mINP.numpy()[0]
  110. return metric_dict
  111. class Recallk(nn.Layer):
  112. def __init__(self, topk=(1, 5)):
  113. super().__init__()
  114. assert isinstance(topk, (int, list, tuple))
  115. if isinstance(topk, int):
  116. topk = [topk]
  117. self.topk = topk
  118. def forward(self, similarities_matrix, query_img_id, gallery_img_id,
  119. keep_mask):
  120. metric_dict = dict()
  121. #get cmc
  122. choosen_indices = paddle.argsort(
  123. similarities_matrix, axis=1, descending=True)
  124. gallery_labels_transpose = paddle.transpose(gallery_img_id, [1, 0])
  125. gallery_labels_transpose = paddle.broadcast_to(
  126. gallery_labels_transpose,
  127. shape=[
  128. choosen_indices.shape[0], gallery_labels_transpose.shape[1]
  129. ])
  130. choosen_label = paddle.index_sample(gallery_labels_transpose,
  131. choosen_indices)
  132. equal_flag = paddle.equal(choosen_label, query_img_id)
  133. if keep_mask is not None:
  134. keep_mask = paddle.index_sample(
  135. keep_mask.astype('float32'), choosen_indices)
  136. equal_flag = paddle.logical_and(equal_flag,
  137. keep_mask.astype('bool'))
  138. equal_flag = paddle.cast(equal_flag, 'float32')
  139. real_query_num = paddle.sum(equal_flag, axis=1)
  140. real_query_num = paddle.sum(
  141. paddle.greater_than(real_query_num, paddle.to_tensor(0.)).astype(
  142. "float32"))
  143. acc_sum = paddle.cumsum(equal_flag, axis=1)
  144. mask = paddle.greater_than(acc_sum,
  145. paddle.to_tensor(0.)).astype("float32")
  146. all_cmc = (paddle.sum(mask, axis=0) / real_query_num).numpy()
  147. for k in self.topk:
  148. metric_dict["recall{}".format(k)] = all_cmc[k - 1]
  149. return metric_dict
  150. class Precisionk(nn.Layer):
  151. def __init__(self, topk=(1, 5)):
  152. super().__init__()
  153. assert isinstance(topk, (int, list, tuple))
  154. if isinstance(topk, int):
  155. topk = [topk]
  156. self.topk = topk
  157. def forward(self, similarities_matrix, query_img_id, gallery_img_id,
  158. keep_mask):
  159. metric_dict = dict()
  160. #get cmc
  161. choosen_indices = paddle.argsort(
  162. similarities_matrix, axis=1, descending=True)
  163. gallery_labels_transpose = paddle.transpose(gallery_img_id, [1, 0])
  164. gallery_labels_transpose = paddle.broadcast_to(
  165. gallery_labels_transpose,
  166. shape=[
  167. choosen_indices.shape[0], gallery_labels_transpose.shape[1]
  168. ])
  169. choosen_label = paddle.index_sample(gallery_labels_transpose,
  170. choosen_indices)
  171. equal_flag = paddle.equal(choosen_label, query_img_id)
  172. if keep_mask is not None:
  173. keep_mask = paddle.index_sample(
  174. keep_mask.astype('float32'), choosen_indices)
  175. equal_flag = paddle.logical_and(equal_flag,
  176. keep_mask.astype('bool'))
  177. equal_flag = paddle.cast(equal_flag, 'float32')
  178. Ns = paddle.arange(gallery_img_id.shape[0]) + 1
  179. equal_flag_cumsum = paddle.cumsum(equal_flag, axis=1)
  180. Precision_at_k = (paddle.mean(equal_flag_cumsum, axis=0) / Ns).numpy()
  181. for k in self.topk:
  182. metric_dict["precision@{}".format(k)] = Precision_at_k[k - 1]
  183. return metric_dict
  184. class DistillationTopkAcc(TopkAcc):
  185. def __init__(self, model_key, feature_key=None, topk=(1, 5)):
  186. super().__init__(topk=topk)
  187. self.model_key = model_key
  188. self.feature_key = feature_key
  189. def forward(self, x, label):
  190. x = x[self.model_key]
  191. if self.feature_key is not None:
  192. x = x[self.feature_key]
  193. return super().forward(x, label)
  194. class GoogLeNetTopkAcc(TopkAcc):
  195. def __init__(self, topk=(1, 5)):
  196. super().__init__()
  197. assert isinstance(topk, (int, list, tuple))
  198. if isinstance(topk, int):
  199. topk = [topk]
  200. self.topk = topk
  201. def forward(self, x, label):
  202. return super().forward(x[0], label)
  203. class MutiLabelMetric(object):
  204. def __init__(self):
  205. pass
  206. def _multi_hot_encode(self, logits, threshold=0.5):
  207. return binarize(logits, threshold=threshold)
  208. def __call__(self, output):
  209. output = F.sigmoid(output)
  210. preds = self._multi_hot_encode(logits=output.numpy(), threshold=0.5)
  211. return preds
  212. class HammingDistance(MutiLabelMetric):
  213. """
  214. Soft metric based label for multilabel classification
  215. Returns:
  216. The smaller the return value is, the better model is.
  217. """
  218. def __init__(self):
  219. super().__init__()
  220. def __call__(self, output, target):
  221. preds = super().__call__(output)
  222. metric_dict = dict()
  223. metric_dict["HammingDistance"] = paddle.to_tensor(
  224. hamming_loss(target, preds))
  225. return metric_dict
  226. class AccuracyScore(MutiLabelMetric):
  227. """
  228. Hard metric for multilabel classification
  229. Args:
  230. base: ["sample", "label"], default="sample"
  231. if "sample", return metric score based sample,
  232. if "label", return metric score based label.
  233. Returns:
  234. accuracy:
  235. """
  236. def __init__(self, base="label"):
  237. super().__init__()
  238. assert base in ["sample", "label"
  239. ], 'must be one of ["sample", "label"]'
  240. self.base = base
  241. def __call__(self, output, target):
  242. preds = super().__call__(output)
  243. metric_dict = dict()
  244. if self.base == "sample":
  245. accuracy = accuracy_metric(target, preds)
  246. elif self.base == "label":
  247. mcm = multilabel_confusion_matrix(target, preds)
  248. tns = mcm[:, 0, 0]
  249. fns = mcm[:, 1, 0]
  250. tps = mcm[:, 1, 1]
  251. fps = mcm[:, 0, 1]
  252. accuracy = (sum(tps) + sum(tns)) / (
  253. sum(tps) + sum(tns) + sum(fns) + sum(fps))
  254. precision = sum(tps) / (sum(tps) + sum(fps))
  255. recall = sum(tps) / (sum(tps) + sum(fns))
  256. F1 = 2 * (accuracy * recall) / (accuracy + recall)
  257. metric_dict["AccuracyScore"] = paddle.to_tensor(accuracy)
  258. return metric_dict