classification.py 5.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. from __future__ import division
  16. from __future__ import print_function
  17. import time
  18. import platform
  19. import paddle
  20. from paddlex.ppcls.utils.misc import AverageMeter
  21. from paddlex.ppcls.utils import logger
  22. def classification_eval(engine, epoch_id=0):
  23. output_info = dict()
  24. time_info = {
  25. "batch_cost": AverageMeter(
  26. "batch_cost", '.5f', postfix=" s,"),
  27. "reader_cost": AverageMeter(
  28. "reader_cost", ".5f", postfix=" s,"),
  29. }
  30. print_batch_step = engine.config["Global"]["print_batch_step"]
  31. metric_key = None
  32. tic = time.time()
  33. accum_samples = 0
  34. total_samples = len(
  35. engine.eval_dataloader.
  36. dataset) if not engine.use_dali else engine.eval_dataloader.size
  37. max_iter = len(engine.eval_dataloader) - 1 if platform.system(
  38. ) == "Windows" else len(engine.eval_dataloader)
  39. for iter_id, batch in enumerate(engine.eval_dataloader):
  40. if iter_id >= max_iter:
  41. break
  42. if iter_id == 5:
  43. for key in time_info:
  44. time_info[key].reset()
  45. if engine.use_dali:
  46. batch = [
  47. paddle.to_tensor(batch[0]['data']),
  48. paddle.to_tensor(batch[0]['label'])
  49. ]
  50. time_info["reader_cost"].update(time.time() - tic)
  51. batch_size = batch[0].shape[0]
  52. batch[0] = paddle.to_tensor(batch[0]).astype("float32")
  53. if not engine.config["Global"].get("use_multilabel", False):
  54. batch[1] = batch[1].reshape([-1, 1]).astype("int64")
  55. # image input
  56. out = engine.model(batch[0])
  57. # calc loss
  58. if engine.eval_loss_func is not None:
  59. loss_dict = engine.eval_loss_func(out, batch[1])
  60. for key in loss_dict:
  61. if key not in output_info:
  62. output_info[key] = AverageMeter(key, '7.5f')
  63. output_info[key].update(loss_dict[key].numpy()[0], batch_size)
  64. # just for DistributedBatchSampler issue: repeat sampling
  65. current_samples = batch_size * paddle.distributed.get_world_size()
  66. accum_samples += current_samples
  67. # calc metric
  68. if engine.eval_metric_func is not None:
  69. if paddle.distributed.get_world_size() > 1:
  70. label_list = []
  71. paddle.distributed.all_gather(label_list, batch[1])
  72. labels = paddle.concat(label_list, 0)
  73. if isinstance(out, dict):
  74. out = out["logits"]
  75. if isinstance(out, list):
  76. pred = []
  77. for x in out:
  78. pred_list = []
  79. paddle.distributed.all_gather(pred_list, x)
  80. pred_x = paddle.concat(pred_list, 0)
  81. pred.append(pred_x)
  82. else:
  83. pred_list = []
  84. paddle.distributed.all_gather(pred_list, out)
  85. pred = paddle.concat(pred_list, 0)
  86. if accum_samples > total_samples and not engine.use_dali:
  87. pred = pred[:total_samples + current_samples -
  88. accum_samples]
  89. labels = labels[:total_samples + current_samples -
  90. accum_samples]
  91. current_samples = total_samples + current_samples - accum_samples
  92. metric_dict = engine.eval_metric_func(pred, labels)
  93. else:
  94. metric_dict = engine.eval_metric_func(out, batch[1])
  95. for key in metric_dict:
  96. if metric_key is None:
  97. metric_key = key
  98. if key not in output_info:
  99. output_info[key] = AverageMeter(key, '7.5f')
  100. output_info[key].update(metric_dict[key].numpy()[0],
  101. current_samples)
  102. time_info["batch_cost"].update(time.time() - tic)
  103. if iter_id % print_batch_step == 0:
  104. time_msg = "s, ".join([
  105. "{}: {:.5f}".format(key, time_info[key].avg)
  106. for key in time_info
  107. ])
  108. ips_msg = "ips: {:.5f} images/sec".format(
  109. batch_size / time_info["batch_cost"].avg)
  110. metric_msg = ", ".join([
  111. "{}: {:.5f}".format(key, output_info[key].val)
  112. for key in output_info
  113. ])
  114. logger.info("[Eval][Epoch {}][Iter: {}/{}]{}, {}, {}".format(
  115. epoch_id, iter_id,
  116. len(engine.eval_dataloader), metric_msg, time_msg, ips_msg))
  117. tic = time.time()
  118. if engine.use_dali:
  119. engine.eval_dataloader.reset()
  120. metric_msg = ", ".join([
  121. "{}: {:.5f}".format(key, output_info[key].avg) for key in output_info
  122. ])
  123. logger.info("[Eval][Epoch {}][Avg]{}".format(epoch_id, metric_msg))
  124. # do not try to save best eval.model
  125. if engine.eval_metric_func is None:
  126. return -1
  127. # return 1st metric in the dict
  128. return output_info[metric_key].avg