val.py 7.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. import numpy as np
  16. import time
  17. import paddle
  18. import paddle.nn.functional as F
  19. from paddlex.paddleseg.utils import metrics, TimeAverager, calculate_eta, logger, progbar
  20. from paddlex.paddleseg.core import infer
  21. np.set_printoptions(suppress=True)
  22. def evaluate(model,
  23. eval_dataset,
  24. aug_eval=False,
  25. scales=1.0,
  26. flip_horizontal=True,
  27. flip_vertical=False,
  28. is_slide=False,
  29. stride=None,
  30. crop_size=None,
  31. num_workers=0,
  32. print_detail=True):
  33. """
  34. Launch evalution.
  35. Args:
  36. model(nn.Layer): A sementic segmentation model.
  37. eval_dataset (paddle.io.Dataset): Used to read and process validation datasets.
  38. aug_eval (bool, optional): Whether to use mulit-scales and flip augment for evaluation. Default: False.
  39. scales (list|float, optional): Scales for augment. It is valid when `aug_eval` is True. Default: 1.0.
  40. flip_horizontal (bool, optional): Whether to use flip horizontally augment. It is valid when `aug_eval` is True. Default: True.
  41. flip_vertical (bool, optional): Whether to use flip vertically augment. It is valid when `aug_eval` is True. Default: False.
  42. is_slide (bool, optional): Whether to evaluate by sliding window. Default: False.
  43. stride (tuple|list, optional): The stride of sliding window, the first is width and the second is height.
  44. It should be provided when `is_slide` is True.
  45. crop_size (tuple|list, optional): The crop size of sliding window, the first is width and the second is height.
  46. It should be provided when `is_slide` is True.
  47. num_workers (int, optional): Num workers for data loader. Default: 0.
  48. print_detail (bool, optional): Whether to print detailed information about the evaluation process. Default: True.
  49. Returns:
  50. float: The mIoU of validation datasets.
  51. float: The accuracy of validation datasets.
  52. """
  53. model.eval()
  54. nranks = paddle.distributed.ParallelEnv().nranks
  55. local_rank = paddle.distributed.ParallelEnv().local_rank
  56. if nranks > 1:
  57. # Initialize parallel environment if not done.
  58. if not paddle.distributed.parallel.parallel_helper._is_parallel_ctx_initialized(
  59. ):
  60. paddle.distributed.init_parallel_env()
  61. batch_sampler = paddle.io.DistributedBatchSampler(
  62. eval_dataset, batch_size=1, shuffle=False, drop_last=False)
  63. loader = paddle.io.DataLoader(
  64. eval_dataset,
  65. batch_sampler=batch_sampler,
  66. num_workers=num_workers,
  67. return_list=True, )
  68. total_iters = len(loader)
  69. intersect_area_all = 0
  70. pred_area_all = 0
  71. label_area_all = 0
  72. if print_detail:
  73. logger.info("Start evaluating (total_samples: {}, total_iters: {})...".
  74. format(len(eval_dataset), total_iters))
  75. progbar_val = progbar.Progbar(target=total_iters, verbose=1)
  76. reader_cost_averager = TimeAverager()
  77. batch_cost_averager = TimeAverager()
  78. batch_start = time.time()
  79. with paddle.no_grad():
  80. for iter, (im, label) in enumerate(loader):
  81. reader_cost_averager.record(time.time() - batch_start)
  82. label = label.astype('int64')
  83. ori_shape = label.shape[-2:]
  84. if aug_eval:
  85. pred = infer.aug_inference(
  86. model,
  87. im,
  88. ori_shape=ori_shape,
  89. transforms=eval_dataset.transforms.transforms,
  90. scales=scales,
  91. flip_horizontal=flip_horizontal,
  92. flip_vertical=flip_vertical,
  93. is_slide=is_slide,
  94. stride=stride,
  95. crop_size=crop_size)
  96. else:
  97. pred = infer.inference(
  98. model,
  99. im,
  100. ori_shape=ori_shape,
  101. transforms=eval_dataset.transforms.transforms,
  102. is_slide=is_slide,
  103. stride=stride,
  104. crop_size=crop_size)
  105. intersect_area, pred_area, label_area = metrics.calculate_area(
  106. pred,
  107. label,
  108. eval_dataset.num_classes,
  109. ignore_index=eval_dataset.ignore_index)
  110. # Gather from all ranks
  111. if nranks > 1:
  112. intersect_area_list = []
  113. pred_area_list = []
  114. label_area_list = []
  115. paddle.distributed.all_gather(intersect_area_list,
  116. intersect_area)
  117. paddle.distributed.all_gather(pred_area_list, pred_area)
  118. paddle.distributed.all_gather(label_area_list, label_area)
  119. # Some image has been evaluated and should be eliminated in last iter
  120. if (iter + 1) * nranks > len(eval_dataset):
  121. valid = len(eval_dataset) - iter * nranks
  122. intersect_area_list = intersect_area_list[:valid]
  123. pred_area_list = pred_area_list[:valid]
  124. label_area_list = label_area_list[:valid]
  125. for i in range(len(intersect_area_list)):
  126. intersect_area_all = intersect_area_all + intersect_area_list[
  127. i]
  128. pred_area_all = pred_area_all + pred_area_list[i]
  129. label_area_all = label_area_all + label_area_list[i]
  130. else:
  131. intersect_area_all = intersect_area_all + intersect_area
  132. pred_area_all = pred_area_all + pred_area
  133. label_area_all = label_area_all + label_area
  134. batch_cost_averager.record(
  135. time.time() - batch_start, num_samples=len(label))
  136. batch_cost = batch_cost_averager.get_average()
  137. reader_cost = reader_cost_averager.get_average()
  138. if local_rank == 0 and print_detail:
  139. progbar_val.update(iter + 1, [('batch_cost', batch_cost),
  140. ('reader cost', reader_cost)])
  141. reader_cost_averager.reset()
  142. batch_cost_averager.reset()
  143. batch_start = time.time()
  144. class_iou, miou = metrics.mean_iou(intersect_area_all, pred_area_all,
  145. label_area_all)
  146. class_acc, acc = metrics.accuracy(intersect_area_all, pred_area_all)
  147. kappa = metrics.kappa(intersect_area_all, pred_area_all, label_area_all)
  148. if print_detail:
  149. logger.info(
  150. "[EVAL] #Images: {} mIoU: {:.4f} Acc: {:.4f} Kappa: {:.4f} ".
  151. format(len(eval_dataset), miou, acc, kappa))
  152. logger.info("[EVAL] Class IoU: \n" + str(np.round(class_iou, 4)))
  153. logger.info("[EVAL] Class Acc: \n" + str(np.round(class_acc, 4)))
  154. return miou, acc, class_iou, class_acc, kappa