evaluator.py 4.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149
  1. # !/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. ################################################################################
  4. #
  5. # Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved
  6. #
  7. ################################################################################
  8. """
  9. Author: PaddlePaddle Authors
  10. """
  11. import os
  12. from pathlib import Path
  13. from abc import ABC, abstractmethod
  14. from .build_model import build_model
  15. from .utils.device import get_device
  16. from ...utils.misc import AutoRegisterABCMetaClass
  17. from ...utils.config import AttrDict
  18. from ...utils.logging import *
  19. def build_evaluater(config: AttrDict) -> "BaseEvaluator":
  20. """build model evaluater
  21. Args:
  22. config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
  23. Returns:
  24. BaseEvaluator: the evaluater, which is subclass of BaseEvaluator.
  25. """
  26. model_name = config.Global.model
  27. return BaseEvaluator.get(model_name)(config)
  28. class BaseEvaluator(ABC, metaclass=AutoRegisterABCMetaClass):
  29. """ Base Model Evaluator """
  30. __is_base = True
  31. def __init__(self, config):
  32. """Initialize the instance.
  33. Args:
  34. config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
  35. """
  36. self.global_config = config.Global
  37. self.eval_config = config.Evaluate
  38. config_path = self.get_config_path(self.eval_config.weight_path)
  39. if not config_path.exists():
  40. warning(
  41. f"The config file(`{config_path}`) related to weight file(`{self.eval_config.weight_path}`) is not exist, use default instead."
  42. )
  43. config_path = None
  44. self.pdx_config, self.pdx_model = build_model(
  45. self.global_config.model, config_path=config_path)
  46. def get_config_path(self, weight_path):
  47. """
  48. get config path
  49. Args:
  50. weight_path (str): The path to the weight
  51. Returns:
  52. config_path (str): The path to the config
  53. """
  54. config_path = Path(weight_path).parent / "config.yaml"
  55. return config_path
  56. def check_return(self, metrics: dict) -> bool:
  57. """check evaluation metrics
  58. Args:
  59. metrics (dict): evaluation output metrics
  60. Returns:
  61. bool: whether the format of evaluation metrics is legal
  62. """
  63. if not isinstance(metrics, dict):
  64. return False
  65. for metric in metrics:
  66. val = metrics[metric]
  67. if not isinstance(val, float):
  68. return False
  69. return True
  70. def __call__(self) -> dict:
  71. """execute model training
  72. Returns:
  73. dict: the evaluation metrics
  74. """
  75. metrics = self.eval()
  76. assert self.check_return(
  77. metrics
  78. ), f"The return value({metrics}) of Evaluator.eval() is illegal!"
  79. return {"metrics": metrics}
  80. def dump_config(self, config_file_path=None):
  81. """dump the config
  82. Args:
  83. config_file_path (str, optional): the path to save dumped config.
  84. Defaults to None, means that save in `Global.output` as `config.yaml`.
  85. """
  86. if config_file_path is None:
  87. config_file_path = os.path.join(self.global_config.output,
  88. "config.yaml")
  89. self.pdx_config.dump(config_file_path)
  90. def eval(self):
  91. """firstly, update evaluation config, then evaluate model, finally return the evaluation result
  92. """
  93. self.update_config()
  94. # self.dump_config()
  95. evaluate_result = self.pdx_model.evaluate(**self.get_eval_kwargs())
  96. assert evaluate_result.returncode == 0, f"Encountered an unexpected error({evaluate_result.returncode}) in \
  97. evaling!"
  98. return evaluate_result.metrics
  99. def get_device(self, using_gpu_number: int=None) -> str:
  100. """get device setting from config
  101. Args:
  102. using_gpu_number (int, optional): specify GPU number to use.
  103. Defaults to None, means that base on config setting.
  104. Returns:
  105. str: device setting, such as: `gpu:0,1`, `cpu`.
  106. """
  107. return get_device(
  108. self.global_config.device, using_gpu_number=using_gpu_number)
  109. @abstractmethod
  110. def update_config(self):
  111. """update evalution config
  112. """
  113. raise NotImplementedError
  114. @abstractmethod
  115. def get_eval_kwargs(self):
  116. """get key-value arguments of model evalution function
  117. """
  118. raise NotImplementedError