evaluator.py 2.0 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758
  1. # !/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. ################################################################################
  4. #
  5. # Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved
  6. #
  7. ################################################################################
  8. """
  9. Author: PaddlePaddle Authors
  10. """
  11. import tarfile
  12. from pathlib import Path
  13. from ..base.evaluator import BaseEvaluator
  14. from .support_models import SUPPORT_MODELS
  15. class TSFCEvaluator(BaseEvaluator):
  16. """ TS Forecast Model Evaluator """
  17. support_models = SUPPORT_MODELS
  18. def update_config(self):
  19. """update evalution config
  20. """
  21. self.pdx_config.update_dataset(self.global_config.dataset_dir,
  22. "TSDataset")
  23. def get_eval_kwargs(self) -> dict:
  24. """get key-value arguments of model evalution function
  25. Returns:
  26. dict: the arguments of evaluation function.
  27. """
  28. return {
  29. "weight_path": self.eval_config.weight_path,
  30. "device": self.get_device(using_gpu_number=1)
  31. }
  32. def uncompress_tar_file(self):
  33. """unpackage the tar file containing training outputs and update weight path
  34. """
  35. if tarfile.is_tarfile(self.eval_config.weight_path):
  36. dest_path = Path(self.eval_config.weight_path).parent
  37. with tarfile.open(self.eval_config.weight_path, 'r') as tar:
  38. tar.extractall(path=dest_path)
  39. self.eval_config.weight_path = dest_path.joinpath(
  40. "best_accuracy.pdparams/best_model/model.pdparams")
  41. def eval(self):
  42. """firstly, update evaluation config, then evaluate model, finally return the evaluation result
  43. """
  44. self.uncompress_tar_file()
  45. self.update_config()
  46. evaluate_result = self.pdx_model.evaluate(**self.get_eval_kwargs())
  47. assert evaluate_result.returncode == 0, f"Encountered an unexpected error({evaluate_result.returncode}) in \
  48. evaling!"
  49. return evaluate_result.metrics