evaluator.py 2.1 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859
  1. # !/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. ################################################################################
  4. #
  5. # Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved
  6. #
  7. ################################################################################
  8. """
  9. Author: PaddlePaddle Authors
  10. """
  11. import tarfile
  12. from pathlib import Path
  13. from ..base.evaluator import BaseEvaluator
  14. from .support_models import SUPPORT_MODELS
  15. class TSADEvaluator(BaseEvaluator):
  16. """ TS Anomaly Detection Model Evaluator """
  17. support_models = SUPPORT_MODELS
  18. def update_config(self):
  19. """update evalution config
  20. """
  21. self.pdx_config.update_dataset(self.global_config.dataset_dir,
  22. "TSADDataset")
  23. self.pdx_config.update_weights(self.eval_config.weight_path)
  24. def uncompress_tar_file(self):
  25. """unpackage the tar file containing training outputs and update weight path
  26. """
  27. if tarfile.is_tarfile(self.eval_config.weight_path):
  28. dest_path = Path(self.eval_config.weight_path).parent
  29. with tarfile.open(self.eval_config.weight_path, 'r') as tar:
  30. tar.extractall(path=dest_path)
  31. self.eval_config.weight_path = dest_path.joinpath(
  32. "best_accuracy.pdparams/best_model/model.pdparams")
  33. def eval(self):
  34. """firstly, update evaluation config, then evaluate model, finally return the evaluation result
  35. """
  36. self.uncompress_tar_file()
  37. self.update_config()
  38. evaluate_result = self.pdx_model.evaluate(**self.get_eval_kwargs())
  39. assert evaluate_result.returncode == 0, f"Encountered an unexpected error({evaluate_result.returncode}) in \
  40. evaling!"
  41. return evaluate_result.metrics
  42. def get_eval_kwargs(self) -> dict:
  43. """get key-value arguments of model evalution function
  44. Returns:
  45. dict: the arguments of evaluation function.
  46. """
  47. return {
  48. "weight_path": self.eval_config.weight_path,
  49. "device": self.get_device(using_gpu_number=1)
  50. }