evaluator.py 2.0 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556
  1. # copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import tarfile
  15. from pathlib import Path
  16. from ..base import BaseEvaluator
  17. from .model_list import MODELS
  18. class TSFCEvaluator(BaseEvaluator):
  19. """TS Forecast Model Evaluator"""
  20. entities = MODELS
  21. def update_config(self):
  22. """update evalution config"""
  23. self.pdx_config.update_dataset(self.global_config.dataset_dir, "TSDataset")
  24. def get_eval_kwargs(self) -> dict:
  25. """get key-value arguments of model evalution function
  26. Returns:
  27. dict: the arguments of evaluation function.
  28. """
  29. return {
  30. "weight_path": self.eval_config.weight_path,
  31. "device": self.get_device(using_device_number=1),
  32. }
  33. def uncompress_tar_file(self):
  34. """unpackage the tar file containing training outputs and update weight path"""
  35. if tarfile.is_tarfile(self.eval_config.weight_path):
  36. dest_path = Path(self.eval_config.weight_path).parent
  37. with tarfile.open(self.eval_config.weight_path, "r") as tar:
  38. tar.extractall(path=dest_path)
  39. self.eval_config.weight_path = dest_path.joinpath(
  40. "best_accuracy.pdparams/best_model/model.pdparams"
  41. )
  42. def evaluate(self):
  43. """firstly, update evaluation config, then evaluate model, finally return the evaluation result"""
  44. self.uncompress_tar_file()
  45. return super().evaluate()