trainer.py 3.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111
  1. # copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. from abc import ABC, abstractmethod
  16. from pathlib import Path
  17. from .build_model import build_model
  18. from ...utils.device import get_device
  19. from ...utils.misc import AutoRegisterABCMetaClass
  20. from ...utils.config import AttrDict
  21. def build_trainer(config: AttrDict) -> "BaseTrainer":
  22. """build model trainer
  23. Args:
  24. config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
  25. Returns:
  26. BaseTrainer: the trainer, which is subclass of BaseTrainer.
  27. """
  28. model_name = config.Global.model
  29. return BaseTrainer.get(model_name)(config)
  30. class BaseTrainer(ABC, metaclass=AutoRegisterABCMetaClass):
  31. """ Base Model Trainer """
  32. __is_base = True
  33. def __init__(self, config: AttrDict):
  34. """Initialize the instance.
  35. Args:
  36. config (AttrDict): PaddleX pipeline config, which is loaded from pipeline yaml file.
  37. """
  38. self.global_config = config.Global
  39. self.train_config = config.Train
  40. self.deamon = self.build_deamon(self.global_config)
  41. self.pdx_config, self.pdx_model = build_model(self.global_config.model)
  42. def __call__(self, *args, **kwargs):
  43. """execute model training
  44. """
  45. os.makedirs(self.global_config.output, exist_ok=True)
  46. self.train(*args, **kwargs)
  47. self.deamon.stop()
  48. def dump_config(self, config_file_path: str=None):
  49. """dump the config
  50. Args:
  51. config_file_path (str, optional): the path to save dumped config. Defaults to None,
  52. means that save in `Global.output` as `config.yaml`.
  53. """
  54. if config_file_path is None:
  55. config_file_path = os.path.join(self.global_config.output,
  56. "config.yaml")
  57. self.pdx_config.dump(config_file_path)
  58. def train(self):
  59. """firstly, update and dump train config, then train model
  60. """
  61. self.update_config()
  62. self.dump_config()
  63. train_result = self.pdx_model.train(**self.get_train_kwargs())
  64. assert train_result.returncode == 0, f"Encountered an unexpected error({train_result.returncode}) in \
  65. training!"
  66. def get_device(self, using_device_number: int=None) -> str:
  67. """get device setting from config
  68. Args:
  69. using_device_number (int, optional): specify device number to use. Defaults to None,
  70. means that base on config setting.
  71. Returns:
  72. str: device setting, such as: `gpu:0,1`, `npu:0,1` `cpu`.
  73. """
  74. return get_device(
  75. self.global_config.device, using_device_number=using_device_number)
  76. @abstractmethod
  77. def build_deamon(self):
  78. """build deamon thread for saving training outputs timely
  79. """
  80. raise NotImplementedError
  81. @abstractmethod
  82. def update_config(self):
  83. """update training config
  84. """
  85. raise NotImplementedError
  86. @abstractmethod
  87. def get_train_kwargs(self):
  88. """get key-value arguments of model training function
  89. """
  90. raise NotImplementedError