config.py 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505
  1. # !/usr/bin/env python3
  2. # -*- coding: UTF-8 -*-
  3. ################################################################################
  4. #
  5. # Copyright (c) 2024 Baidu.com, Inc. All Rights Reserved
  6. #
  7. ################################################################################
  8. """
  9. Author: PaddlePaddle Authors
  10. """
  11. import yaml
  12. from paddleclas.ppcls.utils.config import get_config, override_config
  13. from ...base import BaseConfig
  14. from ....utils.misc import abspath
  15. class ClsConfig(BaseConfig):
  16. """Image Classification Task Config"""
  17. def update(self, list_like_obj: list):
  18. """update self
  19. Args:
  20. list_like_obj (list): list of pairs(key0.key1.idx.key2=value), such as:
  21. [
  22. 'topk=2',
  23. 'VALID.transforms.1.ResizeImage.resize_short=300'
  24. ]
  25. """
  26. dict_ = override_config(self.dict, list_like_obj)
  27. self.reset_from_dict(dict_)
  28. def load(self, config_file_path: str):
  29. """load config from yaml file
  30. Args:
  31. config_file_path (str): the path of yaml file.
  32. Raises:
  33. TypeError: the content of yaml file `config_file_path` error.
  34. """
  35. dict_ = yaml.load(open(config_file_path, 'rb'), Loader=yaml.Loader)
  36. if not isinstance(dict_, dict):
  37. raise TypeError
  38. self.reset_from_dict(dict_)
  39. def dump(self, config_file_path: str):
  40. """dump self to yaml file
  41. Args:
  42. config_file_path (str): the path to save self as yaml file.
  43. """
  44. with open(config_file_path, 'w', encoding='utf-8') as f:
  45. yaml.dump(self.dict, f, default_flow_style=False, sort_keys=False)
  46. def update_dataset(
  47. self,
  48. dataset_path: str,
  49. dataset_type: str=None,
  50. *,
  51. train_list_path: str=None, ):
  52. """update dataset settings
  53. Args:
  54. dataset_path (str): the root path of dataset.
  55. dataset_type (str, optional): dataset type. Defaults to None.
  56. train_list_path (str, optional): the path of train dataset annotation file . Defaults to None.
  57. Raises:
  58. ValueError: the dataset_type error.
  59. """
  60. dataset_path = abspath(dataset_path)
  61. if dataset_type is None:
  62. dataset_type = 'ClsDataset'
  63. if train_list_path:
  64. train_list_path = f"{train_list_path}"
  65. else:
  66. train_list_path = f"{dataset_path}/train.txt"
  67. if dataset_type in ['ClsDataset']:
  68. ds_cfg = [
  69. f'DataLoader.Train.dataset.name={dataset_type}',
  70. f'DataLoader.Train.dataset.image_root={dataset_path}',
  71. f'DataLoader.Train.dataset.cls_label_path={train_list_path}',
  72. f'DataLoader.Eval.dataset.name={dataset_type}',
  73. f'DataLoader.Eval.dataset.image_root={dataset_path}',
  74. f'DataLoader.Eval.dataset.cls_label_path={dataset_path}/val.txt',
  75. f'Infer.PostProcess.class_id_map_file={dataset_path}/label.txt'
  76. ]
  77. else:
  78. raise ValueError(f"{repr(dataset_type)} is not supported.")
  79. self.update(ds_cfg)
  80. def update_batch_size(self, batch_size: int, mode: str='train'):
  81. """update batch size setting
  82. Args:
  83. batch_size (int): the batch size number to set.
  84. mode (str, optional): the mode that to be set batch size, must be one of 'train', 'eval', 'test'.
  85. Defaults to 'train'.
  86. Raises:
  87. ValueError: `mode` error.
  88. """
  89. if mode == 'train':
  90. _cfg = [f'DataLoader.Train.sampler.batch_size={batch_size}']
  91. elif mode == 'eval':
  92. _cfg = [f'DataLoader.Eval.sampler.batch_size={batch_size}']
  93. elif mode == 'test':
  94. _cfg = [f'DataLoader.Infer.batch_size={batch_size}']
  95. else:
  96. raise ValueError("The input `mode` should be train, eval or test.")
  97. self.update(_cfg)
  98. def update_learning_rate(self, learning_rate: float):
  99. """update learning rate
  100. Args:
  101. learning_rate (float): the learning rate value to set.
  102. """
  103. _cfg = [f'Optimizer.lr.learning_rate={learning_rate}']
  104. self.update(_cfg)
  105. def update_warmup_epochs(self, warmup_epochs: int):
  106. """update warmup epochs
  107. Args:
  108. warmup_epochs (int): the warmup epochs value to set.
  109. """
  110. _cfg = [f'Optimizer.lr.warmup_epoch={warmup_epochs}']
  111. self.update(_cfg)
  112. def update_pretrained_weights(self, pretrained_model: str):
  113. """update pretrained weight path
  114. Args:
  115. pretrained_model (str): the local path or url of pretrained weight file to set.
  116. """
  117. assert isinstance(
  118. pretrained_model, (str, None)
  119. ), "The 'pretrained_model' should be a string, indicating the path to the '*.pdparams' file, or 'None', \
  120. indicating that no pretrained model to be used."
  121. if pretrained_model and not pretrained_model.startswith(
  122. ('http://', 'https://')):
  123. pretrained_model = abspath(
  124. pretrained_model.replace(".pdparams", ""))
  125. self.update([f'Global.pretrained_model={pretrained_model}'])
  126. def update_num_classes(self, num_classes: int):
  127. """update classes number
  128. Args:
  129. num_classes (int): the classes number value to set.
  130. """
  131. update_str_list = [f'Arch.class_num={num_classes}']
  132. if self._get_arch_name() == "DistillationModel":
  133. update_str_list.append(
  134. f"Arch.models.0.Teacher.class_num={num_classes}")
  135. update_str_list.append(
  136. f"Arch.models.1.Student.class_num={num_classes}")
  137. self.update(update_str_list)
  138. def _update_slim_config(self, slim_config_path: str):
  139. """update slim settings
  140. Args:
  141. slim_config_path (str): the path to slim config yaml file.
  142. """
  143. slim_config = yaml.load(
  144. open(slim_config_path, 'rb'), Loader=yaml.Loader)['Slim']
  145. self.update([f'Slim={slim_config}'])
  146. def _update_amp(self, amp: None | str):
  147. """update AMP settings
  148. Args:
  149. amp (None | str): the AMP settings.
  150. Raises:
  151. ValueError: AMP setting `amp` error, missing field `AMP`.
  152. """
  153. if amp is None or amp == 'OFF':
  154. if 'AMP' in self.dict:
  155. self._dict.pop('AMP')
  156. else:
  157. if 'AMP' not in self.dict:
  158. raise ValueError("Config must have AMP information.")
  159. _cfg = ['AMP.use_amp=True', f'AMP.level={amp}']
  160. self.update(_cfg)
  161. def update_num_workers(self, num_workers: int):
  162. """update workers number of train and eval dataloader
  163. Args:
  164. num_workers (int): the value of train and eval dataloader workers number to set.
  165. """
  166. _cfg = [
  167. f'DataLoader.Train.loader.num_workers={num_workers}',
  168. f'DataLoader.Eval.loader.num_workers={num_workers}',
  169. ]
  170. self.update(_cfg)
  171. def enable_shared_memory(self):
  172. """enable shared memory setting of train and eval dataloader
  173. """
  174. _cfg = [
  175. f'DataLoader.Train.loader.use_shared_memory=True',
  176. f'DataLoader.Eval.loader.use_shared_memory=True',
  177. ]
  178. self.update(_cfg)
  179. def disable_shared_memory(self):
  180. """disable shared memory setting of train and eval dataloader
  181. """
  182. _cfg = [
  183. f'DataLoader.Train.loader.use_shared_memory=False',
  184. f'DataLoader.Eval.loader.use_shared_memory=False',
  185. ]
  186. self.update(_cfg)
  187. def _update_device(self, device: str):
  188. """update device setting
  189. Args:
  190. device (str): the running device to set
  191. """
  192. device = device.split(':')[0]
  193. _cfg = [f'Global.device={device}']
  194. self.update(_cfg)
  195. def update_label_dict_path(self, dict_path: str):
  196. """update label dict file path
  197. Args:
  198. dict_path (str): the path of label dict file to set
  199. """
  200. _cfg = [f'PostProcess.Topk.class_id_map_file={abspath(dict_path)}', ]
  201. self.update(_cfg)
  202. def _update_to_static(self, dy2st: bool):
  203. """update config to set dynamic to static mode
  204. Args:
  205. dy2st (bool): whether or not to use the dynamic to static mode.
  206. """
  207. self.update([f'Global.to_static={dy2st}'])
  208. def _update_use_vdl(self, use_vdl: bool):
  209. """update config to set VisualDL
  210. Args:
  211. use_vdl (bool): whether or not to use VisualDL.
  212. """
  213. self.update([f'Global.use_visualdl={use_vdl}'])
  214. def _update_epochs(self, epochs: int):
  215. """update epochs setting
  216. Args:
  217. epochs (int): the epochs number value to set
  218. """
  219. self.update([f'Global.epochs={epochs}'])
  220. def _update_checkpoints(self, resume_path: None | str):
  221. """update checkpoint setting
  222. Args:
  223. resume_path (None | str): the resume training setting. if is `None`, train from scratch, otherwise,
  224. train from checkpoint file that path is `.pdparams` file.
  225. """
  226. if resume_path is not None:
  227. resume_path = resume_path.replace(".pdparams", "")
  228. self.update([f'Global.checkpoints={resume_path}'])
  229. def _update_output_dir(self, save_dir: str):
  230. """update output directory
  231. Args:
  232. save_dir (str): the path to save outputs.
  233. """
  234. self.update([f'Global.output_dir={abspath(save_dir)}'])
  235. def update_log_interval(self, log_interval: int):
  236. """update log interval(steps)
  237. Args:
  238. log_interval (int): the log interval value to set.
  239. """
  240. self.update([f'Global.print_batch_step={log_interval}'])
  241. def update_eval_interval(self, eval_interval: int):
  242. """update eval interval(epochs)
  243. Args:
  244. eval_interval (int): the eval interval value to set.
  245. """
  246. self.update([f'Global.eval_interval={eval_interval}'])
  247. def update_save_interval(self, save_interval: int):
  248. """update eval interval(epochs)
  249. Args:
  250. save_interval (int): the save interval value to set.
  251. """
  252. self.update([f'Global.save_interval={save_interval}'])
  253. def _update_predict_img(self, infer_img: str, infer_list: str=None):
  254. """update image to be predicted
  255. Args:
  256. infer_img (str): the path to image that to be predicted.
  257. infer_list (str, optional): the path to file that images. Defaults to None.
  258. """
  259. if infer_list:
  260. self.update([f'Infer.infer_list={infer_list}'])
  261. self.update([f'Infer.infer_imgs={infer_img}'])
  262. def _update_save_inference_dir(self, save_inference_dir: str):
  263. """update directory path to save inference model files
  264. Args:
  265. save_inference_dir (str): the directory path to set.
  266. """
  267. self.update(
  268. [f'Global.save_inference_dir={abspath(save_inference_dir)}'])
  269. def _update_inference_model_dir(self, model_dir: str):
  270. """update inference model directory
  271. Args:
  272. model_dir (str): the directory path of inference model fils that used to predict.
  273. """
  274. self.update([f'Global.inference_model_dir={abspath(model_dir)}'])
  275. def _update_infer_img(self, infer_img: str):
  276. """update path of image that would be predict
  277. Args:
  278. infer_img (str): the image path.
  279. """
  280. self.update([f'Global.infer_imgs={infer_img}'])
  281. def _update_infer_device(self, device: str):
  282. """update the device used in predicting
  283. Args:
  284. device (str): the running device setting
  285. """
  286. self.update([f'Global.use_gpu={device.split(":")[0]=="gpu"}'])
  287. def _update_enable_mkldnn(self, enable_mkldnn: bool):
  288. """update whether to enable MKLDNN
  289. Args:
  290. enable_mkldnn (bool): `True` is enable, otherwise is disable.
  291. """
  292. self.update([f'Global.enable_mkldnn={enable_mkldnn}'])
  293. def _update_infer_img_shape(self, img_shape: str):
  294. """update image cropping shape in the preprocessing
  295. Args:
  296. img_shape (str): the shape of cropping in the preprocessing,
  297. i.e. `PreProcess.transform_ops.1.CropImage.size`.
  298. """
  299. self.update([f'PreProcess.transform_ops.1.CropImage.size={img_shape}'])
  300. def _update_save_predict_result(self, save_dir: str):
  301. """update directory that save predicting output
  302. Args:
  303. save_dir (str): the dicrectory path that save predicting output.
  304. """
  305. self.update([f'Infer.save_dir={save_dir}'])
  306. def update_model(self, **kwargs):
  307. """update model settings
  308. """
  309. for k in kwargs:
  310. v = kwargs[k]
  311. self.update([f'Arch.{k}={v}'])
  312. def update_teacher_model(self, **kwargs):
  313. """update teacher model settings
  314. """
  315. for k in kwargs:
  316. v = kwargs[k]
  317. self.update([f'Arch.models.0.Teacher.{k}={v}'])
  318. def update_student_model(self, **kwargs):
  319. """update student model settings
  320. """
  321. for k in kwargs:
  322. v = kwargs[k]
  323. self.update([f'Arch.models.1.Student.{k}={v}'])
  324. def get_epochs_iters(self) -> int:
  325. """get epochs
  326. Returns:
  327. int: the epochs value, i.e., `Global.epochs` in config.
  328. """
  329. return self.dict['Global']['epochs']
  330. def get_log_interval(self) -> int:
  331. """get log interval(steps)
  332. Returns:
  333. int: the log interval value, i.e., `Global.print_batch_step` in config.
  334. """
  335. return self.dict['Global']['print_batch_step']
  336. def get_eval_interval(self) -> int:
  337. """get eval interval(epochs)
  338. Returns:
  339. int: the eval interval value, i.e., `Global.eval_interval` in config.
  340. """
  341. return self.dict['Global']['eval_interval']
  342. def get_save_interval(self) -> int:
  343. """get save interval(epochs)
  344. Returns:
  345. int: the save interval value, i.e., `Global.save_interval` in config.
  346. """
  347. return self.dict['Global']['save_interval']
  348. def get_learning_rate(self) -> float:
  349. """get learning rate
  350. Returns:
  351. float: the learning rate value, i.e., `Optimizer.lr.learning_rate` in config.
  352. """
  353. return self.dict['Optimizer']['lr']['learning_rate']
  354. def get_warmup_epochs(self) -> int:
  355. """get warmup epochs
  356. Returns:
  357. int: the warmup epochs value, i.e., `Optimizer.lr.warmup_epochs` in config.
  358. """
  359. return self.dict['Optimizer']['lr']['warmup_epoch']
  360. def get_label_dict_path(self) -> str:
  361. """get label dict file path
  362. Returns:
  363. str: the label dict file path, i.e., `PostProcess.Topk.class_id_map_file` in config.
  364. """
  365. return self.dict['PostProcess']['Topk']['class_id_map_file']
  366. def get_batch_size(self, mode='train') -> int:
  367. """get batch size
  368. Args:
  369. mode (str, optional): the mode that to be get batch size value, must be one of 'train', 'eval', 'test'.
  370. Defaults to 'train'.
  371. Returns:
  372. int: the batch size value of `mode`, i.e., `DataLoader.{mode}.sampler.batch_size` in config.
  373. """
  374. return self.dict['DataLoader']['Train']['sampler']['batch_size']
  375. def get_qat_epochs_iters(self) -> int:
  376. """get qat epochs
  377. Returns:
  378. int: the epochs value.
  379. """
  380. return self.get_epochs_iters()
  381. def get_qat_learning_rate(self) -> float:
  382. """get qat learning rate
  383. Returns:
  384. float: the learning rate value.
  385. """
  386. return self.get_learning_rate()
  387. def _get_arch_name(self) -> str:
  388. """get architecture name of model
  389. Returns:
  390. str: the model arch name, i.e., `Arch.name` in config.
  391. """
  392. return self.dict["Arch"]["name"]
  393. def _get_dataset_root(self) -> str:
  394. """get root directory of dataset, i.e. `DataLoader.Train.dataset.image_root`
  395. Returns:
  396. str: the root directory of dataset
  397. """
  398. return self.dict["DataLoader"]["Train"]['dataset']['image_root']
  399. def get_train_save_dir(self) -> str:
  400. """get the directory to save output
  401. Returns:
  402. str: the directory to save output
  403. """
  404. return self['Global']['output_dir']