config.py 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359
  1. # Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import codecs
  15. import os
  16. from typing import Any, Dict, Generic
  17. import paddle
  18. import yaml
  19. from paddlex.paddleseg.cvlibs import manager
  20. from paddlex.paddleseg.utils import logger
  21. class Config(object):
  22. '''
  23. Training configuration parsing. The only yaml/yml file is supported.
  24. The following hyper-parameters are available in the config file:
  25. batch_size: The number of samples per gpu.
  26. iters: The total training steps.
  27. train_dataset: A training data config including type/data_root/transforms/mode.
  28. For data type, please refer to paddleseg.datasets.
  29. For specific transforms, please refer to paddleseg.transforms.transforms.
  30. val_dataset: A validation data config including type/data_root/transforms/mode.
  31. optimizer: A optimizer config, but currently PaddleSeg only supports sgd with momentum in config file.
  32. In addition, weight_decay could be set as a regularization.
  33. learning_rate: A learning rate config. If decay is configured, learning _rate value is the starting learning rate,
  34. where only poly decay is supported using the config file. In addition, decay power and end_lr are tuned experimentally.
  35. loss: A loss config. Multi-loss config is available. The loss type order is consistent with the seg model outputs,
  36. where the coef term indicates the weight of corresponding loss. Note that the number of coef must be the same as the number of
  37. model outputs, and there could be only one loss type if using the same loss type among the outputs, otherwise the number of
  38. loss type must be consistent with coef.
  39. model: A model config including type/backbone and model-dependent arguments.
  40. For model type, please refer to paddleseg.models.
  41. For backbone, please refer to paddleseg.models.backbones.
  42. Args:
  43. path (str) : The path of config file, supports yaml format only.
  44. Examples:
  45. from paddlex.paddleseg.cvlibs.config import Config
  46. # Create a cfg object with yaml file path.
  47. cfg = Config(yaml_cfg_path)
  48. # Parsing the argument when its property is used.
  49. train_dataset = cfg.train_dataset
  50. # the argument of model should be parsed after dataset,
  51. # since the model builder uses some properties in dataset.
  52. model = cfg.model
  53. ...
  54. '''
  55. def __init__(self,
  56. path: str,
  57. learning_rate: float=None,
  58. batch_size: int=None,
  59. iters: int=None):
  60. if not path:
  61. raise ValueError('Please specify the configuration file path.')
  62. if not os.path.exists(path):
  63. raise FileNotFoundError('File {} does not exist'.format(path))
  64. self._model = None
  65. self._losses = None
  66. if path.endswith('yml') or path.endswith('yaml'):
  67. self.dic = self._parse_from_yaml(path)
  68. else:
  69. raise RuntimeError('Config file should in yaml format!')
  70. self.update(
  71. learning_rate=learning_rate, batch_size=batch_size, iters=iters)
  72. def _update_dic(self, dic, base_dic):
  73. """
  74. Update config from dic based base_dic
  75. """
  76. base_dic = base_dic.copy()
  77. for key, val in dic.items():
  78. if isinstance(val, dict) and key in base_dic:
  79. base_dic[key] = self._update_dic(val, base_dic[key])
  80. else:
  81. base_dic[key] = val
  82. dic = base_dic
  83. return dic
  84. def _parse_from_yaml(self, path: str):
  85. '''Parse a yaml file and build config'''
  86. with codecs.open(path, 'r', 'utf-8') as file:
  87. dic = yaml.load(file, Loader=yaml.FullLoader)
  88. if '_base_' in dic:
  89. cfg_dir = os.path.dirname(path)
  90. base_path = dic.pop('_base_')
  91. base_path = os.path.join(cfg_dir, base_path)
  92. base_dic = self._parse_from_yaml(base_path)
  93. dic = self._update_dic(dic, base_dic)
  94. return dic
  95. def update(self,
  96. learning_rate: float=None,
  97. batch_size: int=None,
  98. iters: int=None):
  99. '''Update config'''
  100. if learning_rate:
  101. if 'lr_scheduler' in self.dic:
  102. self.dic['lr_scheduler']['learning_rate'] = learning_rate
  103. else:
  104. self.dic['learning_rate']['value'] = learning_rate
  105. if batch_size:
  106. self.dic['batch_size'] = batch_size
  107. if iters:
  108. self.dic['iters'] = iters
  109. @property
  110. def batch_size(self) -> int:
  111. return self.dic.get('batch_size', 1)
  112. @property
  113. def iters(self) -> int:
  114. iters = self.dic.get('iters')
  115. if not iters:
  116. raise RuntimeError('No iters specified in the configuration file.')
  117. return iters
  118. @property
  119. def lr_scheduler(self) -> paddle.optimizer.lr.LRScheduler:
  120. if 'lr_scheduler' not in self.dic:
  121. raise RuntimeError(
  122. 'No `lr_scheduler` specified in the configuration file.')
  123. params = self.dic.get('lr_scheduler')
  124. lr_type = params.pop('type')
  125. if lr_type == 'PolynomialDecay':
  126. params.setdefault('decay_steps', self.iters)
  127. params.setdefault('end_lr', 0)
  128. params.setdefault('power', 0.9)
  129. return getattr(paddle.optimizer.lr, lr_type)(**params)
  130. @property
  131. def learning_rate(self) -> paddle.optimizer.lr.LRScheduler:
  132. logger.warning(
  133. '''`learning_rate` in configuration file will be deprecated, please use `lr_scheduler` instead. E.g
  134. lr_scheduler:
  135. type: PolynomialDecay
  136. learning_rate: 0.01''')
  137. _learning_rate = self.dic.get('learning_rate', {}).get('value')
  138. if not _learning_rate:
  139. raise RuntimeError(
  140. 'No learning rate specified in the configuration file.')
  141. args = self.decay_args
  142. decay_type = args.pop('type')
  143. if decay_type == 'poly':
  144. lr = _learning_rate
  145. return paddle.optimizer.lr.PolynomialDecay(lr, **args)
  146. elif decay_type == 'piecewise':
  147. values = _learning_rate
  148. return paddle.optimizer.lr.PiecewiseDecay(values=values, **args)
  149. else:
  150. raise RuntimeError('Only poly and piecewise decay support.')
  151. @property
  152. def optimizer(self) -> paddle.optimizer.Optimizer:
  153. if 'lr_scheduler' in self.dic:
  154. lr = self.lr_scheduler
  155. else:
  156. lr = self.learning_rate
  157. args = self.optimizer_args
  158. optimizer_type = args.pop('type')
  159. if optimizer_type == 'sgd':
  160. return paddle.optimizer.Momentum(
  161. lr, parameters=self.model.parameters(), **args)
  162. elif optimizer_type == 'adam':
  163. return paddle.optimizer.Adam(
  164. lr, parameters=self.model.parameters(), **args)
  165. else:
  166. raise RuntimeError('Only sgd and adam optimizer support.')
  167. @property
  168. def optimizer_args(self) -> dict:
  169. args = self.dic.get('optimizer', {}).copy()
  170. if args['type'] == 'sgd':
  171. args.setdefault('momentum', 0.9)
  172. return args
  173. @property
  174. def decay_args(self) -> dict:
  175. args = self.dic.get('learning_rate', {}).get(
  176. 'decay', {'type': 'poly',
  177. 'power': 0.9}).copy()
  178. if args['type'] == 'poly':
  179. args.setdefault('decay_steps', self.iters)
  180. args.setdefault('end_lr', 0)
  181. return args
  182. @property
  183. def loss(self) -> dict:
  184. args = self.dic.get('loss', {}).copy()
  185. if 'types' in args and 'coef' in args:
  186. len_types = len(args['types'])
  187. len_coef = len(args['coef'])
  188. if len_types != len_coef:
  189. if len_types == 1:
  190. args['types'] = args['types'] * len_coef
  191. else:
  192. raise ValueError(
  193. 'The length of types should equal to coef or equal to 1 in loss config, but they are {} and {}.'
  194. .format(len_types, len_coef))
  195. else:
  196. raise ValueError(
  197. 'Loss config should contain keys of "types" and "coef"')
  198. if not self._losses:
  199. self._losses = dict()
  200. for key, val in args.items():
  201. if key == 'types':
  202. self._losses['types'] = []
  203. for item in args['types']:
  204. if item['type'] != 'MixedLoss':
  205. item['ignore_index'] = \
  206. self.train_dataset.ignore_index
  207. self._losses['types'].append(self._load_object(item))
  208. else:
  209. self._losses[key] = val
  210. if len(self._losses['coef']) != len(self._losses['types']):
  211. raise RuntimeError(
  212. 'The length of coef should equal to types in loss config: {} != {}.'
  213. .format(
  214. len(self._losses['coef']), len(self._losses['types'])))
  215. return self._losses
  216. @property
  217. def model(self) -> paddle.nn.Layer:
  218. model_cfg = self.dic.get('model').copy()
  219. if not model_cfg:
  220. raise RuntimeError('No model specified in the configuration file.')
  221. if not 'num_classes' in model_cfg:
  222. num_classes = None
  223. if self.train_dataset_config:
  224. if hasattr(self.train_dataset_class, 'NUM_CLASSES'):
  225. num_classes = self.train_dataset_class.NUM_CLASSES
  226. elif hasattr(self.train_dataset, 'num_classes'):
  227. num_classes = self.train_dataset.num_classes
  228. elif self.val_dataset_config:
  229. if hasattr(self.val_dataset_class, 'NUM_CLASSES'):
  230. num_classes = self.val_dataset_class.NUM_CLASSES
  231. elif hasattr(self.val_dataset, 'num_classes'):
  232. num_classes = self.val_dataset.num_classes
  233. if not num_classes:
  234. raise ValueError(
  235. '`num_classes` is not found. Please set it in model, train_dataset or val_dataset'
  236. )
  237. model_cfg['num_classes'] = num_classes
  238. if not self._model:
  239. self._model = self._load_object(model_cfg)
  240. return self._model
  241. @property
  242. def train_dataset_config(self) -> Dict:
  243. return self.dic.get('train_dataset', {}).copy()
  244. @property
  245. def val_dataset_config(self) -> Dict:
  246. return self.dic.get('val_dataset', {}).copy()
  247. @property
  248. def train_dataset_class(self) -> Generic:
  249. dataset_type = self.train_dataset_config['type']
  250. return self._load_component(dataset_type)
  251. @property
  252. def val_dataset_class(self) -> Generic:
  253. dataset_type = self.val_dataset_config['type']
  254. return self._load_component(dataset_type)
  255. @property
  256. def train_dataset(self) -> paddle.io.Dataset:
  257. _train_dataset = self.train_dataset_config
  258. if not _train_dataset:
  259. return None
  260. return self._load_object(_train_dataset)
  261. @property
  262. def val_dataset(self) -> paddle.io.Dataset:
  263. _val_dataset = self.val_dataset_config
  264. if not _val_dataset:
  265. return None
  266. return self._load_object(_val_dataset)
  267. def _load_component(self, com_name: str) -> Any:
  268. com_list = [
  269. manager.MODELS, manager.BACKBONES, manager.DATASETS,
  270. manager.TRANSFORMS, manager.LOSSES
  271. ]
  272. for com in com_list:
  273. if com_name in com.components_dict:
  274. return com[com_name]
  275. else:
  276. raise RuntimeError(
  277. 'The specified component was not found {}.'.format(com_name))
  278. def _load_object(self, cfg: dict) -> Any:
  279. cfg = cfg.copy()
  280. if 'type' not in cfg:
  281. raise RuntimeError('No object information in {}.'.format(cfg))
  282. component = self._load_component(cfg.pop('type'))
  283. params = {}
  284. for key, val in cfg.items():
  285. if self._is_meta_type(val):
  286. params[key] = self._load_object(val)
  287. elif isinstance(val, list):
  288. params[key] = [
  289. self._load_object(item)
  290. if self._is_meta_type(item) else item for item in val
  291. ]
  292. else:
  293. params[key] = val
  294. return component(**params)
  295. @property
  296. def export_config(self) -> Dict:
  297. return self.dic.get('export', {})
  298. def _is_meta_type(self, item: Any) -> bool:
  299. return isinstance(item, dict) and 'type' in item
  300. def __str__(self) -> str:
  301. return yaml.dump(self.dic)