pp_option.py 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358
  1. # copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. from typing import Dict, List
  16. from ...utils import logging
  17. from ...utils.device import (
  18. check_supported_device_type,
  19. get_default_device,
  20. parse_device,
  21. set_env_for_device,
  22. )
  23. from .new_ir_blacklist import NEWIR_BLOCKLIST
  24. from .trt_blacklist import TRT_BLOCKLIST
  25. class PaddlePredictorOption(object):
  26. """Paddle Inference Engine Option"""
  27. # NOTE: TRT modes start with `trt_`
  28. SUPPORT_RUN_MODE = (
  29. "paddle",
  30. "trt_fp32",
  31. "trt_fp16",
  32. "trt_int8",
  33. "mkldnn",
  34. "mkldnn_bf16",
  35. )
  36. SUPPORT_DEVICE = ("gpu", "cpu", "npu", "xpu", "mlu", "dcu", "gcu")
  37. def __init__(self, model_name=None, **kwargs):
  38. super().__init__()
  39. self.model_name = model_name
  40. self._cfg = {}
  41. self._init_option(**kwargs)
  42. self._changed = False
  43. @property
  44. def changed(self):
  45. return self._changed
  46. @changed.setter
  47. def changed(self, v):
  48. assert isinstance(v, bool)
  49. self._changed = v
  50. def _init_option(self, **kwargs):
  51. for k, v in kwargs.items():
  52. if self._has_setter(k):
  53. setattr(self, k, v)
  54. else:
  55. raise Exception(
  56. f"{k} is not supported to set! The supported option is: {self._get_settable_attributes()}"
  57. )
  58. for k, v in self._get_default_config().items():
  59. self._cfg.setdefault(k, v)
  60. def _get_default_config(self):
  61. """get default config"""
  62. device_type, device_ids = parse_device(get_default_device())
  63. return {
  64. "run_mode": "paddle",
  65. "device_type": device_type,
  66. "device_id": None if device_ids is None else device_ids[0],
  67. "cpu_threads": 8,
  68. "delete_pass": [],
  69. "enable_new_ir": True if self.model_name not in NEWIR_BLOCKLIST else False,
  70. "trt_max_workspace_size": 1 << 30, # only for trt
  71. "trt_max_batch_size": 32, # only for trt
  72. "trt_min_subgraph_size": 3, # only for trt
  73. "trt_use_static": True, # only for trt
  74. "trt_use_calib_mode": False, # only for trt
  75. "trt_use_dynamic_shapes": True, # only for trt
  76. "trt_collect_shape_range_info": True, # only for trt
  77. "trt_discard_cached_shape_range_info": False, # only for trt
  78. "trt_dynamic_shapes": None, # only for trt
  79. "trt_dynamic_shape_input_data": None, # only for trt
  80. "trt_shape_range_info_path": None, # only for trt
  81. "trt_allow_rebuild_at_runtime": True, # only for trt
  82. }
  83. def _update(self, k, v):
  84. self._cfg[k] = v
  85. self.changed = True
  86. @property
  87. def run_mode(self):
  88. return self._cfg["run_mode"]
  89. @run_mode.setter
  90. def run_mode(self, run_mode: str):
  91. """set run mode"""
  92. if run_mode not in self.SUPPORT_RUN_MODE:
  93. support_run_mode_str = ", ".join(self.SUPPORT_RUN_MODE)
  94. raise ValueError(
  95. f"`run_mode` must be {support_run_mode_str}, but received {repr(run_mode)}."
  96. )
  97. # TRT Blocklist
  98. if run_mode.startswith("trt") and self.model_name in TRT_BLOCKLIST:
  99. logging.warning(
  100. f"The model({self.model_name}) is not supported to run in trt mode! Using `paddle` instead!"
  101. )
  102. run_mode = "paddle"
  103. self._update("run_mode", run_mode)
  104. @property
  105. def device_type(self):
  106. return self._cfg["device_type"]
  107. @device_type.setter
  108. def device_type(self, device_type):
  109. check_supported_device_type(device_type, self.model_name)
  110. self._update("device_type", device_type)
  111. @property
  112. def device_id(self):
  113. return self._cfg["device_id"]
  114. @device_id.setter
  115. def device_id(self, device_id):
  116. self._update("device_id", device_id)
  117. @property
  118. def cpu_threads(self):
  119. return self._cfg["cpu_threads"]
  120. @cpu_threads.setter
  121. def cpu_threads(self, cpu_threads):
  122. """set cpu threads"""
  123. if not isinstance(cpu_threads, int) or cpu_threads < 1:
  124. raise Exception()
  125. self._update("cpu_threads", cpu_threads)
  126. @property
  127. def delete_pass(self):
  128. return self._cfg["delete_pass"]
  129. @delete_pass.setter
  130. def delete_pass(self, delete_pass):
  131. self._update("delete_pass", delete_pass)
  132. @property
  133. def enable_new_ir(self):
  134. return self._cfg["enable_new_ir"]
  135. @enable_new_ir.setter
  136. def enable_new_ir(self, enable_new_ir: bool):
  137. """set run mode"""
  138. self._update("enable_new_ir", enable_new_ir)
  139. @property
  140. def trt_max_workspace_size(self):
  141. return self._cfg["trt_max_workspace_size"]
  142. @trt_max_workspace_size.setter
  143. def trt_max_workspace_size(self, trt_max_workspace_size):
  144. self._update("trt_max_workspace_size", trt_max_workspace_size)
  145. @property
  146. def trt_max_batch_size(self):
  147. return self._cfg["trt_max_batch_size"]
  148. @trt_max_batch_size.setter
  149. def trt_max_batch_size(self, trt_max_batch_size):
  150. self._update("trt_max_batch_size", trt_max_batch_size)
  151. @property
  152. def trt_min_subgraph_size(self):
  153. return self._cfg["trt_min_subgraph_size"]
  154. @trt_min_subgraph_size.setter
  155. def trt_min_subgraph_size(self, trt_min_subgraph_size: int):
  156. """set min subgraph size"""
  157. if not isinstance(trt_min_subgraph_size, int):
  158. raise Exception()
  159. self._update("trt_min_subgraph_size", trt_min_subgraph_size)
  160. @property
  161. def trt_use_static(self):
  162. return self._cfg["trt_use_static"]
  163. @trt_use_static.setter
  164. def trt_use_static(self, trt_use_static):
  165. """set trt use static"""
  166. self._update("trt_use_static", trt_use_static)
  167. @property
  168. def trt_use_calib_mode(self):
  169. return self._cfg["trt_use_calib_mode"]
  170. @trt_use_calib_mode.setter
  171. def trt_use_calib_mode(self, trt_use_calib_mode):
  172. """set trt calib mode"""
  173. self._update("trt_use_calib_mode", trt_use_calib_mode)
  174. @property
  175. def trt_use_dynamic_shapes(self):
  176. return self._cfg["trt_use_dynamic_shapes"]
  177. @trt_use_dynamic_shapes.setter
  178. def trt_use_dynamic_shapes(self, trt_use_dynamic_shapes):
  179. self._update("trt_use_dynamic_shapes", trt_use_dynamic_shapes)
  180. @property
  181. def trt_collect_shape_range_info(self):
  182. return self._cfg["trt_collect_shape_range_info"]
  183. @trt_collect_shape_range_info.setter
  184. def trt_collect_shape_range_info(self, trt_collect_shape_range_info):
  185. self._update("trt_collect_shape_range_info", trt_collect_shape_range_info)
  186. @property
  187. def trt_discard_cached_shape_range_info(self):
  188. return self._cfg["trt_discard_cached_shape_range_info"]
  189. @trt_discard_cached_shape_range_info.setter
  190. def trt_discard_cached_shape_range_info(self, trt_discard_cached_shape_range_info):
  191. self._update(
  192. "trt_discard_cached_shape_range_info", trt_discard_cached_shape_range_info
  193. )
  194. @property
  195. def trt_dynamic_shapes(self):
  196. return self._cfg["trt_dynamic_shapes"]
  197. @trt_dynamic_shapes.setter
  198. def trt_dynamic_shapes(self, trt_dynamic_shapes: Dict[str, List[List[int]]]):
  199. assert isinstance(trt_dynamic_shapes, dict)
  200. for input_k in trt_dynamic_shapes:
  201. assert isinstance(trt_dynamic_shapes[input_k], list)
  202. self._update("trt_dynamic_shapes", trt_dynamic_shapes)
  203. @property
  204. def trt_dynamic_shape_input_data(self):
  205. return self._cfg["trt_dynamic_shape_input_data"]
  206. @trt_dynamic_shape_input_data.setter
  207. def trt_dynamic_shape_input_data(
  208. self, trt_dynamic_shape_input_data: Dict[str, List[float]]
  209. ):
  210. self._update("trt_dynamic_shape_input_data", trt_dynamic_shape_input_data)
  211. @property
  212. def trt_shape_range_info_path(self):
  213. return self._cfg["trt_shape_range_info_path"]
  214. @trt_shape_range_info_path.setter
  215. def trt_shape_range_info_path(self, trt_shape_range_info_path: str):
  216. """set shape info filename"""
  217. self._update("trt_shape_range_info_path", trt_shape_range_info_path)
  218. @property
  219. def trt_allow_rebuild_at_runtime(self):
  220. return self._cfg["trt_allow_rebuild_at_runtime"]
  221. @trt_allow_rebuild_at_runtime.setter
  222. def trt_allow_rebuild_at_runtime(self, trt_allow_rebuild_at_runtime):
  223. self._update("trt_allow_rebuild_at_runtime", trt_allow_rebuild_at_runtime)
  224. # For backward compatibility
  225. # TODO: Issue deprecation warnings
  226. @property
  227. def min_subgraph_size(self):
  228. return self.trt_min_subgraph_size
  229. @min_subgraph_size.setter
  230. def min_subgraph_size(self, min_subgraph_size):
  231. self.trt_min_subgraph_size = min_subgraph_size
  232. @property
  233. def shape_info_filename(self):
  234. return self.trt_shape_range_info_path
  235. @shape_info_filename.setter
  236. def shape_info_filename(self, shape_info_filename):
  237. self.trt_shape_range_info_path = shape_info_filename
  238. @property
  239. def trt_calib_mode(self):
  240. return self.trt_use_calib_mode
  241. @trt_calib_mode.setter
  242. def trt_calib_mode(self, trt_calib_mode):
  243. self.trt_use_calib_mode = trt_calib_mode
  244. @property
  245. def batch_size(self):
  246. return self.trt_max_batch_size
  247. @batch_size.setter
  248. def batch_size(self, batch_size):
  249. self.trt_max_batch_size = batch_size
  250. def set_device(self, device: str):
  251. """set device"""
  252. if not device:
  253. return
  254. device_type, device_ids = parse_device(device)
  255. if device_type not in self.SUPPORT_DEVICE:
  256. support_run_mode_str = ", ".join(self.SUPPORT_DEVICE)
  257. raise ValueError(
  258. f"The device type must be one of {support_run_mode_str}, but received {repr(device_type)}."
  259. )
  260. self.device_type = device_type
  261. device_id = device_ids[0] if device_ids is not None else None
  262. self.device_id = device_id
  263. set_env_for_device(device)
  264. if device_type not in ("cpu"):
  265. if device_ids is None or len(device_ids) > 1:
  266. logging.debug(f"The device ID has been set to {device_id}.")
  267. # XXX(gaotingquan): set flag to accelerate inference in paddle 3.0b2
  268. if device_type in ("gpu", "cpu"):
  269. os.environ["FLAGS_enable_pir_api"] = "1"
  270. def get_support_run_mode(self):
  271. """get supported run mode"""
  272. return self.SUPPORT_RUN_MODE
  273. def get_support_device(self):
  274. """get supported device"""
  275. return self.SUPPORT_DEVICE
  276. def __str__(self):
  277. return ", ".join([f"{k}: {v}" for k, v in self._cfg.items()])
  278. def __getattr__(self, key):
  279. if key not in self._cfg:
  280. raise Exception(f"The key ({key}) is not found in cfg: \n {self._cfg}")
  281. return self._cfg.get(key)
  282. def __eq__(self, obj):
  283. if isinstance(obj, PaddlePredictorOption):
  284. return obj._cfg == self._cfg
  285. return False
  286. def _has_setter(self, attr):
  287. prop = getattr(self.__class__, attr, None)
  288. return isinstance(prop, property) and prop.fset is not None
  289. def _get_settable_attributes(self):
  290. return [
  291. name
  292. for name, prop in vars(self.__class__).items()
  293. if isinstance(prop, property) and prop.fset is not None
  294. ]