static_infer.py 34 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import abc
  15. import importlib.util
  16. import subprocess
  17. from pathlib import Path
  18. from typing import List, Sequence
  19. import lazy_paddle as paddle
  20. import numpy as np
  21. from ....utils import logging
  22. from ....utils.device import constr_device
  23. from ....utils.flags import DEBUG, INFER_BENCHMARK_USE_NEW_INFER_API, USE_PIR_TRT
  24. from ...utils.benchmark import benchmark, set_inference_operations
  25. from ...utils.hpi import (
  26. HPIConfig,
  27. OMConfig,
  28. ONNXRuntimeConfig,
  29. OpenVINOConfig,
  30. TensorRTConfig,
  31. get_model_paths,
  32. suggest_inference_backend_and_config,
  33. )
  34. from ...utils.pp_option import PaddlePredictorOption
  35. from ...utils.trt_config import DISABLE_TRT_HALF_OPS_CONFIG
  36. CACHE_DIR = ".cache"
  37. INFERENCE_OPERATIONS = [
  38. "PaddleCopyToDevice",
  39. "PaddleCopyToHost",
  40. "PaddleModelInfer",
  41. "PaddleInferChainLegacy",
  42. "MultiBackendInfer",
  43. ]
  44. set_inference_operations(INFERENCE_OPERATIONS)
  45. # XXX: Better use Paddle Inference API to do this
  46. def _pd_dtype_to_np_dtype(pd_dtype):
  47. if pd_dtype == paddle.inference.DataType.FLOAT64:
  48. return np.float64
  49. elif pd_dtype == paddle.inference.DataType.FLOAT32:
  50. return np.float32
  51. elif pd_dtype == paddle.inference.DataType.INT64:
  52. return np.int64
  53. elif pd_dtype == paddle.inference.DataType.INT32:
  54. return np.int32
  55. elif pd_dtype == paddle.inference.DataType.UINT8:
  56. return np.uint8
  57. elif pd_dtype == paddle.inference.DataType.INT8:
  58. return np.int8
  59. else:
  60. raise TypeError(f"Unsupported data type: {pd_dtype}")
  61. # old trt
  62. def _collect_trt_shape_range_info(
  63. model_file,
  64. model_params,
  65. gpu_id,
  66. shape_range_info_path,
  67. dynamic_shapes,
  68. dynamic_shape_input_data,
  69. ):
  70. dynamic_shape_input_data = dynamic_shape_input_data or {}
  71. config = paddle.inference.Config(model_file, model_params)
  72. config.enable_use_gpu(100, gpu_id)
  73. config.collect_shape_range_info(shape_range_info_path)
  74. # TODO: Add other needed options
  75. config.disable_glog_info()
  76. predictor = paddle.inference.create_predictor(config)
  77. input_names = predictor.get_input_names()
  78. for name in dynamic_shapes:
  79. if name not in input_names:
  80. raise ValueError(
  81. f"Invalid input name {repr(name)} found in `dynamic_shapes`"
  82. )
  83. for name in input_names:
  84. if name not in dynamic_shapes:
  85. raise ValueError(f"Input name {repr(name)} not found in `dynamic_shapes`")
  86. for name in dynamic_shape_input_data:
  87. if name not in input_names:
  88. raise ValueError(
  89. f"Invalid input name {repr(name)} found in `dynamic_shape_input_data`"
  90. )
  91. # It would be better to check if the shapes are valid.
  92. min_arrs, opt_arrs, max_arrs = {}, {}, {}
  93. for name, candidate_shapes in dynamic_shapes.items():
  94. # XXX: Currently we have no way to get the data type of the tensor
  95. # without creating an input handle.
  96. handle = predictor.get_input_handle(name)
  97. dtype = _pd_dtype_to_np_dtype(handle.type())
  98. min_shape, opt_shape, max_shape = candidate_shapes
  99. if name in dynamic_shape_input_data:
  100. min_arrs[name] = np.array(
  101. dynamic_shape_input_data[name][0], dtype=dtype
  102. ).reshape(min_shape)
  103. opt_arrs[name] = np.array(
  104. dynamic_shape_input_data[name][1], dtype=dtype
  105. ).reshape(opt_shape)
  106. max_arrs[name] = np.array(
  107. dynamic_shape_input_data[name][2], dtype=dtype
  108. ).reshape(max_shape)
  109. else:
  110. min_arrs[name] = np.ones(min_shape, dtype=dtype)
  111. opt_arrs[name] = np.ones(opt_shape, dtype=dtype)
  112. max_arrs[name] = np.ones(max_shape, dtype=dtype)
  113. # `opt_arrs` is used twice to ensure it is the most frequently used.
  114. for arrs in [min_arrs, opt_arrs, opt_arrs, max_arrs]:
  115. for name, arr in arrs.items():
  116. handle = predictor.get_input_handle(name)
  117. handle.reshape(arr.shape)
  118. handle.copy_from_cpu(arr)
  119. predictor.run()
  120. # HACK: The shape range info will be written to the file only when
  121. # `predictor` is garbage collected. It works in CPython, but it is
  122. # definitely a bad idea to count on the implementation-dependent behavior of
  123. # a garbage collector. Is there a more explicit and deterministic way to
  124. # handle this?
  125. # HACK: Manually delete the predictor to trigger its destructor, ensuring that the shape_range_info file would be saved.
  126. del predictor
  127. # pir trt
  128. def _convert_trt(
  129. trt_cfg_setting,
  130. pp_model_file,
  131. pp_params_file,
  132. trt_save_path,
  133. device_id,
  134. dynamic_shapes,
  135. dynamic_shape_input_data,
  136. ):
  137. from paddle.tensorrt.export import Input, TensorRTConfig, convert
  138. def _set_trt_config():
  139. for attr_name in trt_cfg_setting:
  140. assert hasattr(
  141. trt_config, attr_name
  142. ), f"The `{type(trt_config)}` don't have the attribute `{attr_name}`!"
  143. setattr(trt_config, attr_name, trt_cfg_setting[attr_name])
  144. def _get_predictor(model_file, params_file):
  145. # HACK
  146. config = paddle.inference.Config(str(model_file), str(params_file))
  147. config.enable_use_gpu(100, device_id)
  148. # NOTE: Disable oneDNN to circumvent a bug in Paddle Inference
  149. config.disable_mkldnn()
  150. config.disable_glog_info()
  151. return paddle.inference.create_predictor(config)
  152. dynamic_shape_input_data = dynamic_shape_input_data or {}
  153. predictor = _get_predictor(pp_model_file, pp_params_file)
  154. input_names = predictor.get_input_names()
  155. for name in dynamic_shapes:
  156. if name not in input_names:
  157. raise ValueError(
  158. f"Invalid input name {repr(name)} found in `dynamic_shapes`"
  159. )
  160. for name in input_names:
  161. if name not in dynamic_shapes:
  162. raise ValueError(f"Input name {repr(name)} not found in `dynamic_shapes`")
  163. for name in dynamic_shape_input_data:
  164. if name not in input_names:
  165. raise ValueError(
  166. f"Invalid input name {repr(name)} found in `dynamic_shape_input_data`"
  167. )
  168. trt_inputs = []
  169. for name, candidate_shapes in dynamic_shapes.items():
  170. # XXX: Currently we have no way to get the data type of the tensor
  171. # without creating an input handle.
  172. handle = predictor.get_input_handle(name)
  173. dtype = _pd_dtype_to_np_dtype(handle.type())
  174. min_shape, opt_shape, max_shape = candidate_shapes
  175. if name in dynamic_shape_input_data:
  176. min_arr = np.array(dynamic_shape_input_data[name][0], dtype=dtype).reshape(
  177. min_shape
  178. )
  179. opt_arr = np.array(dynamic_shape_input_data[name][1], dtype=dtype).reshape(
  180. opt_shape
  181. )
  182. max_arr = np.array(dynamic_shape_input_data[name][2], dtype=dtype).reshape(
  183. max_shape
  184. )
  185. else:
  186. min_arr = np.ones(min_shape, dtype=dtype)
  187. opt_arr = np.ones(opt_shape, dtype=dtype)
  188. max_arr = np.ones(max_shape, dtype=dtype)
  189. # refer to: https://github.com/PolaKuma/Paddle/blob/3347f225bc09f2ec09802a2090432dd5cb5b6739/test/tensorrt/test_converter_model_resnet50.py
  190. trt_input = Input((min_arr, opt_arr, max_arr))
  191. trt_inputs.append(trt_input)
  192. # Create TensorRTConfig
  193. trt_config = TensorRTConfig(inputs=trt_inputs)
  194. _set_trt_config()
  195. trt_config.save_model_dir = str(trt_save_path)
  196. pp_model_path = str(pp_model_file.with_suffix(""))
  197. convert(pp_model_path, trt_config)
  198. def _sort_inputs(inputs, names):
  199. # NOTE: Adjust input tensors to match the sorted sequence.
  200. indices = sorted(range(len(names)), key=names.__getitem__)
  201. inputs = [inputs[indices.index(i)] for i in range(len(inputs))]
  202. return inputs
  203. def _concatenate(*callables):
  204. def _chain(x):
  205. for c in callables:
  206. x = c(x)
  207. return x
  208. return _chain
  209. @benchmark.timeit
  210. class PaddleCopyToDevice:
  211. def __init__(self, device_type, device_id):
  212. self.device_type = device_type
  213. self.device_id = device_id
  214. def __call__(self, arrs):
  215. device_id = [self.device_id] if self.device_id is not None else self.device_id
  216. device = constr_device(self.device_type, device_id)
  217. paddle_tensors = [paddle.to_tensor(i, place=device) for i in arrs]
  218. return paddle_tensors
  219. @benchmark.timeit
  220. class PaddleCopyToHost:
  221. def __call__(self, paddle_tensors):
  222. arrs = [i.numpy() for i in paddle_tensors]
  223. return arrs
  224. @benchmark.timeit
  225. class PaddleModelInfer:
  226. def __init__(self, predictor):
  227. super().__init__()
  228. self.predictor = predictor
  229. def __call__(self, x):
  230. return self.predictor.run(x)
  231. # FIXME: Name might be misleading
  232. @benchmark.timeit
  233. class PaddleInferChainLegacy:
  234. def __init__(self, predictor):
  235. self.predictor = predictor
  236. input_names = self.predictor.get_input_names()
  237. self.input_handles = []
  238. self.output_handles = []
  239. for input_name in input_names:
  240. input_handle = self.predictor.get_input_handle(input_name)
  241. self.input_handles.append(input_handle)
  242. output_names = self.predictor.get_output_names()
  243. for output_name in output_names:
  244. output_handle = self.predictor.get_output_handle(output_name)
  245. self.output_handles.append(output_handle)
  246. def __call__(self, x):
  247. for input_, input_handle in zip(x, self.input_handles):
  248. input_handle.reshape(input_.shape)
  249. input_handle.copy_from_cpu(input_)
  250. self.predictor.run()
  251. outputs = [o.copy_to_cpu() for o in self.output_handles]
  252. return outputs
  253. class StaticInfer(metaclass=abc.ABCMeta):
  254. @abc.abstractmethod
  255. def __call__(self, x: Sequence[np.ndarray]) -> List[np.ndarray]:
  256. raise NotImplementedError
  257. class PaddleInfer(StaticInfer):
  258. def __init__(
  259. self,
  260. model_dir: str,
  261. model_file_prefix: str,
  262. option: PaddlePredictorOption,
  263. ) -> None:
  264. super().__init__()
  265. self.model_dir = model_dir
  266. self.model_file_prefix = model_file_prefix
  267. self._option = option
  268. self.predictor = self._create()
  269. if INFER_BENCHMARK_USE_NEW_INFER_API:
  270. device_type = self._option.device_type
  271. device_type = "gpu" if device_type == "dcu" else device_type
  272. copy_to_device = PaddleCopyToDevice(device_type, self._option.device_id)
  273. copy_to_host = PaddleCopyToHost()
  274. model_infer = PaddleModelInfer(self.predictor)
  275. self.infer = _concatenate(copy_to_device, model_infer, copy_to_host)
  276. else:
  277. self.infer = PaddleInferChainLegacy(self.predictor)
  278. def __call__(self, x: Sequence[np.ndarray]) -> List[np.ndarray]:
  279. names = self.predictor.get_input_names()
  280. if len(names) != len(x):
  281. raise ValueError(
  282. f"The number of inputs does not match the model: {len(names)} vs {len(x)}"
  283. )
  284. # TODO:
  285. # Ensure that input tensors follow the model's input sequence without sorting.
  286. x = _sort_inputs(x, names)
  287. x = list(map(np.ascontiguousarray, x))
  288. pred = self.infer(x)
  289. return pred
  290. def _create(
  291. self,
  292. ):
  293. """_create"""
  294. model_paths = get_model_paths(self.model_dir, self.model_file_prefix)
  295. if "paddle" not in model_paths:
  296. raise RuntimeError("No valid PaddlePaddle model found")
  297. model_file, params_file = model_paths["paddle"]
  298. if (
  299. self._option.model_name == "LaTeX_OCR_rec"
  300. and self._option.device_type == "cpu"
  301. ):
  302. import cpuinfo
  303. if (
  304. "GenuineIntel" in cpuinfo.get_cpu_info().get("vendor_id_raw", "")
  305. and self._option.run_mode != "mkldnn"
  306. ):
  307. logging.warning(
  308. "Now, the `LaTeX_OCR_rec` model only support `mkldnn` mode when running on Intel CPU devices. So using `mkldnn` instead."
  309. )
  310. self._option.run_mode = "mkldnn"
  311. logging.debug("`run_mode` updated to 'mkldnn'")
  312. if self._option.device_type == "cpu" and self._option.device_id is not None:
  313. self._option.device_id = None
  314. logging.debug("`device_id` has been set to None")
  315. if (
  316. self._option.device_type in ("gpu", "dcu")
  317. and self._option.device_id is None
  318. ):
  319. self._option.device_id = 0
  320. logging.debug("`device_id` has been set to 0")
  321. # for TRT
  322. if self._option.run_mode.startswith("trt"):
  323. assert self._option.device_type == "gpu"
  324. cache_dir = self.model_dir / CACHE_DIR / "paddle"
  325. config = self._configure_trt(
  326. model_file,
  327. params_file,
  328. cache_dir,
  329. )
  330. config.exp_disable_mixed_precision_ops({"feed", "fetch"})
  331. config.enable_use_gpu(100, self._option.device_id)
  332. # for Native Paddle and MKLDNN
  333. else:
  334. config = paddle.inference.Config(str(model_file), str(params_file))
  335. if self._option.device_type == "gpu":
  336. config.exp_disable_mixed_precision_ops({"feed", "fetch"})
  337. from paddle.inference import PrecisionType
  338. precision = (
  339. PrecisionType.Half
  340. if self._option.run_mode == "paddle_fp16"
  341. else PrecisionType.Float32
  342. )
  343. config.enable_use_gpu(100, self._option.device_id, precision)
  344. if hasattr(config, "enable_new_ir"):
  345. config.enable_new_ir(self._option.enable_new_ir)
  346. if hasattr(config, "enable_new_executor"):
  347. config.enable_new_executor()
  348. config.set_optimization_level(3)
  349. elif self._option.device_type == "npu":
  350. config.enable_custom_device("npu")
  351. if hasattr(config, "enable_new_executor"):
  352. config.enable_new_executor()
  353. elif self._option.device_type == "xpu":
  354. if hasattr(config, "enable_new_executor"):
  355. config.enable_new_executor()
  356. elif self._option.device_type == "mlu":
  357. config.enable_custom_device("mlu")
  358. if hasattr(config, "enable_new_executor"):
  359. config.enable_new_executor()
  360. elif self._option.device_type == "gcu":
  361. from paddle_custom_device.gcu import passes as gcu_passes
  362. gcu_passes.setUp()
  363. config.enable_custom_device("gcu")
  364. if hasattr(config, "enable_new_executor"):
  365. config.enable_new_ir()
  366. config.enable_new_executor()
  367. else:
  368. pass_builder = config.pass_builder()
  369. name = "PaddleX_" + self._option.model_name
  370. gcu_passes.append_passes_for_legacy_ir(pass_builder, name)
  371. elif self._option.device_type == "dcu":
  372. config.enable_use_gpu(100, self._option.device_id)
  373. if hasattr(config, "enable_new_executor"):
  374. config.enable_new_executor()
  375. # XXX: is_compiled_with_rocm() must be True on dcu platform ?
  376. if paddle.is_compiled_with_rocm():
  377. # Delete unsupported passes in dcu
  378. config.delete_pass("conv2d_add_act_fuse_pass")
  379. config.delete_pass("conv2d_add_fuse_pass")
  380. else:
  381. assert self._option.device_type == "cpu"
  382. config.disable_gpu()
  383. if "mkldnn" in self._option.run_mode:
  384. try:
  385. config.enable_mkldnn()
  386. if "bf16" in self._option.run_mode:
  387. config.enable_mkldnn_bfloat16()
  388. except Exception:
  389. logging.warning(
  390. "MKL-DNN is not available. We will disable MKL-DNN."
  391. )
  392. config.set_mkldnn_cache_capacity(-1)
  393. else:
  394. if hasattr(config, "disable_mkldnn"):
  395. config.disable_mkldnn()
  396. config.set_cpu_math_library_num_threads(self._option.cpu_threads)
  397. if hasattr(config, "enable_new_ir"):
  398. config.enable_new_ir(self._option.enable_new_ir)
  399. if hasattr(config, "enable_new_executor"):
  400. config.enable_new_executor()
  401. config.set_optimization_level(3)
  402. config.enable_memory_optim()
  403. for del_p in self._option.delete_pass:
  404. config.delete_pass(del_p)
  405. # Disable paddle inference logging
  406. if not DEBUG:
  407. config.disable_glog_info()
  408. predictor = paddle.inference.create_predictor(config)
  409. return predictor
  410. def _configure_trt(self, model_file, params_file, cache_dir):
  411. # TODO: Support calibration
  412. if USE_PIR_TRT:
  413. trt_save_path = cache_dir / "trt" / self.model_file_prefix
  414. _convert_trt(
  415. self._option.trt_cfg_setting,
  416. model_file,
  417. params_file,
  418. trt_save_path,
  419. self._option.device_id,
  420. self._option.trt_dynamic_shapes,
  421. self._option.trt_dynamic_shape_input_data,
  422. )
  423. model_file = trt_save_path.with_suffix(".json")
  424. params_file = trt_save_path.with_suffix(".pdiparams")
  425. config = paddle.inference.Config(str(model_file), str(params_file))
  426. else:
  427. config = paddle.inference.Config(str(model_file), str(params_file))
  428. config.set_optim_cache_dir(str(cache_dir / "optim_cache"))
  429. # call enable_use_gpu() first to use TensorRT engine
  430. config.enable_use_gpu(100, self._option.device_id)
  431. for func_name in self._option.trt_cfg_setting:
  432. assert hasattr(
  433. config, func_name
  434. ), f"The `{type(config)}` don't have function `{func_name}`!"
  435. args = self._option.trt_cfg_setting[func_name]
  436. if isinstance(args, list):
  437. getattr(config, func_name)(*args)
  438. else:
  439. getattr(config, func_name)(**args)
  440. if self._option.trt_use_dynamic_shapes:
  441. if self._option.trt_collect_shape_range_info:
  442. # NOTE: We always use a shape range info file.
  443. if self._option.trt_shape_range_info_path is not None:
  444. trt_shape_range_info_path = Path(
  445. self._option.trt_shape_range_info_path
  446. )
  447. else:
  448. trt_shape_range_info_path = cache_dir / "shape_range_info.pbtxt"
  449. should_collect_shape_range_info = True
  450. if not trt_shape_range_info_path.exists():
  451. trt_shape_range_info_path.parent.mkdir(
  452. parents=True, exist_ok=True
  453. )
  454. logging.info(
  455. f"Shape range info will be collected into {trt_shape_range_info_path}"
  456. )
  457. elif self._option.trt_discard_cached_shape_range_info:
  458. trt_shape_range_info_path.unlink()
  459. logging.info(
  460. f"The shape range info file ({trt_shape_range_info_path}) has been removed, and the shape range info will be re-collected."
  461. )
  462. else:
  463. logging.info(
  464. f"A shape range info file ({trt_shape_range_info_path}) already exists. There is no need to collect the info again."
  465. )
  466. should_collect_shape_range_info = False
  467. if should_collect_shape_range_info:
  468. _collect_trt_shape_range_info(
  469. str(model_file),
  470. str(params_file),
  471. self._option.device_id,
  472. str(trt_shape_range_info_path),
  473. self._option.trt_dynamic_shapes,
  474. self._option.trt_dynamic_shape_input_data,
  475. )
  476. if (
  477. self._option.model_name in DISABLE_TRT_HALF_OPS_CONFIG
  478. and self._option.run_mode == "trt_fp16"
  479. ):
  480. paddle.inference.InternalUtils.disable_tensorrt_half_ops(
  481. config, DISABLE_TRT_HALF_OPS_CONFIG[self._option.model_name]
  482. )
  483. config.enable_tuned_tensorrt_dynamic_shape(
  484. str(trt_shape_range_info_path),
  485. self._option.trt_allow_rebuild_at_runtime,
  486. )
  487. else:
  488. if self._option.trt_dynamic_shapes is not None:
  489. min_shapes, opt_shapes, max_shapes = {}, {}, {}
  490. for (
  491. key,
  492. shapes,
  493. ) in self._option.trt_dynamic_shapes.items():
  494. min_shapes[key] = shapes[0]
  495. opt_shapes[key] = shapes[1]
  496. max_shapes[key] = shapes[2]
  497. config.set_trt_dynamic_shape_info(
  498. min_shapes, max_shapes, opt_shapes
  499. )
  500. else:
  501. raise RuntimeError("No dynamic shape information provided")
  502. return config
  503. # FIXME: Name might be misleading
  504. @benchmark.timeit
  505. class MultiBackendInfer(object):
  506. def __init__(self, ui_runtime):
  507. super().__init__()
  508. self.ui_runtime = ui_runtime
  509. # The time consumed by the wrapper code will also be taken into account.
  510. def __call__(self, x):
  511. outputs = self.ui_runtime.infer(x)
  512. return outputs
  513. # TODO: It would be better to refactor the code to make `HPInfer` a higher-level
  514. # class that uses `PaddleInfer`.
  515. class HPInfer(StaticInfer):
  516. def __init__(
  517. self,
  518. model_dir: str,
  519. model_file_prefix: str,
  520. config: HPIConfig,
  521. ) -> None:
  522. super().__init__()
  523. self._model_dir = model_dir
  524. self._model_file_prefix = model_file_prefix
  525. self._config = config
  526. backend, backend_config = self._determine_backend_and_config()
  527. if backend == "paddle":
  528. self._use_paddle = True
  529. self._paddle_infer = self._build_paddle_infer(backend_config)
  530. else:
  531. self._use_paddle = False
  532. ui_runtime = self._build_ui_runtime(backend, backend_config)
  533. self._multi_backend_infer = MultiBackendInfer(ui_runtime)
  534. num_inputs = ui_runtime.num_inputs()
  535. self._input_names = [
  536. ui_runtime.get_input_info(i).name for i in range(num_inputs)
  537. ]
  538. @property
  539. def model_dir(self) -> str:
  540. return self._model_dir
  541. @property
  542. def model_file_prefix(self) -> str:
  543. return self._model_file_prefix
  544. @property
  545. def config(self) -> HPIConfig:
  546. return self._config
  547. def __call__(self, x: Sequence[np.ndarray]) -> List[np.ndarray]:
  548. if self._use_paddle:
  549. return self._call_paddle_infer(x)
  550. else:
  551. return self._call_multi_backend_infer(x)
  552. def _call_paddle_infer(self, x):
  553. return self._paddle_infer(x)
  554. def _call_multi_backend_infer(self, x):
  555. num_inputs = len(self._input_names)
  556. if len(x) != num_inputs:
  557. raise ValueError(f"Expected {num_inputs} inputs but got {len(x)} instead")
  558. x = _sort_inputs(x, self._input_names)
  559. inputs = {}
  560. for name, input_ in zip(self._input_names, x):
  561. inputs[name] = np.ascontiguousarray(input_)
  562. return self._multi_backend_infer(inputs)
  563. def _determine_backend_and_config(self):
  564. from ultra_infer import (
  565. is_built_with_om,
  566. is_built_with_openvino,
  567. is_built_with_ort,
  568. is_built_with_trt,
  569. )
  570. model_paths = get_model_paths(self._model_dir, self._model_file_prefix)
  571. is_onnx_model_available = "onnx" in model_paths
  572. # TODO: Give a warning if Paddle2ONNX is not available but can be used
  573. # to select a better backend.
  574. if self._config.auto_paddle2onnx:
  575. if self._check_paddle2onnx():
  576. is_onnx_model_available = (
  577. is_onnx_model_available or "paddle" in model_paths
  578. )
  579. else:
  580. logging.debug(
  581. "Paddle2ONNX is not available. Automatic model conversion will not be performed."
  582. )
  583. available_backends = []
  584. if "paddle" in model_paths:
  585. available_backends.append("paddle")
  586. if is_built_with_openvino() and is_onnx_model_available:
  587. available_backends.append("openvino")
  588. if is_built_with_ort() and is_onnx_model_available:
  589. available_backends.append("onnxruntime")
  590. if is_built_with_trt() and is_onnx_model_available:
  591. available_backends.append("tensorrt")
  592. if is_built_with_om() and "om" in model_paths:
  593. available_backends.append("om")
  594. if not available_backends:
  595. raise RuntimeError("No inference backend is available")
  596. if (
  597. self._config.backend is not None
  598. and self._config.backend not in available_backends
  599. ):
  600. raise RuntimeError(
  601. f"Inference backend {repr(self._config.backend)} is unavailable"
  602. )
  603. if self._config.auto_config:
  604. # Should we use the strategy pattern here to allow extensible
  605. # strategies?
  606. ret = suggest_inference_backend_and_config(
  607. self._config, available_backends=available_backends
  608. )
  609. if ret[0] is None:
  610. # Should I use a custom exception?
  611. raise RuntimeError(
  612. f"No inference backend and configuration could be suggested. Reason: {ret[1]}"
  613. )
  614. backend, backend_config = ret
  615. else:
  616. backend = self._config.backend
  617. if backend is None:
  618. raise RuntimeError(
  619. "When automatic configuration is not used, the inference backend must be specified manually."
  620. )
  621. backend_config = self._config.backend_config or {}
  622. if backend == "paddle" and not backend_config:
  623. logging.warning(
  624. "The Paddle Inference backend is selected with the default configuration. This may not provide optimal performance."
  625. )
  626. return backend, backend_config
  627. def _build_paddle_infer(self, backend_config):
  628. kwargs = {
  629. "device_type": self._config.device_type,
  630. "device_id": self._config.device_id,
  631. **backend_config,
  632. }
  633. # TODO: This is probably redundant. Can we reuse the code in the
  634. # predictor class?
  635. paddle_info = self._config.hpi_info.backend_configs.paddle_infer
  636. if paddle_info is not None:
  637. if (
  638. kwargs.get("trt_dynamic_shapes") is None
  639. and paddle_info.trt_dynamic_shapes is not None
  640. ):
  641. trt_dynamic_shapes = paddle_info.trt_dynamic_shapes
  642. logging.debug("TensorRT dynamic shapes set to %s", trt_dynamic_shapes)
  643. kwargs["trt_dynamic_shapes"] = trt_dynamic_shapes
  644. if (
  645. kwargs.get("trt_dynamic_shape_input_data") is None
  646. and paddle_info.trt_dynamic_shape_input_data is not None
  647. ):
  648. trt_dynamic_shape_input_data = paddle_info.trt_dynamic_shape_input_data
  649. logging.debug(
  650. "TensorRT dynamic shape input data set to %s",
  651. trt_dynamic_shape_input_data,
  652. )
  653. kwargs["trt_dynamic_shape_input_data"] = trt_dynamic_shape_input_data
  654. pp_option = PaddlePredictorOption(self._config.pdx_model_name, **kwargs)
  655. logging.info("Using Paddle Inference backend")
  656. logging.info("Paddle predictor option: %s", pp_option)
  657. return PaddleInfer(self._model_dir, self._model_file_prefix, option=pp_option)
  658. def _build_ui_runtime(self, backend, backend_config, ui_option=None):
  659. from ultra_infer import ModelFormat, Runtime, RuntimeOption
  660. if ui_option is None:
  661. ui_option = RuntimeOption()
  662. if self._config.device_type == "cpu":
  663. pass
  664. elif self._config.device_type == "gpu":
  665. ui_option.use_gpu(self._config.device_id or 0)
  666. elif self._config.device_type == "npu":
  667. ui_option.use_ascend(self._config.device_id or 0)
  668. else:
  669. raise RuntimeError(
  670. f"Unsupported device type {repr(self._config.device_type)}"
  671. )
  672. model_paths = get_model_paths(self.model_dir, self.model_file_prefix)
  673. if backend in ("openvino", "onnxruntime", "tensorrt"):
  674. # XXX: This introduces side effects.
  675. if "onnx" not in model_paths:
  676. if self._config.auto_paddle2onnx:
  677. if "paddle" not in model_paths:
  678. raise RuntimeError("PaddlePaddle model required")
  679. # The CLI is used here since there is currently no API.
  680. logging.info(
  681. "Automatically converting PaddlePaddle model to ONNX format"
  682. )
  683. subprocess.check_call(
  684. [
  685. "paddlex",
  686. "--paddle2onnx",
  687. "--paddle_model_dir",
  688. self._model_dir,
  689. "--onnx_model_dir",
  690. self._model_dir,
  691. ]
  692. )
  693. model_paths = get_model_paths(
  694. self.model_dir, self.model_file_prefix
  695. )
  696. assert "onnx" in model_paths
  697. else:
  698. raise RuntimeError("ONNX model required")
  699. ui_option.set_model_path(str(model_paths["onnx"]), "", ModelFormat.ONNX)
  700. elif backend == "om":
  701. if "om" not in model_paths:
  702. raise RuntimeError("OM model required")
  703. ui_option.set_model_path(str(model_paths["om"]), "", ModelFormat.OM)
  704. else:
  705. raise ValueError(f"Unsupported inference backend {repr(backend)}")
  706. if backend == "openvino":
  707. backend_config = OpenVINOConfig.model_validate(backend_config)
  708. ui_option.use_openvino_backend()
  709. ui_option.set_cpu_thread_num(backend_config.cpu_num_threads)
  710. elif backend == "onnxruntime":
  711. backend_config = ONNXRuntimeConfig.model_validate(backend_config)
  712. ui_option.use_ort_backend()
  713. ui_option.set_cpu_thread_num(backend_config.cpu_num_threads)
  714. elif backend == "tensorrt":
  715. if (
  716. backend_config.get("use_dynamic_shapes", True)
  717. and backend_config.get("dynamic_shapes") is None
  718. ):
  719. trt_info = self._config.hpi_info.backend_configs.tensorrt
  720. if trt_info is not None and trt_info.dynamic_shapes is not None:
  721. trt_dynamic_shapes = trt_info.dynamic_shapes
  722. logging.debug(
  723. "TensorRT dynamic shapes set to %s", trt_dynamic_shapes
  724. )
  725. backend_config = {
  726. **backend_config,
  727. "dynamic_shapes": trt_dynamic_shapes,
  728. }
  729. backend_config = TensorRTConfig.model_validate(backend_config)
  730. ui_option.use_trt_backend()
  731. cache_dir = self.model_dir / CACHE_DIR / "tensorrt"
  732. cache_dir.mkdir(parents=True, exist_ok=True)
  733. ui_option.trt_option.serialize_file = str(cache_dir / "trt_serialized.trt")
  734. if backend_config.precision == "FP16":
  735. ui_option.trt_option.enable_fp16 = True
  736. if not backend_config.use_dynamic_shapes:
  737. raise RuntimeError(
  738. "TensorRT static shape inference is currently not supported"
  739. )
  740. if backend_config.dynamic_shapes is not None:
  741. if not Path(ui_option.trt_option.serialize_file).exists():
  742. for name, shapes in backend_config.dynamic_shapes.items():
  743. ui_option.trt_option.set_shape(name, *shapes)
  744. else:
  745. logging.warning(
  746. "TensorRT dynamic shapes will be loaded from the file."
  747. )
  748. elif backend == "om":
  749. backend_config = OMConfig.model_validate(backend_config)
  750. ui_option.use_om_backend()
  751. else:
  752. raise ValueError(f"Unsupported inference backend {repr(backend)}")
  753. logging.info("Inference backend: %s", backend)
  754. logging.info("Inference backend config: %s", backend_config)
  755. ui_runtime = Runtime(ui_option)
  756. return ui_runtime
  757. def _check_paddle2onnx(self):
  758. # HACK
  759. return importlib.util.find_spec("paddle2onnx") is not None