pipeline.py 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import math
  15. from typing import Any, Dict, List, Optional, Tuple, Union
  16. import numpy as np
  17. from ....utils import logging
  18. from ...common.batch_sampler import ImageBatchSampler
  19. from ...common.reader import ReadImage
  20. from ...models.object_detection.result import DetResult
  21. from ...utils.hpi import HPIConfig
  22. from ...utils.pp_option import PaddlePredictorOption
  23. from ..base import BasePipeline
  24. from ..components import CropByBoxes
  25. from ..doc_preprocessor.result import DocPreprocessorResult
  26. from ..ocr.result import OCRResult
  27. from .result import SingleTableRecognitionResult, TableRecognitionResult
  28. from .table_recognition_post_processing import get_table_recognition_res
  29. from .utils import get_neighbor_boxes_idx
  30. class TableRecognitionPipeline(BasePipeline):
  31. """Table Recognition Pipeline"""
  32. entities = ["table_recognition"]
  33. def __init__(
  34. self,
  35. config: Dict,
  36. device: str = None,
  37. pp_option: PaddlePredictorOption = None,
  38. use_hpip: bool = False,
  39. hpi_config: Optional[Union[Dict[str, Any], HPIConfig]] = None,
  40. ) -> None:
  41. """Initializes the layout parsing pipeline.
  42. Args:
  43. config (Dict): Configuration dictionary containing various settings.
  44. device (str, optional): Device to run the predictions on. Defaults to None.
  45. pp_option (PaddlePredictorOption, optional): PaddlePredictor options. Defaults to None.
  46. use_hpip (bool, optional): Whether to use the high-performance
  47. inference plugin (HPIP) by default. Defaults to False.
  48. hpi_config (Optional[Union[Dict[str, Any], HPIConfig]], optional):
  49. The default high-performance inference configuration dictionary.
  50. Defaults to None.
  51. """
  52. super().__init__(
  53. device=device, pp_option=pp_option, use_hpip=use_hpip, hpi_config=hpi_config
  54. )
  55. self.use_doc_preprocessor = config.get("use_doc_preprocessor", True)
  56. if self.use_doc_preprocessor:
  57. doc_preprocessor_config = config.get("SubPipelines", {}).get(
  58. "DocPreprocessor",
  59. {
  60. "pipeline_config_error": "config error for doc_preprocessor_pipeline!"
  61. },
  62. )
  63. self.doc_preprocessor_pipeline = self.create_pipeline(
  64. doc_preprocessor_config
  65. )
  66. self.use_layout_detection = config.get("use_layout_detection", True)
  67. if self.use_layout_detection:
  68. layout_det_config = config.get("SubModules", {}).get(
  69. "LayoutDetection",
  70. {"model_config_error": "config error for layout_det_model!"},
  71. )
  72. self.layout_det_model = self.create_model(layout_det_config)
  73. table_structure_config = config.get("SubModules", {}).get(
  74. "TableStructureRecognition",
  75. {"model_config_error": "config error for table_structure_model!"},
  76. )
  77. self.table_structure_model = self.create_model(table_structure_config)
  78. self.use_ocr_model = config.get("use_ocr_model", True)
  79. if self.use_ocr_model:
  80. general_ocr_config = config.get("SubPipelines", {}).get(
  81. "GeneralOCR",
  82. {"pipeline_config_error": "config error for general_ocr_pipeline!"},
  83. )
  84. self.general_ocr_pipeline = self.create_pipeline(general_ocr_config)
  85. else:
  86. self.general_ocr_config_bak = config.get("SubPipelines", {}).get(
  87. "GeneralOCR", None
  88. )
  89. self._crop_by_boxes = CropByBoxes()
  90. self.batch_sampler = ImageBatchSampler(batch_size=1)
  91. self.img_reader = ReadImage(format="BGR")
  92. def get_model_settings(
  93. self,
  94. use_doc_orientation_classify: Optional[bool],
  95. use_doc_unwarping: Optional[bool],
  96. use_layout_detection: Optional[bool],
  97. use_ocr_model: Optional[bool],
  98. ) -> dict:
  99. """
  100. Get the model settings based on the provided parameters or default values.
  101. Args:
  102. use_doc_orientation_classify (Optional[bool]): Whether to use document orientation classification.
  103. use_doc_unwarping (Optional[bool]): Whether to use document unwarping.
  104. use_layout_detection (Optional[bool]): Whether to use layout detection.
  105. use_ocr_model (Optional[bool]): Whether to use OCR model.
  106. Returns:
  107. dict: A dictionary containing the model settings.
  108. """
  109. if use_doc_orientation_classify is None and use_doc_unwarping is None:
  110. use_doc_preprocessor = self.use_doc_preprocessor
  111. else:
  112. if use_doc_orientation_classify is True or use_doc_unwarping is True:
  113. use_doc_preprocessor = True
  114. else:
  115. use_doc_preprocessor = False
  116. if use_layout_detection is None:
  117. use_layout_detection = self.use_layout_detection
  118. if use_ocr_model is None:
  119. use_ocr_model = self.use_ocr_model
  120. return dict(
  121. use_doc_preprocessor=use_doc_preprocessor,
  122. use_layout_detection=use_layout_detection,
  123. use_ocr_model=use_ocr_model,
  124. )
  125. def check_model_settings_valid(
  126. self,
  127. model_settings: Dict,
  128. overall_ocr_res: OCRResult,
  129. layout_det_res: DetResult,
  130. ) -> bool:
  131. """
  132. Check if the input parameters are valid based on the initialized models.
  133. Args:
  134. model_settings (Dict): A dictionary containing input parameters.
  135. overall_ocr_res (OCRResult): Overall OCR result obtained after running the OCR pipeline.
  136. The overall OCR result with convert_points_to_boxes information.
  137. layout_det_res (DetResult): The layout detection result.
  138. Returns:
  139. bool: True if all required models are initialized according to input parameters, False otherwise.
  140. """
  141. if model_settings["use_doc_preprocessor"] and not self.use_doc_preprocessor:
  142. logging.error(
  143. "Set use_doc_preprocessor, but the models for doc preprocessor are not initialized."
  144. )
  145. return False
  146. if model_settings["use_layout_detection"]:
  147. if layout_det_res is not None:
  148. logging.error(
  149. "The layout detection model has already been initialized, please set use_layout_detection=False"
  150. )
  151. return False
  152. if not self.use_layout_detection:
  153. logging.error(
  154. "Set use_layout_detection, but the models for layout detection are not initialized."
  155. )
  156. return False
  157. if model_settings["use_ocr_model"]:
  158. if overall_ocr_res is not None:
  159. logging.error(
  160. "The OCR models have already been initialized, please set use_ocr_model=False"
  161. )
  162. return False
  163. if not self.use_ocr_model:
  164. logging.error(
  165. "Set use_ocr_model, but the models for OCR are not initialized."
  166. )
  167. return False
  168. else:
  169. if overall_ocr_res is None:
  170. logging.error("Set use_ocr_model=False, but no OCR results were found.")
  171. return False
  172. return True
  173. def predict_doc_preprocessor_res(
  174. self, image_array: np.ndarray, input_params: dict
  175. ) -> Tuple[DocPreprocessorResult, np.ndarray]:
  176. """
  177. Preprocess the document image based on input parameters.
  178. Args:
  179. image_array (np.ndarray): The input image array.
  180. input_params (dict): Dictionary containing preprocessing parameters.
  181. Returns:
  182. tuple[DocPreprocessorResult, np.ndarray]: A tuple containing the preprocessing
  183. result dictionary and the processed image array.
  184. """
  185. if input_params["use_doc_preprocessor"]:
  186. use_doc_orientation_classify = input_params["use_doc_orientation_classify"]
  187. use_doc_unwarping = input_params["use_doc_unwarping"]
  188. doc_preprocessor_res = next(
  189. self.doc_preprocessor_pipeline(
  190. image_array,
  191. use_doc_orientation_classify=use_doc_orientation_classify,
  192. use_doc_unwarping=use_doc_unwarping,
  193. )
  194. )
  195. doc_preprocessor_image = doc_preprocessor_res["output_img"]
  196. else:
  197. doc_preprocessor_res = {}
  198. doc_preprocessor_image = image_array
  199. return doc_preprocessor_res, doc_preprocessor_image
  200. def split_ocr_bboxes_by_table_cells(self, ori_img, cells_bboxes):
  201. """
  202. Splits OCR bounding boxes by table cells and retrieves text.
  203. Args:
  204. ori_img (ndarray): The original image from which text regions will be extracted.
  205. cells_bboxes (list or ndarray): Detected cell bounding boxes to extract text from.
  206. Returns:
  207. list: A list containing the recognized texts from each cell.
  208. """
  209. # Check if cells_bboxes is a list and convert it if not.
  210. if not isinstance(cells_bboxes, list):
  211. cells_bboxes = cells_bboxes.tolist()
  212. texts_list = [] # Initialize a list to store the recognized texts.
  213. # Process each bounding box provided in cells_bboxes.
  214. for i in range(len(cells_bboxes)):
  215. # Extract and round up the coordinates of the bounding box.
  216. x1, y1, x2, y2 = [math.ceil(k) for k in cells_bboxes[i]]
  217. # Perform OCR on the defined region of the image and get the recognized text.
  218. rec_te = next(self.general_ocr_pipeline(ori_img[y1:y2, x1:x2, :]))
  219. # Concatenate the texts and append them to the texts_list.
  220. texts_list.append("".join(rec_te["rec_texts"]))
  221. # Return the list of recognized texts from each cell.
  222. return texts_list
  223. def split_ocr_bboxes_by_table_cells(self, ori_img, cells_bboxes):
  224. """
  225. Splits OCR bounding boxes by table cells and retrieves text.
  226. Args:
  227. ori_img (ndarray): The original image from which text regions will be extracted.
  228. cells_bboxes (list or ndarray): Detected cell bounding boxes to extract text from.
  229. Returns:
  230. list: A list containing the recognized texts from each cell.
  231. """
  232. # Check if cells_bboxes is a list and convert it if not.
  233. if not isinstance(cells_bboxes, list):
  234. cells_bboxes = cells_bboxes.tolist()
  235. texts_list = [] # Initialize a list to store the recognized texts.
  236. # Process each bounding box provided in cells_bboxes.
  237. for i in range(len(cells_bboxes)):
  238. # Extract and round up the coordinates of the bounding box.
  239. x1, y1, x2, y2 = [math.ceil(k) for k in cells_bboxes[i]]
  240. # Perform OCR on the defined region of the image and get the recognized text.
  241. rec_te = next(self.general_ocr_pipeline(ori_img[y1:y2, x1:x2, :]))
  242. # Concatenate the texts and append them to the texts_list.
  243. texts_list.append("".join(rec_te["rec_texts"]))
  244. # Return the list of recognized texts from each cell.
  245. return texts_list
  246. def predict_single_table_recognition_res(
  247. self,
  248. image_array: np.ndarray,
  249. overall_ocr_res: OCRResult,
  250. table_box: list,
  251. use_table_cells_ocr_results: bool = False,
  252. flag_find_nei_text: bool = True,
  253. cell_sort_by_y_projection: bool = False,
  254. ) -> SingleTableRecognitionResult:
  255. """
  256. Predict table recognition results from an image array, layout detection results, and OCR results.
  257. Args:
  258. image_array (np.ndarray): The input image represented as a numpy array.
  259. overall_ocr_res (OCRResult): Overall OCR result obtained after running the OCR pipeline.
  260. The overall OCR results containing text recognition information.
  261. table_box (list): The table box coordinates.
  262. use_table_cells_ocr_results (bool): whether to use OCR results with cells.
  263. flag_find_nei_text (bool): Whether to find neighboring text.
  264. cell_sort_by_y_projection (bool): Whether to sort the matched OCR boxes by y-projection.
  265. Returns:
  266. SingleTableRecognitionResult: single table recognition result.
  267. """
  268. table_structure_pred = next(self.table_structure_model(image_array))
  269. if use_table_cells_ocr_results == True:
  270. table_cells_result = list(
  271. map(lambda arr: arr.tolist(), table_structure_pred["bbox"])
  272. )
  273. table_cells_result = [
  274. [rect[0], rect[1], rect[4], rect[5]] for rect in table_cells_result
  275. ]
  276. cells_texts_list = self.split_ocr_bboxes_by_table_cells(
  277. image_array, table_cells_result
  278. )
  279. else:
  280. cells_texts_list = []
  281. single_table_recognition_res = get_table_recognition_res(
  282. table_box,
  283. table_structure_pred,
  284. overall_ocr_res,
  285. cells_texts_list,
  286. use_table_cells_ocr_results,
  287. cell_sort_by_y_projection=cell_sort_by_y_projection,
  288. )
  289. neighbor_text = ""
  290. if flag_find_nei_text:
  291. match_idx_list = get_neighbor_boxes_idx(
  292. overall_ocr_res["rec_boxes"], table_box
  293. )
  294. if len(match_idx_list) > 0:
  295. for idx in match_idx_list:
  296. neighbor_text += overall_ocr_res["rec_texts"][idx] + "; "
  297. single_table_recognition_res["neighbor_texts"] = neighbor_text
  298. return single_table_recognition_res
  299. def predict(
  300. self,
  301. input: Union[str, List[str], np.ndarray, List[np.ndarray]],
  302. use_doc_orientation_classify: Optional[bool] = None,
  303. use_doc_unwarping: Optional[bool] = None,
  304. use_layout_detection: Optional[bool] = None,
  305. use_ocr_model: Optional[bool] = None,
  306. overall_ocr_res: Optional[OCRResult] = None,
  307. layout_det_res: Optional[DetResult] = None,
  308. text_det_limit_side_len: Optional[int] = None,
  309. text_det_limit_type: Optional[str] = None,
  310. text_det_thresh: Optional[float] = None,
  311. text_det_box_thresh: Optional[float] = None,
  312. text_det_unclip_ratio: Optional[float] = None,
  313. text_rec_score_thresh: Optional[float] = None,
  314. use_table_cells_ocr_results: bool = False,
  315. cell_sort_by_y_projection: Optional[bool] = None,
  316. **kwargs,
  317. ) -> TableRecognitionResult:
  318. """
  319. This function predicts the layout parsing result for the given input.
  320. Args:
  321. input (Union[str, list[str], np.ndarray, list[np.ndarray]]): The input image(s) of pdf(s) to be processed.
  322. use_layout_detection (bool): Whether to use layout detection.
  323. use_doc_orientation_classify (bool): Whether to use document orientation classification.
  324. use_doc_unwarping (bool): Whether to use document unwarping.
  325. overall_ocr_res (OCRResult): The overall OCR result with convert_points_to_boxes information.
  326. It will be used if it is not None and use_ocr_model is False.
  327. layout_det_res (DetResult): The layout detection result.
  328. It will be used if it is not None and use_layout_detection is False.
  329. use_table_cells_ocr_results (bool): whether to use OCR results with cells.
  330. cell_sort_by_y_projection (bool): Whether to sort the matched OCR boxes by y-projection.
  331. **kwargs: Additional keyword arguments.
  332. Returns:
  333. TableRecognitionResult: The predicted table recognition result.
  334. """
  335. model_settings = self.get_model_settings(
  336. use_doc_orientation_classify,
  337. use_doc_unwarping,
  338. use_layout_detection,
  339. use_ocr_model,
  340. )
  341. if cell_sort_by_y_projection is None:
  342. cell_sort_by_y_projection = False
  343. if not self.check_model_settings_valid(
  344. model_settings, overall_ocr_res, layout_det_res
  345. ):
  346. yield {"error": "the input params for model settings are invalid!"}
  347. for img_id, batch_data in enumerate(self.batch_sampler(input)):
  348. image_array = self.img_reader(batch_data.instances)[0]
  349. if model_settings["use_doc_preprocessor"]:
  350. doc_preprocessor_res = next(
  351. self.doc_preprocessor_pipeline(
  352. image_array,
  353. use_doc_orientation_classify=use_doc_orientation_classify,
  354. use_doc_unwarping=use_doc_unwarping,
  355. )
  356. )
  357. else:
  358. doc_preprocessor_res = {"output_img": image_array}
  359. doc_preprocessor_image = doc_preprocessor_res["output_img"]
  360. if model_settings["use_ocr_model"]:
  361. overall_ocr_res = next(
  362. self.general_ocr_pipeline(
  363. doc_preprocessor_image,
  364. text_det_limit_side_len=text_det_limit_side_len,
  365. text_det_limit_type=text_det_limit_type,
  366. text_det_thresh=text_det_thresh,
  367. text_det_box_thresh=text_det_box_thresh,
  368. text_det_unclip_ratio=text_det_unclip_ratio,
  369. text_rec_score_thresh=text_rec_score_thresh,
  370. )
  371. )
  372. elif use_table_cells_ocr_results == True:
  373. assert self.general_ocr_config_bak != None
  374. self.general_ocr_pipeline = self.create_pipeline(
  375. self.general_ocr_config_bak
  376. )
  377. table_res_list = []
  378. table_region_id = 1
  379. if not model_settings["use_layout_detection"] and layout_det_res is None:
  380. layout_det_res = {}
  381. img_height, img_width = doc_preprocessor_image.shape[:2]
  382. table_box = [0, 0, img_width - 1, img_height - 1]
  383. single_table_rec_res = self.predict_single_table_recognition_res(
  384. doc_preprocessor_image,
  385. overall_ocr_res,
  386. table_box,
  387. use_table_cells_ocr_results,
  388. flag_find_nei_text=False,
  389. cell_sort_by_y_projection=cell_sort_by_y_projection,
  390. )
  391. single_table_rec_res["table_region_id"] = table_region_id
  392. table_res_list.append(single_table_rec_res)
  393. table_region_id += 1
  394. else:
  395. if model_settings["use_layout_detection"]:
  396. layout_det_res = next(self.layout_det_model(doc_preprocessor_image))
  397. for box_info in layout_det_res["boxes"]:
  398. if box_info["label"].lower() in ["table"]:
  399. crop_img_info = self._crop_by_boxes(image_array, [box_info])
  400. crop_img_info = crop_img_info[0]
  401. table_box = crop_img_info["box"]
  402. single_table_rec_res = (
  403. self.predict_single_table_recognition_res(
  404. crop_img_info["img"],
  405. overall_ocr_res,
  406. table_box,
  407. use_table_cells_ocr_results,
  408. cell_sort_by_y_projection=cell_sort_by_y_projection,
  409. )
  410. )
  411. single_table_rec_res["table_region_id"] = table_region_id
  412. table_res_list.append(single_table_rec_res)
  413. table_region_id += 1
  414. single_img_res = {
  415. "input_path": batch_data.input_paths[0],
  416. "page_index": batch_data.page_indexes[0],
  417. "doc_preprocessor_res": doc_preprocessor_res,
  418. "layout_det_res": layout_det_res,
  419. "overall_ocr_res": overall_ocr_res,
  420. "table_res_list": table_res_list,
  421. "model_settings": model_settings,
  422. }
  423. yield TableRecognitionResult(single_img_res)