pipeline_v4.py 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868
  1. # copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from typing import Any, Dict, Optional, Union, List, Tuple
  15. import re
  16. import cv2
  17. import json
  18. import base64
  19. import numpy as np
  20. import copy
  21. from .pipeline_base import PP_ChatOCR_Pipeline
  22. from ...common.reader import ReadImage
  23. from ...common.batch_sampler import ImageBatchSampler
  24. from ....utils import logging
  25. from ...utils.pp_option import PaddlePredictorOption
  26. from ..layout_parsing.result import LayoutParsingResult
  27. from ..components.chat_server import BaseChat
  28. class PP_ChatOCRv4_Pipeline(PP_ChatOCR_Pipeline):
  29. """PP-ChatOCRv4 Pipeline"""
  30. entities = ["PP-ChatOCRv4-doc"]
  31. def __init__(
  32. self,
  33. config: Dict,
  34. device: str = None,
  35. pp_option: PaddlePredictorOption = None,
  36. use_hpip: bool = False,
  37. initial_predictor: bool = True,
  38. ) -> None:
  39. """Initializes the pp-chatocrv3-doc pipeline.
  40. Args:
  41. config (Dict): Configuration dictionary containing various settings.
  42. device (str, optional): Device to run the predictions on. Defaults to None.
  43. pp_option (PaddlePredictorOption, optional): PaddlePredictor options. Defaults to None.
  44. use_hpip (bool, optional): Whether to use high-performance inference (hpip) for prediction. Defaults to False.
  45. use_layout_parsing (bool, optional): Whether to use layout parsing. Defaults to True.
  46. initial_predictor (bool, optional): Whether to initialize the predictor. Defaults to True.
  47. """
  48. super().__init__(device=device, pp_option=pp_option, use_hpip=use_hpip)
  49. self.pipeline_name = config["pipeline_name"]
  50. self.config = config
  51. self.use_layout_parser = config.get("use_layout_parser", True)
  52. self.use_mllm_predict = config.get("use_mllm_predict", True)
  53. self.layout_parsing_pipeline = None
  54. self.chat_bot = None
  55. self.retriever = None
  56. self.mllm_chat_bot = None
  57. if initial_predictor:
  58. self.inintial_visual_predictor(config)
  59. self.inintial_chat_predictor(config)
  60. self.inintial_mllm_predictor(config)
  61. self.batch_sampler = ImageBatchSampler(batch_size=1)
  62. self.img_reader = ReadImage(format="BGR")
  63. self.table_structure_len_max = 500
  64. def inintial_visual_predictor(self, config: dict) -> None:
  65. """
  66. Initializes the visual predictor with the given configuration.
  67. Args:
  68. config (dict): The configuration dictionary containing the necessary
  69. parameters for initializing the predictor.
  70. Returns:
  71. None
  72. """
  73. self.use_layout_parser = config.get("use_layout_parser", True)
  74. if self.use_layout_parser:
  75. layout_parsing_config = config.get("SubPipelines", {}).get(
  76. "LayoutParser",
  77. {"pipeline_config_error": "config error for layout_parsing_pipeline!"},
  78. )
  79. self.layout_parsing_pipeline = self.create_pipeline(layout_parsing_config)
  80. return
  81. def inintial_retriever_predictor(self, config: dict) -> None:
  82. """
  83. Initializes the retriever predictor with the given configuration.
  84. Args:
  85. config (dict): The configuration dictionary containing the necessary
  86. parameters for initializing the predictor.
  87. Returns:
  88. None
  89. """
  90. from .. import create_retriever
  91. retriever_config = config.get("SubModules", {}).get(
  92. "LLM_Retriever",
  93. {"retriever_config_error": "config error for llm retriever!"},
  94. )
  95. self.retriever = create_retriever(retriever_config)
  96. def inintial_chat_predictor(self, config: dict) -> None:
  97. """
  98. Initializes the chat predictor with the given configuration.
  99. Args:
  100. config (dict): The configuration dictionary containing the necessary
  101. parameters for initializing the predictor.
  102. Returns:
  103. None
  104. """
  105. from .. import create_chat_bot
  106. chat_bot_config = config.get("SubModules", {}).get(
  107. "LLM_Chat",
  108. {"chat_bot_config_error": "config error for llm chat bot!"},
  109. )
  110. self.chat_bot = create_chat_bot(chat_bot_config)
  111. from .. import create_prompt_engineering
  112. text_pe_config = (
  113. config.get("SubModules", {})
  114. .get("PromptEngneering", {})
  115. .get(
  116. "KIE_CommonText",
  117. {"pe_config_error": "config error for text_pe!"},
  118. )
  119. )
  120. self.text_pe = create_prompt_engineering(text_pe_config)
  121. table_pe_config = (
  122. config.get("SubModules", {})
  123. .get("PromptEngneering", {})
  124. .get(
  125. "KIE_Table",
  126. {"pe_config_error": "config error for table_pe!"},
  127. )
  128. )
  129. self.table_pe = create_prompt_engineering(table_pe_config)
  130. return
  131. def inintial_mllm_predictor(self, config: dict) -> None:
  132. """
  133. Initializes the predictor with the given configuration.
  134. Args:
  135. config (dict): The configuration dictionary containing the necessary
  136. parameters for initializing the predictor.
  137. Returns:
  138. None
  139. """
  140. from .. import create_chat_bot, create_prompt_engineering
  141. self.use_mllm_predict = config.get("use_mllm_predict", True)
  142. if self.use_mllm_predict:
  143. mllm_chat_bot_config = config.get("SubModules", {}).get(
  144. "MLLM_Chat",
  145. {"mllm_chat_bot_config": "config error for mllm chat bot!"},
  146. )
  147. self.mllm_chat_bot = create_chat_bot(mllm_chat_bot_config)
  148. ensemble_pe_config = (
  149. config.get("SubModules", {})
  150. .get("PromptEngneering", {})
  151. .get(
  152. "Ensemble",
  153. {"pe_config_error": "config error for ensemble_pe!"},
  154. )
  155. )
  156. self.ensemble_pe = create_prompt_engineering(ensemble_pe_config)
  157. return
  158. def decode_visual_result(self, layout_parsing_result: LayoutParsingResult) -> dict:
  159. """
  160. Decodes the visual result from the layout parsing result.
  161. Args:
  162. layout_parsing_result (LayoutParsingResult): The result of layout parsing.
  163. Returns:
  164. dict: The decoded visual information.
  165. """
  166. text_paragraphs_ocr_res = layout_parsing_result["text_paragraphs_ocr_res"]
  167. seal_res_list = layout_parsing_result["seal_res_list"]
  168. normal_text_dict = {}
  169. for seal_res in seal_res_list:
  170. for text in seal_res["rec_texts"]:
  171. layout_type = "印章"
  172. if layout_type not in normal_text_dict:
  173. normal_text_dict[layout_type] = f"{text}"
  174. else:
  175. normal_text_dict[layout_type] += f"\n {text}"
  176. for text in text_paragraphs_ocr_res["rec_texts"]:
  177. layout_type = "words in text block"
  178. if layout_type not in normal_text_dict:
  179. normal_text_dict[layout_type] = text
  180. else:
  181. normal_text_dict[layout_type] += f"\n {text}"
  182. table_res_list = layout_parsing_result["table_res_list"]
  183. table_text_list = []
  184. table_html_list = []
  185. table_nei_text_list = []
  186. for table_res in table_res_list:
  187. table_html_list.append(table_res["pred_html"])
  188. single_table_text = " ".join(table_res["table_ocr_pred"]["rec_texts"])
  189. table_text_list.append(single_table_text)
  190. table_nei_text_list.append(table_res["neighbor_texts"])
  191. visual_info = {}
  192. visual_info["normal_text_dict"] = normal_text_dict
  193. visual_info["table_text_list"] = table_text_list
  194. visual_info["table_html_list"] = table_html_list
  195. visual_info["table_nei_text_list"] = table_nei_text_list
  196. return visual_info
  197. # Function to perform visual prediction on input images
  198. def visual_predict(
  199. self,
  200. input: Union[str, List[str], np.ndarray, List[np.ndarray]],
  201. use_doc_orientation_classify: Optional[bool] = None,
  202. use_doc_unwarping: Optional[bool] = None,
  203. use_general_ocr: Optional[bool] = None,
  204. use_seal_recognition: Optional[bool] = None,
  205. use_table_recognition: Optional[bool] = None,
  206. text_det_limit_side_len: Optional[int] = None,
  207. text_det_limit_type: Optional[str] = None,
  208. text_det_thresh: Optional[float] = None,
  209. text_det_box_thresh: Optional[float] = None,
  210. text_det_unclip_ratio: Optional[float] = None,
  211. text_rec_score_thresh: Optional[float] = None,
  212. seal_det_limit_side_len: Optional[int] = None,
  213. seal_det_limit_type: Optional[str] = None,
  214. seal_det_thresh: Optional[float] = None,
  215. seal_det_box_thresh: Optional[float] = None,
  216. seal_det_unclip_ratio: Optional[float] = None,
  217. seal_rec_score_thresh: Optional[float] = None,
  218. **kwargs,
  219. ) -> dict:
  220. """
  221. This function takes an input image or a list of images and performs various visual
  222. prediction tasks such as document orientation classification, document unwarping,
  223. general OCR, seal recognition, and table recognition based on the provided flags.
  224. Args:
  225. input (Union[str, list[str], np.ndarray, list[np.ndarray]]): Input image path, list of image paths,
  226. numpy array of an image, or list of numpy arrays.
  227. use_doc_orientation_classify (bool): Flag to use document orientation classification.
  228. use_doc_unwarping (bool): Flag to use document unwarping.
  229. use_general_ocr (bool): Flag to use general OCR.
  230. use_seal_recognition (bool): Flag to use seal recognition.
  231. use_table_recognition (bool): Flag to use table recognition.
  232. **kwargs: Additional keyword arguments.
  233. Returns:
  234. dict: A dictionary containing the layout parsing result and visual information.
  235. """
  236. if self.use_layout_parser == False:
  237. logging.error("The models for layout parser are not initialized.")
  238. yield {"error": "The models for layout parser are not initialized."}
  239. if self.layout_parsing_pipeline is None:
  240. logging.warning(
  241. "The layout parsing pipeline is not initialized, will initialize it now."
  242. )
  243. self.inintial_visual_predictor(self.config)
  244. for layout_parsing_result in self.layout_parsing_pipeline.predict(
  245. input,
  246. use_doc_orientation_classify=use_doc_orientation_classify,
  247. use_doc_unwarping=use_doc_unwarping,
  248. use_general_ocr=use_general_ocr,
  249. use_seal_recognition=use_seal_recognition,
  250. use_table_recognition=use_table_recognition,
  251. text_det_limit_side_len=text_det_limit_side_len,
  252. text_det_limit_type=text_det_limit_type,
  253. text_det_thresh=text_det_thresh,
  254. text_det_box_thresh=text_det_box_thresh,
  255. text_det_unclip_ratio=text_det_unclip_ratio,
  256. text_rec_score_thresh=text_rec_score_thresh,
  257. seal_det_box_thresh=seal_det_box_thresh,
  258. seal_det_limit_side_len=seal_det_limit_side_len,
  259. seal_det_limit_type=seal_det_limit_type,
  260. seal_det_thresh=seal_det_thresh,
  261. seal_det_unclip_ratio=seal_det_unclip_ratio,
  262. seal_rec_score_thresh=seal_rec_score_thresh,
  263. ):
  264. visual_info = self.decode_visual_result(layout_parsing_result)
  265. visual_predict_res = {
  266. "layout_parsing_result": layout_parsing_result,
  267. "visual_info": visual_info,
  268. }
  269. yield visual_predict_res
  270. def save_visual_info_list(self, visual_info: dict, save_path: str) -> None:
  271. """
  272. Save the visual info list to the specified file path.
  273. Args:
  274. visual_info (dict): The visual info result, which can be a single object or a list of objects.
  275. save_path (str): The file path to save the visual info list.
  276. Returns:
  277. None
  278. """
  279. if not isinstance(visual_info, list):
  280. visual_info_list = [visual_info]
  281. else:
  282. visual_info_list = visual_info
  283. with open(save_path, "w") as fout:
  284. fout.write(json.dumps(visual_info_list, ensure_ascii=False) + "\n")
  285. return
  286. def load_visual_info_list(self, data_path: str) -> List[dict]:
  287. """
  288. Loads visual info list from a JSON file.
  289. Args:
  290. data_path (str): The path to the JSON file containing visual info.
  291. Returns:
  292. list[dict]: A list of dict objects parsed from the JSON file.
  293. """
  294. with open(data_path, "r") as fin:
  295. data = fin.readline()
  296. visual_info_list = json.loads(data)
  297. return visual_info_list
  298. def merge_visual_info_list(
  299. self, visual_info_list: List[dict]
  300. ) -> Tuple[list, list, list, list]:
  301. """
  302. Merge visual info lists.
  303. Args:
  304. visual_info_list (list[dict]): A list of visual info results.
  305. Returns:
  306. tuple[list, list, list, list]: A tuple containing four lists, one for normal text dicts,
  307. one for table text lists, one for table HTML lists.
  308. one for table neighbor texts.
  309. """
  310. all_normal_text_list = []
  311. all_table_text_list = []
  312. all_table_html_list = []
  313. all_table_nei_text_list = []
  314. for single_visual_info in visual_info_list:
  315. normal_text_dict = single_visual_info["normal_text_dict"]
  316. for key in normal_text_dict:
  317. normal_text_dict[key] = normal_text_dict[key].replace("\n", "")
  318. table_text_list = single_visual_info["table_text_list"]
  319. table_html_list = single_visual_info["table_html_list"]
  320. table_nei_text_list = single_visual_info["table_nei_text_list"]
  321. all_normal_text_list.append(normal_text_dict)
  322. all_table_text_list.extend(table_text_list)
  323. all_table_html_list.extend(table_html_list)
  324. all_table_nei_text_list.extend(table_nei_text_list)
  325. return (
  326. all_normal_text_list,
  327. all_table_text_list,
  328. all_table_html_list,
  329. all_table_nei_text_list,
  330. )
  331. def build_vector(
  332. self,
  333. visual_info: dict,
  334. min_characters: int = 3500,
  335. llm_request_interval: float = 1.0,
  336. flag_save_bytes_vector: bool = False,
  337. retriever_config: dict = None,
  338. ) -> dict:
  339. """
  340. Build a vector representation from visual information.
  341. Args:
  342. visual_info (dict): The visual information input, can be a single instance or a list of instances.
  343. min_characters (int): The minimum number of characters required for text processing, defaults to 3500.
  344. llm_request_interval (float): The interval between LLM requests, defaults to 1.0.
  345. flag_save_bytes_vector (bool): Whether to save the vector as bytes, defaults to False.
  346. retriever_config (dict): The configuration for the retriever, defaults to None.
  347. Returns:
  348. dict: A dictionary containing the vector info and a flag indicating if the text is too short.
  349. """
  350. if not isinstance(visual_info, list):
  351. visual_info_list = [visual_info]
  352. else:
  353. visual_info_list = visual_info
  354. if retriever_config is not None:
  355. from .. import create_retriever
  356. retriever = create_retriever(retriever_config)
  357. else:
  358. if self.retriever is None:
  359. logging.warning(
  360. "The retriever is not initialized,will initialize it now."
  361. )
  362. self.inintial_retriever_predictor(self.config)
  363. retriever = self.retriever
  364. all_visual_info = self.merge_visual_info_list(visual_info_list)
  365. (
  366. all_normal_text_list,
  367. all_table_text_list,
  368. all_table_html_list,
  369. all_table_nei_text_list,
  370. ) = all_visual_info
  371. vector_info = {}
  372. all_items = []
  373. for i, normal_text_dict in enumerate(all_normal_text_list):
  374. for type, text in normal_text_dict.items():
  375. all_items += [f"{type}:{text}\n"]
  376. for table_html, table_text, table_nei_text in zip(
  377. all_table_html_list, all_table_text_list, all_table_nei_text_list
  378. ):
  379. if len(table_html) > min_characters - self.table_structure_len_max:
  380. all_items += [f"table:{table_text}\t{table_nei_text}"]
  381. all_text_str = "".join(all_items)
  382. vector_info["flag_save_bytes_vector"] = False
  383. if len(all_text_str) > min_characters:
  384. vector_info["flag_too_short_text"] = False
  385. vector_info["vector"] = retriever.generate_vector_database(all_items)
  386. if flag_save_bytes_vector:
  387. vector_info["vector"] = retriever.encode_vector_store_to_bytes(
  388. vector_info["vector"]
  389. )
  390. vector_info["flag_save_bytes_vector"] = True
  391. else:
  392. vector_info["flag_too_short_text"] = True
  393. vector_info["vector"] = all_items
  394. return vector_info
  395. def save_vector(self, vector_info: dict, save_path: str) -> None:
  396. with open(save_path, "w") as fout:
  397. fout.write(json.dumps(vector_info, ensure_ascii=False) + "\n")
  398. return
  399. def load_vector(self, data_path: str) -> dict:
  400. vector_info = None
  401. if self.retriever is None:
  402. logging.warning("The retriever is not initialized,will initialize it now.")
  403. self.inintial_retriever_predictor(self.config)
  404. with open(data_path, "r") as fin:
  405. data = fin.readline()
  406. vector_info = json.loads(data)
  407. if (
  408. "flag_too_short_text" not in vector_info
  409. or "flag_save_bytes_vector" not in vector_info
  410. or "vector" not in vector_info
  411. ):
  412. logging.error("Invalid vector info.")
  413. return {"error": "Invalid vector info when load vector!"}
  414. if vector_info["flag_save_bytes_vector"]:
  415. vector_info["vector"] = self.retriever.decode_vector_store_from_bytes(
  416. vector_info["vector"]
  417. )
  418. return vector_info
  419. def format_key(self, key_list: Union[str, List[str]]) -> List[str]:
  420. """
  421. Formats the key list.
  422. Args:
  423. key_list (str|list[str]): A string or a list of strings representing the keys.
  424. Returns:
  425. list[str]: A list of formatted keys.
  426. """
  427. if key_list == "":
  428. return []
  429. if isinstance(key_list, list):
  430. key_list = [key.replace("\xa0", " ") for key in key_list]
  431. return key_list
  432. if isinstance(key_list, str):
  433. key_list = re.sub(r"[\t\n\r\f\v]", "", key_list)
  434. key_list = key_list.replace(",", ",").split(",")
  435. return key_list
  436. return []
  437. def mllm_pred(
  438. self,
  439. input: Union[str, np.ndarray],
  440. key_list,
  441. **kwargs,
  442. ) -> dict:
  443. if self.use_mllm_predict == False:
  444. logging.error("MLLM prediction is disabled.")
  445. return {"mllm_res": "Error:MLLM prediction is disabled!"}
  446. key_list = self.format_key(key_list)
  447. if len(key_list) == 0:
  448. return {"mllm_res": "Error:输入的key_list无效!"}
  449. if isinstance(input, list):
  450. logging.error("Input is a list, but it's not supported here.")
  451. return {"mllm_res": "Error:Input is a list, but it's not supported here!"}
  452. image_array_list = self.img_reader([input])
  453. if (
  454. isinstance(input, str)
  455. and input.endswith(".pdf")
  456. and len(image_array_list) > 1
  457. ):
  458. logging.error("The input with PDF should have only one page.")
  459. return {"mllm_res": "Error:The input with PDF should have only one page!"}
  460. if self.mllm_chat_bot is None:
  461. logging.warning(
  462. "The MLLM chat bot is not initialized,will initialize it now."
  463. )
  464. self.inintial_mllm_predictor(self.config)
  465. for image_array in image_array_list:
  466. assert len(image_array.shape) == 3
  467. image_string = cv2.imencode(".jpg", image_array)[1].tostring()
  468. image_base64 = base64.b64encode(image_string).decode("utf-8")
  469. result = {}
  470. for key in key_list:
  471. prompt = (
  472. str(key)
  473. + "\n请用图片中完整出现的内容回答,可以是单词、短语或句子,针对问题回答尽可能详细和完整,并保持格式、单位、符号和标点都与图片中的文字内容完全一致。"
  474. )
  475. mllm_chat_bot_result = self.mllm_chat_bot.generate_chat_results(
  476. prompt=prompt, image=image_base64
  477. )
  478. if mllm_chat_bot_result is None:
  479. return {"mllm_res": "大模型调用失败"}
  480. result[key] = mllm_chat_bot_result
  481. return {"mllm_res": result}
  482. def generate_and_merge_chat_results(
  483. self,
  484. chat_bot: BaseChat,
  485. prompt: str,
  486. key_list: list,
  487. final_results: dict,
  488. failed_results: list,
  489. ) -> None:
  490. """
  491. Generate and merge chat results into the final results dictionary.
  492. Args:
  493. prompt (str): The input prompt for the chat bot.
  494. key_list (list): A list of keys to track which results to merge.
  495. final_results (dict): The dictionary to store the final merged results.
  496. failed_results (list): A list of failed results to avoid merging.
  497. Returns:
  498. None
  499. """
  500. llm_result = chat_bot.generate_chat_results(prompt)
  501. if llm_result is None:
  502. logging.error(
  503. "chat bot error: \n [prompt:]\n %s\n [result:] %s\n"
  504. % (prompt, self.chat_bot.ERROR_MASSAGE)
  505. )
  506. return
  507. llm_result = self.chat_bot.fix_llm_result_format(llm_result)
  508. for key, value in llm_result.items():
  509. if value not in failed_results and key in key_list:
  510. key_list.remove(key)
  511. final_results[key] = value
  512. return
  513. def get_related_normal_text(
  514. self,
  515. retriever_config: dict,
  516. use_vector_retrieval: bool,
  517. vector_info: dict,
  518. key_list: List[str],
  519. all_normal_text_list: list,
  520. min_characters: int,
  521. ) -> str:
  522. """
  523. Retrieve related normal text based on vector retrieval or all normal text list.
  524. Args:
  525. retriever_config (dict): Configuration for the retriever.
  526. use_vector_retrieval (bool): Whether to use vector retrieval.
  527. vector_info (dict): Dictionary containing vector information.
  528. key_list (list[str]): List of keys to generate question keys.
  529. all_normal_text_list (list): List of normal text.
  530. min_characters (int): The minimum number of characters required for text processing, defaults to 3500.
  531. Returns:
  532. str: Related normal text.
  533. """
  534. if use_vector_retrieval and vector_info is not None:
  535. if retriever_config is not None:
  536. from .. import create_retriever
  537. retriever = create_retriever(retriever_config)
  538. else:
  539. if self.retriever is None:
  540. logging.warning(
  541. "The retriever is not initialized,will initialize it now."
  542. )
  543. self.inintial_retriever_predictor(self.config)
  544. retriever = self.retriever
  545. question_key_list = [f"{key}" for key in key_list]
  546. vector = vector_info["vector"]
  547. if not vector_info["flag_too_short_text"]:
  548. related_text = retriever.similarity_retrieval(
  549. question_key_list, vector, topk=50, min_characters=min_characters
  550. )
  551. else:
  552. if len(vector) > 0:
  553. related_text = "".join(vector)
  554. else:
  555. related_text = ""
  556. else:
  557. all_items = []
  558. for i, normal_text_dict in enumerate(all_normal_text_list):
  559. for type, text in normal_text_dict.items():
  560. all_items += [f"{type}:{text}\n"]
  561. related_text = "".join(all_items)
  562. if len(related_text) > min_characters:
  563. logging.warning(
  564. "The input text content is too long, the large language model may truncate it."
  565. )
  566. return related_text
  567. def ensemble_ocr_llm_mllm(
  568. self, key_list: List[str], ocr_llm_predict_dict: dict, mllm_predict_dict: dict
  569. ) -> dict:
  570. """
  571. Ensemble OCR_LLM and LMM predictions based on given key list.
  572. Args:
  573. key_list (list[str]): List of keys to retrieve predictions.
  574. ocr_llm_predict_dict (dict): Dictionary containing OCR LLM predictions.
  575. mllm_predict_dict (dict): Dictionary containing mLLM predictions.
  576. Returns:
  577. dict: A dictionary with final predictions.
  578. """
  579. final_predict_dict = {}
  580. for key in key_list:
  581. predict = ""
  582. ocr_llm_predict = ""
  583. mllm_predict = ""
  584. if key in ocr_llm_predict_dict:
  585. ocr_llm_predict = ocr_llm_predict_dict[key]
  586. if key in mllm_predict_dict:
  587. mllm_predict = mllm_predict_dict[key]
  588. if ocr_llm_predict != "" and mllm_predict != "":
  589. prompt = self.ensemble_pe.generate_prompt(
  590. key, ocr_llm_predict, mllm_predict
  591. )
  592. llm_result = self.chat_bot.generate_chat_results(prompt)
  593. if llm_result is not None:
  594. llm_result = self.chat_bot.fix_llm_result_format(llm_result)
  595. if key in llm_result:
  596. tmp = llm_result[key]
  597. if "B" in tmp:
  598. predict = mllm_predict
  599. else:
  600. predict = ocr_llm_predict
  601. else:
  602. predict = ocr_llm_predict
  603. elif key in ocr_llm_predict_dict:
  604. predict = ocr_llm_predict_dict[key]
  605. elif key in mllm_predict_dict:
  606. predict = mllm_predict_dict[key]
  607. if predict != "":
  608. final_predict_dict[key] = predict
  609. return final_predict_dict
  610. def chat(
  611. self,
  612. key_list: Union[str, List[str]],
  613. visual_info: dict,
  614. use_vector_retrieval: bool = True,
  615. vector_info: dict = None,
  616. min_characters: int = 3500,
  617. text_task_description: str = None,
  618. text_output_format: str = None,
  619. text_rules_str: str = None,
  620. text_few_shot_demo_text_content: str = None,
  621. text_few_shot_demo_key_value_list: str = None,
  622. table_task_description: str = None,
  623. table_output_format: str = None,
  624. table_rules_str: str = None,
  625. table_few_shot_demo_text_content: str = None,
  626. table_few_shot_demo_key_value_list: str = None,
  627. mllm_predict_info: dict = None,
  628. mllm_integration_strategy: str = "integration",
  629. chat_bot_config: dict = None,
  630. retriever_config: dict = None,
  631. ) -> dict:
  632. """
  633. Generates chat results based on the provided key list and visual information.
  634. Args:
  635. key_list (Union[str, list[str]]): A single key or a list of keys to extract information.
  636. visual_info (dict): The visual information result.
  637. use_vector_retrieval (bool): Whether to use vector retrieval.
  638. vector_info (dict): The vector information for retrieval.
  639. min_characters (int): The minimum number of characters required for text processing, defaults to 3500.
  640. text_task_description (str): The description of the text task.
  641. text_output_format (str): The output format for text results.
  642. text_rules_str (str): The rules for generating text results.
  643. text_few_shot_demo_text_content (str): The text content for few-shot demos.
  644. text_few_shot_demo_key_value_list (str): The key-value list for few-shot demos.
  645. table_task_description (str): The description of the table task.
  646. table_output_format (str): The output format for table results.
  647. table_rules_str (str): The rules for generating table results.
  648. table_few_shot_demo_text_content (str): The text content for table few-shot demos.
  649. table_few_shot_demo_key_value_list (str): The key-value list for table few-shot demos.
  650. mllm_predict_dict (dict): The dictionary of mLLM predicts.
  651. mllm_integration_strategy (str): The integration strategy of mLLM and LLM, defaults to "integration", options are "integration", "llm_only" and "mllm_only".
  652. chat_bot_config (dict): The parameters for LLM chatbot, including api_type, api_key... refer to config file for more details.
  653. retriever_config (dict): The parameters for LLM retriever, including api_type, api_key... refer to config file for more details.
  654. Returns:
  655. dict: A dictionary containing the chat results.
  656. """
  657. key_list = self.format_key(key_list)
  658. key_list_ori = key_list.copy()
  659. if len(key_list) == 0:
  660. return {"chat_res": "Error:输入的key_list无效!"}
  661. if not isinstance(visual_info, list):
  662. visual_info_list = [visual_info]
  663. else:
  664. visual_info_list = visual_info
  665. if self.chat_bot is None:
  666. logging.warning(
  667. "The LLM chat bot is not initialized,will initialize it now."
  668. )
  669. self.inintial_chat_predictor(self.config)
  670. if chat_bot_config is not None:
  671. from .. import create_chat_bot
  672. chat_bot = create_chat_bot(chat_bot_config)
  673. else:
  674. chat_bot = self.chat_bot
  675. all_visual_info = self.merge_visual_info_list(visual_info_list)
  676. (
  677. all_normal_text_list,
  678. all_table_text_list,
  679. all_table_html_list,
  680. all_table_nei_text_list,
  681. ) = all_visual_info
  682. final_results = {}
  683. failed_results = ["大模型调用失败", "未知", "未找到关键信息", "None", ""]
  684. if len(key_list) > 0:
  685. related_text = self.get_related_normal_text(
  686. retriever_config,
  687. use_vector_retrieval,
  688. vector_info,
  689. key_list,
  690. all_normal_text_list,
  691. min_characters,
  692. )
  693. if len(related_text) > 0:
  694. prompt = self.text_pe.generate_prompt(
  695. related_text,
  696. key_list,
  697. task_description=text_task_description,
  698. output_format=text_output_format,
  699. rules_str=text_rules_str,
  700. few_shot_demo_text_content=text_few_shot_demo_text_content,
  701. few_shot_demo_key_value_list=text_few_shot_demo_key_value_list,
  702. )
  703. self.generate_and_merge_chat_results(
  704. chat_bot, prompt, key_list, final_results, failed_results
  705. )
  706. if len(key_list) > 0:
  707. for table_html, table_text, table_nei_text in zip(
  708. all_table_html_list, all_table_text_list, all_table_nei_text_list
  709. ):
  710. if len(table_html) <= min_characters - self.table_structure_len_max:
  711. for table_info in [table_html]:
  712. if len(key_list) > 0:
  713. if len(table_nei_text) > 0:
  714. table_info = (
  715. table_info + "\n 表格周围文字:" + table_nei_text
  716. )
  717. prompt = self.table_pe.generate_prompt(
  718. table_info,
  719. key_list,
  720. task_description=table_task_description,
  721. output_format=table_output_format,
  722. rules_str=table_rules_str,
  723. few_shot_demo_text_content=table_few_shot_demo_text_content,
  724. few_shot_demo_key_value_list=table_few_shot_demo_key_value_list,
  725. )
  726. self.generate_and_merge_chat_results(
  727. chat_bot,
  728. prompt,
  729. key_list,
  730. final_results,
  731. failed_results,
  732. )
  733. if (
  734. self.use_mllm_predict
  735. and mllm_integration_strategy != "llm_only"
  736. and mllm_predict_info is not None
  737. ):
  738. if mllm_integration_strategy == "integration":
  739. final_predict_dict = self.ensemble_ocr_llm_mllm(
  740. key_list_ori, final_results, mllm_predict_info
  741. )
  742. elif mllm_integration_strategy == "mllm_only":
  743. final_predict_dict = mllm_predict_info
  744. else:
  745. return {
  746. "chat_res": f"Error:Unsupported mllm_integration_strategy {mllm_integration_strategy}, only support 'integration', 'llm_only' and 'mllm_only'!"
  747. }
  748. else:
  749. final_predict_dict = final_results
  750. return {"chat_res": final_predict_dict}
  751. def predict(self, *args, **kwargs) -> None:
  752. logging.error(
  753. "PP-ChatOCRv4-doc Pipeline do not support to call `predict()` directly! Please invoke `visual_predict`, `build_vector`, `chat` sequentially to obtain the result."
  754. )
  755. return