pdf_parse_by_txt.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488
  1. import time
  2. # from anyio import Path
  3. from magic_pdf.libs.commons import fitz, get_delta_time, get_img_s3_client, get_docx_model_output
  4. import json
  5. import os
  6. import math
  7. from loguru import logger
  8. from magic_pdf.layout.bbox_sort import (
  9. prepare_bboxes_for_layout_split,
  10. )
  11. from magic_pdf.layout.layout_sort import LAYOUT_UNPROC, get_bboxes_layout, get_columns_cnt_of_layout, sort_text_block
  12. from magic_pdf.libs.drop_reason import DropReason
  13. from magic_pdf.libs.markdown_utils import escape_special_markdown_char
  14. from magic_pdf.libs.safe_filename import sanitize_filename
  15. from magic_pdf.libs.vis_utils import draw_bbox_on_page, draw_layout_bbox_on_page
  16. from magic_pdf.pre_proc.detect_images import parse_images
  17. from magic_pdf.pre_proc.detect_tables import parse_tables # 获取tables的bbox
  18. from magic_pdf.pre_proc.detect_equation import parse_equations # 获取equations的bbox
  19. from magic_pdf.pre_proc.detect_header import parse_headers # 获取headers的bbox
  20. from magic_pdf.pre_proc.detect_page_number import parse_pageNos # 获取pageNos的bbox
  21. from magic_pdf.pre_proc.detect_footnote import parse_footnotes_by_model, parse_footnotes_by_rule # 获取footnotes的bbox
  22. from magic_pdf.pre_proc.detect_footer_by_model import parse_footers # 获取footers的bbox
  23. from magic_pdf.post_proc.detect_para import (
  24. ParaProcessPipeline,
  25. TitleDetectionException,
  26. TitleLevelException,
  27. ParaSplitException,
  28. ParaMergeException,
  29. DenseSingleLineBlockException,
  30. )
  31. from magic_pdf.pre_proc.main_text_font import get_main_text_font
  32. from magic_pdf.pre_proc.remove_colored_strip_bbox import remove_colored_strip_textblock
  33. from magic_pdf.pre_proc.remove_footer_header import remove_headder_footer_one_page
  34. '''
  35. from para.para_pipeline import ParaProcessPipeline
  36. from para.exceptions import (
  37. TitleDetectionException,
  38. TitleLevelException,
  39. ParaSplitException,
  40. ParaMergeException,
  41. DenseSingleLineBlockException,
  42. )
  43. '''
  44. from magic_pdf.libs.commons import read_file, join_path
  45. from magic_pdf.libs.pdf_image_tools import save_images_by_bboxes
  46. from magic_pdf.post_proc.remove_footnote import merge_footnote_blocks, remove_footnote_blocks
  47. from magic_pdf.pre_proc.citationmarker_remove import remove_citation_marker
  48. from magic_pdf.pre_proc.equations_replace import combine_chars_to_pymudict, remove_chars_in_text_blocks, replace_equations_in_textblock
  49. from magic_pdf.pre_proc.pdf_pre_filter import pdf_filter
  50. from magic_pdf.pre_proc.detect_footer_header_by_statistics import drop_footer_header
  51. from magic_pdf.pre_proc.construct_page_dict import construct_page_component
  52. from magic_pdf.pre_proc.fix_image import combine_images, fix_image_vertical, fix_seperated_image, include_img_title
  53. from magic_pdf.post_proc.pdf_post_filter import pdf_post_filter
  54. from magic_pdf.pre_proc.remove_rotate_bbox import get_side_boundry, remove_rotate_side_textblock, remove_side_blank_block
  55. from magic_pdf.pre_proc.resolve_bbox_conflict import check_text_block_horizontal_overlap, resolve_bbox_overlap_conflict
  56. from magic_pdf.pre_proc.fix_table import fix_table_text_block, fix_tables, include_table_title
  57. from magic_pdf.pre_proc.solve_line_alien import solve_inline_too_large_interval
  58. denseSingleLineBlockException_msg = DenseSingleLineBlockException().message
  59. titleDetectionException_msg = TitleDetectionException().message
  60. titleLevelException_msg = TitleLevelException().message
  61. paraSplitException_msg = ParaSplitException().message
  62. paraMergeException_msg = ParaMergeException().message
  63. def parse_pdf_by_txt(
  64. pdf_bytes,
  65. pdf_model_output,
  66. start_page_id=0,
  67. end_page_id=None,
  68. debug_mode=False,
  69. ):
  70. save_tmp_path = os.path.join(os.path.dirname(__file__), "../..", "tmp", "unittest")
  71. md_bookname_save_path = ""
  72. book_name = sanitize_filename(book_name)
  73. if debug_mode:
  74. save_path = join_path(save_tmp_path, "md")
  75. pdf_local_path = join_path(save_tmp_path, "download-pdfs", book_name)
  76. if not os.path.exists(os.path.dirname(pdf_local_path)):
  77. # 如果目录不存在,创建它
  78. os.makedirs(os.path.dirname(pdf_local_path))
  79. md_bookname_save_path = join_path(save_tmp_path, "md", book_name)
  80. if not os.path.exists(md_bookname_save_path):
  81. # 如果目录不存在,创建它
  82. os.makedirs(md_bookname_save_path)
  83. with open(pdf_local_path + ".pdf", "wb") as pdf_file:
  84. pdf_file.write(pdf_bytes)
  85. pdf_docs = fitz.open("pdf", pdf_bytes)
  86. pdf_info_dict = {}
  87. img_s3_client = get_img_s3_client(save_path, image_s3_config) # 更改函数名和参数,避免歧义
  88. # img_s3_client = "img_s3_client" #不创建这个对象,直接用字符串占位
  89. start_time = time.time()
  90. """通过统计pdf全篇文字,识别正文字体"""
  91. main_text_font = get_main_text_font(pdf_docs)
  92. end_page_id = end_page_id if end_page_id else len(pdf_docs) - 1
  93. for page_id in range(start_page_id, end_page_id + 1):
  94. page = pdf_docs[page_id]
  95. page_width = page.rect.width
  96. page_height = page.rect.height
  97. if debug_mode:
  98. time_now = time.time()
  99. logger.info(f"page_id: {page_id}, last_page_cost_time: {get_delta_time(start_time)}")
  100. start_time = time_now
  101. """
  102. # 通过一个规则,过滤掉单页超过1500非junkimg的pdf
  103. # 对单页面非重复id的img数量做统计,如果当前页超过1500则直接return need_drop
  104. """
  105. page_imgs = page.get_images()
  106. # 去除对junkimg的依赖,简化逻辑
  107. if len(page_imgs) > 1500: # 如果当前页超过1500张图片,直接跳过
  108. logger.warning(f"page_id: {page_id}, img_counts: {len(page_imgs)}, drop this pdf: {book_name}")
  109. result = {"need_drop": True, "drop_reason": DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}
  110. if not debug_mode:
  111. return result
  112. # img_counts = 0
  113. # for img in page_imgs:
  114. # img_bojid = img[0]
  115. # if img_bojid in junk_img_bojids: # 判断这个图片在不在junklist中
  116. # continue # 如果在junklist就不用管了,跳过
  117. # else:
  118. # recs = page.get_image_rects(img, transform=True)
  119. # if recs: # 如果这张图在当前页面有展示
  120. # img_counts += 1
  121. # if img_counts >= 1500: # 如果去除了junkimg的影响,单页img仍然超过1500的话,就排除当前pdf
  122. # logger.warning(
  123. # f"page_id: {page_id}, img_counts: {img_counts}, drop this pdf: {book_name}, drop_reason: {DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}"
  124. # )
  125. # result = {"need_drop": True, "drop_reason": DropReason.HIGH_COMPUTATIONAL_lOAD_BY_IMGS}
  126. # if not debug_mode:
  127. # return result
  128. """
  129. ==================================================================================================================================
  130. 首先获取基本的block数据,对pdf进行分解,获取图片、表格、公式、text的bbox
  131. """
  132. # 解析pdf原始文本block
  133. text_raw_blocks = page.get_text(
  134. "dict",
  135. flags=fitz.TEXTFLAGS_TEXT,
  136. )["blocks"]
  137. model_output_json = get_docx_model_output(pdf_model_output, page_id)
  138. # 解析图片
  139. image_bboxes = parse_images(page_id, page, model_output_json)
  140. image_bboxes = fix_image_vertical(image_bboxes, text_raw_blocks) # 修正图片的位置
  141. image_bboxes = fix_seperated_image(image_bboxes) # 合并有边重合的图片
  142. image_bboxes = include_img_title(text_raw_blocks, image_bboxes) # 向图片上方和下方寻找title,使用规则进行匹配,暂时只支持英文规则
  143. """此时image_bboxes中可能出现这种情况,水平并列的2个图片,下方分别有各自的子标题,2个子标题下方又有大标题(形如Figxxx),会出现2个图片的bbox都包含了这个大标题,这种情况需要把图片合并"""
  144. image_bboxes = combine_images(image_bboxes) # 合并图片
  145. # 解析表格并对table_bboxes进行位置的微调,防止表格周围的文字被截断
  146. table_bboxes = parse_tables(page_id, page, model_output_json)
  147. table_bboxes = fix_tables(page, table_bboxes, include_table_title=True, scan_line_num=2) # 修正
  148. table_bboxes = fix_table_text_block(text_raw_blocks, table_bboxes) # 修正与text block的关系,某些table修正与pymupdf获取到的table内textblock没有完全包含,因此要进行一次修正。
  149. #debug_show_bbox(pdf_docs, page_id, table_bboxes, [], [b['bbox'] for b in text_raw_blocks], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
  150. table_bboxes = include_table_title(text_raw_blocks, table_bboxes) # 向table上方和下方寻找title,使用规则进行匹配,暂时只支持英文规则
  151. # 解析公式
  152. equations_inline_bboxes, equations_interline_bboxes = parse_equations(page_id, page, model_output_json)
  153. """
  154. ==================================================================================================================================
  155. 进入预处理-1阶段
  156. -------------------
  157. # # 解析标题
  158. # title_bboxs = parse_titles(page_id, page, model_output_json)
  159. # # 评估Layout是否规整、简单
  160. # isSimpleLayout_flag, fullColumn_cnt, subColumn_cnt, curPage_loss = evaluate_pdf_layout(page_id, page, model_output_json)
  161. 接下来开始进行预处理过程
  162. """
  163. """去掉每页的页码、页眉、页脚"""
  164. page_no_bboxs = parse_pageNos(page_id, page, model_output_json)
  165. header_bboxs = parse_headers(page_id, page, model_output_json)
  166. footer_bboxs = parse_footers(page_id, page, model_output_json)
  167. image_bboxes, table_bboxes, remain_text_blocks, removed_hdr_foot_txt_block, removed_hdr_foot_img_block, removed_hdr_foot_table = remove_headder_footer_one_page(text_raw_blocks, image_bboxes, table_bboxes, header_bboxs, footer_bboxs, page_no_bboxs, page_width, page_height)
  168. """去除页面上半部分长条色块内的文本块"""
  169. remain_text_blocks, removed_colored_narrow_strip_background_text_block = remove_colored_strip_textblock(remain_text_blocks, page)
  170. #debug_show_bbox(pdf_docs, page_id, footnote_bboxes_by_model, [b['bbox'] for b in remain_text_blocks], header_bboxs, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 7)
  171. """去掉旋转的文字:水印、垂直排列的文字"""
  172. remain_text_blocks, removed_non_horz_text_block = remove_rotate_side_textblock(
  173. remain_text_blocks, page_width, page_height
  174. ) # 去掉水印,非水平文字
  175. remain_text_blocks, removed_empty_side_block = remove_side_blank_block(remain_text_blocks, page_width, page_height) # 删除页面四周可能会留下的完全空白的textblock,这种block形成原因未知
  176. """出现在图片、表格上的文字块去掉,把层叠的图片单独分离出来,不参与layout的计算"""
  177. (
  178. image_bboxes,
  179. table_bboxes,
  180. equations_interline_bboxes,
  181. equations_inline_bboxes,
  182. remain_text_blocks,
  183. text_block_on_image_removed,
  184. images_overlap_backup,
  185. interline_eq_temp_text_block
  186. ) = resolve_bbox_overlap_conflict(
  187. image_bboxes, table_bboxes, equations_interline_bboxes, equations_inline_bboxes, remain_text_blocks
  188. )
  189. # """去掉footnote, 从文字和图片中"""
  190. # # 通过模型识别到的footnote
  191. # footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path,
  192. # debug_mode=debug_mode)
  193. # # 通过规则识别到的footnote
  194. # footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id)
  195. """
  196. ==================================================================================================================================
  197. """
  198. if debug_mode: # debugmode截图到本地
  199. save_path = join_path(save_tmp_path, "md")
  200. # 把图、表、公式都进行截图,保存到存储上,返回图片路径作为内容
  201. image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info = save_images_by_bboxes(
  202. book_name,
  203. page_id,
  204. page,
  205. save_path,
  206. image_bboxes,
  207. images_overlap_backup,
  208. table_bboxes,
  209. equations_inline_bboxes,
  210. equations_interline_bboxes,
  211. # 传入img_s3_client
  212. img_s3_client,
  213. ) # 只要表格和图片的截图
  214. """"以下进入到公式替换环节 """
  215. char_level_text_blocks = page.get_text("rawdict", flags=fitz.TEXTFLAGS_TEXT)['blocks']
  216. remain_text_blocks = combine_chars_to_pymudict(remain_text_blocks, char_level_text_blocks)# 合并chars
  217. remain_text_blocks = replace_equations_in_textblock(remain_text_blocks, inline_eq_info, interline_eq_info)
  218. remain_text_blocks = remove_citation_marker(remain_text_blocks) # 公式替换之后去角标,防止公式无法替换成功。但是这样也会带来个问题就是把角标当公式。各有优劣。
  219. remain_text_blocks = remove_chars_in_text_blocks(remain_text_blocks) # 减少中间态数据体积
  220. #debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in inline_eq_info], [b['bbox'] for b in interline_eq_info], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 3)
  221. """去掉footnote, 从文字和图片中(先去角标再去footnote试试)"""
  222. # 通过模型识别到的footnote
  223. footnote_bboxes_by_model = parse_footnotes_by_model(page_id, page, model_output_json, md_bookname_save_path, debug_mode=debug_mode)
  224. # 通过规则识别到的footnote
  225. footnote_bboxes_by_rule = parse_footnotes_by_rule(remain_text_blocks, page_height, page_id, main_text_font)
  226. """进入pdf过滤器,去掉一些不合理的pdf"""
  227. is_good_pdf, err = pdf_filter(page, remain_text_blocks, table_bboxes, image_bboxes)
  228. if not is_good_pdf:
  229. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {err}")
  230. if not debug_mode:
  231. return err
  232. """
  233. ==================================================================================================================================
  234. 进行版面布局切分和过滤
  235. """
  236. """在切分之前,先检查一下bbox是否有左右重叠的情况,如果有,那么就认为这个pdf暂时没有能力处理好,这种左右重叠的情况大概率是由于pdf里的行间公式、表格没有被正确识别出来造成的 """
  237. is_text_block_horz_overlap = check_text_block_horizontal_overlap(remain_text_blocks, header_bboxs, footer_bboxs)
  238. if is_text_block_horz_overlap:
  239. # debug_show_bbox(pdf_docs, page_id, [b['bbox'] for b in remain_text_blocks], [], [], join_path(save_path, book_name, f"{book_name}_debug.pdf"), 0)
  240. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.TEXT_BLCOK_HOR_OVERLAP}")
  241. result = {"need_drop": True, "drop_reason": DropReason.TEXT_BLCOK_HOR_OVERLAP}
  242. if not debug_mode:
  243. return result
  244. """统一格式化成一个数据结构用于计算layout"""
  245. page_y0 = 0 if len(header_bboxs) == 0 else max([b[3] for b in header_bboxs])
  246. page_y1 = page_height if len(footer_bboxs) == 0 else min([b[1] for b in footer_bboxs])
  247. left_x, right_x = get_side_boundry(removed_non_horz_text_block, page_width, page_height)
  248. page_boundry = [math.floor(left_x), page_y0 + 1, math.ceil(right_x), page_y1 - 1]
  249. # 返回的是一个数组,每个元素[x0, y0, x1, y1, block_content, idx_x, idx_y], 初始时候idx_x, idx_y都是None. 对于图片、公式来说,block_content是图片的地址, 对于段落来说,block_content是段落的内容
  250. all_bboxes = prepare_bboxes_for_layout_split(
  251. image_info, image_backup_info, table_info, inline_eq_info, interline_eq_info, remain_text_blocks, page_boundry, page)
  252. #debug_show_bbox(pdf_docs, page_id, [], [], all_bboxes, join_path(save_path, book_name, f"{book_name}_debug.pdf"), 1)
  253. """page_y0, page_y1能够过滤掉页眉和页脚,不会算作layout内"""
  254. layout_bboxes, layout_tree = get_bboxes_layout(all_bboxes, page_boundry, page_id)
  255. if len(remain_text_blocks)>0 and len(all_bboxes)>0 and len(layout_bboxes)==0:
  256. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}")
  257. result = {"need_drop": True, "drop_reason": DropReason.CAN_NOT_DETECT_PAGE_LAYOUT}
  258. if not debug_mode:
  259. return result
  260. """以下去掉复杂的布局和超过2列的布局"""
  261. if any([lay["layout_label"] == LAYOUT_UNPROC for lay in layout_bboxes]): # 复杂的布局
  262. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.COMPLICATED_LAYOUT}")
  263. result = {"need_drop": True, "drop_reason": DropReason.COMPLICATED_LAYOUT}
  264. if not debug_mode:
  265. return result
  266. layout_column_width = get_columns_cnt_of_layout(layout_tree)
  267. if layout_column_width > 2: # 去掉超过2列的布局pdf
  268. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {DropReason.TOO_MANY_LAYOUT_COLUMNS}")
  269. result = {
  270. "need_drop": True,
  271. "drop_reason": DropReason.TOO_MANY_LAYOUT_COLUMNS,
  272. "extra_info": {"column_cnt": layout_column_width},
  273. }
  274. if not debug_mode:
  275. return result
  276. """
  277. ==================================================================================================================================
  278. 构造出下游需要的数据结构
  279. """
  280. remain_text_blocks = remain_text_blocks + interline_eq_temp_text_block # 把计算layout时候临时删除的行间公式再放回去,防止行间公式替换的时候丢失。
  281. removed_text_blocks = []
  282. removed_text_blocks.extend(removed_hdr_foot_txt_block)
  283. # removed_text_blocks.extend(removed_footnote_text_block)
  284. removed_text_blocks.extend(text_block_on_image_removed)
  285. removed_text_blocks.extend(removed_non_horz_text_block)
  286. removed_text_blocks.extend(removed_colored_narrow_strip_background_text_block)
  287. removed_images = []
  288. # removed_images.extend(footnote_imgs)
  289. removed_images.extend(removed_hdr_foot_img_block)
  290. images_backup = []
  291. images_backup.extend(image_backup_info)
  292. remain_text_blocks = escape_special_markdown_char(remain_text_blocks) # 转义span里的text
  293. sorted_text_remain_text_block = sort_text_block(remain_text_blocks, layout_bboxes)
  294. footnote_bboxes_tmp = []
  295. footnote_bboxes_tmp.extend(footnote_bboxes_by_model)
  296. footnote_bboxes_tmp.extend(footnote_bboxes_by_rule)
  297. page_info = construct_page_component(
  298. page_id,
  299. image_info,
  300. table_info,
  301. sorted_text_remain_text_block,
  302. layout_bboxes,
  303. inline_eq_info,
  304. interline_eq_info,
  305. page.get_text("dict", flags=fitz.TEXTFLAGS_TEXT)["blocks"],
  306. removed_text_blocks=removed_text_blocks,
  307. removed_image_blocks=removed_images,
  308. images_backup=images_backup,
  309. droped_table_block=[],
  310. table_backup=[],
  311. layout_tree=layout_tree,
  312. page_w=page.rect.width,
  313. page_h=page.rect.height,
  314. footnote_bboxes_tmp=footnote_bboxes_tmp
  315. )
  316. pdf_info_dict[f"page_{page_id}"] = page_info
  317. # end page for
  318. '''计算后处理阶段耗时'''
  319. start_time = time.time()
  320. """
  321. ==================================================================================================================================
  322. 去掉页眉和页脚,这里需要用到一定的统计量,所以放到最后
  323. 页眉和页脚主要从文本box和图片box中去除,位于页面的四周。
  324. 下面函数会直接修改pdf_info_dict,从文字块中、图片中删除属于页眉页脚的内容,删除内容做相对应记录
  325. """
  326. # 去页眉页脚
  327. header, footer = drop_footer_header(pdf_info_dict)
  328. """对单个layout内footnote和他下面的所有textbbox合并"""
  329. for page_key, page_info in pdf_info_dict.items():
  330. page_info = merge_footnote_blocks(page_info, main_text_font)
  331. page_info = remove_footnote_blocks(page_info)
  332. pdf_info_dict[page_key] = page_info
  333. """进入pdf后置过滤器,去掉一些不合理的pdf"""
  334. i = 0
  335. for page_info in pdf_info_dict.values():
  336. is_good_pdf, err = pdf_post_filter(page_info)
  337. if not is_good_pdf:
  338. logger.warning(f"page_id: {i}, drop this pdf: {book_name}, reason: {err}")
  339. if not debug_mode:
  340. return err
  341. i += 1
  342. if debug_mode:
  343. params_file_save_path = join_path(save_tmp_path, "md", book_name, "preproc_out.json")
  344. page_draw_rect_save_path = join_path(save_tmp_path, "md", book_name, "layout.pdf")
  345. # dir_path = os.path.dirname(page_draw_rect_save_path)
  346. # if not os.path.exists(dir_path):
  347. # # 如果目录不存在,创建它
  348. # os.makedirs(dir_path)
  349. with open(params_file_save_path, "w", encoding="utf-8") as f:
  350. json.dump(pdf_info_dict, f, ensure_ascii=False, indent=4)
  351. # 先检测本地 page_draw_rect_save_path 是否存在,如果存在则删除
  352. if os.path.exists(page_draw_rect_save_path):
  353. os.remove(page_draw_rect_save_path)
  354. # 绘制bbox和layout到pdf
  355. draw_bbox_on_page(pdf_docs, pdf_info_dict, page_draw_rect_save_path)
  356. draw_layout_bbox_on_page(pdf_docs, pdf_info_dict, header, footer, page_draw_rect_save_path)
  357. if debug_mode:
  358. # 打印后处理阶段耗时
  359. logger.info(f"post_processing_time: {get_delta_time(start_time)}")
  360. """
  361. ==================================================================================================================================
  362. 进入段落处理-2阶段
  363. """
  364. # 处理行内文字间距较大问题
  365. pdf_info_dict = solve_inline_too_large_interval(pdf_info_dict)
  366. start_time = time.time()
  367. para_process_pipeline = ParaProcessPipeline()
  368. def _deal_with_text_exception(error_info):
  369. logger.warning(f"page_id: {page_id}, drop this pdf: {book_name}, reason: {error_info}")
  370. if error_info == denseSingleLineBlockException_msg:
  371. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.DENSE_SINGLE_LINE_BLOCK}")
  372. result = {"need_drop": True, "drop_reason": DropReason.DENSE_SINGLE_LINE_BLOCK}
  373. return result
  374. if error_info == titleDetectionException_msg:
  375. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.TITLE_DETECTION_FAILED}")
  376. result = {"need_drop": True, "drop_reason": DropReason.TITLE_DETECTION_FAILED}
  377. return result
  378. elif error_info == titleLevelException_msg:
  379. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.TITLE_LEVEL_FAILED}")
  380. result = {"need_drop": True, "drop_reason": DropReason.TITLE_LEVEL_FAILED}
  381. return result
  382. elif error_info == paraSplitException_msg:
  383. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.PARA_SPLIT_FAILED}")
  384. result = {"need_drop": True, "drop_reason": DropReason.PARA_SPLIT_FAILED}
  385. return result
  386. elif error_info == paraMergeException_msg:
  387. logger.warning(f"Drop this pdf: {book_name}, reason: {DropReason.PARA_MERGE_FAILED}")
  388. result = {"need_drop": True, "drop_reason": DropReason.PARA_MERGE_FAILED}
  389. return result
  390. if debug_mode:
  391. input_pdf_file = f"{pdf_local_path}.pdf"
  392. output_dir = f"{save_path}/{book_name}"
  393. output_pdf_file = f"{output_dir}/pdf_annos.pdf"
  394. """
  395. Call the para_process_pipeline function to process the pdf_info_dict.
  396. Parameters:
  397. para_debug_mode: str or None
  398. If para_debug_mode is None, the para_process_pipeline will not keep any intermediate results.
  399. If para_debug_mode is "simple", the para_process_pipeline will only keep the annos on the pdf and the final results as a json file.
  400. If para_debug_mode is "full", the para_process_pipeline will keep all the intermediate results generated during each step.
  401. """
  402. pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(
  403. pdf_info_dict,
  404. para_debug_mode="simple",
  405. input_pdf_path=input_pdf_file,
  406. output_pdf_path=output_pdf_file,
  407. )
  408. # 打印段落处理阶段耗时
  409. logger.info(f"para_process_time: {get_delta_time(start_time)}")
  410. # debug的时候不return drop信息
  411. if error_info is not None:
  412. _deal_with_text_exception(error_info)
  413. return pdf_info_dict
  414. else:
  415. pdf_info_dict, error_info = para_process_pipeline.para_process_pipeline(pdf_info_dict)
  416. if error_info is not None:
  417. return _deal_with_text_exception(error_info)
  418. return pdf_info_dict