title_processor.py 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015
  1. import os
  2. import sys
  3. import re
  4. import numpy as np
  5. from libs.nlp_utils import NLPModels
  6. from para.commons import *
  7. if sys.version_info[0] >= 3:
  8. sys.stdout.reconfigure(encoding="utf-8") # type: ignore
  9. class TitleProcessor:
  10. def __init__(self, *doc_statistics) -> None:
  11. if len(doc_statistics) > 0:
  12. self.doc_statistics = doc_statistics[0]
  13. self.nlp_model = NLPModels()
  14. self.MAX_TITLE_LEVEL = 3
  15. self.numbered_title_pattern = r"""
  16. ^ # 行首
  17. ( # 开始捕获组
  18. [\(\(]\d+[\)\)] # 括号内数字,支持中文和英文括号,例如:(1) 或 (1)
  19. |\d+[\)\)]\s # 数字后跟右括号和空格,支持中文和英文括号,例如:2) 或 2)
  20. |[\(\(][A-Z][\)\)] # 括号内大写字母,支持中文和英文括号,例如:(A) 或 (A)
  21. |[A-Z][\)\)]\s # 大写字母后跟右括号和空格,例如:A) 或 A)
  22. |[\(\(][IVXLCDM]+[\)\)] # 括号内罗马数字,支持中文和英文括号,例如:(I) 或 (I)
  23. |[IVXLCDM]+[\)\)]\s # 罗马数字后跟右括号和空格,例如:I) 或 I)
  24. |\d+(\.\d+)*\s # 数字或复合数字编号后跟空格,例如:1. 或 3.2.1
  25. |[一二三四五六七八九十百千]+[、\s] # 中文序号后跟顿号和空格,例如:一、
  26. |[\(|\(][一二三四五六七八九十百千]+[\)|\)]\s* # 中文括号内中文序号后跟空格,例如:(一)
  27. |[A-Z]\.\d+(\.\d+)?\s # 大写字母后跟点和数字,例如:A.1 或 A.1.1
  28. |[\(\(][a-z][\)\)] # 括号内小写字母,支持中文和英文括号,例如:(a) 或 (a)
  29. |[a-z]\)\s # 小写字母后跟右括号和空格,例如:a)
  30. |[A-Z]-\s # 大写字母后跟短横线和空格,例如:A-
  31. |\w+:\s # 英文序号词后跟冒号和空格,例如:First:
  32. |第[一二三四五六七八九十百千]+[章节部分条款]\s # 以“第”开头的中文标题后跟空格
  33. |[IVXLCDM]+\. # 罗马数字后跟点,例如:I.
  34. |\d+\.\s # 单个数字后跟点和空格,例如:1.
  35. ) # 结束捕获组
  36. .+ # 标题的其余部分
  37. """
  38. def _is_potential_title(
  39. self,
  40. curr_line,
  41. prev_line,
  42. prev_line_is_title,
  43. next_line,
  44. avg_char_width,
  45. avg_char_height,
  46. median_font_size,
  47. ):
  48. """
  49. This function checks if the line is a potential title.
  50. Parameters
  51. ----------
  52. curr_line : dict
  53. current line
  54. prev_line : dict
  55. previous line
  56. next_line : dict
  57. next line
  58. avg_char_width : float
  59. average of char widths
  60. avg_char_height : float
  61. average of line heights
  62. Returns
  63. -------
  64. bool
  65. True if the line is a potential title, False otherwise.
  66. """
  67. def __is_line_centered(line_bbox, page_bbox, avg_char_width):
  68. """
  69. This function checks if the line is centered on the page
  70. Parameters
  71. ----------
  72. line_bbox : list
  73. bbox of the line
  74. page_bbox : list
  75. bbox of the page
  76. avg_char_width : float
  77. average of char widths
  78. Returns
  79. -------
  80. bool
  81. True if the line is centered on the page, False otherwise.
  82. """
  83. horizontal_ratio = 0.5
  84. horizontal_thres = horizontal_ratio * avg_char_width
  85. x0, _, x1, _ = line_bbox
  86. _, _, page_x1, _ = page_bbox
  87. return abs((x0 + x1) / 2 - page_x1 / 2) < horizontal_thres
  88. def __is_bold_font_line(line):
  89. """
  90. Check if a line contains any bold font style.
  91. """
  92. def _is_bold_span(span):
  93. # if span text is empty or only contains space, return False
  94. if not span["text"].strip():
  95. return False
  96. return bool(span["flags"] & 2**4) # Check if the font is bold
  97. for span in line["spans"]:
  98. if not _is_bold_span(span):
  99. return False
  100. return True
  101. def __is_italic_font_line(line):
  102. """
  103. Check if a line contains any italic font style.
  104. """
  105. def __is_italic_span(span):
  106. return bool(span["flags"] & 2**1) # Check if the font is italic
  107. for span in line["spans"]:
  108. if not __is_italic_span(span):
  109. return False
  110. return True
  111. def __is_punctuation_heavy(line_text):
  112. """
  113. Check if the line contains a high ratio of punctuation marks, which may indicate
  114. that the line is not a title.
  115. Parameters:
  116. line_text (str): Text of the line.
  117. Returns:
  118. bool: True if the line is heavy with punctuation, False otherwise.
  119. """
  120. # Pattern for common title format like "X.Y. Title"
  121. pattern = r"\b\d+\.\d+\..*\b"
  122. # If the line matches the title format, return False
  123. if re.match(pattern, line_text.strip()):
  124. return False
  125. # Find all punctuation marks in the line
  126. punctuation_marks = re.findall(r"[^\w\s]", line_text)
  127. number_of_punctuation_marks = len(punctuation_marks)
  128. text_length = len(line_text)
  129. if text_length == 0:
  130. return False
  131. punctuation_ratio = number_of_punctuation_marks / text_length
  132. if punctuation_ratio >= 0.1:
  133. return True
  134. return False
  135. def __has_mixed_font_styles(spans, strict_mode=False):
  136. """
  137. This function checks if the line has mixed font styles, the strict mode will compare the font types
  138. Parameters
  139. ----------
  140. spans : list
  141. spans of the line
  142. strict_mode : bool
  143. True for strict mode, the font types will be fully compared
  144. False for non-strict mode, the font types will be compared by the most longest common prefix
  145. Returns
  146. -------
  147. bool
  148. True if the line has mixed font styles, False otherwise.
  149. """
  150. if strict_mode:
  151. font_styles = set()
  152. for span in spans:
  153. font_style = span["font"].lower()
  154. font_styles.add(font_style)
  155. return len(font_styles) > 1
  156. else: # non-strict mode
  157. font_styles = []
  158. for span in spans:
  159. font_style = span["font"].lower()
  160. font_styles.append(font_style)
  161. if len(font_styles) > 1:
  162. longest_common_prefix = os.path.commonprefix(font_styles)
  163. if len(longest_common_prefix) > 0:
  164. return False
  165. else:
  166. return True
  167. else:
  168. return False
  169. def __is_different_font_type_from_neighbors(curr_line_font_type, prev_line_font_type, next_line_font_type):
  170. """
  171. This function checks if the current line has a different font type from the previous and next lines
  172. Parameters
  173. ----------
  174. curr_line_font_type : str
  175. font type of the current line
  176. prev_line_font_type : str
  177. font type of the previous line
  178. next_line_font_type : str
  179. font type of the next line
  180. Returns
  181. -------
  182. bool
  183. True if the current line has a different font type from the previous and next lines, False otherwise.
  184. """
  185. return all(
  186. curr_line_font_type != other_font_type.lower()
  187. for other_font_type in [prev_line_font_type, next_line_font_type]
  188. if other_font_type is not None
  189. )
  190. def __is_larger_font_size_from_neighbors(curr_line_font_size, prev_line_font_size, next_line_font_size):
  191. """
  192. This function checks if the current line has a larger font size than the previous and next lines
  193. Parameters
  194. ----------
  195. curr_line_font_size : float
  196. font size of the current line
  197. prev_line_font_size : float
  198. font size of the previous line
  199. next_line_font_size : float
  200. font size of the next line
  201. Returns
  202. -------
  203. bool
  204. True if the current line has a larger font size than the previous and next lines, False otherwise.
  205. """
  206. return all(
  207. curr_line_font_size > other_font_size * 1.2
  208. for other_font_size in [prev_line_font_size, next_line_font_size]
  209. if other_font_size is not None
  210. )
  211. def __is_similar_to_pre_line(curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size):
  212. """
  213. This function checks if the current line is similar to the previous line
  214. Parameters
  215. ----------
  216. curr_line : dict
  217. current line
  218. prev_line : dict
  219. previous line
  220. Returns
  221. -------
  222. bool
  223. True if the current line is similar to the previous line, False otherwise.
  224. """
  225. if curr_line_font_type == prev_line_font_type and curr_line_font_size == prev_line_font_size:
  226. return True
  227. else:
  228. return False
  229. def __is_same_font_type_of_docAvg(curr_line_font_type):
  230. """
  231. This function checks if the current line has the same font type as the document average font type
  232. Parameters
  233. ----------
  234. curr_line_font_type : str
  235. font type of the current line
  236. Returns
  237. -------
  238. bool
  239. True if the current line has the same font type as the document average font type, False otherwise.
  240. """
  241. doc_most_common_font_type = safe_get(self.doc_statistics, "most_common_font_type", "").lower()
  242. doc_second_most_common_font_type = safe_get(self.doc_statistics, "second_most_common_font_type", "").lower()
  243. return curr_line_font_type.lower() in [doc_most_common_font_type, doc_second_most_common_font_type]
  244. def __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio: float = 1):
  245. """
  246. This function checks if the current line has a large enough font size
  247. Parameters
  248. ----------
  249. curr_line_font_size : float
  250. font size of the current line
  251. ratio : float
  252. ratio of the current line font size to the document average font size
  253. Returns
  254. -------
  255. bool
  256. True if the current line has a large enough font size, False otherwise.
  257. """
  258. doc_most_common_font_size = safe_get(self.doc_statistics, "most_common_font_size", 0)
  259. doc_second_most_common_font_size = safe_get(self.doc_statistics, "second_most_common_font_size", 0)
  260. doc_avg_font_size = min(doc_most_common_font_size, doc_second_most_common_font_size)
  261. return curr_line_font_size >= doc_avg_font_size * ratio
  262. def __is_sufficient_spacing_above_and_below(
  263. curr_line_bbox,
  264. prev_line_bbox,
  265. next_line_bbox,
  266. avg_char_height,
  267. median_font_size,
  268. ):
  269. """
  270. This function checks if the current line has sufficient spacing above and below
  271. Parameters
  272. ----------
  273. curr_line_bbox : list
  274. bbox of the current line
  275. prev_line_bbox : list
  276. bbox of the previous line
  277. next_line_bbox : list
  278. bbox of the next line
  279. avg_char_width : float
  280. average of char widths
  281. avg_char_height : float
  282. average of line heights
  283. Returns
  284. -------
  285. bool
  286. True if the current line has sufficient spacing above and below, False otherwise.
  287. """
  288. vertical_ratio = 1.25
  289. vertical_thres = vertical_ratio * median_font_size
  290. _, y0, _, y1 = curr_line_bbox
  291. sufficient_spacing_above = False
  292. if prev_line_bbox:
  293. vertical_spacing_above = min(y0 - prev_line_bbox[1], y1 - prev_line_bbox[3])
  294. sufficient_spacing_above = vertical_spacing_above > vertical_thres
  295. else:
  296. sufficient_spacing_above = True
  297. sufficient_spacing_below = False
  298. if next_line_bbox:
  299. vertical_spacing_below = min(next_line_bbox[1] - y0, next_line_bbox[3] - y1)
  300. sufficient_spacing_below = vertical_spacing_below > vertical_thres
  301. else:
  302. sufficient_spacing_below = True
  303. return (sufficient_spacing_above, sufficient_spacing_below)
  304. def __is_word_list_line_by_rules(curr_line_text):
  305. """
  306. This function checks if the current line is a word list
  307. Parameters
  308. ----------
  309. curr_line_text : str
  310. text of the current line
  311. Returns
  312. -------
  313. bool
  314. True if the current line is a name list, False otherwise.
  315. """
  316. # name_list_pattern = r"([a-zA-Z][a-zA-Z\s]{0,20}[a-zA-Z]|[\u4e00-\u9fa5·]{2,16})(?=[,,;;\s]|$)"
  317. name_list_pattern = r"(?<![\u4e00-\u9fa5])([A-Z][a-z]{0,19}\s[A-Z][a-z]{0,19}|[\u4e00-\u9fa5]{2,6})(?=[,,;;\s]|$)"
  318. compiled_pattern = re.compile(name_list_pattern)
  319. if compiled_pattern.search(curr_line_text):
  320. return True
  321. else:
  322. return False
  323. # """
  324. def __get_text_catgr_by_nlp(curr_line_text):
  325. """
  326. This function checks if the current line is a name list using nlp model, such as spacy
  327. Parameters
  328. ----------
  329. curr_line_text : str
  330. text of the current line
  331. Returns
  332. -------
  333. bool
  334. True if the current line is a name list, False otherwise.
  335. """
  336. result = self.nlp_model.detect_entity_catgr_using_nlp(curr_line_text)
  337. return result
  338. # """
  339. def __is_numbered_title(curr_line_text):
  340. """
  341. This function checks if the current line is a numbered list
  342. Parameters
  343. ----------
  344. curr_line_text : str
  345. text of the current line
  346. Returns
  347. -------
  348. bool
  349. True if the current line is a numbered list, False otherwise.
  350. """
  351. compiled_pattern = re.compile(self.numbered_title_pattern, re.VERBOSE)
  352. if compiled_pattern.search(curr_line_text):
  353. return True
  354. else:
  355. return False
  356. def __is_end_with_ending_puncs(line_text):
  357. """
  358. This function checks if the current line ends with a ending punctuation mark
  359. Parameters
  360. ----------
  361. line_text : str
  362. text of the current line
  363. Returns
  364. -------
  365. bool
  366. True if the current line ends with a punctuation mark, False otherwise.
  367. """
  368. end_puncs = [".", "?", "!", "。", "?", "!", "…"]
  369. line_text = line_text.rstrip()
  370. if line_text[-1] in end_puncs:
  371. return True
  372. return False
  373. def __contains_only_no_meaning_symbols(line_text):
  374. """
  375. This function checks if the current line contains only symbols that have no meaning, if so, it is not a title.
  376. Situation contains:
  377. 1. Only have punctuation marks
  378. 2. Only have other non-meaning symbols
  379. Parameters
  380. ----------
  381. line_text : str
  382. text of the current line
  383. Returns
  384. -------
  385. bool
  386. True if the current line contains only symbols that have no meaning, False otherwise.
  387. """
  388. punctuation_marks = re.findall(r"[^\w\s]", line_text) # find all punctuation marks
  389. number_of_punctuation_marks = len(punctuation_marks)
  390. text_length = len(line_text)
  391. if text_length == 0:
  392. return False
  393. punctuation_ratio = number_of_punctuation_marks / text_length
  394. if punctuation_ratio >= 0.9:
  395. return True
  396. return False
  397. def __is_equation(line_text):
  398. """
  399. This function checks if the current line is an equation.
  400. Parameters
  401. ----------
  402. line_text : str
  403. Returns
  404. -------
  405. bool
  406. True if the current line is an equation, False otherwise.
  407. """
  408. equation_reg = r"\$.*?\\overline.*?\$" # to match interline equations
  409. if re.search(equation_reg, line_text):
  410. return True
  411. else:
  412. return False
  413. def __is_title_by_len(text, max_length=200):
  414. """
  415. This function checks if the current line is a title by length.
  416. Parameters
  417. ----------
  418. text : str
  419. text of the current line
  420. max_length : int
  421. max length of the title
  422. Returns
  423. -------
  424. bool
  425. True if the current line is a title, False otherwise.
  426. """
  427. text = text.strip()
  428. return len(text) <= max_length
  429. def __compute_line_font_type_and_size(curr_line):
  430. """
  431. This function computes the font type and font size of the line.
  432. Parameters
  433. ----------
  434. line : dict
  435. line
  436. Returns
  437. -------
  438. font_type : str
  439. font type of the line
  440. font_size : float
  441. font size of the line
  442. """
  443. spans = curr_line["spans"]
  444. max_accumulated_length = 0
  445. max_span_font_size = curr_line["spans"][0]["size"] # default value, float type
  446. max_span_font_type = curr_line["spans"][0]["font"].lower() # default value, string type
  447. for span in spans:
  448. if span["text"].isspace():
  449. continue
  450. span_length = span["bbox"][2] - span["bbox"][0]
  451. if span_length > max_accumulated_length:
  452. max_accumulated_length = span_length
  453. max_span_font_size = span["size"]
  454. max_span_font_type = span["font"].lower()
  455. return max_span_font_type, max_span_font_size
  456. """
  457. Title detecting main Process.
  458. """
  459. """
  460. Basic features about the current line.
  461. """
  462. curr_line_bbox = curr_line["bbox"]
  463. curr_line_text = curr_line["text"]
  464. curr_line_font_type, curr_line_font_size = __compute_line_font_type_and_size(curr_line)
  465. if len(curr_line_text.strip()) == 0: # skip empty lines
  466. return False
  467. prev_line_bbox = prev_line["bbox"] if prev_line else None
  468. if prev_line:
  469. prev_line_font_type, prev_line_font_size = __compute_line_font_type_and_size(prev_line)
  470. else:
  471. prev_line_font_type, prev_line_font_size = None, None
  472. next_line_bbox = next_line["bbox"] if next_line else None
  473. if next_line:
  474. next_line_font_type, next_line_font_size = __compute_line_font_type_and_size(next_line)
  475. else:
  476. next_line_font_type, next_line_font_size = None, None
  477. """
  478. Aggregated features about the current line.
  479. """
  480. is_italc_font = __is_italic_font_line(curr_line)
  481. is_bold_font = __is_bold_font_line(curr_line)
  482. is_font_size_little_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=0.8)
  483. is_font_size_not_less_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1)
  484. is_much_larger_font_than_doc_avg = __is_font_size_not_less_than_docAvg(curr_line_font_size, ratio=1.6)
  485. is_not_same_font_type_of_docAvg = not __is_same_font_type_of_docAvg(curr_line_font_type)
  486. is_potential_title_font = is_bold_font or is_font_size_not_less_than_doc_avg or is_not_same_font_type_of_docAvg
  487. is_mix_font_styles_strict = __has_mixed_font_styles(curr_line["spans"], strict_mode=True)
  488. is_mix_font_styles_loose = __has_mixed_font_styles(curr_line["spans"], strict_mode=False)
  489. is_punctuation_heavy = __is_punctuation_heavy(curr_line_text)
  490. is_word_list_line_by_rules = __is_word_list_line_by_rules(curr_line_text)
  491. is_person_or_org_list_line_by_nlp = __get_text_catgr_by_nlp(curr_line_text) in ["PERSON", "GPE", "ORG"]
  492. is_font_size_larger_than_neighbors = __is_larger_font_size_from_neighbors(
  493. curr_line_font_size, prev_line_font_size, next_line_font_size
  494. )
  495. is_font_type_diff_from_neighbors = __is_different_font_type_from_neighbors(
  496. curr_line_font_type, prev_line_font_type, next_line_font_type
  497. )
  498. has_sufficient_spaces_above, has_sufficient_spaces_below = __is_sufficient_spacing_above_and_below(
  499. curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_height, median_font_size
  500. )
  501. is_similar_to_pre_line = __is_similar_to_pre_line(
  502. curr_line_font_type, prev_line_font_type, curr_line_font_size, prev_line_font_size
  503. )
  504. """
  505. Further aggregated features about the current line.
  506. Attention:
  507. Features that start with __ are for internal use.
  508. """
  509. __is_line_left_aligned_from_neighbors = is_line_left_aligned_from_neighbors(
  510. curr_line_bbox, prev_line_bbox, next_line_bbox, avg_char_width
  511. )
  512. __is_font_diff_from_neighbors = is_font_size_larger_than_neighbors or is_font_type_diff_from_neighbors
  513. is_a_left_inline_title = (
  514. is_mix_font_styles_strict and __is_line_left_aligned_from_neighbors and __is_font_diff_from_neighbors
  515. )
  516. is_title_by_check_prev_line = prev_line is None and has_sufficient_spaces_above and is_potential_title_font
  517. is_title_by_check_next_line = next_line is None and has_sufficient_spaces_below and is_potential_title_font
  518. is_title_by_check_pre_and_next_line = (
  519. (prev_line is not None or next_line is not None)
  520. and has_sufficient_spaces_above
  521. and has_sufficient_spaces_below
  522. and is_potential_title_font
  523. )
  524. is_numbered_title = __is_numbered_title(curr_line_text) and (
  525. (has_sufficient_spaces_above or prev_line is None) and (has_sufficient_spaces_below or next_line is None)
  526. )
  527. is_not_end_with_ending_puncs = not __is_end_with_ending_puncs(curr_line_text)
  528. is_not_only_no_meaning_symbols = not __contains_only_no_meaning_symbols(curr_line_text)
  529. is_equation = __is_equation(curr_line_text)
  530. is_title_by_len = __is_title_by_len(curr_line_text)
  531. """
  532. Decide if the line is a title.
  533. """
  534. # is_title = False
  535. # if prev_line_is_title:
  536. is_title = (
  537. is_not_end_with_ending_puncs # not end with ending punctuation marks
  538. and is_not_only_no_meaning_symbols # not only have no meaning symbols
  539. and is_title_by_len # is a title by length, default max length is 200
  540. and not is_equation # an interline equation should never be a title
  541. and is_potential_title_font # is a potential title font, which is bold or larger than the document average font size or not the same font type as the document average font type
  542. and (
  543. (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
  544. or (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
  545. or (
  546. is_much_larger_font_than_doc_avg
  547. and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
  548. )
  549. or (
  550. is_font_size_little_less_than_doc_avg
  551. and is_bold_font
  552. and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
  553. )
  554. ) # not the same font type as the document average font type, which includes the most common font type and the second most common font type
  555. and (
  556. (
  557. not is_person_or_org_list_line_by_nlp
  558. and (
  559. is_much_larger_font_than_doc_avg
  560. or (is_not_same_font_type_of_docAvg and is_font_size_not_less_than_doc_avg)
  561. )
  562. )
  563. or (
  564. not (is_word_list_line_by_rules and is_person_or_org_list_line_by_nlp)
  565. and not is_a_left_inline_title
  566. and not is_punctuation_heavy
  567. and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
  568. )
  569. or (
  570. is_person_or_org_list_line_by_nlp
  571. and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
  572. and (is_bold_font and is_much_larger_font_than_doc_avg and is_not_same_font_type_of_docAvg)
  573. )
  574. or (is_numbered_title and not is_a_left_inline_title)
  575. )
  576. )
  577. # ) or (is_similar_to_pre_line and prev_line_is_title)
  578. is_name_or_org_list_to_be_removed = (
  579. (is_person_or_org_list_line_by_nlp)
  580. and is_punctuation_heavy
  581. and (is_title_by_check_prev_line or is_title_by_check_next_line or is_title_by_check_pre_and_next_line)
  582. ) and not is_title
  583. if is_name_or_org_list_to_be_removed:
  584. is_author_or_org_list = True
  585. # print curr_line_text to check
  586. # print_yellow(f"Text of is_author_or_org_list: {curr_line_text}")
  587. else:
  588. is_author_or_org_list = False
  589. """
  590. # print reason why the line is a title
  591. if is_title:
  592. print_green("This line is a title.")
  593. print_green("↓" * 10)
  594. print()
  595. print("curr_line_text: ", curr_line_text)
  596. print()
  597. # print reason why the line is not a title
  598. line_text = curr_line_text.strip()
  599. test_text = "Career/Personal Life"
  600. text_content_condition = line_text == test_text
  601. if not is_title and text_content_condition: # Print specific line
  602. # if not is_title: # Print each line
  603. print_red("This line is not a title.")
  604. print_red("↓" * 10)
  605. print()
  606. print("curr_line_text: ", curr_line_text)
  607. print()
  608. if is_not_end_with_ending_puncs:
  609. print_green(f"is_not_end_with_ending_puncs")
  610. else:
  611. print_red(f"is_end_with_ending_puncs")
  612. if is_not_only_no_meaning_symbols:
  613. print_green(f"is_not_only_no_meaning_symbols")
  614. else:
  615. print_red(f"is_only_no_meaning_symbols")
  616. if is_title_by_len:
  617. print_green(f"is_title_by_len: {is_title_by_len}")
  618. else:
  619. print_red(f"is_not_title_by_len: {is_title_by_len}")
  620. if is_equation:
  621. print_red(f"is_equation")
  622. else:
  623. print_green(f"is_not_equation")
  624. if is_potential_title_font:
  625. print_green(f"is_potential_title_font")
  626. else:
  627. print_red(f"is_not_potential_title_font")
  628. if is_punctuation_heavy:
  629. print_red("is_punctuation_heavy")
  630. else:
  631. print_green("is_not_punctuation_heavy")
  632. if is_bold_font:
  633. print_green(f"is_bold_font")
  634. else:
  635. print_red(f"is_not_bold_font")
  636. if is_font_size_not_less_than_doc_avg:
  637. print_green(f"is_larger_font_than_doc_avg")
  638. else:
  639. print_red(f"is_not_larger_font_than_doc_avg")
  640. if is_much_larger_font_than_doc_avg:
  641. print_green(f"is_much_larger_font_than_doc_avg")
  642. else:
  643. print_red(f"is_not_much_larger_font_than_doc_avg")
  644. if is_not_same_font_type_of_docAvg:
  645. print_green(f"is_not_same_font_type_of_docAvg")
  646. else:
  647. print_red(f"is_same_font_type_of_docAvg")
  648. if is_word_list_line_by_rules:
  649. print_red("is_word_list_line_by_rules")
  650. else:
  651. print_green("is_not_name_list_by_rules")
  652. if is_person_or_org_list_line_by_nlp:
  653. print_red("is_person_or_org_list_line_by_nlp")
  654. else:
  655. print_green("is_not_person_or_org_list_line_by_nlp")
  656. if not is_numbered_title:
  657. print_red("is_not_numbered_title")
  658. else:
  659. print_green("is_numbered_title")
  660. if is_a_left_inline_title:
  661. print_red("is_a_left_inline_title")
  662. else:
  663. print_green("is_not_a_left_inline_title")
  664. if not is_title_by_check_prev_line:
  665. print_red("is_not_title_by_check_prev_line")
  666. else:
  667. print_green("is_title_by_check_prev_line")
  668. if not is_title_by_check_next_line:
  669. print_red("is_not_title_by_check_next_line")
  670. else:
  671. print_green("is_title_by_check_next_line")
  672. if not is_title_by_check_pre_and_next_line:
  673. print_red("is_not_title_by_check_pre_and_next_line")
  674. else:
  675. print_green("is_title_by_check_pre_and_next_line")
  676. # print_green("Common features:")
  677. # print_green("↓" * 10)
  678. # print(f" curr_line_font_type: {curr_line_font_type}")
  679. # print(f" curr_line_font_size: {curr_line_font_size}")
  680. # print()
  681. """
  682. return is_title, is_author_or_org_list
  683. def _detect_block_title(self, input_block):
  684. """
  685. Use the functions 'is_potential_title' to detect titles of each paragraph block.
  686. If a line is a title, then the value of key 'is_title' of the line will be set to True.
  687. """
  688. raw_lines = input_block["lines"]
  689. prev_line_is_title_flag = False
  690. for i, curr_line in enumerate(raw_lines):
  691. prev_line = raw_lines[i - 1] if i > 0 else None
  692. next_line = raw_lines[i + 1] if i < len(raw_lines) - 1 else None
  693. blk_avg_char_width = input_block["avg_char_width"]
  694. blk_avg_char_height = input_block["avg_char_height"]
  695. blk_media_font_size = input_block["median_font_size"]
  696. is_title, is_author_or_org_list = self._is_potential_title(
  697. curr_line,
  698. prev_line,
  699. prev_line_is_title_flag,
  700. next_line,
  701. blk_avg_char_width,
  702. blk_avg_char_height,
  703. blk_media_font_size,
  704. )
  705. if is_title:
  706. curr_line["is_title"] = is_title
  707. prev_line_is_title_flag = True
  708. else:
  709. curr_line["is_title"] = False
  710. prev_line_is_title_flag = False
  711. if is_author_or_org_list:
  712. curr_line["is_author_or_org_list"] = is_author_or_org_list
  713. else:
  714. curr_line["is_author_or_org_list"] = False
  715. return input_block
  716. def batch_process_blocks_detect_titles(self, pdf_dic):
  717. """
  718. This function batch process the blocks to detect titles.
  719. Parameters
  720. ----------
  721. pdf_dict : dict
  722. result dictionary
  723. Returns
  724. -------
  725. pdf_dict : dict
  726. result dictionary
  727. """
  728. num_titles = 0
  729. for page_id, blocks in pdf_dic.items():
  730. if page_id.startswith("page_"):
  731. para_blocks = []
  732. if "para_blocks" in blocks.keys():
  733. para_blocks = blocks["para_blocks"]
  734. all_single_line_blocks = []
  735. for block in para_blocks:
  736. if len(block["lines"]) == 1:
  737. all_single_line_blocks.append(block)
  738. new_para_blocks = []
  739. if not len(all_single_line_blocks) == len(para_blocks): # Not all blocks are single line blocks.
  740. for para_block in para_blocks:
  741. new_block = self._detect_block_title(para_block)
  742. new_para_blocks.append(new_block)
  743. num_titles += sum([line.get("is_title", 0) for line in new_block["lines"]])
  744. else: # All blocks are single line blocks.
  745. for para_block in para_blocks:
  746. new_para_blocks.append(para_block)
  747. num_titles += sum([line.get("is_title", 0) for line in para_block["lines"]])
  748. para_blocks = new_para_blocks
  749. blocks["para_blocks"] = para_blocks
  750. for para_block in para_blocks:
  751. all_titles = all(safe_get(line, "is_title", False) for line in para_block["lines"])
  752. para_text_len = sum([len(line["text"]) for line in para_block["lines"]])
  753. if (
  754. all_titles and para_text_len < 200
  755. ): # total length of the paragraph is less than 200, more than this should not be a title
  756. para_block["is_block_title"] = 1
  757. else:
  758. para_block["is_block_title"] = 0
  759. all_name_or_org_list_to_be_removed = all(
  760. safe_get(line, "is_author_or_org_list", False) for line in para_block["lines"]
  761. )
  762. if all_name_or_org_list_to_be_removed and page_id == "page_0":
  763. para_block["is_block_an_author_or_org_list"] = 1
  764. else:
  765. para_block["is_block_an_author_or_org_list"] = 0
  766. pdf_dic["statistics"]["num_titles"] = num_titles
  767. return pdf_dic
  768. def __determine_size_based_level(self, title_blocks):
  769. """
  770. This function determines the title level based on the font size of the title.
  771. Parameters
  772. ----------
  773. title_blocks : list
  774. Returns
  775. -------
  776. title_blocks : list
  777. """
  778. font_sizes = np.array([safe_get(tb["block"], "block_font_size", 0) for tb in title_blocks])
  779. # Use the mean and std of font sizes to remove extreme values
  780. mean_font_size = np.mean(font_sizes)
  781. std_font_size = np.std(font_sizes)
  782. min_extreme_font_size = mean_font_size - std_font_size # type: ignore
  783. max_extreme_font_size = mean_font_size + std_font_size # type: ignore
  784. # Compute the threshold for title level
  785. middle_font_sizes = font_sizes[(font_sizes > min_extreme_font_size) & (font_sizes < max_extreme_font_size)]
  786. if middle_font_sizes.size > 0:
  787. middle_mean_font_size = np.mean(middle_font_sizes)
  788. level_threshold = middle_mean_font_size
  789. else:
  790. level_threshold = mean_font_size
  791. for tb in title_blocks:
  792. title_block = tb["block"]
  793. title_font_size = safe_get(title_block, "block_font_size", 0)
  794. current_level = 1 # Initialize title level, the biggest level is 1
  795. # print(f"Before adjustment by font size, {current_level}")
  796. if title_font_size >= max_extreme_font_size:
  797. current_level = 1
  798. elif title_font_size <= min_extreme_font_size:
  799. current_level = 3
  800. elif float(title_font_size) >= float(level_threshold):
  801. current_level = 2
  802. else:
  803. current_level = 3
  804. # print(f"After adjustment by font size, {current_level}")
  805. title_block["block_title_level"] = current_level
  806. return title_blocks
  807. def batch_process_blocks_recog_title_level(self, pdf_dic):
  808. title_blocks = []
  809. # Collect all titles
  810. for page_id, blocks in pdf_dic.items():
  811. if page_id.startswith("page_"):
  812. para_blocks = blocks.get("para_blocks", [])
  813. for block in para_blocks:
  814. if block.get("is_block_title"):
  815. title_obj = {"page_id": page_id, "block": block}
  816. title_blocks.append(title_obj)
  817. # Determine title level
  818. if title_blocks:
  819. # Determine title level based on font size
  820. title_blocks = self.__determine_size_based_level(title_blocks)
  821. return pdf_dic