utils.py 90 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. __all__ = [
  15. "get_sub_regions_ocr_res",
  16. "get_layout_ordering",
  17. "get_single_block_parsing_res",
  18. "get_show_color",
  19. "sorted_layout_boxes",
  20. ]
  21. import re
  22. from copy import deepcopy
  23. from typing import Any, Dict, List, Optional, Tuple, Union
  24. import numpy as np
  25. from PIL import Image
  26. from ...models.object_detection.result import DetResult
  27. from ..ocr.result import OCRResult
  28. def get_overlap_boxes_idx(src_boxes: np.ndarray, ref_boxes: np.ndarray) -> List:
  29. """
  30. Get the indices of source boxes that overlap with reference boxes based on a specified threshold.
  31. Args:
  32. src_boxes (np.ndarray): A 2D numpy array of source bounding boxes.
  33. ref_boxes (np.ndarray): A 2D numpy array of reference bounding boxes.
  34. Returns:
  35. match_idx_list (list): A list of indices of source boxes that overlap with reference boxes.
  36. """
  37. match_idx_list = []
  38. src_boxes_num = len(src_boxes)
  39. if src_boxes_num > 0 and len(ref_boxes) > 0:
  40. for rno in range(len(ref_boxes)):
  41. ref_box = ref_boxes[rno]
  42. x1 = np.maximum(ref_box[0], src_boxes[:, 0])
  43. y1 = np.maximum(ref_box[1], src_boxes[:, 1])
  44. x2 = np.minimum(ref_box[2], src_boxes[:, 2])
  45. y2 = np.minimum(ref_box[3], src_boxes[:, 3])
  46. pub_w = x2 - x1
  47. pub_h = y2 - y1
  48. match_idx = np.where((pub_w > 3) & (pub_h > 3))[0]
  49. match_idx_list.extend(match_idx)
  50. return match_idx_list
  51. def get_sub_regions_ocr_res(
  52. overall_ocr_res: OCRResult,
  53. object_boxes: List,
  54. flag_within: bool = True,
  55. return_match_idx: bool = False,
  56. ) -> OCRResult:
  57. """
  58. Filters OCR results to only include text boxes within specified object boxes based on a flag.
  59. Args:
  60. overall_ocr_res (OCRResult): The original OCR result containing all text boxes.
  61. object_boxes (list): A list of bounding boxes for the objects of interest.
  62. flag_within (bool): If True, only include text boxes within the object boxes. If False, exclude text boxes within the object boxes.
  63. return_match_idx (bool): If True, return the list of matching indices.
  64. Returns:
  65. OCRResult: A filtered OCR result containing only the relevant text boxes.
  66. """
  67. sub_regions_ocr_res = {}
  68. sub_regions_ocr_res["rec_polys"] = []
  69. sub_regions_ocr_res["rec_texts"] = []
  70. sub_regions_ocr_res["rec_scores"] = []
  71. sub_regions_ocr_res["rec_boxes"] = []
  72. overall_text_boxes = overall_ocr_res["rec_boxes"]
  73. match_idx_list = get_overlap_boxes_idx(overall_text_boxes, object_boxes)
  74. match_idx_list = list(set(match_idx_list))
  75. for box_no in range(len(overall_text_boxes)):
  76. if flag_within:
  77. if box_no in match_idx_list:
  78. flag_match = True
  79. else:
  80. flag_match = False
  81. else:
  82. if box_no not in match_idx_list:
  83. flag_match = True
  84. else:
  85. flag_match = False
  86. if flag_match:
  87. sub_regions_ocr_res["rec_polys"].append(
  88. overall_ocr_res["rec_polys"][box_no]
  89. )
  90. sub_regions_ocr_res["rec_texts"].append(
  91. overall_ocr_res["rec_texts"][box_no]
  92. )
  93. sub_regions_ocr_res["rec_scores"].append(
  94. overall_ocr_res["rec_scores"][box_no]
  95. )
  96. sub_regions_ocr_res["rec_boxes"].append(
  97. overall_ocr_res["rec_boxes"][box_no]
  98. )
  99. for key in ["rec_polys", "rec_scores", "rec_boxes"]:
  100. sub_regions_ocr_res[key] = np.array(sub_regions_ocr_res[key])
  101. return (
  102. (sub_regions_ocr_res, match_idx_list)
  103. if return_match_idx
  104. else sub_regions_ocr_res
  105. )
  106. def sorted_layout_boxes(res, w):
  107. """
  108. Sort text boxes in order from top to bottom, left to right
  109. Args:
  110. res: List of dictionaries containing layout information.
  111. w: Width of image.
  112. Returns:
  113. List of dictionaries containing sorted layout information.
  114. """
  115. num_boxes = len(res)
  116. if num_boxes == 1:
  117. return res
  118. # Sort on the y axis first or sort it on the x axis
  119. sorted_boxes = sorted(res, key=lambda x: (x["block_bbox"][1], x["block_bbox"][0]))
  120. _boxes = list(sorted_boxes)
  121. new_res = []
  122. res_left = []
  123. res_right = []
  124. i = 0
  125. while True:
  126. if i >= num_boxes:
  127. break
  128. # Check that the bbox is on the left
  129. elif (
  130. _boxes[i]["block_bbox"][0] < w / 4
  131. and _boxes[i]["block_bbox"][2] < 3 * w / 5
  132. ):
  133. res_left.append(_boxes[i])
  134. i += 1
  135. elif _boxes[i]["block_bbox"][0] > 2 * w / 5:
  136. res_right.append(_boxes[i])
  137. i += 1
  138. else:
  139. new_res += res_left
  140. new_res += res_right
  141. new_res.append(_boxes[i])
  142. res_left = []
  143. res_right = []
  144. i += 1
  145. res_left = sorted(res_left, key=lambda x: (x["block_bbox"][1]))
  146. res_right = sorted(res_right, key=lambda x: (x["block_bbox"][1]))
  147. if res_left:
  148. new_res += res_left
  149. if res_right:
  150. new_res += res_right
  151. return new_res
  152. def _calculate_overlap_area_div_minbox_area_ratio(
  153. bbox1: Union[list, tuple],
  154. bbox2: Union[list, tuple],
  155. ) -> float:
  156. """
  157. Calculate the ratio of the overlap area between bbox1 and bbox2
  158. to the area of the smaller bounding box.
  159. Args:
  160. bbox1 (list or tuple): Coordinates of the first bounding box [x_min, y_min, x_max, y_max].
  161. bbox2 (list or tuple): Coordinates of the second bounding box [x_min, y_min, x_max, y_max].
  162. Returns:
  163. float: The ratio of the overlap area to the area of the smaller bounding box.
  164. """
  165. bbox1 = list(map(int, bbox1))
  166. bbox2 = list(map(int, bbox2))
  167. x_left = max(bbox1[0], bbox2[0])
  168. y_top = max(bbox1[1], bbox2[1])
  169. x_right = min(bbox1[2], bbox2[2])
  170. y_bottom = min(bbox1[3], bbox2[3])
  171. if x_right <= x_left or y_bottom <= y_top:
  172. return 0.0
  173. intersection_area = (x_right - x_left) * (y_bottom - y_top)
  174. area_bbox1 = (bbox1[2] - bbox1[0]) * (bbox1[3] - bbox1[1])
  175. area_bbox2 = (bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1])
  176. min_box_area = min(area_bbox1, area_bbox2)
  177. if min_box_area <= 0:
  178. return 0.0
  179. return intersection_area / min_box_area
  180. def _whether_y_overlap_exceeds_threshold(
  181. bbox1: Union[list, tuple],
  182. bbox2: Union[list, tuple],
  183. overlap_ratio_threshold: float = 0.6,
  184. ) -> bool:
  185. """
  186. Determines whether the vertical overlap between two bounding boxes exceeds a given threshold.
  187. Args:
  188. bbox1 (list or tuple): The first bounding box defined as (left, top, right, bottom).
  189. bbox2 (list or tuple): The second bounding box defined as (left, top, right, bottom).
  190. overlap_ratio_threshold (float): The threshold ratio to determine if the overlap is significant.
  191. Defaults to 0.6.
  192. Returns:
  193. bool: True if the vertical overlap divided by the minimum height of the two bounding boxes
  194. exceeds the overlap_ratio_threshold, otherwise False.
  195. """
  196. _, y1_0, _, y1_1 = bbox1
  197. _, y2_0, _, y2_1 = bbox2
  198. overlap = max(0, min(y1_1, y2_1) - max(y1_0, y2_0))
  199. min_height = min(y1_1 - y1_0, y2_1 - y2_0)
  200. return (overlap / min_height) > overlap_ratio_threshold
  201. def _adjust_span_text(span: List[str], prepend: bool = False, append: bool = False):
  202. """
  203. Adjust the text of a span by prepending or appending a newline.
  204. Args:
  205. span (list): A list where the second element is the text of the span.
  206. prepend (bool): If True, prepend a newline to the text.
  207. append (bool): If True, append a newline to the text.
  208. Returns:
  209. None: The function modifies the span in place.
  210. """
  211. if prepend:
  212. span[1] = "\n" + span[1]
  213. if append:
  214. span[1] = span[1] + "\n"
  215. return span
  216. def _format_line(
  217. line: List[List[Union[List[int], str]]],
  218. layout_min: int,
  219. layout_max: int,
  220. is_reference: bool = False,
  221. ) -> None:
  222. """
  223. Format a line of text spans based on layout constraints.
  224. Args:
  225. line (list): A list of spans, where each span is a list containing a bounding box and text.
  226. layout_min (int): The minimum x-coordinate of the layout bounding box.
  227. layout_max (int): The maximum x-coordinate of the layout bounding box.
  228. is_reference (bool): A flag indicating whether the line is a reference line, which affects formatting rules.
  229. Returns:
  230. None: The function modifies the line in place.
  231. """
  232. first_span = line[0]
  233. end_span = line[-1]
  234. if not is_reference:
  235. if first_span[0][0] - layout_min > 10:
  236. first_span = _adjust_span_text(first_span, prepend=True)
  237. if layout_max - end_span[0][2] > 10:
  238. end_span = _adjust_span_text(end_span, append=True)
  239. else:
  240. if first_span[0][0] - layout_min < 5:
  241. first_span = _adjust_span_text(first_span, prepend=True)
  242. if layout_max - end_span[0][2] > 20:
  243. end_span = _adjust_span_text(end_span, append=True)
  244. line[0] = first_span
  245. line[-1] = end_span
  246. return line
  247. def split_boxes_if_x_contained(boxes, offset=1e-5):
  248. """
  249. Check if there is any complete containment in the x-direction
  250. between the bounding boxes and split the containing box accordingly.
  251. Args:
  252. boxes (list of lists): Each element is a list containing an ndarray of length 4, a description, and a label.
  253. offset (float): A small offset value to ensure that the split boxes are not too close to the original boxes.
  254. Returns:
  255. A new list of boxes, including split boxes, with the same `rec_text` and `label` attributes.
  256. """
  257. def is_x_contained(box_a, box_b):
  258. """Check if box_a completely contains box_b in the x-direction."""
  259. return box_a[0][0] <= box_b[0][0] and box_a[0][2] >= box_b[0][2]
  260. new_boxes = []
  261. for i in range(len(boxes)):
  262. box_a = boxes[i]
  263. is_split = False
  264. for j in range(len(boxes)):
  265. if i == j:
  266. continue
  267. box_b = boxes[j]
  268. if is_x_contained(box_a, box_b):
  269. is_split = True
  270. # Split box_a based on the x-coordinates of box_b
  271. if box_a[0][0] < box_b[0][0]:
  272. w = box_b[0][0] - offset - box_a[0][0]
  273. if w > 1:
  274. new_boxes.append(
  275. [
  276. np.array(
  277. [
  278. box_a[0][0],
  279. box_a[0][1],
  280. box_b[0][0] - offset,
  281. box_a[0][3],
  282. ]
  283. ),
  284. box_a[1],
  285. box_a[2],
  286. ]
  287. )
  288. if box_a[0][2] > box_b[0][2]:
  289. w = box_a[0][2] - box_b[0][2] + offset
  290. if w > 1:
  291. box_a = [
  292. np.array(
  293. [
  294. box_b[0][2] + offset,
  295. box_a[0][1],
  296. box_a[0][2],
  297. box_a[0][3],
  298. ]
  299. ),
  300. box_a[1],
  301. box_a[2],
  302. ]
  303. if j == len(boxes) - 1 and is_split:
  304. new_boxes.append(box_a)
  305. if not is_split:
  306. new_boxes.append(box_a)
  307. return new_boxes
  308. def _sort_line_by_x_projection(
  309. input_img: np.ndarray,
  310. general_ocr_pipeline: Any,
  311. line: List[List[Union[List[int], str]]],
  312. ) -> None:
  313. """
  314. Sort a line of text spans based on their vertical position within the layout bounding box.
  315. Args:
  316. input_img (ndarray): The input image used for OCR.
  317. general_ocr_pipeline (Any): The general OCR pipeline used for text recognition.
  318. line (list): A list of spans, where each span is a list containing a bounding box and text.
  319. Returns:
  320. list: The sorted line of text spans.
  321. """
  322. splited_boxes = split_boxes_if_x_contained(line)
  323. splited_lines = []
  324. if len(line) != len(splited_boxes):
  325. splited_boxes.sort(key=lambda span: span[0][0])
  326. text_rec_model = general_ocr_pipeline.text_rec_model
  327. for span in splited_boxes:
  328. if span[2] == "text":
  329. crop_img = input_img[
  330. int(span[0][1]) : int(span[0][3]),
  331. int(span[0][0]) : int(span[0][2]),
  332. ]
  333. span[1] = next(text_rec_model([crop_img]))["rec_text"]
  334. splited_lines.append(span)
  335. else:
  336. splited_lines = line
  337. return splited_lines
  338. def _sort_ocr_res_by_y_projection(
  339. input_img: np.ndarray,
  340. general_ocr_pipeline: Any,
  341. label: Any,
  342. block_bbox: Tuple[int, int, int, int],
  343. ocr_res: Dict[str, List[Any]],
  344. line_height_iou_threshold: float = 0.7,
  345. ) -> Dict[str, List[Any]]:
  346. """
  347. Sorts OCR results based on their spatial arrangement, grouping them into lines and blocks.
  348. Args:
  349. input_img (ndarray): The input image used for OCR.
  350. general_ocr_pipeline (Any): The general OCR pipeline used for text recognition.
  351. label (Any): The label associated with the OCR results. It's not used in the function but might be
  352. relevant for other parts of the calling context.
  353. block_bbox (Tuple[int, int, int, int]): A tuple representing the layout bounding box, defined as
  354. (left, top, right, bottom).
  355. ocr_res (Dict[str, List[Any]]): A dictionary containing OCR results with the following keys:
  356. - "boxes": A list of bounding boxes, each defined as [left, top, right, bottom].
  357. - "rec_texts": A corresponding list of recognized text strings for each box.
  358. line_height_iou_threshold (float): The threshold for determining whether two boxes belong to
  359. the same line based on their vertical overlap. Defaults to 0.7.
  360. Returns:
  361. Dict[str, List[Any]]: A dictionary with the same structure as `ocr_res`, but with boxes and texts sorted
  362. and grouped into lines and blocks.
  363. """
  364. assert (
  365. ocr_res["boxes"] and ocr_res["rec_texts"]
  366. ), "OCR results must contain 'boxes' and 'rec_texts'"
  367. boxes = ocr_res["boxes"]
  368. rec_texts = ocr_res["rec_texts"]
  369. rec_labels = ocr_res["rec_labels"]
  370. x_min, _, x_max, _ = block_bbox
  371. inline_x_min = min([box[0] for box in boxes])
  372. inline_x_max = max([box[2] for box in boxes])
  373. spans = list(zip(boxes, rec_texts, rec_labels))
  374. spans.sort(key=lambda span: span[0][1])
  375. spans = [list(span) for span in spans]
  376. lines = []
  377. current_line = [spans[0]]
  378. current_y0, current_y1 = spans[0][0][1], spans[0][0][3]
  379. for span in spans[1:]:
  380. y0, y1 = span[0][1], span[0][3]
  381. if _whether_y_overlap_exceeds_threshold(
  382. (0, current_y0, 0, current_y1),
  383. (0, y0, 0, y1),
  384. line_height_iou_threshold,
  385. ):
  386. current_line.append(span)
  387. current_y0 = min(current_y0, y0)
  388. current_y1 = max(current_y1, y1)
  389. else:
  390. lines.append(current_line)
  391. current_line = [span]
  392. current_y0, current_y1 = y0, y1
  393. if current_line:
  394. lines.append(current_line)
  395. new_lines = []
  396. for line in lines:
  397. line.sort(key=lambda span: span[0][0])
  398. ocr_labels = [span[2] for span in line]
  399. if "formula" in ocr_labels:
  400. line = _sort_line_by_x_projection(input_img, general_ocr_pipeline, line)
  401. if label == "reference":
  402. line = _format_line(line, inline_x_min, inline_x_max, is_reference=True)
  403. elif label != "content":
  404. line = _format_line(line, x_min, x_max)
  405. new_lines.append(line)
  406. ocr_res["boxes"] = [span[0] for line in new_lines for span in line]
  407. if label == "content":
  408. ocr_res["rec_texts"] = [
  409. "".join(f"{span[1]} " for span in line).rstrip() for line in new_lines
  410. ]
  411. else:
  412. ocr_res["rec_texts"] = [span[1] + " " for line in new_lines for span in line]
  413. return ocr_res, len(new_lines)
  414. def _process_text(input_text: str) -> str:
  415. """
  416. Process the input text to handle spaces.
  417. The function removes multiple consecutive spaces between Chinese characters and ensures that
  418. only a single space is retained between Chinese and non-Chinese characters.
  419. Args:
  420. input_text (str): The text to be processed.
  421. Returns:
  422. str: The processed text with properly formatted spaces.
  423. """
  424. def handle_spaces_(text: str) -> str:
  425. """
  426. Handle spaces in the text by removing multiple consecutive spaces and inserting a single space
  427. between Chinese and non-Chinese characters.
  428. Args:
  429. text (str): The text to handle spaces for.
  430. Returns:
  431. str: The text with properly formatted spaces.
  432. """
  433. spaces = re.finditer(r"\s+", text)
  434. processed_text = list(text)
  435. for space in reversed(list(spaces)):
  436. start, end = space.span()
  437. prev_char = processed_text[start - 1] if start > 0 else ""
  438. next_char = processed_text[end] if end < len(processed_text) else ""
  439. is_prev_chinese = (
  440. re.match(r"[\u4e00-\u9fff]", prev_char) if prev_char else False
  441. )
  442. is_next_chinese = (
  443. re.match(r"[\u4e00-\u9fff]", next_char) if next_char else False
  444. )
  445. if is_prev_chinese and is_next_chinese:
  446. processed_text[start:end] = []
  447. else:
  448. processed_text[start:end] = [" "]
  449. return "".join(processed_text)
  450. text_without_spaces = handle_spaces_(input_text)
  451. final_text = re.sub(r"\s+", " ", text_without_spaces).strip()
  452. return final_text
  453. def get_single_block_parsing_res(
  454. general_ocr_pipeline: Any,
  455. overall_ocr_res: OCRResult,
  456. layout_det_res: DetResult,
  457. table_res_list: list,
  458. seal_res_list: list,
  459. ) -> OCRResult:
  460. """
  461. Extract structured information from OCR and layout detection results.
  462. Args:
  463. overall_ocr_res (OCRResult): An object containing the overall OCR results, including detected text boxes and recognized text. The structure is expected to have:
  464. - "input_img": The image on which OCR was performed.
  465. - "dt_boxes": A list of detected text box coordinates.
  466. - "rec_texts": A list of recognized text corresponding to the detected boxes.
  467. layout_det_res (DetResult): An object containing the layout detection results, including detected layout boxes and their labels. The structure is expected to have:
  468. - "boxes": A list of dictionaries with keys "coordinate" for box coordinates and "block_label" for the type of content.
  469. table_res_list (list): A list of table detection results, where each item is a dictionary containing:
  470. - "block_bbox": The bounding box of the table layout.
  471. - "pred_html": The predicted HTML representation of the table.
  472. seal_res_list (List): A list of seal detection results. The details of each item depend on the specific application context.
  473. Returns:
  474. list: A list of structured boxes where each item is a dictionary containing:
  475. - "block_label": The label of the content (e.g., 'table', 'chart', 'image').
  476. - The label as a key with either table HTML or image data and text.
  477. - "block_bbox": The coordinates of the layout box.
  478. """
  479. single_block_layout_parsing_res = []
  480. input_img = overall_ocr_res["doc_preprocessor_res"]["output_img"]
  481. seal_index = 0
  482. with_doc_title = False
  483. max_block_area = 0.0
  484. paragraph_title_indexs = []
  485. layout_det_res_list, _ = _remove_overlap_blocks(
  486. deepcopy(layout_det_res["boxes"]),
  487. threshold=0.5,
  488. smaller=True,
  489. )
  490. for box_idx, box_info in enumerate(layout_det_res_list):
  491. block_bbox = box_info["coordinate"]
  492. label = box_info["label"]
  493. rec_res = {"boxes": [], "rec_texts": [], "rec_labels": [], "flag": False}
  494. seg_start_coordinate = float("inf")
  495. seg_end_coordinate = float("-inf")
  496. num_of_lines = 1
  497. if label == "doc_title":
  498. with_doc_title = True
  499. elif label == "paragraph_title":
  500. paragraph_title_indexs.append(box_idx)
  501. block_area = (block_bbox[2] - block_bbox[0]) * (block_bbox[3] - block_bbox[1])
  502. max_block_area = max(max_block_area, block_area)
  503. if label == "table":
  504. for table_res in table_res_list:
  505. if len(table_res["cell_box_list"]) == 0:
  506. continue
  507. if (
  508. _calculate_overlap_area_div_minbox_area_ratio(
  509. block_bbox, table_res["cell_box_list"][0]
  510. )
  511. > 0.5
  512. ):
  513. single_block_layout_parsing_res.append(
  514. {
  515. "block_label": label,
  516. "block_content": table_res["pred_html"],
  517. "block_bbox": block_bbox,
  518. },
  519. )
  520. break
  521. elif label == "seal":
  522. if len(seal_res_list) > 0:
  523. single_block_layout_parsing_res.append(
  524. {
  525. "block_label": label,
  526. "block_content": _process_text(
  527. ", ".join(seal_res_list[seal_index]["rec_texts"])
  528. ),
  529. "block_bbox": block_bbox,
  530. },
  531. )
  532. seal_index += 1
  533. else:
  534. overall_text_boxes = overall_ocr_res["rec_boxes"]
  535. for box_no in range(len(overall_text_boxes)):
  536. if (
  537. _calculate_overlap_area_div_minbox_area_ratio(
  538. block_bbox, overall_text_boxes[box_no]
  539. )
  540. > 0.5
  541. ):
  542. rec_res["boxes"].append(overall_text_boxes[box_no])
  543. rec_res["rec_texts"].append(
  544. overall_ocr_res["rec_texts"][box_no],
  545. )
  546. rec_res["rec_labels"].append(
  547. overall_ocr_res["rec_labels"][box_no],
  548. )
  549. rec_res["flag"] = True
  550. if rec_res["flag"]:
  551. rec_res, num_of_lines = _sort_ocr_res_by_y_projection(
  552. input_img, general_ocr_pipeline, label, block_bbox, rec_res, 0.7
  553. )
  554. seg_start_coordinate = rec_res["boxes"][0][0]
  555. seg_end_coordinate = rec_res["boxes"][-1][2]
  556. if label == "formula":
  557. rec_res["rec_texts"] = [
  558. rec_res_text.replace("$", "")
  559. for rec_res_text in rec_res["rec_texts"]
  560. ]
  561. if label in ["chart", "image"]:
  562. x_min, y_min, x_max, y_max = list(map(int, block_bbox))
  563. img_path = f"imgs/img_in_table_box_{x_min}_{y_min}_{x_max}_{y_max}.jpg"
  564. img = Image.fromarray(input_img[y_min:y_max, x_min:x_max, ::-1])
  565. single_block_layout_parsing_res.append(
  566. {
  567. "block_label": label,
  568. "block_content": _process_text("".join(rec_res["rec_texts"])),
  569. "block_image": {img_path: img},
  570. "block_bbox": block_bbox,
  571. },
  572. )
  573. else:
  574. if label in ["doc_title"]:
  575. content = " ".join(rec_res["rec_texts"])
  576. elif label in ["content"]:
  577. content = "\n".join(rec_res["rec_texts"])
  578. else:
  579. content = "".join(rec_res["rec_texts"])
  580. if label != "reference":
  581. content = _process_text(content)
  582. single_block_layout_parsing_res.append(
  583. {
  584. "block_label": label,
  585. "block_content": content,
  586. "block_bbox": block_bbox,
  587. "seg_start_coordinate": seg_start_coordinate,
  588. "seg_end_coordinate": seg_end_coordinate,
  589. "num_of_lines": num_of_lines,
  590. "block_area": block_area,
  591. },
  592. )
  593. if (
  594. not with_doc_title
  595. and len(paragraph_title_indexs) == 1
  596. and single_block_layout_parsing_res[paragraph_title_indexs[0]].get(
  597. "block_area", 0
  598. )
  599. > max_block_area * 0.3
  600. ):
  601. single_block_layout_parsing_res[paragraph_title_indexs[0]][
  602. "block_label"
  603. ] = "doc_title"
  604. if len(layout_det_res_list) == 0:
  605. for ocr_rec_box, ocr_rec_text in zip(
  606. overall_ocr_res["rec_boxes"], overall_ocr_res["rec_texts"]
  607. ):
  608. single_block_layout_parsing_res.append(
  609. {
  610. "block_label": "text",
  611. "block_content": ocr_rec_text,
  612. "block_bbox": ocr_rec_box,
  613. "seg_start_coordinate": ocr_rec_box[0],
  614. "seg_end_coordinate": ocr_rec_box[2],
  615. },
  616. )
  617. single_block_layout_parsing_res = get_layout_ordering(
  618. single_block_layout_parsing_res,
  619. no_mask_labels=[
  620. "text",
  621. "formula",
  622. "algorithm",
  623. "reference",
  624. "content",
  625. "abstract",
  626. ],
  627. )
  628. return single_block_layout_parsing_res
  629. def _projection_by_bboxes(boxes: np.ndarray, axis: int) -> np.ndarray:
  630. """
  631. Generate a 1D projection histogram from bounding boxes along a specified axis.
  632. Args:
  633. boxes: A (N, 4) array of bounding boxes defined by [x_min, y_min, x_max, y_max].
  634. axis: Axis for projection; 0 for horizontal (x-axis), 1 for vertical (y-axis).
  635. Returns:
  636. A 1D numpy array representing the projection histogram based on bounding box intervals.
  637. """
  638. assert axis in [0, 1]
  639. max_length = np.max(boxes[:, axis::2])
  640. projection = np.zeros(max_length, dtype=int)
  641. # Increment projection histogram over the interval defined by each bounding box
  642. for start, end in boxes[:, axis::2]:
  643. projection[start:end] += 1
  644. return projection
  645. def _split_projection_profile(arr_values: np.ndarray, min_value: float, min_gap: float):
  646. """
  647. Split the projection profile into segments based on specified thresholds.
  648. Args:
  649. arr_values: 1D array representing the projection profile.
  650. min_value: Minimum value threshold to consider a profile segment significant.
  651. min_gap: Minimum gap width to consider a separation between segments.
  652. Returns:
  653. A tuple of start and end indices for each segment that meets the criteria.
  654. """
  655. # Identify indices where the projection exceeds the minimum value
  656. significant_indices = np.where(arr_values > min_value)[0]
  657. if not len(significant_indices):
  658. return
  659. # Calculate gaps between significant indices
  660. index_diffs = significant_indices[1:] - significant_indices[:-1]
  661. gap_indices = np.where(index_diffs > min_gap)[0]
  662. # Determine start and end indices of segments
  663. segment_starts = np.insert(
  664. significant_indices[gap_indices + 1],
  665. 0,
  666. significant_indices[0],
  667. )
  668. segment_ends = np.append(
  669. significant_indices[gap_indices],
  670. significant_indices[-1] + 1,
  671. )
  672. return segment_starts, segment_ends
  673. def _recursive_yx_cut(
  674. boxes: np.ndarray, indices: List[int], res: List[int], min_gap: int = 1
  675. ):
  676. """
  677. Recursively project and segment bounding boxes, starting with Y-axis and followed by X-axis.
  678. Args:
  679. boxes: A (N, 4) array representing bounding boxes.
  680. indices: List of indices indicating the original position of boxes.
  681. res: List to store indices of the final segmented bounding boxes.
  682. min_gap (int): Minimum gap width to consider a separation between segments on the X-axis. Defaults to 1.
  683. Returns:
  684. None: This function modifies the `res` list in place.
  685. """
  686. assert len(boxes) == len(
  687. indices
  688. ), "The length of boxes and indices must be the same."
  689. # Sort by y_min for Y-axis projection
  690. y_sorted_indices = boxes[:, 1].argsort()
  691. y_sorted_boxes = boxes[y_sorted_indices]
  692. y_sorted_indices = np.array(indices)[y_sorted_indices]
  693. # Perform Y-axis projection
  694. y_projection = _projection_by_bboxes(boxes=y_sorted_boxes, axis=1)
  695. y_intervals = _split_projection_profile(y_projection, 0, 1)
  696. if not y_intervals:
  697. return
  698. # Process each segment defined by Y-axis projection
  699. for y_start, y_end in zip(*y_intervals):
  700. # Select boxes within the current y interval
  701. y_interval_indices = (y_start <= y_sorted_boxes[:, 1]) & (
  702. y_sorted_boxes[:, 1] < y_end
  703. )
  704. y_boxes_chunk = y_sorted_boxes[y_interval_indices]
  705. y_indices_chunk = y_sorted_indices[y_interval_indices]
  706. # Sort by x_min for X-axis projection
  707. x_sorted_indices = y_boxes_chunk[:, 0].argsort()
  708. x_sorted_boxes_chunk = y_boxes_chunk[x_sorted_indices]
  709. x_sorted_indices_chunk = y_indices_chunk[x_sorted_indices]
  710. # Perform X-axis projection
  711. x_projection = _projection_by_bboxes(boxes=x_sorted_boxes_chunk, axis=0)
  712. x_intervals = _split_projection_profile(x_projection, 0, min_gap)
  713. if not x_intervals:
  714. continue
  715. # If X-axis cannot be further segmented, add current indices to results
  716. if len(x_intervals[0]) == 1:
  717. res.extend(x_sorted_indices_chunk)
  718. continue
  719. # Recursively process each segment defined by X-axis projection
  720. for x_start, x_end in zip(*x_intervals):
  721. x_interval_indices = (x_start <= x_sorted_boxes_chunk[:, 0]) & (
  722. x_sorted_boxes_chunk[:, 0] < x_end
  723. )
  724. _recursive_yx_cut(
  725. x_sorted_boxes_chunk[x_interval_indices],
  726. x_sorted_indices_chunk[x_interval_indices],
  727. res,
  728. )
  729. def _recursive_xy_cut(
  730. boxes: np.ndarray, indices: List[int], res: List[int], min_gap: int = 1
  731. ):
  732. """
  733. Recursively performs X-axis projection followed by Y-axis projection to segment bounding boxes.
  734. Args:
  735. boxes: A (N, 4) array representing bounding boxes with [x_min, y_min, x_max, y_max].
  736. indices: A list of indices representing the position of boxes in the original data.
  737. res: A list to store indices of bounding boxes that meet the criteria.
  738. min_gap (int): Minimum gap width to consider a separation between segments on the X-axis. Defaults to 1.
  739. Returns:
  740. None: This function modifies the `res` list in place.
  741. """
  742. # Ensure boxes and indices have the same length
  743. assert len(boxes) == len(
  744. indices
  745. ), "The length of boxes and indices must be the same."
  746. # Sort by x_min to prepare for X-axis projection
  747. x_sorted_indices = boxes[:, 0].argsort()
  748. x_sorted_boxes = boxes[x_sorted_indices]
  749. x_sorted_indices = np.array(indices)[x_sorted_indices]
  750. # Perform X-axis projection
  751. x_projection = _projection_by_bboxes(boxes=x_sorted_boxes, axis=0)
  752. x_intervals = _split_projection_profile(x_projection, 0, 1)
  753. if not x_intervals:
  754. return
  755. # Process each segment defined by X-axis projection
  756. for x_start, x_end in zip(*x_intervals):
  757. # Select boxes within the current x interval
  758. x_interval_indices = (x_start <= x_sorted_boxes[:, 0]) & (
  759. x_sorted_boxes[:, 0] < x_end
  760. )
  761. x_boxes_chunk = x_sorted_boxes[x_interval_indices]
  762. x_indices_chunk = x_sorted_indices[x_interval_indices]
  763. # Sort selected boxes by y_min to prepare for Y-axis projection
  764. y_sorted_indices = x_boxes_chunk[:, 1].argsort()
  765. y_sorted_boxes_chunk = x_boxes_chunk[y_sorted_indices]
  766. y_sorted_indices_chunk = x_indices_chunk[y_sorted_indices]
  767. # Perform Y-axis projection
  768. y_projection = _projection_by_bboxes(boxes=y_sorted_boxes_chunk, axis=1)
  769. y_intervals = _split_projection_profile(y_projection, 0, min_gap)
  770. if not y_intervals:
  771. continue
  772. # If Y-axis cannot be further segmented, add current indices to results
  773. if len(y_intervals[0]) == 1:
  774. res.extend(y_sorted_indices_chunk)
  775. continue
  776. # Recursively process each segment defined by Y-axis projection
  777. for y_start, y_end in zip(*y_intervals):
  778. y_interval_indices = (y_start <= y_sorted_boxes_chunk[:, 1]) & (
  779. y_sorted_boxes_chunk[:, 1] < y_end
  780. )
  781. _recursive_xy_cut(
  782. y_sorted_boxes_chunk[y_interval_indices],
  783. y_sorted_indices_chunk[y_interval_indices],
  784. res,
  785. )
  786. def sort_by_xycut(
  787. block_bboxes: Union[np.ndarray, List[List[int]]],
  788. direction: int = 0,
  789. min_gap: int = 1,
  790. ) -> List[int]:
  791. """
  792. Sort bounding boxes using recursive XY cut method based on the specified direction.
  793. Args:
  794. block_bboxes (Union[np.ndarray, List[List[int]]]): An array or list of bounding boxes,
  795. where each box is represented as
  796. [x_min, y_min, x_max, y_max].
  797. direction (int): Direction for the initial cut. Use 1 for Y-axis first and 0 for X-axis first.
  798. Defaults to 0.
  799. min_gap (int): Minimum gap width to consider a separation between segments. Defaults to 1.
  800. Returns:
  801. List[int]: A list of indices representing the order of sorted bounding boxes.
  802. """
  803. block_bboxes = np.asarray(block_bboxes).astype(int)
  804. res = []
  805. if direction == 1:
  806. _recursive_yx_cut(
  807. block_bboxes,
  808. np.arange(len(block_bboxes)).tolist(),
  809. res,
  810. min_gap,
  811. )
  812. else:
  813. _recursive_xy_cut(
  814. block_bboxes,
  815. np.arange(len(block_bboxes)).tolist(),
  816. res,
  817. min_gap,
  818. )
  819. return res
  820. def gather_imgs(original_img, layout_det_objs):
  821. imgs_in_doc = []
  822. for det_obj in layout_det_objs:
  823. if det_obj["label"] in ("image", "chart"):
  824. x_min, y_min, x_max, y_max = list(map(int, det_obj["coordinate"]))
  825. img_path = f"imgs/img_in_table_box_{x_min}_{y_min}_{x_max}_{y_max}.jpg"
  826. img = Image.fromarray(original_img[y_min:y_max, x_min:x_max, ::-1])
  827. imgs_in_doc.append(
  828. {
  829. "path": img_path,
  830. "img": img,
  831. "coordinate": (x_min, y_min, x_max, y_max),
  832. "score": det_obj["score"],
  833. }
  834. )
  835. return imgs_in_doc
  836. def _get_minbox_if_overlap_by_ratio(
  837. bbox1: Union[List[int], Tuple[int, int, int, int]],
  838. bbox2: Union[List[int], Tuple[int, int, int, int]],
  839. ratio: float,
  840. smaller: bool = True,
  841. ) -> Optional[Union[List[int], Tuple[int, int, int, int]]]:
  842. """
  843. Determine if the overlap area between two bounding boxes exceeds a given ratio
  844. and return the smaller (or larger) bounding box based on the `smaller` flag.
  845. Args:
  846. bbox1 (Union[List[int], Tuple[int, int, int, int]]): Coordinates of the first bounding box [x_min, y_min, x_max, y_max].
  847. bbox2 (Union[List[int], Tuple[int, int, int, int]]): Coordinates of the second bounding box [x_min, y_min, x_max, y_max].
  848. ratio (float): The overlap ratio threshold.
  849. smaller (bool): If True, return the smaller bounding box; otherwise, return the larger one.
  850. Returns:
  851. Optional[Union[List[int], Tuple[int, int, int, int]]]:
  852. The selected bounding box or None if the overlap ratio is not exceeded.
  853. """
  854. # Calculate the areas of both bounding boxes
  855. area1 = (bbox1[2] - bbox1[0]) * (bbox1[3] - bbox1[1])
  856. area2 = (bbox2[2] - bbox2[0]) * (bbox2[3] - bbox2[1])
  857. # Calculate the overlap ratio using a helper function
  858. overlap_ratio = _calculate_overlap_area_div_minbox_area_ratio(bbox1, bbox2)
  859. # Check if the overlap ratio exceeds the threshold
  860. if overlap_ratio > ratio:
  861. if (area1 <= area2 and smaller) or (area1 >= area2 and not smaller):
  862. return 1
  863. else:
  864. return 2
  865. return None
  866. def _remove_overlap_blocks(
  867. blocks: List[Dict[str, List[int]]], threshold: float = 0.65, smaller: bool = True
  868. ) -> Tuple[List[Dict[str, List[int]]], List[Dict[str, List[int]]]]:
  869. """
  870. Remove overlapping blocks based on a specified overlap ratio threshold.
  871. Args:
  872. blocks (List[Dict[str, List[int]]]): List of block dictionaries, each containing a 'block_bbox' key.
  873. threshold (float): Ratio threshold to determine significant overlap.
  874. smaller (bool): If True, the smaller block in overlap is removed.
  875. Returns:
  876. Tuple[List[Dict[str, List[int]]], List[Dict[str, List[int]]]]:
  877. A tuple containing the updated list of blocks and a list of dropped blocks.
  878. """
  879. dropped_blocks = []
  880. dropped_indexes = set()
  881. # Iterate over each pair of blocks to find overlaps
  882. for i, block1 in enumerate(blocks):
  883. for j in range(i + 1, len(blocks)):
  884. block2 = blocks[j]
  885. # Skip blocks that are already marked for removal
  886. if i in dropped_indexes or j in dropped_indexes:
  887. continue
  888. # Check for overlap and determine which block to remove
  889. overlap_box_index = _get_minbox_if_overlap_by_ratio(
  890. block1["coordinate"],
  891. block2["coordinate"],
  892. threshold,
  893. smaller=smaller,
  894. )
  895. if overlap_box_index is not None:
  896. # Determine which block to remove based on overlap_box_index
  897. if overlap_box_index == 1:
  898. drop_index = i
  899. else:
  900. drop_index = j
  901. dropped_indexes.add(drop_index)
  902. # Remove marked blocks from the original list
  903. for index in sorted(dropped_indexes, reverse=True):
  904. dropped_blocks.append(blocks[index])
  905. del blocks[index]
  906. return blocks, dropped_blocks
  907. def _get_text_median_width(blocks: List[Dict[str, any]]) -> float:
  908. """
  909. Calculate the median width of blocks labeled as "text".
  910. Args:
  911. blocks (List[Dict[str, any]]): List of block dictionaries, each containing a 'block_bbox' and 'label'.
  912. Returns:
  913. float: The median width of text blocks, or infinity if no text blocks are found.
  914. """
  915. widths = [
  916. block["block_bbox"][2] - block["block_bbox"][0]
  917. for block in blocks
  918. if block.get("block_label") == "text"
  919. ]
  920. return np.median(widths) if widths else float("inf")
  921. def _get_layout_property(
  922. blocks: List[Dict[str, any]],
  923. median_width: float,
  924. no_mask_labels: List[str],
  925. threshold: float = 0.8,
  926. ) -> Tuple[List[Dict[str, any]], bool]:
  927. """
  928. Determine the layout (single or double column) of text blocks.
  929. Args:
  930. blocks (List[Dict[str, any]]): List of block dictionaries containing 'label' and 'block_bbox'.
  931. median_width (float): Median width of text blocks.
  932. no_mask_labels (List[str]): Labels of blocks to be considered for layout analysis.
  933. threshold (float): Threshold for determining layout overlap.
  934. Returns:
  935. Tuple[List[Dict[str, any]], bool]: Updated list of blocks with layout information and a boolean
  936. indicating if the double layout area is greater than the single layout area.
  937. """
  938. blocks.sort(
  939. key=lambda x: (
  940. x["block_bbox"][0],
  941. (x["block_bbox"][2] - x["block_bbox"][0]),
  942. ),
  943. )
  944. check_single_layout = {}
  945. page_min_x, page_max_x = float("inf"), 0
  946. double_label_area = 0
  947. single_label_area = 0
  948. for i, block in enumerate(blocks):
  949. page_min_x = min(page_min_x, block["block_bbox"][0])
  950. page_max_x = max(page_max_x, block["block_bbox"][2])
  951. page_width = page_max_x - page_min_x
  952. for i, block in enumerate(blocks):
  953. if block["block_label"] not in no_mask_labels:
  954. continue
  955. x_min_i, _, x_max_i, _ = block["block_bbox"]
  956. layout_length = x_max_i - x_min_i
  957. cover_count, cover_with_threshold_count = 0, 0
  958. match_block_with_threshold_indexes = []
  959. for j, other_block in enumerate(blocks):
  960. if i == j or other_block["block_label"] not in no_mask_labels:
  961. continue
  962. x_min_j, _, x_max_j, _ = other_block["block_bbox"]
  963. x_match_min, x_match_max = max(
  964. x_min_i,
  965. x_min_j,
  966. ), min(x_max_i, x_max_j)
  967. match_block_iou = (x_match_max - x_match_min) / (x_max_j - x_min_j)
  968. if match_block_iou > 0:
  969. cover_count += 1
  970. if match_block_iou > threshold:
  971. cover_with_threshold_count += 1
  972. match_block_with_threshold_indexes.append(
  973. (j, match_block_iou),
  974. )
  975. x_min_i = x_match_max
  976. if x_min_i >= x_max_i:
  977. break
  978. if (
  979. layout_length > median_width * 1.3
  980. and (cover_with_threshold_count >= 2 or cover_count >= 2)
  981. ) or layout_length > 0.6 * page_width:
  982. # if layout_length > median_width * 1.3 and (cover_with_threshold_count >= 2):
  983. block["layout"] = "double"
  984. double_label_area += (block["block_bbox"][2] - block["block_bbox"][0]) * (
  985. block["block_bbox"][3] - block["block_bbox"][1]
  986. )
  987. else:
  988. block["layout"] = "single"
  989. check_single_layout[i] = match_block_with_threshold_indexes
  990. # Check single-layout block
  991. for i, single_layout in check_single_layout.items():
  992. if single_layout:
  993. index, match_iou = single_layout[-1]
  994. if match_iou > 0.9 and blocks[index]["layout"] == "double":
  995. blocks[i]["layout"] = "double"
  996. double_label_area += (
  997. blocks[i]["block_bbox"][2] - blocks[i]["block_bbox"][0]
  998. ) * (blocks[i]["block_bbox"][3] - blocks[i]["block_bbox"][1])
  999. else:
  1000. single_label_area += (
  1001. blocks[i]["block_bbox"][2] - blocks[i]["block_bbox"][0]
  1002. ) * (blocks[i]["block_bbox"][3] - blocks[i]["block_bbox"][1])
  1003. return blocks, (double_label_area > single_label_area)
  1004. def _get_bbox_direction(input_bbox: List[float], ratio: float = 1.0) -> bool:
  1005. """
  1006. Determine if a bounding box is horizontal or vertical.
  1007. Args:
  1008. input_bbox (List[float]): Bounding box [x_min, y_min, x_max, y_max].
  1009. ratio (float): Ratio for determining orientation. Default is 1.0.
  1010. Returns:
  1011. bool: True if the bounding box is considered horizontal, False if vertical.
  1012. """
  1013. width = input_bbox[2] - input_bbox[0]
  1014. height = input_bbox[3] - input_bbox[1]
  1015. return width * ratio >= height
  1016. def _get_projection_iou(
  1017. input_bbox: List[float], match_bbox: List[float], is_horizontal: bool = True
  1018. ) -> float:
  1019. """
  1020. Calculate the IoU of lines between two bounding boxes.
  1021. Args:
  1022. input_bbox (List[float]): First bounding box [x_min, y_min, x_max, y_max].
  1023. match_bbox (List[float]): Second bounding box [x_min, y_min, x_max, y_max].
  1024. is_horizontal (bool): Whether to compare horizontally or vertically.
  1025. Returns:
  1026. float: Line IoU. Returns 0 if there is no overlap.
  1027. """
  1028. if is_horizontal:
  1029. x_match_min = max(input_bbox[0], match_bbox[0])
  1030. x_match_max = min(input_bbox[2], match_bbox[2])
  1031. overlap = max(0, x_match_max - x_match_min)
  1032. input_width = min(input_bbox[2] - input_bbox[0], match_bbox[2] - match_bbox[0])
  1033. else:
  1034. y_match_min = max(input_bbox[1], match_bbox[1])
  1035. y_match_max = min(input_bbox[3], match_bbox[3])
  1036. overlap = max(0, y_match_max - y_match_min)
  1037. input_width = min(input_bbox[3] - input_bbox[1], match_bbox[3] - match_bbox[1])
  1038. return overlap / input_width if input_width > 0 else 0.0
  1039. def _get_sub_category(
  1040. blocks: List[Dict[str, Any]], title_labels: List[str]
  1041. ) -> Tuple[List[Dict[str, Any]], List[float]]:
  1042. """
  1043. Determine the layout of title and text blocks and collect pre_cuts.
  1044. Args:
  1045. blocks (List[Dict[str, Any]]): List of block dictionaries.
  1046. title_labels (List[str]): List of labels considered as titles.
  1047. Returns:
  1048. List[Dict[str, Any]]: Updated list of blocks with title-text layout information.
  1049. Dict[float]: Dict of pre_cuts coordinates.
  1050. """
  1051. sub_title_labels = ["paragraph_title"]
  1052. vision_labels = ["image", "table", "chart", "figure"]
  1053. vision_title_labels = ["figure_title", "chart_title", "table_title"]
  1054. all_labels = title_labels + sub_title_labels + vision_labels + vision_title_labels
  1055. special_pre_cut_labels = sub_title_labels
  1056. # single doc title is irregular,pre cut not applicable
  1057. num_doc_title = 0
  1058. for block in blocks:
  1059. if block["block_label"] == "doc_title":
  1060. num_doc_title += 1
  1061. if num_doc_title == 2:
  1062. special_pre_cut_labels = title_labels + sub_title_labels
  1063. break
  1064. if len(blocks) == 0:
  1065. return blocks, {}
  1066. min_x = min(block["block_bbox"][0] for block in blocks)
  1067. min_y = min(block["block_bbox"][1] for block in blocks)
  1068. max_x = max(block["block_bbox"][2] for block in blocks)
  1069. max_y = max(block["block_bbox"][3] for block in blocks)
  1070. region_bbox = (min_x, min_y, max_x, max_y)
  1071. region_x_center = (region_bbox[0] + region_bbox[2]) / 2
  1072. region_y_center = (region_bbox[1] + region_bbox[3]) / 2
  1073. region_width = region_bbox[2] - region_bbox[0]
  1074. region_height = region_bbox[3] - region_bbox[1]
  1075. pre_cuts = {}
  1076. for i, block1 in enumerate(blocks):
  1077. block1.setdefault("title_text", [])
  1078. block1.setdefault("sub_title", [])
  1079. block1.setdefault("vision_footnote", [])
  1080. block1.setdefault("sub_label", block1["block_label"])
  1081. if block1["block_label"] not in all_labels:
  1082. continue
  1083. bbox1 = block1["block_bbox"]
  1084. x1, y1, x2, y2 = bbox1
  1085. is_horizontal_1 = _get_bbox_direction(block1["block_bbox"])
  1086. left_up_title_text_distance = float("inf")
  1087. left_up_title_text_index = -1
  1088. left_up_title_text_direction = None
  1089. right_down_title_text_distance = float("inf")
  1090. right_down_title_text_index = -1
  1091. right_down_title_text_direction = None
  1092. # pre-cuts
  1093. # Condition 1: Length is greater than half of the layout region
  1094. if is_horizontal_1:
  1095. block_length = x2 - x1
  1096. required_length = region_width / 2
  1097. else:
  1098. block_length = y2 - y1
  1099. required_length = region_height / 2
  1100. if block1["block_label"] in special_pre_cut_labels:
  1101. length_condition = True
  1102. else:
  1103. length_condition = block_length > required_length
  1104. # Condition 2: Centered check (must be within ±20 in both horizontal and vertical directions)
  1105. block_x_center = (x1 + x2) / 2
  1106. block_y_center = (y1 + y2) / 2
  1107. tolerance_len = block_length // 5
  1108. if block1["block_label"] in special_pre_cut_labels:
  1109. tolerance_len = block_length // 10
  1110. if is_horizontal_1:
  1111. is_centered = abs(block_x_center - region_x_center) <= tolerance_len
  1112. else:
  1113. is_centered = abs(block_y_center - region_y_center) <= tolerance_len
  1114. # Condition 3: Check for surrounding text
  1115. has_left_text = False
  1116. has_right_text = False
  1117. has_above_text = False
  1118. has_below_text = False
  1119. for block2 in blocks:
  1120. if block2["block_label"] != "text":
  1121. continue
  1122. bbox2 = block2["block_bbox"]
  1123. x1_2, y1_2, x2_2, y2_2 = bbox2
  1124. if is_horizontal_1:
  1125. if x2_2 <= x1 and not (y2_2 <= y1 or y1_2 >= y2):
  1126. has_left_text = True
  1127. if x1_2 >= x2 and not (y2_2 <= y1 or y1_2 >= y2):
  1128. has_right_text = True
  1129. else:
  1130. if y2_2 <= y1 and not (x2_2 <= x1 or x1_2 >= x2):
  1131. has_above_text = True
  1132. if y1_2 >= y2 and not (x2_2 <= x1 or x1_2 >= x2):
  1133. has_below_text = True
  1134. if (is_horizontal_1 and has_left_text and has_right_text) or (
  1135. not is_horizontal_1 and has_above_text and has_below_text
  1136. ):
  1137. break
  1138. no_text_on_sides = (
  1139. not (has_left_text or has_right_text)
  1140. if is_horizontal_1
  1141. else not (has_above_text or has_below_text)
  1142. )
  1143. # Add coordinates if all conditions are met
  1144. if is_centered and length_condition and no_text_on_sides:
  1145. if is_horizontal_1:
  1146. pre_cuts.setdefault("y", []).append(y1)
  1147. else:
  1148. pre_cuts.setdefault("x", []).append(x1)
  1149. for j, block2 in enumerate(blocks):
  1150. if i == j:
  1151. continue
  1152. bbox2 = block2["block_bbox"]
  1153. x1_prime, y1_prime, x2_prime, y2_prime = bbox2
  1154. is_horizontal_2 = _get_bbox_direction(bbox2)
  1155. match_block_iou = _get_projection_iou(
  1156. bbox2,
  1157. bbox1,
  1158. is_horizontal_1,
  1159. )
  1160. def distance_(is_horizontal, is_left_up):
  1161. if is_horizontal:
  1162. if is_left_up:
  1163. return (y1 - y2_prime + 2) // 5 + x1_prime / 5000
  1164. else:
  1165. return (y1_prime - y2 + 2) // 5 + x1_prime / 5000
  1166. else:
  1167. if is_left_up:
  1168. return (x1 - x2_prime + 2) // 5 + y1_prime / 5000
  1169. else:
  1170. return (x1_prime - x2 + 2) // 5 + y1_prime / 5000
  1171. block_iou_threshold = 0.1
  1172. if block1["block_label"] in sub_title_labels:
  1173. block_iou_threshold = 0.5
  1174. if is_horizontal_1:
  1175. if match_block_iou >= block_iou_threshold:
  1176. left_up_distance = distance_(True, True)
  1177. right_down_distance = distance_(True, False)
  1178. if (
  1179. y2_prime <= y1
  1180. and left_up_distance <= left_up_title_text_distance
  1181. ):
  1182. left_up_title_text_distance = left_up_distance
  1183. left_up_title_text_index = j
  1184. left_up_title_text_direction = is_horizontal_2
  1185. elif (
  1186. y1_prime > y2
  1187. and right_down_distance < right_down_title_text_distance
  1188. ):
  1189. right_down_title_text_distance = right_down_distance
  1190. right_down_title_text_index = j
  1191. right_down_title_text_direction = is_horizontal_2
  1192. else:
  1193. if match_block_iou >= block_iou_threshold:
  1194. left_up_distance = distance_(False, True)
  1195. right_down_distance = distance_(False, False)
  1196. if (
  1197. x2_prime <= x1
  1198. and left_up_distance <= left_up_title_text_distance
  1199. ):
  1200. left_up_title_text_distance = left_up_distance
  1201. left_up_title_text_index = j
  1202. left_up_title_text_direction = is_horizontal_2
  1203. elif (
  1204. x1_prime > x2
  1205. and right_down_distance < right_down_title_text_distance
  1206. ):
  1207. right_down_title_text_distance = right_down_distance
  1208. right_down_title_text_index = j
  1209. right_down_title_text_direction = is_horizontal_2
  1210. height = bbox1[3] - bbox1[1]
  1211. width = bbox1[2] - bbox1[0]
  1212. title_text_weight = [0.8, 0.8]
  1213. title_text, sub_title, vision_footnote = [], [], []
  1214. def get_sub_category_(
  1215. title_text_direction,
  1216. title_text_index,
  1217. label,
  1218. is_left_up=True,
  1219. ):
  1220. direction_ = [1, 3] if is_left_up else [2, 4]
  1221. if (
  1222. title_text_direction == is_horizontal_1
  1223. and title_text_index != -1
  1224. and (label == "text" or label == "paragraph_title")
  1225. ):
  1226. bbox2 = blocks[title_text_index]["block_bbox"]
  1227. if is_horizontal_1:
  1228. height1 = bbox2[3] - bbox2[1]
  1229. width1 = bbox2[2] - bbox2[0]
  1230. if label == "text":
  1231. if (
  1232. _nearest_edge_distance(bbox1, bbox2)[0] <= 15
  1233. and block1["block_label"] in vision_labels
  1234. and width1 < width
  1235. and height1 < 0.5 * height
  1236. ):
  1237. blocks[title_text_index]["sub_label"] = "vision_footnote"
  1238. vision_footnote.append(bbox2)
  1239. elif (
  1240. height1 < height * title_text_weight[0]
  1241. and (width1 < width or width1 > 1.5 * width)
  1242. and block1["block_label"] in title_labels
  1243. ):
  1244. blocks[title_text_index]["sub_label"] = "title_text"
  1245. title_text.append((direction_[0], bbox2))
  1246. elif (
  1247. label == "paragraph_title"
  1248. and block1["block_label"] in sub_title_labels
  1249. ):
  1250. sub_title.append(bbox2)
  1251. else:
  1252. height1 = bbox2[3] - bbox2[1]
  1253. width1 = bbox2[2] - bbox2[0]
  1254. if label == "text":
  1255. if (
  1256. _nearest_edge_distance(bbox1, bbox2)[0] <= 15
  1257. and block1["block_label"] in vision_labels
  1258. and height1 < height
  1259. and width1 < 0.5 * width
  1260. ):
  1261. blocks[title_text_index]["sub_label"] = "vision_footnote"
  1262. vision_footnote.append(bbox2)
  1263. elif (
  1264. width1 < width * title_text_weight[1]
  1265. and block1["block_label"] in title_labels
  1266. ):
  1267. blocks[title_text_index]["sub_label"] = "title_text"
  1268. title_text.append((direction_[1], bbox2))
  1269. elif (
  1270. label == "paragraph_title"
  1271. and block1["block_label"] in sub_title_labels
  1272. ):
  1273. sub_title.append(bbox2)
  1274. if (
  1275. is_horizontal_1
  1276. and abs(left_up_title_text_distance - right_down_title_text_distance) * 5
  1277. > height
  1278. ) or (
  1279. not is_horizontal_1
  1280. and abs(left_up_title_text_distance - right_down_title_text_distance) * 5
  1281. > width
  1282. ):
  1283. if left_up_title_text_distance < right_down_title_text_distance:
  1284. get_sub_category_(
  1285. left_up_title_text_direction,
  1286. left_up_title_text_index,
  1287. blocks[left_up_title_text_index]["block_label"],
  1288. True,
  1289. )
  1290. else:
  1291. get_sub_category_(
  1292. right_down_title_text_direction,
  1293. right_down_title_text_index,
  1294. blocks[right_down_title_text_index]["block_label"],
  1295. False,
  1296. )
  1297. else:
  1298. get_sub_category_(
  1299. left_up_title_text_direction,
  1300. left_up_title_text_index,
  1301. blocks[left_up_title_text_index]["block_label"],
  1302. True,
  1303. )
  1304. get_sub_category_(
  1305. right_down_title_text_direction,
  1306. right_down_title_text_index,
  1307. blocks[right_down_title_text_index]["block_label"],
  1308. False,
  1309. )
  1310. if block1["block_label"] in title_labels:
  1311. if blocks[i].get("title_text") == []:
  1312. blocks[i]["title_text"] = title_text
  1313. if block1["block_label"] in sub_title_labels:
  1314. if blocks[i].get("sub_title") == []:
  1315. blocks[i]["sub_title"] = sub_title
  1316. if block1["block_label"] in vision_labels:
  1317. if blocks[i].get("vision_footnote") == []:
  1318. blocks[i]["vision_footnote"] = vision_footnote
  1319. return blocks, pre_cuts
  1320. def get_layout_ordering(
  1321. parsing_res_list: List[Dict[str, Any]],
  1322. no_mask_labels: List[str] = [],
  1323. ) -> None:
  1324. """
  1325. Process layout parsing results to remove overlapping bounding boxes
  1326. and assign an ordering index based on their positions.
  1327. Modifies:
  1328. The 'parsing_res_list' list by adding an 'index' to each block.
  1329. Args:
  1330. parsing_res_list (List[Dict[str, Any]]): List of block dictionaries with 'block_bbox' and 'block_label'.
  1331. no_mask_labels (List[str]): Labels for which overlapping removal is not performed.
  1332. """
  1333. title_text_labels = ["doc_title"]
  1334. title_labels = ["doc_title", "paragraph_title"]
  1335. vision_labels = ["image", "table", "seal", "chart", "figure"]
  1336. vision_title_labels = ["table_title", "chart_title", "figure_title"]
  1337. parsing_res_list, pre_cuts = _get_sub_category(parsing_res_list, title_text_labels)
  1338. parsing_res_by_pre_cuts_list = []
  1339. if len(pre_cuts) > 0:
  1340. block_bboxes = [block["block_bbox"] for block in parsing_res_list]
  1341. for axis, cuts in pre_cuts.items():
  1342. axis_index = 1 if axis == "y" else 0
  1343. max_val = max(bbox[axis_index + 2] for bbox in block_bboxes)
  1344. intervals = []
  1345. prev = 0
  1346. for cut in sorted(cuts):
  1347. intervals.append((prev, cut))
  1348. prev = cut
  1349. intervals.append((prev, max_val))
  1350. for start, end in intervals:
  1351. mask = [
  1352. (bbox[axis_index] >= start) and (bbox[axis_index] < end)
  1353. for bbox in block_bboxes
  1354. ]
  1355. parsing_res_by_pre_cuts_list.append(
  1356. [parsing_res_list[i] for i, m in enumerate(mask) if m]
  1357. )
  1358. else:
  1359. parsing_res_by_pre_cuts_list = [parsing_res_list]
  1360. final_parsing_res_list = []
  1361. num_index = 0
  1362. num_sub_index = 0
  1363. for parsing_res_by_pre_cuts in parsing_res_by_pre_cuts_list:
  1364. doc_flag = False
  1365. median_width = _get_text_median_width(parsing_res_by_pre_cuts)
  1366. parsing_res_by_pre_cuts, projection_direction = _get_layout_property(
  1367. parsing_res_by_pre_cuts,
  1368. median_width,
  1369. no_mask_labels=no_mask_labels,
  1370. threshold=0.3,
  1371. )
  1372. # Convert bounding boxes to float and remove overlaps
  1373. (
  1374. double_text_blocks,
  1375. title_text_blocks,
  1376. title_blocks,
  1377. vision_blocks,
  1378. vision_title_blocks,
  1379. vision_footnote_blocks,
  1380. other_blocks,
  1381. ) = ([], [], [], [], [], [], [])
  1382. drop_indexes = []
  1383. for index, block in enumerate(parsing_res_by_pre_cuts):
  1384. label = block["sub_label"]
  1385. block["block_bbox"] = list(map(int, block["block_bbox"]))
  1386. if label == "doc_title":
  1387. doc_flag = True
  1388. if label in no_mask_labels:
  1389. if block["layout"] == "double":
  1390. double_text_blocks.append(block)
  1391. drop_indexes.append(index)
  1392. elif label == "title_text":
  1393. title_text_blocks.append(block)
  1394. drop_indexes.append(index)
  1395. elif label == "vision_footnote":
  1396. vision_footnote_blocks.append(block)
  1397. drop_indexes.append(index)
  1398. elif label in vision_title_labels:
  1399. vision_title_blocks.append(block)
  1400. drop_indexes.append(index)
  1401. elif label in title_labels:
  1402. title_blocks.append(block)
  1403. drop_indexes.append(index)
  1404. elif label in vision_labels:
  1405. vision_blocks.append(block)
  1406. drop_indexes.append(index)
  1407. else:
  1408. other_blocks.append(block)
  1409. drop_indexes.append(index)
  1410. for index in sorted(drop_indexes, reverse=True):
  1411. del parsing_res_by_pre_cuts[index]
  1412. if len(parsing_res_by_pre_cuts) > 0:
  1413. # single text label
  1414. if (
  1415. len(double_text_blocks) > len(parsing_res_by_pre_cuts)
  1416. or projection_direction
  1417. ):
  1418. parsing_res_by_pre_cuts.extend(title_blocks + double_text_blocks)
  1419. title_blocks = []
  1420. double_text_blocks = []
  1421. block_bboxes = [
  1422. block["block_bbox"] for block in parsing_res_by_pre_cuts
  1423. ]
  1424. block_bboxes.sort(
  1425. key=lambda x: (
  1426. x[0] // max(20, median_width),
  1427. x[1],
  1428. ),
  1429. )
  1430. block_bboxes = np.array(block_bboxes)
  1431. sorted_indices = sort_by_xycut(block_bboxes, direction=1, min_gap=1)
  1432. else:
  1433. block_bboxes = [
  1434. block["block_bbox"] for block in parsing_res_by_pre_cuts
  1435. ]
  1436. block_bboxes.sort(key=lambda x: (x[0] // 20, x[1]))
  1437. block_bboxes = np.array(block_bboxes)
  1438. sorted_indices = sort_by_xycut(block_bboxes, direction=0, min_gap=20)
  1439. sorted_boxes = block_bboxes[sorted_indices].tolist()
  1440. for block in parsing_res_by_pre_cuts:
  1441. block["index"] = num_index + sorted_boxes.index(block["block_bbox"]) + 1
  1442. block["sub_index"] = (
  1443. num_sub_index + sorted_boxes.index(block["block_bbox"]) + 1
  1444. )
  1445. def nearest_match_(input_blocks, distance_type="manhattan", is_add_index=True):
  1446. for block in input_blocks:
  1447. bbox = block["block_bbox"]
  1448. min_distance = float("inf")
  1449. min_distance_config = [
  1450. [float("inf"), float("inf")],
  1451. float("inf"),
  1452. float("inf"),
  1453. ] # for double text
  1454. nearest_gt_index = 0
  1455. for match_block in parsing_res_by_pre_cuts:
  1456. match_bbox = match_block["block_bbox"]
  1457. if distance_type == "nearest_iou_edge_distance":
  1458. distance, min_distance_config = _nearest_iou_edge_distance(
  1459. bbox,
  1460. match_bbox,
  1461. block["sub_label"],
  1462. vision_labels=vision_labels,
  1463. no_mask_labels=no_mask_labels,
  1464. median_width=median_width,
  1465. title_labels=title_labels,
  1466. title_text=block["title_text"],
  1467. sub_title=block["sub_title"],
  1468. min_distance_config=min_distance_config,
  1469. tolerance_len=10,
  1470. )
  1471. elif distance_type == "title_text":
  1472. if (
  1473. match_block["block_label"] in title_labels + ["abstract"]
  1474. and match_block["title_text"] != []
  1475. ):
  1476. iou_left_up = _calculate_overlap_area_div_minbox_area_ratio(
  1477. bbox,
  1478. match_block["title_text"][0][1],
  1479. )
  1480. iou_right_down = (
  1481. _calculate_overlap_area_div_minbox_area_ratio(
  1482. bbox,
  1483. match_block["title_text"][-1][1],
  1484. )
  1485. )
  1486. iou = 1 - max(iou_left_up, iou_right_down)
  1487. distance = _manhattan_distance(bbox, match_bbox) * iou
  1488. else:
  1489. distance = float("inf")
  1490. elif distance_type == "manhattan":
  1491. distance = _manhattan_distance(bbox, match_bbox)
  1492. elif distance_type == "vision_footnote":
  1493. if (
  1494. match_block["block_label"] in vision_labels
  1495. and match_block["vision_footnote"] != []
  1496. ):
  1497. iou_left_up = _calculate_overlap_area_div_minbox_area_ratio(
  1498. bbox,
  1499. match_block["vision_footnote"][0],
  1500. )
  1501. iou_right_down = (
  1502. _calculate_overlap_area_div_minbox_area_ratio(
  1503. bbox,
  1504. match_block["vision_footnote"][-1],
  1505. )
  1506. )
  1507. iou = 1 - max(iou_left_up, iou_right_down)
  1508. distance = _manhattan_distance(bbox, match_bbox) * iou
  1509. else:
  1510. distance = float("inf")
  1511. elif distance_type == "vision_body":
  1512. if (
  1513. match_block["block_label"] in vision_title_labels
  1514. and block["vision_footnote"] != []
  1515. ):
  1516. iou_left_up = _calculate_overlap_area_div_minbox_area_ratio(
  1517. match_bbox,
  1518. block["vision_footnote"][0],
  1519. )
  1520. iou_right_down = (
  1521. _calculate_overlap_area_div_minbox_area_ratio(
  1522. match_bbox,
  1523. block["vision_footnote"][-1],
  1524. )
  1525. )
  1526. iou = 1 - max(iou_left_up, iou_right_down)
  1527. distance = _manhattan_distance(bbox, match_bbox) * iou
  1528. else:
  1529. distance = float("inf")
  1530. # when reference block cross mulitple columns, its order should be after the blocks above it.
  1531. elif distance_type == "append":
  1532. if match_bbox[3] <= bbox[1]:
  1533. distance = -(match_bbox[2] * 10 + match_bbox[3])
  1534. else:
  1535. distance = float("inf")
  1536. else:
  1537. raise NotImplementedError
  1538. if distance < min_distance:
  1539. min_distance = distance
  1540. if is_add_index:
  1541. nearest_gt_index = match_block.get("index", 999)
  1542. else:
  1543. nearest_gt_index = match_block.get("sub_index", 999)
  1544. if is_add_index:
  1545. block["index"] = nearest_gt_index
  1546. else:
  1547. block["sub_index"] = nearest_gt_index
  1548. parsing_res_by_pre_cuts.append(block)
  1549. # double text label
  1550. double_text_blocks.sort(
  1551. key=lambda x: (
  1552. x["block_bbox"][1] // 10,
  1553. x["block_bbox"][0] // median_width,
  1554. x["block_bbox"][1] ** 2 + x["block_bbox"][0] ** 2,
  1555. ),
  1556. )
  1557. # filter the reference blocks from all blocks that cross mulitple columns.
  1558. # they should be ordered using "append".
  1559. double_text_reference_blocks = []
  1560. i = 0
  1561. while i < len(double_text_blocks):
  1562. if double_text_blocks[i]["block_label"] == "reference":
  1563. double_text_reference_blocks.append(double_text_blocks.pop(i))
  1564. else:
  1565. i += 1
  1566. nearest_match_(
  1567. double_text_blocks,
  1568. distance_type="nearest_iou_edge_distance",
  1569. )
  1570. nearest_match_(
  1571. double_text_reference_blocks,
  1572. distance_type="append",
  1573. )
  1574. parsing_res_by_pre_cuts.sort(
  1575. key=lambda x: (x["index"], x["block_bbox"][1], x["block_bbox"][0]),
  1576. )
  1577. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1578. block["index"] = num_index + idx + 1
  1579. block["sub_index"] = num_sub_index + idx + 1
  1580. # title label
  1581. title_blocks.sort(
  1582. key=lambda x: (
  1583. x["block_bbox"][1] // 10,
  1584. x["block_bbox"][0] // median_width,
  1585. x["block_bbox"][1] ** 2 + x["block_bbox"][0] ** 2,
  1586. ),
  1587. )
  1588. nearest_match_(title_blocks, distance_type="nearest_iou_edge_distance")
  1589. if doc_flag:
  1590. text_sort_labels = ["doc_title"]
  1591. text_label_priority = {
  1592. label: priority for priority, label in enumerate(text_sort_labels)
  1593. }
  1594. doc_titles = []
  1595. for i, block in enumerate(parsing_res_by_pre_cuts):
  1596. if block["block_label"] == "doc_title":
  1597. doc_titles.append(
  1598. (i, block["block_bbox"][1], block["block_bbox"][0]),
  1599. )
  1600. doc_titles.sort(key=lambda x: (x[1], x[2]))
  1601. first_doc_title_index = doc_titles[0][0]
  1602. parsing_res_by_pre_cuts[first_doc_title_index]["index"] = 1
  1603. parsing_res_by_pre_cuts.sort(
  1604. key=lambda x: (
  1605. x["index"],
  1606. text_label_priority.get(x["block_label"], 9999),
  1607. x["block_bbox"][1],
  1608. x["block_bbox"][0],
  1609. ),
  1610. )
  1611. else:
  1612. parsing_res_by_pre_cuts.sort(
  1613. key=lambda x: (
  1614. x["index"],
  1615. x["block_bbox"][1],
  1616. x["block_bbox"][0],
  1617. ),
  1618. )
  1619. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1620. block["index"] = num_index + idx + 1
  1621. block["sub_index"] = num_sub_index + idx + 1
  1622. # title-text label
  1623. nearest_match_(title_text_blocks, distance_type="title_text")
  1624. def hor_tb_and_ver_lr(x):
  1625. input_bbox = x["block_bbox"]
  1626. is_horizontal = _get_bbox_direction(input_bbox)
  1627. if is_horizontal:
  1628. return input_bbox[1]
  1629. else:
  1630. return input_bbox[0]
  1631. parsing_res_by_pre_cuts.sort(
  1632. key=lambda x: (x["index"], hor_tb_and_ver_lr(x)),
  1633. )
  1634. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1635. block["index"] = num_index + idx + 1
  1636. block["sub_index"] = num_sub_index + idx + 1
  1637. # image,figure,chart,seal label
  1638. nearest_match_(
  1639. vision_blocks,
  1640. distance_type="nearest_iou_edge_distance",
  1641. is_add_index=False,
  1642. )
  1643. parsing_res_by_pre_cuts.sort(
  1644. key=lambda x: (
  1645. x["sub_index"],
  1646. x["block_bbox"][1],
  1647. x["block_bbox"][0],
  1648. ),
  1649. )
  1650. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1651. block["sub_index"] = num_sub_index + idx + 1
  1652. # image,figure,chart,seal title label
  1653. nearest_match_(
  1654. vision_title_blocks,
  1655. distance_type="nearest_iou_edge_distance",
  1656. is_add_index=False,
  1657. )
  1658. parsing_res_by_pre_cuts.sort(
  1659. key=lambda x: (
  1660. x["sub_index"],
  1661. x["block_bbox"][1],
  1662. x["block_bbox"][0],
  1663. ),
  1664. )
  1665. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1666. block["sub_index"] = num_sub_index + idx + 1
  1667. # vision footnote label
  1668. nearest_match_(
  1669. vision_footnote_blocks,
  1670. distance_type="vision_footnote",
  1671. is_add_index=False,
  1672. )
  1673. text_label_priority = {"vision_footnote": 9999}
  1674. parsing_res_by_pre_cuts.sort(
  1675. key=lambda x: (
  1676. x["sub_index"],
  1677. text_label_priority.get(x["sub_label"], 0),
  1678. x["block_bbox"][1],
  1679. x["block_bbox"][0],
  1680. ),
  1681. )
  1682. for idx, block in enumerate(parsing_res_by_pre_cuts):
  1683. block["sub_index"] = num_sub_index + idx + 1
  1684. # header、footnote、header_image... label
  1685. nearest_match_(other_blocks, distance_type="manhattan", is_add_index=False)
  1686. # add all parsing result
  1687. final_parsing_res_list.extend(parsing_res_by_pre_cuts)
  1688. # update num index
  1689. num_sub_index += len(parsing_res_by_pre_cuts)
  1690. for parsing_res in parsing_res_by_pre_cuts:
  1691. if parsing_res.get("index"):
  1692. num_index += 1
  1693. parsing_res_list = [
  1694. {
  1695. "block_label": parsing_res["block_label"],
  1696. "block_content": parsing_res["block_content"],
  1697. "block_bbox": parsing_res["block_bbox"],
  1698. "block_image": parsing_res.get("block_image", None),
  1699. "sub_label": parsing_res["sub_label"],
  1700. "sub_index": parsing_res["sub_index"],
  1701. "index": parsing_res.get("index", None),
  1702. "seg_start_coordinate": parsing_res.get(
  1703. "seg_start_coordinate", float("inf")
  1704. ),
  1705. "seg_end_coordinate": parsing_res.get("seg_end_coordinate", float("-inf")),
  1706. "num_of_lines": parsing_res.get("num_of_lines", 1),
  1707. }
  1708. for parsing_res in final_parsing_res_list
  1709. ]
  1710. return parsing_res_list
  1711. def _manhattan_distance(
  1712. point1: Tuple[float, float],
  1713. point2: Tuple[float, float],
  1714. weight_x: float = 1.0,
  1715. weight_y: float = 1.0,
  1716. ) -> float:
  1717. """
  1718. Calculate the weighted Manhattan distance between two points.
  1719. Args:
  1720. point1 (Tuple[float, float]): The first point as (x, y).
  1721. point2 (Tuple[float, float]): The second point as (x, y).
  1722. weight_x (float): The weight for the x-axis distance. Default is 1.0.
  1723. weight_y (float): The weight for the y-axis distance. Default is 1.0.
  1724. Returns:
  1725. float: The weighted Manhattan distance between the two points.
  1726. """
  1727. return weight_x * abs(point1[0] - point2[0]) + weight_y * abs(point1[1] - point2[1])
  1728. def _calculate_horizontal_distance(
  1729. input_bbox: List[int],
  1730. match_bbox: List[int],
  1731. height: int,
  1732. disperse: int,
  1733. title_text: List[Tuple[int, List[int]]],
  1734. ) -> float:
  1735. """
  1736. Calculate the horizontal distance between two bounding boxes, considering title text adjustments.
  1737. Args:
  1738. input_bbox (List[int]): The bounding box coordinates [x1, y1, x2, y2] of the input object.
  1739. match_bbox (List[int]): The bounding box coordinates [x1', y1', x2', y2'] of the object to match against.
  1740. height (int): The height of the input bounding box used for normalization.
  1741. disperse (int): The dispersion factor used to normalize the horizontal distance.
  1742. title_text (List[Tuple[int, List[int]]]): A list of tuples containing title text information and their bounding box coordinates.
  1743. Format: [(position_indicator, [x1, y1, x2, y2]), ...].
  1744. Returns:
  1745. float: The calculated horizontal distance taking into account the title text adjustments.
  1746. """
  1747. x1, y1, x2, y2 = input_bbox
  1748. x1_prime, y1_prime, x2_prime, y2_prime = match_bbox
  1749. # Determine vertical distance adjustment based on title text
  1750. if y2 < y1_prime:
  1751. if title_text and title_text[-1][0] == 2:
  1752. y2 += title_text[-1][1][3] - title_text[-1][1][1]
  1753. vertical_adjustment = (y1_prime - y2) * 0.5
  1754. else:
  1755. if title_text and title_text[0][0] == 1:
  1756. y1 -= title_text[0][1][3] - title_text[0][1][1]
  1757. vertical_adjustment = y1 - y2_prime
  1758. # Calculate horizontal distance with adjustments
  1759. horizontal_distance = (
  1760. abs(x2_prime - x1) // disperse
  1761. + vertical_adjustment // height
  1762. + vertical_adjustment / 5000
  1763. )
  1764. return horizontal_distance
  1765. def _calculate_vertical_distance(
  1766. input_bbox: List[int],
  1767. match_bbox: List[int],
  1768. width: int,
  1769. disperse: int,
  1770. title_text: List[Tuple[int, List[int]]],
  1771. ) -> float:
  1772. """
  1773. Calculate the vertical distance between two bounding boxes, considering title text adjustments.
  1774. Args:
  1775. input_bbox (List[int]): The bounding box coordinates [x1, y1, x2, y2] of the input object.
  1776. match_bbox (List[int]): The bounding box coordinates [x1', y1', x2', y2'] of the object to match against.
  1777. width (int): The width of the input bounding box used for normalization.
  1778. disperse (int): The dispersion factor used to normalize the vertical distance.
  1779. title_text (List[Tuple[int, List[int]]]): A list of tuples containing title text information and their bounding box coordinates.
  1780. Format: [(position_indicator, [x1, y1, x2, y2]), ...].
  1781. Returns:
  1782. float: The calculated vertical distance taking into account the title text adjustments.
  1783. """
  1784. x1, y1, x2, y2 = input_bbox
  1785. x1_prime, y1_prime, x2_prime, y2_prime = match_bbox
  1786. # Determine horizontal distance adjustment based on title text
  1787. if x1 > x2_prime:
  1788. if title_text and title_text[0][0] == 3:
  1789. x1 -= title_text[0][1][2] - title_text[0][1][0]
  1790. horizontal_adjustment = (x1 - x2_prime) * 0.5
  1791. else:
  1792. if title_text and title_text[-1][0] == 4:
  1793. x2 += title_text[-1][1][2] - title_text[-1][1][0]
  1794. horizontal_adjustment = x1_prime - x2
  1795. # Calculate vertical distance with adjustments
  1796. vertical_distance = (
  1797. abs(y2_prime - y1) // disperse
  1798. + horizontal_adjustment // width
  1799. + horizontal_adjustment / 5000
  1800. )
  1801. return vertical_distance
  1802. def _nearest_edge_distance(
  1803. input_bbox: List[int],
  1804. match_bbox: List[int],
  1805. weight: List[float] = [1.0, 1.0, 1.0, 1.0],
  1806. label: str = "text",
  1807. no_mask_labels: List[str] = [],
  1808. min_edge_distance_config: List[float] = [],
  1809. tolerance_len: float = 10.0,
  1810. ) -> Tuple[float, List[float]]:
  1811. """
  1812. Calculate the nearest edge distance between two bounding boxes, considering directional weights.
  1813. Args:
  1814. input_bbox (list): The bounding box coordinates [x1, y1, x2, y2] of the input object.
  1815. match_bbox (list): The bounding box coordinates [x1', y1', x2', y2'] of the object to match against.
  1816. weight (list, optional): Directional weights for the edge distances [left, right, up, down]. Defaults to [1, 1, 1, 1].
  1817. label (str, optional): The label/type of the object in the bounding box (e.g., 'text'). Defaults to 'text'.
  1818. no_mask_labels (list, optional): Labels for which no masking is applied when calculating edge distances. Defaults to an empty list.
  1819. min_edge_distance_config (list, optional): Configuration for minimum edge distances [min_edge_distance_x, min_edge_distance_y].
  1820. Defaults to [float('inf'), float('inf')].
  1821. tolerance_len (float, optional): The tolerance length for adjusting edge distances. Defaults to 10.
  1822. Returns:
  1823. Tuple[float, List[float]]: A tuple containing:
  1824. - The calculated minimum edge distance between the bounding boxes.
  1825. - A list with the minimum edge distances in the x and y directions.
  1826. """
  1827. match_bbox_iou = _calculate_overlap_area_div_minbox_area_ratio(
  1828. input_bbox,
  1829. match_bbox,
  1830. )
  1831. if match_bbox_iou > 0 and label not in no_mask_labels:
  1832. return 0, [0, 0]
  1833. if not min_edge_distance_config:
  1834. min_edge_distance_config = [float("inf"), float("inf")]
  1835. min_edge_distance_x, min_edge_distance_y = min_edge_distance_config
  1836. x1, y1, x2, y2 = input_bbox
  1837. x1_prime, y1_prime, x2_prime, y2_prime = match_bbox
  1838. direction_num = 0
  1839. distance_x = float("inf")
  1840. distance_y = float("inf")
  1841. distance = [float("inf")] * 4
  1842. # input_bbox is to the left of match_bbox
  1843. if x2 < x1_prime:
  1844. direction_num += 1
  1845. distance[0] = x1_prime - x2
  1846. if abs(distance[0] - min_edge_distance_x) <= tolerance_len:
  1847. distance_x = min_edge_distance_x * weight[0]
  1848. else:
  1849. distance_x = distance[0] * weight[0]
  1850. # input_bbox is to the right of match_bbox
  1851. elif x1 > x2_prime:
  1852. direction_num += 1
  1853. distance[1] = x1 - x2_prime
  1854. if abs(distance[1] - min_edge_distance_x) <= tolerance_len:
  1855. distance_x = min_edge_distance_x * weight[1]
  1856. else:
  1857. distance_x = distance[1] * weight[1]
  1858. elif match_bbox_iou > 0:
  1859. distance[0] = 0
  1860. distance_x = 0
  1861. # input_bbox is above match_bbox
  1862. if y2 < y1_prime:
  1863. direction_num += 1
  1864. distance[2] = y1_prime - y2
  1865. if abs(distance[2] - min_edge_distance_y) <= tolerance_len:
  1866. distance_y = min_edge_distance_y * weight[2]
  1867. else:
  1868. distance_y = distance[2] * weight[2]
  1869. if label in no_mask_labels:
  1870. distance_y = max(0.1, distance_y) * 10 # for abstract
  1871. # input_bbox is below match_bbox
  1872. elif y1 > y2_prime:
  1873. direction_num += 1
  1874. distance[3] = y1 - y2_prime
  1875. if abs(distance[3] - min_edge_distance_y) <= tolerance_len:
  1876. distance_y = min_edge_distance_y * weight[3]
  1877. else:
  1878. distance_y = distance[3] * weight[3]
  1879. elif match_bbox_iou > 0:
  1880. distance[2] = 0
  1881. distance_y = 0
  1882. if direction_num == 2:
  1883. return (distance_x + distance_y), [
  1884. min(distance[0], distance[1]),
  1885. min(distance[2], distance[3]),
  1886. ]
  1887. else:
  1888. return min(distance_x, distance_y), [
  1889. min(distance[0], distance[1]),
  1890. min(distance[2], distance[3]),
  1891. ]
  1892. def _get_weights(label, horizontal):
  1893. """Define weights based on the label and orientation."""
  1894. if label == "doc_title":
  1895. return (
  1896. [1, 0.1, 0.1, 1] if horizontal else [0.2, 0.1, 1, 1]
  1897. ) # left-down , right-left
  1898. elif label in [
  1899. "paragraph_title",
  1900. "table_title",
  1901. "abstract",
  1902. "image",
  1903. "seal",
  1904. "chart",
  1905. "figure",
  1906. ]:
  1907. return [1, 1, 0.1, 1] # down
  1908. else:
  1909. return [1, 1, 1, 0.1] # up
  1910. def _nearest_iou_edge_distance(
  1911. input_bbox: List[int],
  1912. match_bbox: List[int],
  1913. label: str,
  1914. vision_labels: List[str],
  1915. no_mask_labels: List[str],
  1916. median_width: int = -1,
  1917. title_labels: List[str] = [],
  1918. title_text: List[Tuple[int, List[int]]] = [],
  1919. sub_title: List[List[int]] = [],
  1920. min_distance_config: List[float] = [],
  1921. tolerance_len: float = 10.0,
  1922. ) -> Tuple[float, List[float]]:
  1923. """
  1924. Calculate the nearest IOU edge distance between two bounding boxes, considering label types, title adjustments, and minimum distance configurations.
  1925. This function computes the edge distance between two bounding boxes while considering their overlap (IOU) and various adjustments based on label types,
  1926. title text, and subtitle information. It also applies minimum distance configurations and tolerance adjustments.
  1927. Args:
  1928. input_bbox (List[int]): The bounding box coordinates [x1, y1, x2, y2] of the input object.
  1929. match_bbox (List[int]): The bounding box coordinates [x1', y1', x2', y2'] of the object to match against.
  1930. label (str): The label/type of the object in the bounding box (e.g., 'image', 'text', etc.).
  1931. vision_labels (List[str]): List of labels for vision-related objects (e.g., images, icons).
  1932. no_mask_labels (List[str]): Labels for which no masking is applied when calculating edge distances.
  1933. median_width (int, optional): The median width for title dispersion calculation. Defaults to -1.
  1934. title_labels (List[str], optional): Labels that indicate the object is a title. Defaults to an empty list.
  1935. title_text (List[Tuple[int, List[int]]], optional): Text content associated with title labels, in the format [(position_indicator, [x1, y1, x2, y2]), ...].
  1936. sub_title (List[List[int]], optional): List of subtitle bounding boxes to adjust the input_bbox. Defaults to an empty list.
  1937. min_distance_config (List[float], optional): Configuration for minimum distances [min_edge_distance_config, up_edge_distances_config, total_distance].
  1938. tolerance_len (float, optional): The tolerance length for adjusting edge distances. Defaults to 10.0.
  1939. Returns:
  1940. Tuple[float, List[float]]: A tuple containing:
  1941. - The calculated distance considering IOU and adjustments.
  1942. - The updated minimum distance configuration.
  1943. """
  1944. x1, y1, x2, y2 = input_bbox
  1945. x1_prime, y1_prime, x2_prime, y2_prime = match_bbox
  1946. min_edge_distance_config, up_edge_distances_config, total_distance = (
  1947. min_distance_config
  1948. )
  1949. iou_distance = 0
  1950. if label in vision_labels:
  1951. horizontal1 = horizontal2 = True
  1952. else:
  1953. horizontal1 = _get_bbox_direction(input_bbox)
  1954. horizontal2 = _get_bbox_direction(match_bbox, 3)
  1955. if (
  1956. horizontal1 != horizontal2
  1957. or _get_projection_iou(input_bbox, match_bbox, horizontal1) < 0.01
  1958. ):
  1959. iou_distance = 1
  1960. if label == "doc_title":
  1961. # Calculate distance for titles
  1962. disperse = max(1, median_width)
  1963. tolerance_len = max(tolerance_len, disperse)
  1964. # Adjust input_bbox based on sub_title
  1965. if sub_title:
  1966. for sub in sub_title:
  1967. x1_, y1_, x2_, y2_ = sub
  1968. x1, y1, x2, y2 = (
  1969. min(x1, x1_),
  1970. min(y1, y1_),
  1971. min(x2, x2_),
  1972. max(y2, y2_),
  1973. )
  1974. input_bbox = [x1, y1, x2, y2]
  1975. if title_text:
  1976. for sub in title_text:
  1977. x1_, y1_, x2_, y2_ = sub[1]
  1978. if horizontal1:
  1979. x1, y1, x2, y2 = (
  1980. min(x1, x1_),
  1981. min(y1, y1_),
  1982. min(x2, x2_),
  1983. max(y2, y2_),
  1984. )
  1985. else:
  1986. x1, y1, x2, y2 = (
  1987. min(x1, x1_),
  1988. min(y1, y1_),
  1989. max(x2, x2_),
  1990. min(y2, y2_),
  1991. )
  1992. input_bbox = [x1, y1, x2, y2]
  1993. # Calculate edge distance
  1994. weight = _get_weights(label, horizontal1)
  1995. if label == "abstract":
  1996. tolerance_len *= 2
  1997. edge_distance, edge_distance_config = _nearest_edge_distance(
  1998. input_bbox,
  1999. match_bbox,
  2000. weight,
  2001. label=label,
  2002. no_mask_labels=no_mask_labels,
  2003. min_edge_distance_config=min_edge_distance_config,
  2004. tolerance_len=tolerance_len,
  2005. )
  2006. # Weights for combining distances
  2007. iou_edge_weight = [10**8, 10**4, 1, 0.0001]
  2008. # Calculate up and left edge distances
  2009. up_edge_distance = y1_prime
  2010. left_edge_distance = x1_prime
  2011. if (
  2012. label in no_mask_labels or label in title_labels or label in vision_labels
  2013. ) and y1 > y2_prime:
  2014. up_edge_distance = -y2_prime
  2015. left_edge_distance = -x2_prime
  2016. min_up_edge_distance = up_edge_distances_config
  2017. if abs(min_up_edge_distance - up_edge_distance) <= tolerance_len:
  2018. up_edge_distance = min_up_edge_distance
  2019. # Calculate total distance
  2020. distance = (
  2021. iou_distance * iou_edge_weight[0]
  2022. + edge_distance * iou_edge_weight[1]
  2023. + up_edge_distance * iou_edge_weight[2]
  2024. + left_edge_distance * iou_edge_weight[3]
  2025. )
  2026. # Update minimum distance configuration if a smaller distance is found
  2027. if total_distance > distance:
  2028. edge_distance_config = [
  2029. edge_distance_config[0],
  2030. edge_distance_config[1],
  2031. ]
  2032. min_distance_config = [
  2033. edge_distance_config,
  2034. up_edge_distance,
  2035. distance,
  2036. ]
  2037. return distance, min_distance_config
  2038. def get_show_color(label: str) -> Tuple:
  2039. label_colors = {
  2040. # Medium Blue (from 'titles_list')
  2041. "paragraph_title": (102, 102, 255, 100),
  2042. "doc_title": (255, 248, 220, 100), # Cornsilk
  2043. # Light Yellow (from 'tables_caption_list')
  2044. "table_title": (255, 255, 102, 100),
  2045. # Sky Blue (from 'imgs_caption_list')
  2046. "figure_title": (102, 178, 255, 100),
  2047. "chart_title": (221, 160, 221, 100), # Plum
  2048. "vision_footnote": (144, 238, 144, 100), # Light Green
  2049. # Deep Purple (from 'texts_list')
  2050. "text": (153, 0, 76, 100),
  2051. # Bright Green (from 'interequations_list')
  2052. "formula": (0, 255, 0, 100),
  2053. "abstract": (255, 239, 213, 100), # Papaya Whip
  2054. # Medium Green (from 'lists_list' and 'indexs_list')
  2055. "content": (40, 169, 92, 100),
  2056. # Neutral Gray (from 'dropped_bbox_list')
  2057. "seal": (158, 158, 158, 100),
  2058. # Olive Yellow (from 'tables_body_list')
  2059. "table": (204, 204, 0, 100),
  2060. # Bright Green (from 'imgs_body_list')
  2061. "image": (153, 255, 51, 100),
  2062. # Bright Green (from 'imgs_body_list')
  2063. "figure": (153, 255, 51, 100),
  2064. "chart": (216, 191, 216, 100), # Thistle
  2065. # Pale Yellow-Green (from 'tables_footnote_list')
  2066. "reference": (229, 255, 204, 100),
  2067. "algorithm": (255, 250, 240, 100), # Floral White
  2068. }
  2069. default_color = (158, 158, 158, 100)
  2070. return label_colors.get(label, default_color)