layout_sort.py 33 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733
  1. """
  2. 对pdf上的box进行layout识别,并对内部组成的box进行排序
  3. """
  4. import json
  5. from loguru import logger
  6. from layout.bbox_sort import CONTENT_IDX, CONTENT_TYPE_IDX, X0_EXT_IDX, X0_IDX, X1_EXT_IDX, X1_IDX, Y0_EXT_IDX, Y0_IDX, Y1_EXT_IDX, Y1_IDX, paper_bbox_sort
  7. from layout.layout_det_utils import find_all_left_bbox_direct, find_all_right_bbox_direct, find_bottom_bbox_direct_from_left_edge, find_bottom_bbox_direct_from_right_edge, find_top_bbox_direct_from_left_edge, find_top_bbox_direct_from_right_edge, find_all_top_bbox_direct, find_all_bottom_bbox_direct, get_left_edge_bboxes, get_right_edge_bboxes
  8. from libs.boxbase import get_bbox_in_boundry
  9. LAYOUT_V = "V"
  10. LAYOUT_H = "H"
  11. LAYOUT_UNPROC = "U"
  12. LAYOUT_BAD = "B"
  13. def _is_single_line_text(bbox):
  14. """
  15. 检查bbox里面的文字是否只有一行
  16. """
  17. return True # TODO
  18. box_type = bbox[CONTENT_TYPE_IDX]
  19. if box_type != 'text':
  20. return False
  21. paras = bbox[CONTENT_IDX]["paras"]
  22. text_content = ""
  23. for para_id, para in paras.items(): # 拼装内部的段落文本
  24. is_title = para['is_title']
  25. if is_title!=0:
  26. text_content += f"## {para['text']}"
  27. else:
  28. text_content += para["text"]
  29. text_content += "\n\n"
  30. return bbox[CONTENT_TYPE_IDX] == 'text' and len(text_content.split("\n\n")) <= 1
  31. def _horizontal_split(bboxes:list, boundry:tuple, avg_font_size=20)-> list:
  32. """
  33. 对bboxes进行水平切割
  34. 方法是:找到左侧和右侧都没有被直接遮挡的box,然后进行扩展,之后进行切割
  35. return:
  36. 返回几个大的Layout区域 [[x0, y0, x1, y1, "h|u|v"], ], h代表水平,u代表未探测的,v代表垂直布局
  37. """
  38. sorted_layout_blocks = [] # 这是要最终返回的值
  39. bound_x0, bound_y0, bound_x1, bound_y1 = boundry
  40. all_bboxes = get_bbox_in_boundry(bboxes, boundry)
  41. #all_bboxes = paper_bbox_sort(all_bboxes, abs(bound_x1-bound_x0), abs(bound_y1-bound_x0)) # 大致拍下序, 这个是基于直接遮挡的。
  42. """
  43. 首先在水平方向上扩展独占一行的bbox
  44. """
  45. last_h_split_line_y1 = bound_y0 #记录下上次的水平分割线
  46. for i, bbox in enumerate(all_bboxes):
  47. left_nearest_bbox = find_all_left_bbox_direct(bbox, all_bboxes) # 非扩展线
  48. right_nearest_bbox = find_all_right_bbox_direct(bbox, all_bboxes)
  49. if left_nearest_bbox is None and right_nearest_bbox is None: # 独占一行
  50. """
  51. 然而,如果只是孤立的一行文字,那么就还要满足以下几个条件才可以:
  52. 1. bbox和中心线相交。或者
  53. 2. 上方或者下方也存在同类水平的独占一行的bbox。 或者
  54. 3. TODO 加强条件:这个bbox上方和下方是同一列column,那么就不能算作独占一行
  55. """
  56. # 先检查这个bbox里是否只包含一行文字
  57. is_single_line = _is_single_line_text(bbox)
  58. """
  59. 这里有个点需要注意,当页面内容不是居中的时候,第一次调用传递的是page的boundry,这个时候mid_x就不是中心线了.
  60. 所以这里计算出最紧致的boundry,然后再计算mid_x
  61. """
  62. boundry_real_x0, boundry_real_x1 = min([bbox[X0_IDX] for bbox in all_bboxes]), max([bbox[X1_IDX] for bbox in all_bboxes])
  63. mid_x = (boundry_real_x0+boundry_real_x1)/2
  64. # 检查这个box是否内容在中心线有交
  65. # 必须跨过去2个字符的宽度
  66. is_cross_boundry_mid_line = min(mid_x-bbox[X0_IDX], bbox[X1_IDX]-mid_x) > avg_font_size*2
  67. """
  68. 检查条件2
  69. """
  70. is_belong_to_col = False
  71. """
  72. 检查是否能被上方col吸收,方法是:
  73. 1. 上方非空且不是独占一行的,并且
  74. 2. 从上个水平分割的最大y=y1开始到当前bbox,最左侧的bbox的[min_x0, max_x1],能够覆盖当前box的[x0, x1]
  75. """
  76. """
  77. 以迭代的方式向上找,查找范围是[bound_x0, last_h_sp, bound_x1, bbox[Y0_IDX]]
  78. """
  79. #先确定上方的y0, y0
  80. b_y0, b_y1 = last_h_split_line_y1, bbox[Y0_IDX]
  81. #然后从box开始逐个向上找到所有与box在x上有交集的box
  82. box_to_check = [bound_x0, b_y0, bound_x1, b_y1]
  83. bbox_in_bound_check = get_bbox_in_boundry(all_bboxes, box_to_check)
  84. bboxes_on_top = []
  85. virtual_box = bbox
  86. while True:
  87. b_on_top = find_all_top_bbox_direct(virtual_box, bbox_in_bound_check)
  88. if b_on_top is not None:
  89. bboxes_on_top.append(b_on_top)
  90. virtual_box = [min([virtual_box[X0_IDX], b_on_top[X0_IDX]]), min(virtual_box[Y0_IDX], b_on_top[Y0_IDX]), max([virtual_box[X1_IDX], b_on_top[X1_IDX]]), b_y1]
  91. else:
  92. break
  93. # 随后确定这些box的最小x0, 最大x1
  94. if len(bboxes_on_top)>0 and len(bboxes_on_top) != len(bbox_in_bound_check):# virtual_box可能会膨胀到占满整个区域,这实际上就不能属于一个col了。
  95. min_x0, max_x1 = virtual_box[X0_IDX], virtual_box[X1_IDX]
  96. # 然后采用一种比较粗糙的方法,看min_x0,max_x1是否与位于[bound_x0, last_h_sp, bound_x1, bbox[Y0_IDX]]之间的box有相交
  97. if not any([b[X0_IDX] <= min_x0-1 <= b[X1_IDX] or b[X0_IDX] <= max_x1+1 <= b[X1_IDX] for b in bbox_in_bound_check]):
  98. # 其上,下都不能被扩展成行,暂时只检查一下上方 TODO
  99. top_nearest_bbox = find_all_top_bbox_direct(bbox, bboxes)
  100. bottom_nearest_bbox = find_all_bottom_bbox_direct(bbox, bboxes)
  101. if not any([
  102. top_nearest_bbox is not None and (find_all_left_bbox_direct(top_nearest_bbox, bboxes) is None and find_all_right_bbox_direct(top_nearest_bbox, bboxes) is None),
  103. bottom_nearest_bbox is not None and (find_all_left_bbox_direct(bottom_nearest_bbox, bboxes) is None and find_all_right_bbox_direct(bottom_nearest_bbox, bboxes) is None),
  104. top_nearest_bbox is None or bottom_nearest_bbox is None
  105. ]):
  106. is_belong_to_col = True
  107. # 检查是否能被下方col吸收 TODO
  108. """
  109. 这里为什么没有is_cross_boundry_mid_line的条件呢?
  110. 确实有些杂志左右两栏宽度不是对称的。
  111. """
  112. if not is_belong_to_col or is_cross_boundry_mid_line:
  113. bbox[X0_EXT_IDX] = bound_x0
  114. bbox[Y0_EXT_IDX] = bbox[Y0_IDX]
  115. bbox[X1_EXT_IDX] = bound_x1
  116. bbox[Y1_EXT_IDX] = bbox[Y1_IDX]
  117. last_h_split_line_y1 = bbox[Y1_IDX] # 更新这条线
  118. else:
  119. continue
  120. """
  121. 此时独占一行的被成功扩展到指定的边界上,这个时候利用边界条件合并连续的bbox,成为一个group
  122. 然后合并所有连续水平方向的bbox.
  123. """
  124. all_bboxes.sort(key=lambda x: x[Y0_IDX])
  125. h_bboxes = []
  126. h_bbox_group = []
  127. for bbox in all_bboxes:
  128. if bbox[X0_EXT_IDX] == bound_x0 and bbox[X1_EXT_IDX] == bound_x1:
  129. h_bbox_group.append(bbox)
  130. else:
  131. if len(h_bbox_group)>0:
  132. h_bboxes.append(h_bbox_group)
  133. h_bbox_group = []
  134. # 最后一个group
  135. if len(h_bbox_group)>0:
  136. h_bboxes.append(h_bbox_group)
  137. """
  138. 现在h_bboxes里面是所有的group了,每个group都是一个list
  139. 对h_bboxes里的每个group进行计算放回到sorted_layouts里
  140. """
  141. h_layouts = []
  142. for gp in h_bboxes:
  143. gp.sort(key=lambda x: x[Y0_IDX])
  144. # 然后计算这个group的layout_bbox,也就是最小的x0,y0, 最大的x1,y1
  145. x0, y0, x1, y1 = gp[0][X0_EXT_IDX], gp[0][Y0_EXT_IDX], gp[-1][X1_EXT_IDX], gp[-1][Y1_EXT_IDX]
  146. h_layouts.append([x0, y0, x1, y1, LAYOUT_H]) # 水平的布局
  147. """
  148. 接下来利用这些连续的水平bbox的layout_bbox的y0, y1,从水平上切分开其余的为几个部分
  149. """
  150. h_split_lines = [bound_y0]
  151. for gp in h_bboxes: # gp是一个list[bbox_list]
  152. y0, y1 = gp[0][1], gp[-1][3]
  153. h_split_lines.append(y0)
  154. h_split_lines.append(y1)
  155. h_split_lines.append(bound_y1)
  156. unsplited_bboxes = []
  157. for i in range(0, len(h_split_lines), 2):
  158. start_y0, start_y1 = h_split_lines[i:i+2]
  159. # 然后找出[start_y0, start_y1]之间的其他bbox,这些组成一个未分割板块
  160. bboxes_in_block = [bbox for bbox in all_bboxes if bbox[Y0_IDX]>=start_y0 and bbox[Y1_IDX]<=start_y1]
  161. unsplited_bboxes.append(bboxes_in_block)
  162. # 接着把未处理的加入到h_layouts里
  163. for bboxes_in_block in unsplited_bboxes:
  164. if len(bboxes_in_block) == 0:
  165. continue
  166. x0, y0, x1, y1 = bound_x0, min([bbox[Y0_IDX] for bbox in bboxes_in_block]), bound_x1, max([bbox[Y1_IDX] for bbox in bboxes_in_block])
  167. h_layouts.append([x0, y0, x1, y1, LAYOUT_UNPROC])
  168. h_layouts.sort(key=lambda x: x[1]) # 按照y0排序, 也就是从上到下的顺序
  169. """
  170. 转换成如下格式返回
  171. """
  172. for layout in h_layouts:
  173. sorted_layout_blocks.append({
  174. "layout_bbox": layout[:4],
  175. "layout_label":layout[4],
  176. "sub_layout":[],
  177. })
  178. return sorted_layout_blocks
  179. ###############################################################################################
  180. #
  181. # 垂直方向的处理
  182. #
  183. #
  184. ###############################################################################################
  185. def _vertical_align_split_v1(bboxes:list, boundry:tuple)-> list:
  186. """
  187. 计算垂直方向上的对齐, 并分割bboxes成layout。负责对一列多行的进行列维度分割。
  188. 如果不能完全分割,剩余部分作为layout_lable为u的layout返回
  189. -----------------------
  190. | | |
  191. | | |
  192. | | |
  193. | | |
  194. -------------------------
  195. 此函数会将:以上布局将会切分出来2列
  196. """
  197. sorted_layout_blocks = [] # 这是要最终返回的值
  198. new_boundry = [boundry[0], boundry[1], boundry[2], boundry[3]]
  199. v_blocks = []
  200. """
  201. 先从左到右切分
  202. """
  203. while True:
  204. all_bboxes = get_bbox_in_boundry(bboxes, new_boundry)
  205. left_edge_bboxes = get_left_edge_bboxes(all_bboxes)
  206. if len(left_edge_bboxes) == 0:
  207. break
  208. right_split_line_x1 = max([bbox[X1_IDX] for bbox in left_edge_bboxes])+1
  209. # 然后检查这条线能不与其他bbox的左边界相交或者重合
  210. if any([bbox[X0_IDX] <= right_split_line_x1 <= bbox[X1_IDX] for bbox in all_bboxes]):
  211. # 垂直切分线与某些box发生相交,说明无法完全垂直方向切分。
  212. break
  213. else: # 说明成功分割出一列
  214. # 找到左侧边界最靠左的bbox作为layout的x0
  215. layout_x0 = min([bbox[X0_IDX] for bbox in left_edge_bboxes]) # 这里主要是为了画出来有一定间距
  216. v_blocks.append([layout_x0, new_boundry[1], right_split_line_x1, new_boundry[3], LAYOUT_V])
  217. new_boundry[0] = right_split_line_x1 # 更新边界
  218. """
  219. 再从右到左切, 此时如果还是无法完全切分,那么剩余部分作为layout_lable为u的layout返回
  220. """
  221. unsplited_block = []
  222. while True:
  223. all_bboxes = get_bbox_in_boundry(bboxes, new_boundry)
  224. right_edge_bboxes = get_right_edge_bboxes(all_bboxes)
  225. if len(right_edge_bboxes) == 0:
  226. break
  227. left_split_line_x0 = min([bbox[X0_IDX] for bbox in right_edge_bboxes])-1
  228. # 然后检查这条线能不与其他bbox的左边界相交或者重合
  229. if any([bbox[X0_IDX] <= left_split_line_x0 <= bbox[X1_IDX] for bbox in all_bboxes]):
  230. # 这里是余下的
  231. unsplited_block.append([new_boundry[0], new_boundry[1], new_boundry[2], new_boundry[3], LAYOUT_UNPROC])
  232. break
  233. else:
  234. # 找到右侧边界最靠右的bbox作为layout的x1
  235. layout_x1 = max([bbox[X1_IDX] for bbox in right_edge_bboxes])
  236. v_blocks.append([left_split_line_x0, new_boundry[1], layout_x1, new_boundry[3], LAYOUT_V])
  237. new_boundry[2] = left_split_line_x0 # 更新右边界
  238. """
  239. 最后拼装成layout格式返回
  240. """
  241. for block in v_blocks:
  242. sorted_layout_blocks.append({
  243. "layout_bbox": block[:4],
  244. "layout_label":block[4],
  245. "sub_layout":[],
  246. })
  247. for block in unsplited_block:
  248. sorted_layout_blocks.append({
  249. "layout_bbox": block[:4],
  250. "layout_label":block[4],
  251. "sub_layout":[],
  252. })
  253. # 按照x0排序
  254. sorted_layout_blocks.sort(key=lambda x: x['layout_bbox'][0])
  255. return sorted_layout_blocks
  256. def _vertical_align_split_v2(bboxes:list, boundry:tuple)-> list:
  257. """
  258. 改进的 _vertical_align_split算法,原算法会因为第二列的box由于左侧没有遮挡被认为是左侧的一部分,导致整个layout多列被识别为一列。
  259. 利用从左上角的box开始向下看的方法,不断扩展w_x0, w_x1,直到不能继续向下扩展,或者到达边界下边界。
  260. """
  261. sorted_layout_blocks = [] # 这是要最终返回的值
  262. new_boundry = [boundry[0], boundry[1], boundry[2], boundry[3]]
  263. bad_boxes = [] # 被割中的box
  264. v_blocks = []
  265. while True:
  266. all_bboxes = get_bbox_in_boundry(bboxes, new_boundry)
  267. if len(all_bboxes) == 0:
  268. break
  269. left_top_box = min(all_bboxes, key=lambda x: (x[X0_IDX],x[Y0_IDX]))# 这里应该加强,检查一下必须是在第一列的 TODO
  270. start_box = [left_top_box[X0_IDX], left_top_box[Y0_IDX], left_top_box[X1_IDX], left_top_box[Y1_IDX]]
  271. w_x0, w_x1 = left_top_box[X0_IDX], left_top_box[X1_IDX]
  272. """
  273. 然后沿着这个box线向下找最近的那个box, 然后扩展w_x0, w_x1
  274. 扩展之后,宽度会增加,随后用x=w_x1来检测在边界内是否有box与相交,如果相交,那么就说明不能再扩展了。
  275. 当不能扩展的时候就要看是否到达下边界:
  276. 1. 达到,那么更新左边界继续分下一个列
  277. 2. 没有达到,那么此时开始从右侧切分进入下面的循环里
  278. """
  279. while left_top_box is not None: # 向下去找
  280. virtual_box = [w_x0, left_top_box[Y0_IDX], w_x1, left_top_box[Y1_IDX]]
  281. left_top_box = find_bottom_bbox_direct_from_left_edge(virtual_box, all_bboxes)
  282. if left_top_box:
  283. w_x0, w_x1 = min(virtual_box[X0_IDX], left_top_box[X0_IDX]), max([virtual_box[X1_IDX], left_top_box[X1_IDX]])
  284. # 万一这个初始的box在column中间,那么还要向上看
  285. start_box = [w_x0, start_box[Y0_IDX], w_x1, start_box[Y1_IDX]] # 扩展一下宽度更鲁棒
  286. left_top_box = find_top_bbox_direct_from_left_edge(start_box, all_bboxes)
  287. while left_top_box is not None: # 向上去找
  288. virtual_box = [w_x0, left_top_box[Y0_IDX], w_x1, left_top_box[Y1_IDX]]
  289. left_top_box = find_top_bbox_direct_from_left_edge(virtual_box, all_bboxes)
  290. if left_top_box:
  291. w_x0, w_x1 = min(virtual_box[X0_IDX], left_top_box[X0_IDX]), max([virtual_box[X1_IDX], left_top_box[X1_IDX]])
  292. # 检查相交
  293. if any([bbox[X0_IDX] <= w_x1+1 <= bbox[X1_IDX] for bbox in all_bboxes]):
  294. for b in all_bboxes:
  295. if b[X0_IDX] <= w_x1+1 <= b[X1_IDX]:
  296. bad_boxes.append([b[X0_IDX], b[Y0_IDX], b[X1_IDX], b[Y1_IDX]])
  297. break
  298. else: # 说明成功分割出一列
  299. v_blocks.append([w_x0, new_boundry[1], w_x1, new_boundry[3], LAYOUT_V])
  300. new_boundry[0] = w_x1 # 更新边界
  301. """
  302. 接着开始从右上角的box扫描
  303. """
  304. w_x0 , w_x1 = 0, 0
  305. unsplited_block = []
  306. while True:
  307. all_bboxes = get_bbox_in_boundry(bboxes, new_boundry)
  308. if len(all_bboxes) == 0:
  309. break
  310. # 先找到X1最大的
  311. bbox_list_sorted = sorted(all_bboxes, key=lambda bbox: bbox[X1_IDX], reverse=True)
  312. # Then, find the boxes with the smallest Y0 value
  313. bigest_x1 = bbox_list_sorted[0][X1_IDX]
  314. boxes_with_bigest_x1 = [bbox for bbox in bbox_list_sorted if bbox[X1_IDX] == bigest_x1] # 也就是最靠右的那些
  315. right_top_box = min(boxes_with_bigest_x1, key=lambda bbox: bbox[Y0_IDX]) # y0最小的那个
  316. start_box = [right_top_box[X0_IDX], right_top_box[Y0_IDX], right_top_box[X1_IDX], right_top_box[Y1_IDX]]
  317. w_x0, w_x1 = right_top_box[X0_IDX], right_top_box[X1_IDX]
  318. while right_top_box is not None:
  319. virtual_box = [w_x0, right_top_box[Y0_IDX], w_x1, right_top_box[Y1_IDX]]
  320. right_top_box = find_bottom_bbox_direct_from_right_edge(virtual_box, all_bboxes)
  321. if right_top_box:
  322. w_x0, w_x1 = min([w_x0, right_top_box[X0_IDX]]), max([w_x1, right_top_box[X1_IDX]])
  323. # 在向上扫描
  324. start_box = [w_x0, start_box[Y0_IDX], w_x1, start_box[Y1_IDX]] # 扩展一下宽度更鲁棒
  325. right_top_box = find_top_bbox_direct_from_right_edge(start_box, all_bboxes)
  326. while right_top_box is not None:
  327. virtual_box = [w_x0, right_top_box[Y0_IDX], w_x1, right_top_box[Y1_IDX]]
  328. right_top_box = find_top_bbox_direct_from_right_edge(virtual_box, all_bboxes)
  329. if right_top_box:
  330. w_x0, w_x1 = min([w_x0, right_top_box[X0_IDX]]), max([w_x1, right_top_box[X1_IDX]])
  331. # 检查是否与其他box相交, 垂直切分线与某些box发生相交,说明无法完全垂直方向切分。
  332. if any([bbox[X0_IDX] <= w_x0-1 <= bbox[X1_IDX] for bbox in all_bboxes]):
  333. unsplited_block.append([new_boundry[0], new_boundry[1], new_boundry[2], new_boundry[3], LAYOUT_UNPROC])
  334. for b in all_bboxes:
  335. if b[X0_IDX] <= w_x0-1 <= b[X1_IDX]:
  336. bad_boxes.append([b[X0_IDX], b[Y0_IDX], b[X1_IDX], b[Y1_IDX]])
  337. break
  338. else: # 说明成功分割出一列
  339. v_blocks.append([w_x0, new_boundry[1], w_x1, new_boundry[3], LAYOUT_V])
  340. new_boundry[2] = w_x0
  341. """转换数据结构"""
  342. for block in v_blocks:
  343. sorted_layout_blocks.append({
  344. "layout_bbox": block[:4],
  345. "layout_label":block[4],
  346. "sub_layout":[],
  347. })
  348. for block in unsplited_block:
  349. sorted_layout_blocks.append({
  350. "layout_bbox": block[:4],
  351. "layout_label":block[4],
  352. "sub_layout":[],
  353. "bad_boxes": bad_boxes # 记录下来,这个box是被割中的
  354. })
  355. # 按照x0排序
  356. sorted_layout_blocks.sort(key=lambda x: x['layout_bbox'][0])
  357. return sorted_layout_blocks
  358. def _try_horizontal_mult_column_split(bboxes:list, boundry:tuple)-> list:
  359. """
  360. 尝试水平切分,如果切分不动,那就当一个BAD_LAYOUT返回
  361. ------------------
  362. | | |
  363. ------------------
  364. | | | | <- 这里是此函数要切分的场景
  365. ------------------
  366. | | |
  367. | | |
  368. """
  369. pass
  370. def _vertical_split(bboxes:list, boundry:tuple)-> list:
  371. """
  372. 从垂直方向进行切割,分block
  373. 这个版本里,如果垂直切分不动,那就当一个BAD_LAYOUT返回
  374. --------------------------
  375. | | |
  376. | | |
  377. | |
  378. 这种列是此函数要切分的 -> | |
  379. | |
  380. | | |
  381. | | |
  382. -------------------------
  383. """
  384. sorted_layout_blocks = [] # 这是要最终返回的值
  385. bound_x0, bound_y0, bound_x1, bound_y1 = boundry
  386. all_bboxes = get_bbox_in_boundry(bboxes, boundry)
  387. """
  388. all_bboxes = fix_vertical_bbox_pos(all_bboxes) # 垂直方向解覆盖
  389. all_bboxes = fix_hor_bbox_pos(all_bboxes) # 水平解覆盖
  390. 这两行代码目前先不执行,因为公式检测,表格检测还不是很成熟,导致非常多的textblock参与了运算,时间消耗太大。
  391. 这两行代码的作用是:
  392. 如果遇到互相重叠的bbox, 那么会把面积较小的box进行压缩,从而避免重叠。对布局切分来说带来正反馈。
  393. """
  394. #all_bboxes = paper_bbox_sort(all_bboxes, abs(bound_x1-bound_x0), abs(bound_y1-bound_x0)) # 大致拍下序, 这个是基于直接遮挡的。
  395. """
  396. 首先在垂直方向上扩展独占一行的bbox
  397. """
  398. for bbox in all_bboxes:
  399. top_nearest_bbox = find_all_top_bbox_direct(bbox, all_bboxes) # 非扩展线
  400. bottom_nearest_bbox = find_all_bottom_bbox_direct(bbox, all_bboxes)
  401. if top_nearest_bbox is None and bottom_nearest_bbox is None and not any([b[X0_IDX]<bbox[X1_IDX]<b[X1_IDX] or b[X0_IDX]<bbox[X0_IDX]<b[X1_IDX] for b in all_bboxes]): # 独占一列, 且不和其他重叠
  402. bbox[X0_EXT_IDX] = bbox[X0_IDX]
  403. bbox[Y0_EXT_IDX] = bound_y0
  404. bbox[X1_EXT_IDX] = bbox[X1_IDX]
  405. bbox[Y1_EXT_IDX] = bound_y1
  406. """
  407. 此时独占一列的被成功扩展到指定的边界上,这个时候利用边界条件合并连续的bbox,成为一个group
  408. 然后合并所有连续垂直方向的bbox.
  409. """
  410. all_bboxes.sort(key=lambda x: x[X0_IDX])
  411. # fix: 这里水平方向的列不要合并成一个行,因为需要保证返回给下游的最小block,总是可以无脑从上到下阅读文字。
  412. v_bboxes = []
  413. for box in all_bboxes:
  414. if box[Y0_EXT_IDX] == bound_y0 and box[Y1_EXT_IDX] == bound_y1:
  415. v_bboxes.append(box)
  416. """
  417. 现在v_bboxes里面是所有的group了,每个group都是一个list
  418. 对v_bboxes里的每个group进行计算放回到sorted_layouts里
  419. """
  420. v_layouts = []
  421. for vbox in v_bboxes:
  422. #gp.sort(key=lambda x: x[X0_IDX])
  423. # 然后计算这个group的layout_bbox,也就是最小的x0,y0, 最大的x1,y1
  424. x0, y0, x1, y1 = vbox[X0_EXT_IDX], vbox[Y0_EXT_IDX], vbox[X1_EXT_IDX], vbox[Y1_EXT_IDX]
  425. v_layouts.append([x0, y0, x1, y1, LAYOUT_V]) # 垂直的布局
  426. """
  427. 接下来利用这些连续的垂直bbox的layout_bbox的x0, x1,从垂直上切分开其余的为几个部分
  428. """
  429. v_split_lines = [bound_x0]
  430. for gp in v_bboxes:
  431. x0, x1 = gp[X0_IDX], gp[X1_IDX]
  432. v_split_lines.append(x0)
  433. v_split_lines.append(x1)
  434. v_split_lines.append(bound_x1)
  435. unsplited_bboxes = []
  436. for i in range(0, len(v_split_lines), 2):
  437. start_x0, start_x1 = v_split_lines[i:i+2]
  438. # 然后找出[start_x0, start_x1]之间的其他bbox,这些组成一个未分割板块
  439. bboxes_in_block = [bbox for bbox in all_bboxes if bbox[X0_IDX]>=start_x0 and bbox[X1_IDX]<=start_x1]
  440. unsplited_bboxes.append(bboxes_in_block)
  441. # 接着把未处理的加入到v_layouts里
  442. for bboxes_in_block in unsplited_bboxes:
  443. if len(bboxes_in_block) == 0:
  444. continue
  445. x0, y0, x1, y1 = min([bbox[X0_IDX] for bbox in bboxes_in_block]), bound_y0, max([bbox[X1_IDX] for bbox in bboxes_in_block]), bound_y1
  446. v_layouts.append([x0, y0, x1, y1, LAYOUT_UNPROC]) # 说明这篇区域未能够分析出可靠的版面
  447. v_layouts.sort(key=lambda x: x[0]) # 按照x0排序, 也就是从左到右的顺序
  448. for layout in v_layouts:
  449. sorted_layout_blocks.append({
  450. "layout_bbox": layout[:4],
  451. "layout_label":layout[4],
  452. "sub_layout":[],
  453. })
  454. """
  455. 至此,垂直方向切成了2种类型,其一是独占一列的,其二是未处理的。
  456. 下面对这些未处理的进行垂直方向切分,这个切分要切出来类似“吕”这种类型的垂直方向的布局
  457. """
  458. for i, layout in enumerate(sorted_layout_blocks):
  459. if layout['layout_label'] == LAYOUT_UNPROC:
  460. x0, y0, x1, y1 = layout['layout_bbox']
  461. v_split_layouts = _vertical_align_split_v2(bboxes, [x0, y0, x1, y1])
  462. sorted_layout_blocks[i] = {
  463. "layout_bbox": [x0, y0, x1, y1],
  464. "layout_label": LAYOUT_H,
  465. "sub_layout": v_split_layouts
  466. }
  467. layout['layout_label'] = LAYOUT_H # 被垂线切分成了水平布局
  468. return sorted_layout_blocks
  469. def split_layout(bboxes:list, boundry:tuple, page_num:int)-> list:
  470. """
  471. 把bboxes切割成layout
  472. return:
  473. [
  474. {
  475. "layout_bbox": [x0, y0, x1, y1],
  476. "layout_label":"u|v|h|b", 未处理|垂直|水平|BAD_LAYOUT
  477. "sub_layout": [] #每个元素都是[x0, y0, x1, y1, block_content, idx_x, idx_y, content_type, ext_x0, ext_y0, ext_x1, ext_y1], 并且顺序就是阅读顺序
  478. }
  479. ]
  480. example:
  481. [
  482. {
  483. "layout_bbox": [0, 0, 100, 100],
  484. "layout_label":"u|v|h|b",
  485. "sub_layout":[
  486. ]
  487. },
  488. {
  489. "layout_bbox": [0, 0, 100, 100],
  490. "layout_label":"u|v|h|b",
  491. "sub_layout":[
  492. {
  493. "layout_bbox": [0, 0, 100, 100],
  494. "layout_label":"u|v|h|b",
  495. "content_bboxes":[
  496. [],
  497. [],
  498. []
  499. ]
  500. },
  501. {
  502. "layout_bbox": [0, 0, 100, 100],
  503. "layout_label":"u|v|h|b",
  504. "sub_layout":[
  505. ]
  506. }
  507. }
  508. ]
  509. """
  510. sorted_layouts = [] # 最终返回的结果
  511. boundry_x0, boundry_y0, boundry_x1, boundry_y1 = boundry
  512. if len(bboxes) <=1:
  513. return [
  514. {
  515. "layout_bbox": [boundry_x0, boundry_y0, boundry_x1, boundry_y1],
  516. "layout_label": LAYOUT_V,
  517. "sub_layout":[]
  518. }
  519. ]
  520. """
  521. 接下来按照先水平后垂直的顺序进行切分
  522. """
  523. bboxes = paper_bbox_sort(bboxes, boundry_x1-boundry_x0, boundry_y1-boundry_y0)
  524. sorted_layouts = _horizontal_split(bboxes, boundry) # 通过水平分割出来的layout
  525. for i, layout in enumerate(sorted_layouts):
  526. x0, y0, x1, y1 = layout['layout_bbox']
  527. layout_type = layout['layout_label']
  528. if layout_type == LAYOUT_UNPROC: # 说明是非独占单行的,这些需要垂直切分
  529. v_split_layouts = _vertical_split(bboxes, [x0, y0, x1, y1])
  530. """
  531. 最后这里有个逻辑问题:如果这个函数只分离出来了一个column layout,那么这个layout分割肯定超出了算法能力范围。因为我们假定的是传进来的
  532. box已经把行全部剥离了,所以这里必须十多个列才可以。如果只剥离出来一个layout,并且是多个box,那么就说明这个layout是无法分割的,标记为LAYOUT_UNPROC
  533. """
  534. layout_label = LAYOUT_V
  535. if len(v_split_layouts) == 1:
  536. if len(v_split_layouts[0]['sub_layout']) == 0:
  537. layout_label = LAYOUT_UNPROC
  538. #logger.warning(f"WARNING: pageno={page_num}, 无法分割的layout: ", v_split_layouts)
  539. """
  540. 组合起来最终的layout
  541. """
  542. sorted_layouts[i] = {
  543. "layout_bbox": [x0, y0, x1, y1],
  544. "layout_label": layout_label,
  545. "sub_layout": v_split_layouts
  546. }
  547. layout['layout_label'] = LAYOUT_H
  548. """
  549. 水平和垂直方向都切分完毕了。此时还有一些未处理的,这些未处理的可能是因为水平和垂直方向都无法切分。
  550. 这些最后调用_try_horizontal_mult_block_split做一次水平多个block的联合切分,如果也不能切分最终就当做BAD_LAYOUT返回
  551. """
  552. # TODO
  553. return sorted_layouts
  554. def get_bboxes_layout(all_boxes:list, boundry:tuple, page_id:int):
  555. """
  556. 对利用layout排序之后的box,进行排序
  557. return:
  558. [
  559. {
  560. "layout_bbox": [x0, y0, x1, y1],
  561. "layout_label":"u|v|h|b", 未处理|垂直|水平|BAD_LAYOUT
  562. },
  563. ]
  564. """
  565. def _preorder_traversal(layout):
  566. """
  567. 对sorted_layouts的叶子节点,也就是len(sub_layout)==0的节点进行排序。排序按照前序遍历的顺序,也就是从上到下,从左到右的顺序
  568. """
  569. sorted_layout_blocks = []
  570. for layout in layout:
  571. sub_layout = layout['sub_layout']
  572. if len(sub_layout) == 0:
  573. sorted_layout_blocks.append(layout)
  574. else:
  575. s = _preorder_traversal(sub_layout)
  576. sorted_layout_blocks.extend(s)
  577. return sorted_layout_blocks
  578. # -------------------------------------------------------------------------------------------------------------------------
  579. sorted_layouts = split_layout(all_boxes, boundry, page_id)# 先切分成layout,得到一个Tree
  580. total_sorted_layout_blocks = _preorder_traversal(sorted_layouts)
  581. return total_sorted_layout_blocks, sorted_layouts
  582. def get_columns_cnt_of_layout(layout_tree):
  583. """
  584. 获取一个layout的宽度
  585. """
  586. max_width_list = [0] # 初始化一个元素,防止max,min函数报错
  587. for items in layout_tree: # 针对每一层(横切)计算列数,横着的算一列
  588. layout_type = items['layout_label']
  589. sub_layouts = items['sub_layout']
  590. if len(sub_layouts)==0:
  591. max_width_list.append(1)
  592. else:
  593. if layout_type == LAYOUT_H:
  594. max_width_list.append(1)
  595. else:
  596. width = 0
  597. for l in sub_layouts:
  598. if len(l['sub_layout']) == 0:
  599. width += 1
  600. else:
  601. for lay in l['sub_layout']:
  602. width += get_columns_cnt_of_layout([lay])
  603. max_width_list.append(width)
  604. return max(max_width_list)
  605. def sort_with_layout(bboxes:list, page_width, page_height) -> (list,list):
  606. """
  607. 输入是一个bbox的list.
  608. 获取到输入之后,先进行layout切分,然后对这些bbox进行排序。返回排序后的bboxes
  609. """
  610. new_bboxes = []
  611. for box in bboxes:
  612. # new_bboxes.append([box[0], box[1], box[2], box[3], None, None, None, 'text', None, None, None, None])
  613. new_bboxes.append([box[0], box[1], box[2], box[3], None, None, None, 'text', None, None, None, None, box[4]])
  614. layout_bboxes, _ = get_bboxes_layout(new_bboxes, [0, 0, page_width, page_height], 0)
  615. if any([lay['layout_label']==LAYOUT_UNPROC for lay in layout_bboxes]):
  616. logger.warning(f"drop this pdf, reason: 复杂版面")
  617. return None,None
  618. sorted_bboxes = []
  619. # 利用layout bbox每次框定一些box,然后排序
  620. for layout in layout_bboxes:
  621. lbox = layout['layout_bbox']
  622. bbox_in_layout = get_bbox_in_boundry(new_bboxes, lbox)
  623. sorted_bbox = paper_bbox_sort(bbox_in_layout, lbox[2]-lbox[0], lbox[3]-lbox[1])
  624. sorted_bboxes.extend(sorted_bbox)
  625. return sorted_bboxes, layout_bboxes
  626. def sort_text_block(text_block, layout_bboxes):
  627. """
  628. 对一页的text_block进行排序
  629. """
  630. sorted_text_bbox = []
  631. all_text_bbox = []
  632. # 做一个box=>text的映射
  633. box_to_text = {}
  634. for blk in text_block:
  635. box = blk['bbox']
  636. box_to_text[(box[0], box[1], box[2], box[3])] = blk
  637. all_text_bbox.append(box)
  638. # text_blocks_to_sort = []
  639. # for box in box_to_text.keys():
  640. # text_blocks_to_sort.append([box[0], box[1], box[2], box[3], None, None, None, 'text', None, None, None, None])
  641. # 按照layout_bboxes的顺序,对text_block进行排序
  642. for layout in layout_bboxes:
  643. layout_box = layout['layout_bbox']
  644. text_bbox_in_layout = get_bbox_in_boundry(all_text_bbox, [layout_box[0]-1, layout_box[1]-1, layout_box[2]+1, layout_box[3]+1])
  645. #sorted_bbox = paper_bbox_sort(text_bbox_in_layout, layout_box[2]-layout_box[0], layout_box[3]-layout_box[1])
  646. text_bbox_in_layout.sort(key = lambda x: x[1]) # 一个layout内部的box,按照y0自上而下排序
  647. #sorted_bbox = [[b] for b in text_blocks_to_sort]
  648. for sb in text_bbox_in_layout:
  649. sorted_text_bbox.append(box_to_text[(sb[0], sb[1], sb[2], sb[3])])
  650. return sorted_text_bbox