raw_processor.py 6.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. from para.commons import *
  2. class RawBlockProcessor:
  3. def __init__(self) -> None:
  4. self.y_tolerance = 2
  5. self.pdf_dic = {}
  6. def __span_flags_decomposer(self, span_flags):
  7. """
  8. Make font flags human readable.
  9. Parameters
  10. ----------
  11. self : object
  12. The instance of the class.
  13. span_flags : int
  14. span flags
  15. Returns
  16. -------
  17. l : dict
  18. decomposed flags
  19. """
  20. l = {
  21. "is_superscript": False,
  22. "is_italic": False,
  23. "is_serifed": False,
  24. "is_sans_serifed": False,
  25. "is_monospaced": False,
  26. "is_proportional": False,
  27. "is_bold": False,
  28. }
  29. if span_flags & 2**0:
  30. l["is_superscript"] = True # 表示上标
  31. if span_flags & 2**1:
  32. l["is_italic"] = True # 表示斜体
  33. if span_flags & 2**2:
  34. l["is_serifed"] = True # 表示衬线字体
  35. else:
  36. l["is_sans_serifed"] = True # 表示非衬线字体
  37. if span_flags & 2**3:
  38. l["is_monospaced"] = True # 表示等宽字体
  39. else:
  40. l["is_proportional"] = True # 表示比例字体
  41. if span_flags & 2**4:
  42. l["is_bold"] = True # 表示粗体
  43. return l
  44. def __make_new_lines(self, raw_lines):
  45. """
  46. This function makes new lines.
  47. Parameters
  48. ----------
  49. self : object
  50. The instance of the class.
  51. raw_lines : list
  52. raw lines
  53. Returns
  54. -------
  55. new_lines : list
  56. new lines
  57. """
  58. new_lines = []
  59. new_line = None
  60. for raw_line in raw_lines:
  61. raw_line_bbox = raw_line["bbox"]
  62. raw_line_spans = raw_line["spans"]
  63. raw_line_text = "".join([span["text"] for span in raw_line_spans])
  64. raw_line_dir = raw_line.get("dir", None)
  65. decomposed_line_spans = []
  66. for span in raw_line_spans:
  67. raw_flags = span["flags"]
  68. decomposed_flags = self.__span_flags_decomposer(raw_flags)
  69. span["decomposed_flags"] = decomposed_flags
  70. decomposed_line_spans.append(span)
  71. if new_line is None:
  72. new_line = {
  73. "bbox": raw_line_bbox,
  74. "text": raw_line_text,
  75. "dir": raw_line_dir if raw_line_dir else (0, 0),
  76. "spans": decomposed_line_spans,
  77. }
  78. else:
  79. if (
  80. abs(raw_line_bbox[1] - new_line["bbox"][1]) <= self.y_tolerance
  81. and abs(raw_line_bbox[3] - new_line["bbox"][3]) <= self.y_tolerance
  82. ):
  83. new_line["bbox"] = (
  84. min(new_line["bbox"][0], raw_line_bbox[0]), # left
  85. new_line["bbox"][1], # top
  86. max(new_line["bbox"][2], raw_line_bbox[2]), # right
  87. raw_line_bbox[3], # bottom
  88. )
  89. new_line["text"] += " " + raw_line_text
  90. new_line["spans"].extend(raw_line_spans)
  91. new_line["dir"] = (
  92. new_line["dir"][0] + raw_line_dir[0],
  93. new_line["dir"][1] + raw_line_dir[1],
  94. )
  95. else:
  96. new_lines.append(new_line)
  97. new_line = {
  98. "bbox": raw_line_bbox,
  99. "text": raw_line_text,
  100. "dir": raw_line_dir if raw_line_dir else (0, 0),
  101. "spans": raw_line_spans,
  102. }
  103. if new_line:
  104. new_lines.append(new_line)
  105. return new_lines
  106. def __make_new_block(self, raw_block):
  107. """
  108. This function makes a new block.
  109. Parameters
  110. ----------
  111. self : object
  112. The instance of the class.
  113. ----------
  114. raw_block : dict
  115. a raw block
  116. Returns
  117. -------
  118. new_block : dict
  119. Schema of new_block:
  120. {
  121. "block_id": "block_1",
  122. "bbox": [0, 0, 100, 100],
  123. "text": "This is a block.",
  124. "lines": [
  125. {
  126. "bbox": [0, 0, 100, 100],
  127. "text": "This is a line.",
  128. "spans": [
  129. {
  130. "text": "This is a span.",
  131. "font": "Times New Roman",
  132. "size": 12,
  133. "color": "#000000",
  134. }
  135. ],
  136. }
  137. ],
  138. }
  139. """
  140. new_block = {}
  141. block_id = raw_block["number"]
  142. block_bbox = raw_block["bbox"]
  143. block_text = " ".join(span["text"] for line in raw_block["lines"] for span in line["spans"])
  144. raw_lines = raw_block["lines"]
  145. block_lines = self.__make_new_lines(raw_lines)
  146. new_block["block_id"] = block_id
  147. new_block["bbox"] = block_bbox
  148. new_block["text"] = block_text
  149. new_block["lines"] = block_lines
  150. return new_block
  151. def batch_process_blocks(self, pdf_dic):
  152. """
  153. This function processes the blocks in batch.
  154. Parameters
  155. ----------
  156. self : object
  157. The instance of the class.
  158. ----------
  159. blocks : list
  160. Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/test/preproc_2_parasplit_example.json.
  161. Returns
  162. -------
  163. result_dict : dict
  164. result dictionary
  165. """
  166. for page_id, blocks in pdf_dic.items():
  167. if page_id.startswith("page_"):
  168. para_blocks = []
  169. if "preproc_blocks" in blocks.keys():
  170. input_blocks = blocks["preproc_blocks"]
  171. for raw_block in input_blocks:
  172. new_block = self.__make_new_block(raw_block)
  173. para_blocks.append(new_block)
  174. blocks["para_blocks"] = para_blocks
  175. return pdf_dic