stats.py 9.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269
  1. import sys
  2. from collections import Counter
  3. import numpy as np
  4. from para.commons import *
  5. if sys.version_info[0] >= 3:
  6. sys.stdout.reconfigure(encoding="utf-8") # type: ignore
  7. class BlockStatisticsCalculator:
  8. def __init__(self) -> None:
  9. pass
  10. def __calc_stats_of_new_lines(self, new_lines):
  11. """
  12. This function calculates the paragraph metrics
  13. Parameters
  14. ----------
  15. combined_lines : list
  16. combined lines
  17. Returns
  18. -------
  19. X0 : float
  20. Median of x0 values, which represents the left average boundary of the block
  21. X1 : float
  22. Median of x1 values, which represents the right average boundary of the block
  23. avg_char_width : float
  24. Average of char widths, which represents the average char width of the block
  25. avg_char_height : float
  26. Average of line heights, which represents the average line height of the block
  27. """
  28. x0_values = []
  29. x1_values = []
  30. char_widths = []
  31. char_heights = []
  32. block_font_types = []
  33. block_font_sizes = []
  34. block_directions = []
  35. if len(new_lines) > 0:
  36. for i, line in enumerate(new_lines):
  37. line_bbox = line["bbox"]
  38. line_text = line["text"]
  39. line_spans = line["spans"]
  40. num_chars = len([ch for ch in line_text if not ch.isspace()])
  41. x0_values.append(line_bbox[0])
  42. x1_values.append(line_bbox[2])
  43. if num_chars > 0:
  44. char_width = (line_bbox[2] - line_bbox[0]) / num_chars
  45. char_widths.append(char_width)
  46. for span in line_spans:
  47. block_font_types.append(span["font"])
  48. block_font_sizes.append(span["size"])
  49. if "dir" in line:
  50. block_directions.append(line["dir"])
  51. # line_font_types = [span["font"] for span in line_spans]
  52. char_heights = [span["size"] for span in line_spans]
  53. X0 = np.median(x0_values) if x0_values else 0
  54. X1 = np.median(x1_values) if x1_values else 0
  55. avg_char_width = sum(char_widths) / len(char_widths) if char_widths else 0
  56. avg_char_height = sum(char_heights) / len(char_heights) if char_heights else 0
  57. # max_freq_font_type = max(set(block_font_types), key=block_font_types.count) if block_font_types else None
  58. max_span_length = 0
  59. max_span_font_type = None
  60. for line in new_lines:
  61. line_spans = line["spans"]
  62. for span in line_spans:
  63. span_length = span["bbox"][2] - span["bbox"][0]
  64. if span_length > max_span_length:
  65. max_span_length = span_length
  66. max_span_font_type = span["font"]
  67. max_freq_font_type = max_span_font_type
  68. avg_font_size = sum(block_font_sizes) / len(block_font_sizes) if block_font_sizes else None
  69. avg_dir_horizontal = sum([dir[0] for dir in block_directions]) / len(block_directions) if block_directions else 0
  70. avg_dir_vertical = sum([dir[1] for dir in block_directions]) / len(block_directions) if block_directions else 0
  71. median_font_size = float(np.median(block_font_sizes)) if block_font_sizes else None
  72. return (
  73. X0,
  74. X1,
  75. avg_char_width,
  76. avg_char_height,
  77. max_freq_font_type,
  78. avg_font_size,
  79. (avg_dir_horizontal, avg_dir_vertical),
  80. median_font_size,
  81. )
  82. def __make_new_block(self, input_block):
  83. new_block = {}
  84. raw_lines = input_block["lines"]
  85. stats = self.__calc_stats_of_new_lines(raw_lines)
  86. block_id = input_block["block_id"]
  87. block_bbox = input_block["bbox"]
  88. block_text = input_block["text"]
  89. block_lines = raw_lines
  90. block_avg_left_boundary = stats[0]
  91. block_avg_right_boundary = stats[1]
  92. block_avg_char_width = stats[2]
  93. block_avg_char_height = stats[3]
  94. block_font_type = stats[4]
  95. block_font_size = stats[5]
  96. block_direction = stats[6]
  97. block_median_font_size = stats[7]
  98. new_block["block_id"] = block_id
  99. new_block["bbox"] = block_bbox
  100. new_block["text"] = block_text
  101. new_block["dir"] = block_direction
  102. new_block["X0"] = block_avg_left_boundary
  103. new_block["X1"] = block_avg_right_boundary
  104. new_block["avg_char_width"] = block_avg_char_width
  105. new_block["avg_char_height"] = block_avg_char_height
  106. new_block["block_font_type"] = block_font_type
  107. new_block["block_font_size"] = block_font_size
  108. new_block["lines"] = block_lines
  109. new_block["median_font_size"] = block_median_font_size
  110. return new_block
  111. def batch_process_blocks(self, pdf_dic):
  112. """
  113. This function processes the blocks in batch.
  114. Parameters
  115. ----------
  116. self : object
  117. The instance of the class.
  118. ----------
  119. blocks : list
  120. Input block is a list of raw blocks. Schema can refer to the value of key ""preproc_blocks", demo file is app/pdf_toolbox/test/preproc_2_parasplit_example.json
  121. Returns
  122. -------
  123. result_dict : dict
  124. result dictionary
  125. """
  126. for page_id, blocks in pdf_dic.items():
  127. if page_id.startswith("page_"):
  128. para_blocks = []
  129. if "para_blocks" in blocks.keys():
  130. input_blocks = blocks["para_blocks"]
  131. for input_block in input_blocks:
  132. new_block = self.__make_new_block(input_block)
  133. para_blocks.append(new_block)
  134. blocks["para_blocks"] = para_blocks
  135. return pdf_dic
  136. class DocStatisticsCalculator:
  137. def __init__(self) -> None:
  138. pass
  139. def calc_stats_of_doc(self, pdf_dict):
  140. """
  141. This function computes the statistics of the document
  142. Parameters
  143. ----------
  144. result_dict : dict
  145. result dictionary
  146. Returns
  147. -------
  148. statistics : dict
  149. statistics of the document
  150. """
  151. total_text_length = 0
  152. total_num_blocks = 0
  153. for page_id, blocks in pdf_dict.items():
  154. if page_id.startswith("page_"):
  155. if "para_blocks" in blocks.keys():
  156. para_blocks = blocks["para_blocks"]
  157. for para_block in para_blocks:
  158. total_text_length += len(para_block["text"])
  159. total_num_blocks += 1
  160. avg_text_length = total_text_length / total_num_blocks if total_num_blocks else 0
  161. font_list = []
  162. for page_id, blocks in pdf_dict.items():
  163. if page_id.startswith("page_"):
  164. if "para_blocks" in blocks.keys():
  165. input_blocks = blocks["para_blocks"]
  166. for input_block in input_blocks:
  167. block_text_length = len(input_block.get("text", ""))
  168. if block_text_length < avg_text_length * 0.5:
  169. continue
  170. block_font_type = safe_get(input_block, "block_font_type", "")
  171. block_font_size = safe_get(input_block, "block_font_size", 0)
  172. font_list.append((block_font_type, block_font_size))
  173. font_counter = Counter(font_list)
  174. most_common_font = font_counter.most_common(1)[0] if font_list else (("", 0), 0)
  175. second_most_common_font = font_counter.most_common(2)[1] if len(font_counter) > 1 else (("", 0), 0)
  176. statistics = {
  177. "num_pages": 0,
  178. "num_blocks": 0,
  179. "num_paras": 0,
  180. "num_titles": 0,
  181. "num_header_blocks": 0,
  182. "num_footer_blocks": 0,
  183. "num_watermark_blocks": 0,
  184. "num_vertical_margin_note_blocks": 0,
  185. "most_common_font_type": most_common_font[0][0],
  186. "most_common_font_size": most_common_font[0][1],
  187. "number_of_most_common_font": most_common_font[1],
  188. "second_most_common_font_type": second_most_common_font[0][0],
  189. "second_most_common_font_size": second_most_common_font[0][1],
  190. "number_of_second_most_common_font": second_most_common_font[1],
  191. "avg_text_length": avg_text_length,
  192. }
  193. for page_id, blocks in pdf_dict.items():
  194. if page_id.startswith("page_"):
  195. blocks = pdf_dict[page_id]["para_blocks"]
  196. statistics["num_pages"] += 1
  197. for block_id, block_data in enumerate(blocks):
  198. statistics["num_blocks"] += 1
  199. if "paras" in block_data.keys():
  200. statistics["num_paras"] += len(block_data["paras"])
  201. for line in block_data["lines"]:
  202. if line.get("is_title", 0):
  203. statistics["num_titles"] += 1
  204. if block_data.get("is_header", 0):
  205. statistics["num_header_blocks"] += 1
  206. if block_data.get("is_footer", 0):
  207. statistics["num_footer_blocks"] += 1
  208. if block_data.get("is_watermark", 0):
  209. statistics["num_watermark_blocks"] += 1
  210. if block_data.get("is_vertical_margin_note", 0):
  211. statistics["num_vertical_margin_note_blocks"] += 1
  212. pdf_dict["statistics"] = statistics
  213. return pdf_dict