Browse Source

feat: add timing logs for PDF byte preparation to improve performance monitoring

myhloli 3 weeks ago
parent
commit
4214634de8
1 changed files with 11 additions and 0 deletions
  1. 11 0
      mineru/cli/common.py

+ 11 - 0
mineru/cli/common.py

@@ -3,6 +3,7 @@ import io
 import json
 import os
 import copy
+import time
 from multiprocessing import Pool
 from pathlib import Path
 
@@ -81,6 +82,7 @@ def _convert_pdf_in_process(args):
 
 def _prepare_pdf_bytes(pdf_bytes_list, start_page_id, end_page_id):
     """准备处理PDF字节数据"""
+    start_time = time.time()
     # 准备参数列表
     args_list = [(pdf_bytes, start_page_id, end_page_id) for pdf_bytes in pdf_bytes_list]
 
@@ -88,6 +90,15 @@ def _prepare_pdf_bytes(pdf_bytes_list, start_page_id, end_page_id):
     with Pool(processes=min(len(pdf_bytes_list), min(os.cpu_count() or 1, 4))) as pool:
         result = pool.map(_convert_pdf_in_process, args_list)
 
+    logger.debug(f"Prepare PDF bytes cost: {round(time.time() - start_time, 2)}s")
+
+    start_time = time.time()
+    result = []
+    for pdf_bytes in pdf_bytes_list:
+        new_pdf_bytes = convert_pdf_bytes_to_bytes_by_pypdfium2(pdf_bytes, start_page_id, end_page_id)
+        result.append(new_pdf_bytes)
+    logger.debug(f"Prepare PDF bytes cost: {round(time.time() - start_time, 2)}s")
+
     return result