Browse Source

refactor: optimize page range calculation and enhance logging for image conversion process

myhloli 3 weeks ago
parent
commit
245ae28c27
2 changed files with 11 additions and 6 deletions
  1. 3 0
      mineru/backend/pipeline/pipeline_analyze.py
  2. 8 6
      mineru/utils/pdf_image_tools.py

+ 3 - 0
mineru/backend/pipeline/pipeline_analyze.py

@@ -99,7 +99,10 @@ def doc_analyze(
         _lang = lang_list[pdf_idx]
 
         # 收集每个数据集中的页面
+        # load_images_start = time.time()
         images_list, pdf_doc = load_images_from_pdf(pdf_bytes, image_type=ImageType.PIL)
+        # load_images_time = round(time.time() - load_images_start, 2)
+        # logger.debug(f"load images cost: {load_images_time}, speed: {round(len(images_list) / load_images_time, 3)} images/s")
         all_image_lists.append(images_list)
         all_pdf_docs.append(pdf_doc)
         for page_idx in range(len(images_list)):

+ 8 - 6
mineru/utils/pdf_image_tools.py

@@ -83,10 +83,12 @@ def load_images_from_pdf(
             # 最后一个线程处理剩余所有页面
             range_end = end_page_id
         else:
-            range_end = min(start_page_id + (i + 1) * pages_per_thread - 1, end_page_id)
+            range_end = start_page_id + (i + 1) * pages_per_thread - 1
 
         page_ranges.append((range_start, range_end))
 
+    # logger.debug(f"PDF to images using {actual_threads} processes, page ranges: {page_ranges}")
+
     with ProcessPoolExecutor(max_workers=actual_threads) as executor:
         # 提交所有任务
         futures = []
@@ -134,11 +136,11 @@ def load_images_from_pdf_core(
     pdf_page_num = len(pdf_doc)
     end_page_id = get_end_page_id(end_page_id, pdf_page_num)
 
-    for index in range(0, pdf_page_num):
-        if start_page_id <= index <= end_page_id:
-            page = pdf_doc[index]
-            image_dict = pdf_page_to_image(page, dpi=dpi, image_type=image_type)
-            images_list.append(image_dict)
+    for index in range(start_page_id, end_page_id+1):
+        # logger.debug(f"Converting page {index}/{pdf_page_num} to image")
+        page = pdf_doc[index]
+        image_dict = pdf_page_to_image(page, dpi=dpi, image_type=image_type)
+        images_list.append(image_dict)
 
     pdf_doc.close()