processor_configs.yaml 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238
  1. # ============================================================================
  2. # PDF 批量处理器配置文件
  3. # ============================================================================
  4. # 处理器定义
  5. processors:
  6. # -------------------------------------------------------------------------
  7. # MinerU vLLM 处理器
  8. # 基于 MinerU 的多线程批量处理(支持 PDF 和图片)
  9. # -------------------------------------------------------------------------
  10. yusys_ocr_v4:
  11. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  12. input_arg: "--input"
  13. output_arg: "--output_dir"
  14. scene_arg: "--scene"
  15. extra_args:
  16. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_yusys_v4.yaml"
  17. - "--pages=1-35"
  18. - "--streaming"
  19. - "--debug"
  20. - "--log_level=DEBUG"
  21. output_subdir: "bank_statement_yusys_v4"
  22. log_subdir: "logs/bank_statement_yusys_v4"
  23. venv: "conda activate mineru2"
  24. description: "YUSYS Wired UNET OCR 框架 GLM-OCR"
  25. yusys_ocr_v3:
  26. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  27. input_arg: "--input"
  28. output_arg: "--output_dir"
  29. scene_arg: "--scene"
  30. extra_args:
  31. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_yusys_v3.yaml"
  32. # - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_yusys_v2.yaml"
  33. - "--pages=1-35"
  34. - "--streaming"
  35. # - "--debug"
  36. - "--log_level=INFO"
  37. output_subdir: "bank_statement_yusys_v3"
  38. log_subdir: "logs/bank_statement_yusys_v3"
  39. # output_subdir: "bank_statement_yusys_v2"
  40. # log_subdir: "logs/bank_statement_yusys_v2"
  41. venv: "conda activate mineru2"
  42. description: "YUSYS Wired UNET OCR 框架"
  43. yusys_mineru:
  44. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  45. input_arg: "--input"
  46. output_arg: "--output_dir"
  47. scene_arg: "--scene"
  48. extra_args:
  49. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_mineru_vl.yaml"
  50. # - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_yusys_v2.yaml"
  51. - "--pages=1-35"
  52. - "--streaming"
  53. - "--debug"
  54. - "--log_level=DEBUG"
  55. output_subdir: "bank_statement_mineru_vl"
  56. log_subdir: "logs/bank_statement_mineru_vl"
  57. # output_subdir: "bank_statement_yusys_v2"
  58. # log_subdir: "logs/bank_statement_yusys_v2"
  59. venv: "conda activate mineru2"
  60. description: "YUSYS MinerU OCR 框架"
  61. yusys_ocr_local:
  62. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  63. input_arg: "--input"
  64. output_arg: "--output_dir"
  65. scene_arg: "--scene"
  66. extra_args:
  67. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_yusys_local.yaml"
  68. - "--pages=1-35"
  69. - "--streaming"
  70. - "--debug"
  71. - "--log_level=DEBUG"
  72. output_subdir: "bank_statement_yusys_local"
  73. log_subdir: "logs/bank_statement_yusys_local"
  74. venv: "conda activate mineru2"
  75. description: "YUSYS(local) Wired UNET OCR GLM-OCR"
  76. yusys_paddleocr_local:
  77. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  78. input_arg: "--input"
  79. output_arg: "--output_dir"
  80. scene_arg: "--scene"
  81. extra_args:
  82. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_paddle_vl_local.yaml"
  83. - "--pages=1-35"
  84. - "--streaming"
  85. - "--debug"
  86. - "--log_level=DEBUG"
  87. output_subdir: "bank_statement_yusys_paddleocr_local"
  88. log_subdir: "logs/bank_statement_yusys_paddleocr_local"
  89. venv: "conda activate mineru2"
  90. description: "YUSYS(local) Wired UNET OCR PaddleOCR-VL"
  91. yusys_mineru_local:
  92. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/main_v2.py"
  93. input_arg: "--input"
  94. output_arg: "--output_dir"
  95. scene_arg: "--scene"
  96. extra_args:
  97. - "--config=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/universal_doc_parser/config/bank_statement_mineru_vl_local.yaml"
  98. - "--pages=1-35"
  99. - "--streaming"
  100. - "--debug"
  101. - "--log_level=DEBUG"
  102. output_subdir: "bank_statement_yusys_mineru_local"
  103. log_subdir: "logs/bank_statement_yusys_mineru_local"
  104. venv: "conda activate mineru2"
  105. description: "YUSYS(local) Wired UNET OCR MinerU-VL"
  106. # -------------------------------------------------------------------------
  107. # PaddleOCR-VL 处理器
  108. # -------------------------------------------------------------------------
  109. paddleocr_vl_single_process:
  110. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/paddle_vl_tool/main.py"
  111. input_arg: "--input"
  112. output_arg: "--output_dir"
  113. extra_args:
  114. - "--pipeline=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/paddle_common/config/PaddleOCR-VL-Client.yaml"
  115. - "--device=cpu"
  116. # - "--no-adapter"
  117. output_subdir: "paddleocr_vl_results"
  118. log_subdir: "logs/paddleocr_vl" # 🎯 新增:日志子目录
  119. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  120. description: "PaddleOCR-VL 处理器 - 视觉语言模型OCR"
  121. # -------------------------------------------------------------------------
  122. # PP-StructureV3 本地处理器
  123. # -------------------------------------------------------------------------
  124. ppstructurev3_single_process:
  125. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/ppstructure_tool/main.py"
  126. input_arg: "--input"
  127. output_arg: "--output_dir"
  128. extra_args:
  129. - "--pipeline=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/paddle_common/config/PP-StructureV3-zhch.yaml"
  130. - "--device=cpu"
  131. output_subdir: "ppstructurev3_results"
  132. log_subdir: "logs/ppstructurev3"
  133. venv: "conda activate paddle"
  134. description: "PP-StructureV3 处理器 - 本地处理"
  135. ppstructurev3_gpu:
  136. script: "/home/ubuntu/zhch/PaddleX/zhch/ppstructurev3_single_process.py"
  137. input_arg: "--input_file"
  138. output_arg: "--output_dir"
  139. extra_args:
  140. - "--pipeline=/home/ubuntu/zhch/PaddleX/zhch/my_config/PP-StructureV3.yaml"
  141. output_subdir: "ppstructurev3_gpu_results"
  142. log_subdir: "logs/ppstructurev3_gpu"
  143. venv: "conda activate paddle"
  144. description: "PP-StructureV3 处理器 - GPU加速"
  145. # -------------------------------------------------------------------------
  146. # PP-StructureV3 CPU 处理器
  147. # 明确使用 CPU 处理
  148. # -------------------------------------------------------------------------
  149. ppstructurev3_cpu:
  150. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/ppstructure_tool/main.py"
  151. input_arg: "--input"
  152. output_arg: "--output_dir"
  153. extra_args:
  154. - "--pipeline=/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/paddle_common/config/PP-StructureV3-zhch.yaml"
  155. - "--device=cpu"
  156. output_subdir: "ppstructurev3_cpu_results"
  157. log_subdir: "logs/ppstructurev3_cpu"
  158. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  159. description: "PP-StructureV3 处理器 - CPU处理"
  160. # -------------------------------------------------------------------------
  161. # PP-StructureV3 API 客户端 (默认)
  162. # 通过 HTTP API 调用远程服务
  163. # -------------------------------------------------------------------------
  164. ppstructurev3_single_client:
  165. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/ppstructure_tool/api_client.py"
  166. input_arg: "--input"
  167. output_arg: "--output_dir"
  168. extra_args:
  169. - "--api_url=http://10.192.72.11:20026/layout-parsing"
  170. - "--timeout=300"
  171. output_subdir: "ppstructurev3_client_results"
  172. log_subdir: "logs/ppstructurev3_client"
  173. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  174. description: "PP-StructureV3 HTTP API 客户端 - 远程服务"
  175. # -------------------------------------------------------------------------
  176. # MinerU vLLM 处理器
  177. # 基于 MinerU 的多线程批量处理(支持 PDF 和图片)
  178. # -------------------------------------------------------------------------
  179. mineru_vllm:
  180. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/mineru_vl_tool/main.py"
  181. input_arg: "--input"
  182. output_arg: "--output_dir"
  183. extra_args:
  184. - "--server_url=http://10.192.72.11:20006"
  185. - "--timeout=300"
  186. - "--batch_size=1"
  187. output_subdir: "mineru_vllm_results"
  188. log_subdir: "logs/mineru_vllm"
  189. venv: "conda activate mineru2"
  190. description: "MinerU vLLM 处理器 - 支持PDF和图片"
  191. # -------------------------------------------------------------------------
  192. # DotsOCR vLLM 处理器
  193. # 基于 DotsOCR 的批量处理(支持 PDF 和图片)
  194. # -------------------------------------------------------------------------
  195. dotsocr_vllm:
  196. script: "/Users/zhch158/workspace/repository.git/ocr_platform/ocr_tools/dots.ocr_vl_tool/main.py"
  197. input_arg: "--input"
  198. output_arg: "--output_dir"
  199. extra_args:
  200. - "--ip=10.192.72.11"
  201. - "--port=8101"
  202. - "--model_name=DotsOCR"
  203. - "--prompt_mode=prompt_layout_all_en"
  204. - "--batch_size=1"
  205. - "--max_workers=1"
  206. - "--dpi=200"
  207. output_subdir: "dotsocr_vllm_results"
  208. log_subdir: "logs/dotsocr_vllm"
  209. venv: "conda activate py312"
  210. description: "DotsOCR vLLM 处理器 - 支持PDF和图片"
  211. # ============================================================================
  212. # 全局配置
  213. # ============================================================================
  214. global:
  215. # PDF 文件基础目录
  216. base_dir: "/Users/zhch158/workspace/data/流水分析"
  217. # 默认输出子目录名称(如果处理器未指定)
  218. output_subdir: "results"
  219. # 🎯 新增:全局日志配置
  220. log_dir: "logs" # 全局日志目录(相对于 base_dir)
  221. log_retention_days: 30 # 日志保留天数
  222. log_level: "INFO" # 日志级别: DEBUG, INFO, WARNING, ERROR