processor_configs.yaml 6.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151
  1. # ============================================================================
  2. # PDF 批量处理器配置文件
  3. # ============================================================================
  4. # 处理器定义
  5. processors:
  6. # -------------------------------------------------------------------------
  7. # MinerU vLLM 处理器
  8. # 基于 MinerU 的多线程批量处理(支持 PDF 和图片)
  9. # -------------------------------------------------------------------------
  10. yusys_ocr:
  11. script: "/Users/zhch158/workspace/repository.git/MinerU/zhch/universal_doc_parser/main_v2.py"
  12. input_arg: "--input"
  13. output_arg: "--output_dir"
  14. extra_args:
  15. - "--config=/Users/zhch158/workspace/repository.git/MinerU/zhch/universal_doc_parser/config/bank_statement_yusys_v2.yaml"
  16. - "--pages=1-35"
  17. - "--streaming"
  18. - "--debug"
  19. - "--log_level=DEBUG"
  20. output_subdir: "bank_statement_yusys_v2"
  21. log_subdir: "logs/bank_statement_yusys_v2"
  22. venv: "conda activate mineru2"
  23. description: "YUSYS统一OCR框架"
  24. # -------------------------------------------------------------------------
  25. # PaddleOCR-VL 处理器
  26. # -------------------------------------------------------------------------
  27. paddleocr_vl_single_process:
  28. script: "/Users/zhch158/workspace/repository.git/PaddleX/zhch/paddleocr_vl_single_process.py"
  29. input_arg: "--input_file"
  30. output_arg: "--output_dir"
  31. extra_args:
  32. - "--pipeline=/Users/zhch158/workspace/repository.git/PaddleX/zhch/my_config/PaddleOCR-VL-Client-RT-DETR-H_layout_17cls.yaml"
  33. - "--device=cpu"
  34. # - "--no-adapter"
  35. output_subdir: "paddleocr_vl_results"
  36. log_subdir: "logs/paddleocr_vl" # 🎯 新增:日志子目录
  37. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  38. description: "PaddleOCR-VL 处理器 - 视觉语言模型OCR"
  39. # -------------------------------------------------------------------------
  40. # PP-StructureV3 本地处理器
  41. # -------------------------------------------------------------------------
  42. ppstructurev3_single_process:
  43. script: "/home/ubuntu/zhch/PaddleX/zhch/ppstructurev3_single_process.py"
  44. input_arg: "--input_file"
  45. output_arg: "--output_dir"
  46. extra_args:
  47. - "--pipeline=/home/ubuntu/zhch/PaddleX/zhch/my_config/PP-StructureV3.yaml"
  48. - "--device=cpu"
  49. output_subdir: "ppstructurev3_results"
  50. log_subdir: "logs/ppstructurev3"
  51. venv: "conda activate paddle"
  52. description: "PP-StructureV3 处理器 - 本地处理"
  53. ppstructurev3_gpu:
  54. script: "/home/ubuntu/zhch/PaddleX/zhch/ppstructurev3_single_process.py"
  55. input_arg: "--input_file"
  56. output_arg: "--output_dir"
  57. extra_args:
  58. - "--pipeline=/home/ubuntu/zhch/PaddleX/zhch/my_config/PP-StructureV3.yaml"
  59. output_subdir: "ppstructurev3_gpu_results"
  60. log_subdir: "logs/ppstructurev3_gpu"
  61. venv: "conda activate paddle"
  62. description: "PP-StructureV3 处理器 - GPU加速"
  63. # -------------------------------------------------------------------------
  64. # PP-StructureV3 CPU 处理器
  65. # 明确使用 CPU 处理
  66. # -------------------------------------------------------------------------
  67. ppstructurev3_cpu:
  68. script: "/Users/zhch158/workspace/repository.git/PaddleX/zhch/ppstructurev3_single_process.py"
  69. input_arg: "--input_file"
  70. output_arg: "--output_dir"
  71. extra_args:
  72. - "--pipeline=/Users/zhch158/workspace/repository.git/PaddleX/zhch/my_config/PP-StructureV3-zhch.yaml"
  73. - "--device=cpu"
  74. output_subdir: "ppstructurev3_cpu_results"
  75. log_subdir: "logs/ppstructurev3_cpu"
  76. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  77. description: "PP-StructureV3 处理器 - CPU处理"
  78. # -------------------------------------------------------------------------
  79. # PP-StructureV3 API 客户端 (默认)
  80. # 通过 HTTP API 调用远程服务
  81. # -------------------------------------------------------------------------
  82. ppstructurev3_single_client:
  83. script: "/Users/zhch158/workspace/repository.git/PaddleX/zhch/ppstructurev3_single_client.py"
  84. input_arg: "--input_file"
  85. output_arg: "--output_dir"
  86. extra_args:
  87. - "--api_url=http://10.192.72.11:8111/layout-parsing"
  88. - "--timeout=300"
  89. output_subdir: "ppstructurev3_client_results"
  90. log_subdir: "logs/ppstructurev3_client"
  91. venv: "source /Users/zhch158/workspace/repository.git/PaddleX/paddle_env/bin/activate"
  92. description: "PP-StructureV3 HTTP API 客户端 - 远程服务"
  93. # -------------------------------------------------------------------------
  94. # MinerU vLLM 处理器
  95. # 基于 MinerU 的多线程批量处理(支持 PDF 和图片)
  96. # -------------------------------------------------------------------------
  97. mineru_vllm:
  98. script: "/Users/zhch158/workspace/repository.git/MinerU/zhch/mineru2_vllm_multthreads.py"
  99. input_arg: "--input_file"
  100. output_arg: "--output_dir"
  101. extra_args:
  102. - "--server_url=http://10.192.72.11:8121"
  103. - "--timeout=300"
  104. - "--batch_size=1"
  105. output_subdir: "mineru_vllm_results"
  106. log_subdir: "logs/mineru_vllm"
  107. venv: "conda activate mineru2"
  108. description: "MinerU vLLM 处理器 - 支持PDF和图片"
  109. # -------------------------------------------------------------------------
  110. # DotsOCR vLLM 处理器
  111. # 基于 DotsOCR 的批量处理(支持 PDF 和图片)
  112. # -------------------------------------------------------------------------
  113. dotsocr_vllm:
  114. script: "/Users/zhch158/workspace/repository.git/dots.ocr/zhch/dotsocr_vllm_multthreads.py"
  115. input_arg: "--input_file"
  116. output_arg: "--output_dir"
  117. extra_args:
  118. - "--ip=10.192.72.11"
  119. - "--port=8101"
  120. - "--model_name=DotsOCR"
  121. - "--prompt_mode=prompt_layout_all_en"
  122. - "--batch_size=1"
  123. - "--max_workers=1"
  124. - "--dpi=200"
  125. output_subdir: "dotsocr_vllm_results"
  126. log_subdir: "logs/dotsocr_vllm"
  127. venv: "conda activate py312"
  128. description: "DotsOCR vLLM 处理器 - 支持PDF和图片"
  129. # ============================================================================
  130. # 全局配置
  131. # ============================================================================
  132. global:
  133. # PDF 文件基础目录
  134. base_dir: "/Users/zhch158/workspace/data/流水分析"
  135. # 默认输出子目录名称(如果处理器未指定)
  136. output_subdir: "results"
  137. # 🎯 新增:全局日志配置
  138. log_dir: "logs" # 全局日志目录(相对于 base_dir)
  139. log_retention_days: 30 # 日志保留天数
  140. log_level: "INFO" # 日志级别: DEBUG, INFO, WARNING, ERROR