s3pdf2md.py 1.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445
  1. import os
  2. import sys
  3. from pathlib import Path
  4. import click
  5. import json
  6. from loguru import logger
  7. from libs.commons import join_path, parse_aws_param, parse_bucket_key, read_file
  8. from mkcontent import mk_nlp_markdown
  9. from pdf2md import main
  10. from pdf_parse_by_model import parse_pdf_by_model
  11. @click.command()
  12. @click.option("--pdf-file-path", help="s3上pdf文件的路径")
  13. @click.option("--pdf-name", help="pdf name")
  14. def main_shell(pdf_file_path: str, pdf_name: str):
  15. with open('/mnt/petrelfs/share_data/ouyanglinke/OCR/OCR_validation_dataset_final_rotated_formulafix_highdpi_scihub.json', 'r') as f:
  16. samples = json.load(f)
  17. for sample in samples:
  18. pdf_file_path = sample['s3_path']
  19. pdf_bin_file_profile = "outsider"
  20. pdf_name = sample['pdf_name']
  21. pdf_model_dir = f"s3://llm-pdf-text/eval_1k/layout_res/{pdf_name}"
  22. pdf_model_profile = "langchao"
  23. p = Path(pdf_file_path)
  24. pdf_file_name = p.name # pdf文件名字,含后缀
  25. #pdf_model_dir = join_path(pdf_model_parent_dir, pdf_file_name)
  26. main(
  27. pdf_file_path,
  28. pdf_bin_file_profile,
  29. pdf_model_dir,
  30. pdf_model_profile,
  31. debug_mode=True,
  32. )
  33. if __name__ == "__main__":
  34. main_shell()