test_bench.py 2.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576
  1. """
  2. bench
  3. """
  4. import os
  5. import shutil
  6. import json
  7. from lib import calculate_score
  8. import pytest
  9. from conf import conf
  10. code_path = os.environ.get('GITHUB_WORKSPACE')
  11. pdf_dev_path = conf.conf["pdf_dev_path"]
  12. pdf_res_path = conf.conf["pdf_res_path"]
  13. class TestBench():
  14. """
  15. test bench
  16. """
  17. def ci_ben(self):
  18. """
  19. ci benchmark
  20. """
  21. try:
  22. fr = open(os.path.join(pdf_dev_path, "result.json"), "r", encoding="utf-8")
  23. lines = fr.readlines()
  24. last_line = lines[-1].strip()
  25. last_score = json.loads(last_line)
  26. print ("last_score:", last_score)
  27. last_simscore = last_score["average_sim_score"]
  28. last_editdistance = last_score["average_edit_distance"]
  29. last_bleu = last_score["average_bleu_score"]
  30. except IOError:
  31. print ("result.json not exist")
  32. test_cli()
  33. os.system(f"python lib/pre_clean.py --tool_name mineru --download_dir {pdf_dev_path}")
  34. now_score = get_score()
  35. print ("now_score:", now_score)
  36. now_simscore = now_score["average_sim_score"]
  37. now_editdistance = now_score["average_edit_distance"]
  38. now_bleu = now_score["average_bleu_score"]
  39. assert last_simscore <= now_simscore
  40. assert last_editdistance <= now_editdistance
  41. assert last_bleu <= now_bleu
  42. def get_score():
  43. """
  44. get score
  45. """
  46. data_path = os.path.join(pdf_dev_path, "ci")
  47. score = calculate_score.Scoring(os.path.join(data_path, "result.json"))
  48. score.calculate_similarity_total("mineru", data_path)
  49. res = score.summary_scores()
  50. return res
  51. def test_cli():
  52. """
  53. test pdf-command cli
  54. """
  55. rm_cmd = f"rm -rf {pdf_res_path}"
  56. os.system(rm_cmd)
  57. os.makedirs(pdf_res_path)
  58. cmd = f'magic-pdf pdf-command --pdf {os.path.join(pdf_dev_path, "mineru")}'
  59. os.system(cmd)
  60. for root, dirs, files in os.walk(pdf_res_path):
  61. for magic_file in files:
  62. target_dir = os.path.join(pdf_dev_path, "mineru")
  63. if magic_file.endswith(".md"):
  64. source_file = os.path.join(root, magic_file)
  65. target_file = os.path.join(pdf_dev_path, "mineru", magic_file)
  66. if not os.path.exists(target_dir):
  67. os.makedirs(target_dir)
  68. shutil.copy(source_file, target_file)