test_bench.py 1.8 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162
  1. """
  2. bench
  3. """
  4. import os
  5. import shutil
  6. import json
  7. from lib import calculate_score
  8. import pytest
  9. from conf import conf
  10. code_path = os.environ.get('GITHUB_WORKSPACE')
  11. pdf_dev_path = conf.conf["pdf_dev_path"]
  12. pdf_res_path = conf.conf["pdf_res_path"]
  13. class TestBench():
  14. """
  15. test bench
  16. """
  17. def __init__(self) -> None:
  18. """
  19. init
  20. """
  21. self.last_simscore = 0
  22. self.last_editdistance = 0
  23. self.last_bleu = 0
  24. def test_ci_ben(self):
  25. """
  26. ci benchmark
  27. """
  28. try:
  29. fr = open(os.path.join(pdf_dev_path, "result.json"), "r", encoding="utf-8")
  30. lines = fr.readlines()
  31. last_line = lines[-1].strip()
  32. last_score = json.loads(last_line)
  33. self.last_simscore = last_score["average_sim_score"]
  34. self.last_editdistance = last_score["average_edit_distance"]
  35. self.last_bleu = last_score["average_bleu_score"]
  36. except IOError:
  37. print ("result.json not exist")
  38. os.system(f"python lib/pre_clean.py --tool_name mineru --download_dir {pdf_dev_path}")
  39. now_score = get_score()
  40. print ("now_score:", now_score)
  41. fw = open(os.path.join(pdf_dev_path, "result.json"), "a", encoding="utf-8")
  42. fw.write(json.dumps(now_score) + "\n")
  43. now_simscore = now_score["average_sim_score"]
  44. now_editdistance = now_score["average_edit_distance"]
  45. now_bleu = now_score["average_bleu_score"]
  46. assert self.last_simscore <= now_simscore
  47. assert self.last_editdistance <= now_editdistance
  48. assert self.last_bleu <= now_bleu
  49. def get_score():
  50. """
  51. get score
  52. """
  53. score = calculate_score.Scoring(os.path.join(pdf_dev_path, "result.json"))
  54. score.calculate_similarity_total("mineru", pdf_dev_path)
  55. res = score.summary_scores()
  56. return res