base.py 25 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571
  1. # copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import absolute_import
  15. import paddle.fluid as fluid
  16. import os
  17. import sys
  18. import numpy as np
  19. import time
  20. import math
  21. import yaml
  22. import copy
  23. import json
  24. import functools
  25. import paddlex.utils.logging as logging
  26. from paddlex.utils import seconds_to_hms
  27. from paddlex.utils.utils import EarlyStop
  28. from paddlex.cv.transforms import arrange_transforms
  29. import paddlex
  30. from collections import OrderedDict
  31. from os import path as osp
  32. from paddle.fluid.framework import Program
  33. from .utils.pretrain_weights import get_pretrain_weights
  34. def dict2str(dict_input):
  35. out = ''
  36. for k, v in dict_input.items():
  37. try:
  38. v = round(float(v), 6)
  39. except:
  40. pass
  41. out = out + '{}={}, '.format(k, v)
  42. return out.strip(', ')
  43. class BaseAPI:
  44. def __init__(self, model_type):
  45. self.model_type = model_type
  46. # 现有的CV模型都有这个属性,而这个属且也需要在eval时用到
  47. self.num_classes = None
  48. self.labels = None
  49. self.version = paddlex.__version__
  50. if paddlex.env_info['place'] == 'cpu':
  51. self.places = fluid.cpu_places()
  52. else:
  53. self.places = fluid.cuda_places()
  54. self.exe = fluid.Executor(self.places[0])
  55. self.train_prog = None
  56. self.test_prog = None
  57. self.parallel_train_prog = None
  58. self.train_inputs = None
  59. self.test_inputs = None
  60. self.train_outputs = None
  61. self.test_outputs = None
  62. self.train_data_loader = None
  63. self.eval_metrics = None
  64. # 若模型是从inference model加载进来的,无法调用训练接口进行训练
  65. self.trainable = True
  66. # 是否使用多卡间同步BatchNorm均值和方差
  67. self.sync_bn = False
  68. # 当前模型状态
  69. self.status = 'Normal'
  70. # 已完成迭代轮数,为恢复训练时的起始轮数
  71. self.completed_epochs = 0
  72. def _get_single_card_bs(self, batch_size):
  73. if batch_size % len(self.places) == 0:
  74. return int(batch_size // len(self.places))
  75. else:
  76. raise Exception("Please support correct batch_size, \
  77. which can be divided by available cards({}) in {}"
  78. .format(paddlex.env_info['num'], paddlex.env_info[
  79. 'place']))
  80. def build_program(self):
  81. # 构建训练网络
  82. self.train_inputs, self.train_outputs = self.build_net(mode='train')
  83. self.train_prog = fluid.default_main_program()
  84. startup_prog = fluid.default_startup_program()
  85. # 构建预测网络
  86. self.test_prog = fluid.Program()
  87. with fluid.program_guard(self.test_prog, startup_prog):
  88. with fluid.unique_name.guard():
  89. self.test_inputs, self.test_outputs = self.build_net(
  90. mode='test')
  91. self.test_prog = self.test_prog.clone(for_test=True)
  92. def build_train_data_loader(self, dataset, batch_size):
  93. # 初始化data_loader
  94. if self.train_data_loader is None:
  95. self.train_data_loader = fluid.io.DataLoader.from_generator(
  96. feed_list=list(self.train_inputs.values()),
  97. capacity=64,
  98. use_double_buffer=True,
  99. iterable=True)
  100. batch_size_each_gpu = self._get_single_card_bs(batch_size)
  101. generator = dataset.generator(
  102. batch_size=batch_size_each_gpu, drop_last=True)
  103. self.train_data_loader.set_sample_list_generator(
  104. dataset.generator(batch_size=batch_size_each_gpu),
  105. places=self.places)
  106. def export_quant_model(self,
  107. dataset,
  108. save_dir,
  109. batch_size=1,
  110. batch_num=10,
  111. cache_dir="./temp"):
  112. arrange_transforms(
  113. model_type=self.model_type,
  114. class_name=self.__class__.__name__,
  115. transforms=dataset.transforms,
  116. mode='quant')
  117. dataset.num_samples = batch_size * batch_num
  118. try:
  119. from .slim.post_quantization import PaddleXPostTrainingQuantization
  120. PaddleXPostTrainingQuantization._collect_target_varnames
  121. except:
  122. raise Exception(
  123. "Model Quantization is not available, try to upgrade your paddlepaddle>=1.8.0"
  124. )
  125. is_use_cache_file = True
  126. if cache_dir is None:
  127. is_use_cache_file = False
  128. post_training_quantization = PaddleXPostTrainingQuantization(
  129. executor=self.exe,
  130. dataset=dataset,
  131. program=self.test_prog,
  132. inputs=self.test_inputs,
  133. outputs=self.test_outputs,
  134. batch_size=batch_size,
  135. batch_nums=batch_num,
  136. scope=None,
  137. algo='KL',
  138. quantizable_op_type=["conv2d", "depthwise_conv2d", "mul"],
  139. is_full_quantize=False,
  140. is_use_cache_file=is_use_cache_file,
  141. cache_dir=cache_dir)
  142. post_training_quantization.quantize()
  143. post_training_quantization.save_quantized_model(save_dir)
  144. model_info = self.get_model_info()
  145. model_info['status'] = 'Quant'
  146. # 保存模型输出的变量描述
  147. model_info['_ModelInputsOutputs'] = dict()
  148. model_info['_ModelInputsOutputs']['test_inputs'] = [
  149. [k, v.name] for k, v in self.test_inputs.items()
  150. ]
  151. model_info['_ModelInputsOutputs']['test_outputs'] = [
  152. [k, v.name] for k, v in self.test_outputs.items()
  153. ]
  154. with open(
  155. osp.join(save_dir, 'model.yml'), encoding='utf-8',
  156. mode='w') as f:
  157. yaml.dump(model_info, f)
  158. def net_initialize(self,
  159. startup_prog=None,
  160. pretrain_weights=None,
  161. fuse_bn=False,
  162. save_dir='.',
  163. sensitivities_file=None,
  164. eval_metric_loss=0.05,
  165. resume_checkpoint=None):
  166. if not resume_checkpoint:
  167. pretrain_dir = osp.join(save_dir, 'pretrain')
  168. if not os.path.isdir(pretrain_dir):
  169. if os.path.exists(pretrain_dir):
  170. os.remove(pretrain_dir)
  171. os.makedirs(pretrain_dir)
  172. if pretrain_weights is not None and not os.path.exists(
  173. pretrain_weights):
  174. if self.model_type == 'classifier':
  175. if pretrain_weights not in ['IMAGENET']:
  176. logging.warning(
  177. "Pretrain_weights for classifier should be defined as directory path or parameter file or 'IMAGENET' or None, but it is {}, so we force to set it as 'IMAGENET'".
  178. format(pretrain_weights))
  179. pretrain_weights = 'IMAGENET'
  180. elif self.model_type == 'detector':
  181. if pretrain_weights not in ['IMAGENET', 'COCO']:
  182. logging.warning(
  183. "Pretrain_weights for detector should be defined as directory path or parameter file or 'IMAGENET' or 'COCO' or None, but it is {}, so we force to set it as 'IMAGENET'".
  184. format(pretrain_weights))
  185. pretrain_weights = 'IMAGENET'
  186. elif self.model_type == 'segmenter':
  187. if pretrain_weights not in [
  188. 'IMAGENET', 'COCO', 'CITYSCAPES'
  189. ]:
  190. logging.warning(
  191. "Pretrain_weights for segmenter should be defined as directory path or parameter file or 'IMAGENET' or 'COCO' or 'CITYSCAPES', but it is {}, so we force to set it as 'IMAGENET'".
  192. format(pretrain_weights))
  193. pretrain_weights = 'IMAGENET'
  194. if hasattr(self, 'backbone'):
  195. backbone = self.backbone
  196. else:
  197. backbone = self.__class__.__name__
  198. if backbone == "HRNet":
  199. backbone = backbone + "_W{}".format(self.width)
  200. class_name = self.__class__.__name__
  201. pretrain_weights = get_pretrain_weights(
  202. pretrain_weights, class_name, backbone, pretrain_dir)
  203. if startup_prog is None:
  204. startup_prog = fluid.default_startup_program()
  205. self.exe.run(startup_prog)
  206. if resume_checkpoint:
  207. logging.info(
  208. "Resume checkpoint from {}.".format(resume_checkpoint),
  209. use_color=True)
  210. paddlex.utils.utils.load_pretrain_weights(
  211. self.exe, self.train_prog, resume_checkpoint, resume=True)
  212. if not osp.exists(osp.join(resume_checkpoint, "model.yml")):
  213. raise Exception("There's not model.yml in {}".format(
  214. resume_checkpoint))
  215. with open(osp.join(resume_checkpoint, "model.yml")) as f:
  216. info = yaml.load(f.read(), Loader=yaml.Loader)
  217. self.completed_epochs = info['completed_epochs']
  218. elif pretrain_weights is not None:
  219. logging.info(
  220. "Load pretrain weights from {}.".format(pretrain_weights),
  221. use_color=True)
  222. paddlex.utils.utils.load_pretrain_weights(
  223. self.exe, self.train_prog, pretrain_weights, fuse_bn)
  224. # 进行裁剪
  225. if sensitivities_file is not None:
  226. import paddleslim
  227. from .slim.prune_config import get_sensitivities
  228. sensitivities_file = get_sensitivities(sensitivities_file, self,
  229. save_dir)
  230. from .slim.prune import get_params_ratios, prune_program
  231. logging.info(
  232. "Start to prune program with eval_metric_loss = {}".format(
  233. eval_metric_loss),
  234. use_color=True)
  235. origin_flops = paddleslim.analysis.flops(self.test_prog)
  236. prune_params_ratios = get_params_ratios(
  237. sensitivities_file, eval_metric_loss=eval_metric_loss)
  238. prune_program(self, prune_params_ratios)
  239. current_flops = paddleslim.analysis.flops(self.test_prog)
  240. remaining_ratio = current_flops / origin_flops
  241. logging.info(
  242. "Finish prune program, before FLOPs:{}, after prune FLOPs:{}, remaining ratio:{}"
  243. .format(origin_flops, current_flops, remaining_ratio),
  244. use_color=True)
  245. self.status = 'Prune'
  246. def get_model_info(self):
  247. info = dict()
  248. info['version'] = paddlex.__version__
  249. info['Model'] = self.__class__.__name__
  250. info['_Attributes'] = {'model_type': self.model_type}
  251. if 'self' in self.init_params:
  252. del self.init_params['self']
  253. if '__class__' in self.init_params:
  254. del self.init_params['__class__']
  255. if 'model_name' in self.init_params:
  256. del self.init_params['model_name']
  257. info['_init_params'] = self.init_params
  258. info['_Attributes']['num_classes'] = self.num_classes
  259. info['_Attributes']['labels'] = self.labels
  260. info['_Attributes']['fixed_input_shape'] = self.fixed_input_shape
  261. try:
  262. primary_metric_key = list(self.eval_metrics.keys())[0]
  263. primary_metric_value = float(self.eval_metrics[primary_metric_key])
  264. info['_Attributes']['eval_metrics'] = {
  265. primary_metric_key: primary_metric_value
  266. }
  267. except:
  268. pass
  269. if hasattr(self, 'test_transforms'):
  270. if hasattr(self.test_transforms, 'to_rgb'):
  271. if self.test_transforms.to_rgb:
  272. info['TransformsMode'] = 'RGB'
  273. else:
  274. info['TransformsMode'] = 'BGR'
  275. if self.test_transforms is not None:
  276. info['Transforms'] = list()
  277. for op in self.test_transforms.transforms:
  278. name = op.__class__.__name__
  279. attr = op.__dict__
  280. info['Transforms'].append({name: attr})
  281. info['completed_epochs'] = self.completed_epochs
  282. return info
  283. def save_model(self, save_dir):
  284. if not osp.isdir(save_dir):
  285. if osp.exists(save_dir):
  286. os.remove(save_dir)
  287. os.makedirs(save_dir)
  288. if self.train_prog is not None:
  289. fluid.save(self.train_prog, osp.join(save_dir, 'model'))
  290. else:
  291. fluid.save(self.test_prog, osp.join(save_dir, 'model'))
  292. model_info = self.get_model_info()
  293. model_info['status'] = self.status
  294. with open(
  295. osp.join(save_dir, 'model.yml'), encoding='utf-8',
  296. mode='w') as f:
  297. yaml.dump(model_info, f)
  298. # 评估结果保存
  299. if hasattr(self, 'eval_details'):
  300. with open(osp.join(save_dir, 'eval_details.json'), 'w') as f:
  301. json.dump(self.eval_details, f)
  302. if self.status == 'Prune':
  303. # 保存裁剪的shape
  304. shapes = {}
  305. for block in self.train_prog.blocks:
  306. for param in block.all_parameters():
  307. pd_var = fluid.global_scope().find_var(param.name)
  308. pd_param = pd_var.get_tensor()
  309. shapes[param.name] = np.array(pd_param).shape
  310. with open(
  311. osp.join(save_dir, 'prune.yml'), encoding='utf-8',
  312. mode='w') as f:
  313. yaml.dump(shapes, f)
  314. # 模型保存成功的标志
  315. open(osp.join(save_dir, '.success'), 'w').close()
  316. logging.info("Model saved in {}.".format(save_dir))
  317. def export_inference_model(self, save_dir):
  318. test_input_names = [
  319. var.name for var in list(self.test_inputs.values())
  320. ]
  321. test_outputs = list(self.test_outputs.values())
  322. if self.__class__.__name__ == 'MaskRCNN':
  323. from paddlex.utils.save import save_mask_inference_model
  324. save_mask_inference_model(
  325. dirname=save_dir,
  326. executor=self.exe,
  327. params_filename='__params__',
  328. feeded_var_names=test_input_names,
  329. target_vars=test_outputs,
  330. main_program=self.test_prog)
  331. else:
  332. fluid.io.save_inference_model(
  333. dirname=save_dir,
  334. executor=self.exe,
  335. params_filename='__params__',
  336. feeded_var_names=test_input_names,
  337. target_vars=test_outputs,
  338. main_program=self.test_prog)
  339. model_info = self.get_model_info()
  340. model_info['status'] = 'Infer'
  341. # 保存模型输出的变量描述
  342. model_info['_ModelInputsOutputs'] = dict()
  343. model_info['_ModelInputsOutputs']['test_inputs'] = [
  344. [k, v.name] for k, v in self.test_inputs.items()
  345. ]
  346. model_info['_ModelInputsOutputs']['test_outputs'] = [
  347. [k, v.name] for k, v in self.test_outputs.items()
  348. ]
  349. with open(
  350. osp.join(save_dir, 'model.yml'), encoding='utf-8',
  351. mode='w') as f:
  352. yaml.dump(model_info, f)
  353. # 模型保存成功的标志
  354. open(osp.join(save_dir, '.success'), 'w').close()
  355. logging.info("Model for inference deploy saved in {}.".format(
  356. save_dir))
  357. def train_loop(self,
  358. num_epochs,
  359. train_dataset,
  360. train_batch_size,
  361. eval_dataset=None,
  362. save_interval_epochs=1,
  363. log_interval_steps=10,
  364. save_dir='output',
  365. use_vdl=False,
  366. early_stop=False,
  367. early_stop_patience=5):
  368. if train_dataset.num_samples < train_batch_size:
  369. raise Exception(
  370. 'The amount of training datset must be larger than batch size.')
  371. if not osp.isdir(save_dir):
  372. if osp.exists(save_dir):
  373. os.remove(save_dir)
  374. os.makedirs(save_dir)
  375. if use_vdl:
  376. from visualdl import LogWriter
  377. vdl_logdir = osp.join(save_dir, 'vdl_log')
  378. # 给transform添加arrange操作
  379. arrange_transforms(
  380. model_type=self.model_type,
  381. class_name=self.__class__.__name__,
  382. transforms=train_dataset.transforms,
  383. mode='train')
  384. # 构建train_data_loader
  385. self.build_train_data_loader(
  386. dataset=train_dataset, batch_size=train_batch_size)
  387. if eval_dataset is not None:
  388. self.eval_transforms = eval_dataset.transforms
  389. self.test_transforms = copy.deepcopy(eval_dataset.transforms)
  390. # 获取实时变化的learning rate
  391. lr = self.optimizer._learning_rate
  392. if isinstance(lr, fluid.framework.Variable):
  393. self.train_outputs['lr'] = lr
  394. # 在多卡上跑训练
  395. if self.parallel_train_prog is None:
  396. build_strategy = fluid.compiler.BuildStrategy()
  397. build_strategy.fuse_all_optimizer_ops = False
  398. if paddlex.env_info['place'] != 'cpu' and len(self.places) > 1:
  399. build_strategy.sync_batch_norm = self.sync_bn
  400. exec_strategy = fluid.ExecutionStrategy()
  401. exec_strategy.num_iteration_per_drop_scope = 1
  402. self.parallel_train_prog = fluid.CompiledProgram(
  403. self.train_prog).with_data_parallel(
  404. loss_name=self.train_outputs['loss'].name,
  405. build_strategy=build_strategy,
  406. exec_strategy=exec_strategy)
  407. total_num_steps = math.floor(train_dataset.num_samples /
  408. train_batch_size)
  409. num_steps = 0
  410. time_stat = list()
  411. time_train_one_epoch = None
  412. time_eval_one_epoch = None
  413. total_num_steps_eval = 0
  414. # 模型总共的评估次数
  415. total_eval_times = math.ceil(num_epochs / save_interval_epochs)
  416. # 检测目前仅支持单卡评估,训练数据batch大小与显卡数量之商为验证数据batch大小。
  417. eval_batch_size = train_batch_size
  418. if self.model_type == 'detector':
  419. eval_batch_size = self._get_single_card_bs(train_batch_size)
  420. if eval_dataset is not None:
  421. total_num_steps_eval = math.ceil(eval_dataset.num_samples /
  422. eval_batch_size)
  423. if use_vdl:
  424. # VisualDL component
  425. log_writer = LogWriter(vdl_logdir)
  426. thresh = 0.0001
  427. if early_stop:
  428. earlystop = EarlyStop(early_stop_patience, thresh)
  429. best_accuracy_key = ""
  430. best_accuracy = -1.0
  431. best_model_epoch = -1
  432. start_epoch = self.completed_epochs
  433. for i in range(start_epoch, num_epochs):
  434. records = list()
  435. step_start_time = time.time()
  436. epoch_start_time = time.time()
  437. for step, data in enumerate(self.train_data_loader()):
  438. outputs = self.exe.run(
  439. self.parallel_train_prog,
  440. feed=data,
  441. fetch_list=list(self.train_outputs.values()))
  442. outputs_avg = np.mean(np.array(outputs), axis=1)
  443. records.append(outputs_avg)
  444. # 训练完成剩余时间预估
  445. current_time = time.time()
  446. step_cost_time = current_time - step_start_time
  447. step_start_time = current_time
  448. if len(time_stat) < 20:
  449. time_stat.append(step_cost_time)
  450. else:
  451. time_stat[num_steps % 20] = step_cost_time
  452. # 每间隔log_interval_steps,输出loss信息
  453. num_steps += 1
  454. if num_steps % log_interval_steps == 0:
  455. step_metrics = OrderedDict(
  456. zip(list(self.train_outputs.keys()), outputs_avg))
  457. if use_vdl:
  458. for k, v in step_metrics.items():
  459. log_writer.add_scalar(
  460. 'Metrics/Training(Step): {}'.format(k), v,
  461. num_steps)
  462. # 估算剩余时间
  463. avg_step_time = np.mean(time_stat)
  464. if time_train_one_epoch is not None:
  465. eta = (num_epochs - i - 1) * time_train_one_epoch + (
  466. total_num_steps - step - 1) * avg_step_time
  467. else:
  468. eta = ((num_epochs - i) * total_num_steps - step - 1
  469. ) * avg_step_time
  470. if time_eval_one_epoch is not None:
  471. eval_eta = (
  472. total_eval_times - i // save_interval_epochs
  473. ) * time_eval_one_epoch
  474. else:
  475. eval_eta = (
  476. total_eval_times - i // save_interval_epochs
  477. ) * total_num_steps_eval * avg_step_time
  478. eta_str = seconds_to_hms(eta + eval_eta)
  479. logging.info(
  480. "[TRAIN] Epoch={}/{}, Step={}/{}, {}, time_each_step={}s, eta={}"
  481. .format(i + 1, num_epochs, step + 1, total_num_steps,
  482. dict2str(step_metrics),
  483. round(avg_step_time, 2), eta_str))
  484. train_metrics = OrderedDict(
  485. zip(list(self.train_outputs.keys()), np.mean(
  486. records, axis=0)))
  487. logging.info('[TRAIN] Epoch {} finished, {} .'.format(
  488. i + 1, dict2str(train_metrics)))
  489. time_train_one_epoch = time.time() - epoch_start_time
  490. epoch_start_time = time.time()
  491. # 每间隔save_interval_epochs, 在验证集上评估和对模型进行保存
  492. self.completed_epochs += 1
  493. eval_epoch_start_time = time.time()
  494. if (i + 1) % save_interval_epochs == 0 or i == num_epochs - 1:
  495. current_save_dir = osp.join(save_dir, "epoch_{}".format(i + 1))
  496. if not osp.isdir(current_save_dir):
  497. os.makedirs(current_save_dir)
  498. if eval_dataset is not None and eval_dataset.num_samples > 0:
  499. self.eval_metrics, self.eval_details = self.evaluate(
  500. eval_dataset=eval_dataset,
  501. batch_size=eval_batch_size,
  502. epoch_id=i + 1,
  503. return_details=True)
  504. logging.info('[EVAL] Finished, Epoch={}, {} .'.format(
  505. i + 1, dict2str(self.eval_metrics)))
  506. # 保存最优模型
  507. best_accuracy_key = list(self.eval_metrics.keys())[0]
  508. current_accuracy = self.eval_metrics[best_accuracy_key]
  509. if current_accuracy > best_accuracy:
  510. best_accuracy = current_accuracy
  511. best_model_epoch = i + 1
  512. best_model_dir = osp.join(save_dir, "best_model")
  513. self.save_model(save_dir=best_model_dir)
  514. if use_vdl:
  515. for k, v in self.eval_metrics.items():
  516. if isinstance(v, list):
  517. continue
  518. if isinstance(v, np.ndarray):
  519. if v.size > 1:
  520. continue
  521. log_writer.add_scalar(
  522. "Metrics/Eval(Epoch): {}".format(k), v, i + 1)
  523. self.save_model(save_dir=current_save_dir)
  524. time_eval_one_epoch = time.time() - eval_epoch_start_time
  525. eval_epoch_start_time = time.time()
  526. if best_model_epoch > 0:
  527. logging.info(
  528. 'Current evaluated best model in eval_dataset is epoch_{}, {}={}'
  529. .format(best_model_epoch, best_accuracy_key,
  530. best_accuracy))
  531. if eval_dataset is not None and early_stop:
  532. if earlystop(current_accuracy):
  533. break