Browse Source

Merge pull request #1241 from will-jl944/develop_jf

RESTful 2.0 ready
FlyingQianMM 4 năm trước cách đây
mục cha
commit
ed5d75c83c

+ 3 - 2
paddlex_restful/restful/app.py

@@ -26,6 +26,7 @@ from . import workspace_pb2 as w
 from .utils import CustomEncoder, ShareData, is_pic, get_logger, TaskStatus, get_ip
 from paddlex_restful.restful.dataset.utils import get_encoding
 import numpy as np
+import pickle
 
 app = Flask(__name__)
 CORS(app, supports_credentials=True)
@@ -570,7 +571,7 @@ def task_evaluate():
                 'result'] is not None:
             if 'Confusion_Matrix' in ret['result']:
                 ret['result']['Confusion_Matrix'] = ret['result'][
-                    'Confusion_Matrix'].tolist()
+                    'Confusion_Matrix']
             ret['result'] = CustomEncoder().encode(ret['result'])
             ret['result'] = json.loads(ret['result'])
         ret['evaluate_status'] = ret['evaluate_status'].value
@@ -590,7 +591,7 @@ def task_evaluate_file():
             assert os.path.abspath(ret).startswith(
                 os.path.abspath(SD.workspace_dir)
             ) and ".." not in ret, "Illegal path {}.".format(ret)
-            return send_file(ret)
+            return pickle.load(open(os.path.abspath(ret), 'rb'))
         else:
             from .project.task import get_evaluate_result
             from .project.task import import_evaluate_excel

+ 4 - 9
paddlex_restful/restful/project/evaluate/classification.py

@@ -1,4 +1,4 @@
-# copyright (c) 2021 PaddlePaddle Authors. All Rights Reserve.
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,18 +17,13 @@ import yaml
 import os.path as osp
 import numpy as np
 from sklearn.metrics import confusion_matrix, roc_curve, auc
-from paddlex_restful.restful.dataset.utils import get_encoding
 
 
 class Evaluator(object):
     def __init__(self, model_path, topk=5):
-        model_yml = osp.join(model_path, "model.yml")
-        with open(model_yml, encoding=get_encoding(model_yml)) as f:
+        with open(osp.join(model_path, "model.yml")) as f:
             model_info = yaml.load(f.read(), Loader=yaml.Loader)
-        eval_details_file = osp.join(model_path, 'eval_details.json')
-        with open(
-                eval_details_file, 'r',
-                encoding=get_encoding(eval_details_file)) as f:
+        with open(osp.join(model_path, 'eval_details.json'), 'r') as f:
             eval_details = json.load(f)
         self.topk = topk
 
@@ -116,7 +111,7 @@ class Evaluator(object):
         '''生成评估报告。
         '''
         report = dict()
-        report['Confusion_Matrix'] = self.cal_confusion_matrix()
+        report['Confusion_Matrix'] = self.cal_confusion_matrix().tolist()
         report['PRF1_average'] = {}
         report['PRF1'], report['PRF1_average'][
             'over_all'] = self.cal_precision_recall_F1()

+ 4 - 3
paddlex_restful/restful/project/evaluate/detection.py

@@ -606,7 +606,8 @@ class DetEvaluator(object):
         '''生成评估报告。
         '''
         report = dict()
-        report['Confusion_Matrix'] = copy.deepcopy(self.cal_confusion_matrix())
+        report['Confusion_Matrix'] = copy.deepcopy(self.cal_confusion_matrix()
+                                                   .tolist())
         report['mAP'] = copy.deepcopy(self.cal_map())
         report['PRAP'] = copy.deepcopy(self.cal_precision_recall())
         report['label_list'] = copy.deepcopy(list(self.cname2cid.keys()))
@@ -769,7 +770,7 @@ class InsSegEvaluator(DetEvaluator):
         '''
         report = dict()
         report['BBox_Confusion_Matrix'] = copy.deepcopy(
-            self.cal_confusion_matrix())
+            self.cal_confusion_matrix().tolist())
         report['BBox_mAP'] = copy.deepcopy(self.cal_map())
         report['BBox_PRAP'] = copy.deepcopy(self.cal_precision_recall())
         report['label_list'] = copy.deepcopy(list(self.cname2cid.keys()))
@@ -779,7 +780,7 @@ class InsSegEvaluator(DetEvaluator):
             report['BBox_PRAP'][k]['AP'] = v
 
         report['Mask_Confusion_Matrix'] = copy.deepcopy(
-            self.cal_confusion_matrix_mask())
+            self.cal_confusion_matrix_mask().tolist())
         report['Mask_mAP'] = copy.deepcopy(self.cal_map_mask())
         report['Mask_PRAP'] = copy.deepcopy(self.cal_precision_recall_mask())
         per_ap_mask = copy.deepcopy(self.cal_ap_mask())

+ 1 - 1
paddlex_restful/restful/project/evaluate/segmentation.py

@@ -112,7 +112,7 @@ class Evaluator(object):
             category_iou_dict[self.labels[i]] = category_iou[i]
 
         report = dict()
-        report['Confusion_Matrix'] = self.cal_confusion_matrix()
+        report['Confusion_Matrix'] = self.cal_confusion_matrix().tolist()
         report['Mean_IoU'] = mean_iou
         report['Mean_Acc'] = mean_acc
         report['PRIoU'] = self.cal_precision_recall()

+ 10 - 10
paddlex_restful/restful/project/operate.py

@@ -93,7 +93,7 @@ def _call_paddlex_evaluate_model(task_path,
                                  topk=5,
                                  score_thresh=0.3,
                                  overlap_thresh=0.5):
-    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    evaluate_status_path = osp.join(task_path, 'logs/evaluate')
     sys.stdout = open(
         osp.join(evaluate_status_path, 'out.log'), 'w', encoding='utf-8')
     sys.stderr = open(
@@ -636,7 +636,7 @@ def evaluate_model(task_path,
     output_path = osp.join(task_path, 'output')
     if not osp.exists(osp.join(output_path, 'best_model')):
         raise Exception("未在训练路径{}下发现保存的best_model,无法进行评估".format(output_path))
-    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    evaluate_status_path = osp.join(task_path, 'logs/evaluate')
     safe_clean_folder(evaluate_status_path)
     if epoch is None:
         model_path = osp.join(output_path, 'best_model')
@@ -657,7 +657,7 @@ def get_evaluate_status(task_path):
     Args:
         task_path(str): 训练任务文件夹
     """
-    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    evaluate_status_path = osp.join(task_path, 'logs/evaluate')
     if not osp.exists(evaluate_status_path):
         return None, "No evaluate fold in path {}".format(task_path)
     status, message = get_folder_status(evaluate_status_path, True)
@@ -689,7 +689,7 @@ def get_predict_status(task_path):
         task_path(str): 从predict_path下的'XPRESTART'文件中获取训练的进程id
     """
     from ..utils import list_files
-    predict_status_path = osp.join(task_path, "./logs/predict")
+    predict_status_path = osp.join(task_path, "logs/predict")
     save_dir = osp.join(task_path, "visualized_test_results")
     if not osp.exists(save_dir):
         return None, "任务目录下没有visualized_test_results文件夹,{}".format(
@@ -744,7 +744,7 @@ def predict_test_pics(task_path,
         params_conf_file), "任务无法启动,路径{}下不存在参数配置文件params.pkl".format(task_path)
     with open(params_conf_file, 'rb') as f:
         params = pickle.load(f)
-    predict_status_path = osp.join(task_path, "./logs/predict")
+    predict_status_path = osp.join(task_path, "logs/predict")
     safe_clean_folder(predict_status_path)
     save_dir = osp.join(task_path, 'visualized_test_results')
     safe_clean_folder(save_dir)
@@ -764,7 +764,7 @@ def stop_predict_task(task_path):
         task_path(str): 从predict_path下的'XPRESTART'文件中获取训练的进程id
     """
     from ..utils import list_files
-    predict_status_path = osp.join(task_path, "./logs/predict")
+    predict_status_path = osp.join(task_path, "logs/predict")
     save_dir = osp.join(task_path, "visualized_test_results")
     if not osp.exists(save_dir):
         return None, "任务目录下没有visualized_test_results文件夹,{}".format(
@@ -813,7 +813,7 @@ def get_export_status(task_path):
     Return:
         导出的状态和其他消息.
     """
-    export_status_path = osp.join(task_path, './logs/export')
+    export_status_path = osp.join(task_path, 'logs/export')
     if not osp.exists(export_status_path):
         return None, "{}任务目录下没有export文件夹".format(task_path)
     status, message = get_folder_status(export_status_path, True)
@@ -848,7 +848,7 @@ def export_quant_model(task_path, save_dir, epoch=None):
     output_path = osp.join(task_path, 'output')
     if not osp.exists(osp.join(output_path, 'best_model')):
         raise Exception("未在训练路径{}下发现保存的best_model,导出失败".format(output_path))
-    export_status_path = osp.join(task_path, './logs/export')
+    export_status_path = osp.join(task_path, 'logs/export')
     safe_clean_folder(export_status_path)
 
     params_conf_file = osp.join(task_path, 'params.pkl')
@@ -875,7 +875,7 @@ def export_noquant_model(task_path, save_dir, epoch=None):
     output_path = osp.join(task_path, 'output')
     if not osp.exists(osp.join(output_path, 'best_model')):
         raise Exception("未在训练路径{}下发现保存的best_model,导出失败".format(output_path))
-    export_status_path = osp.join(task_path, './logs/export')
+    export_status_path = osp.join(task_path, 'logs/export')
     safe_clean_folder(export_status_path)
     p = mp.Process(
         target=_call_paddlex_export_infer,
@@ -902,7 +902,7 @@ def stop_export_task(task_path):
     Return:
         the export status and message.
     """
-    export_status_path = osp.join(task_path, './logs/export')
+    export_status_path = osp.join(task_path, 'logs/export')
     if not osp.exists(export_status_path):
         return None, "{}任务目录下没有export文件夹".format(task_path)
     status, message = get_folder_status(export_status_path, True)

+ 6 - 7
paddlex_restful/restful/project/task.py

@@ -334,7 +334,7 @@ def get_eval_all_metrics(data, workspace):
                 epoch_index = int(file_dir.split('_')[-1])
                 yml_file_path = osp.join(epoch_dir, "model.yml")
                 f = open(yml_file_path, 'r', encoding='utf-8')
-                yml_file = yaml.load(f.read())
+                yml_file = yaml.load(f.read(), Loader=yaml.Loader)
                 result = yml_file["_Attributes"]["eval_metrics"]
                 key = list(result.keys())[0]
                 value = result[key]
@@ -382,21 +382,20 @@ def start_train_task(data, workspace, monitored_processes):
     tid = data['tid']
     assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
     path = workspace.tasks[tid].path
-    if 'eval_metric_loss' in data and \
-        data['eval_metric_loss'] is not None:
+    if 'pruned_flops' in data and \
+        data['pruned_flops'] is not None:
         # 裁剪任务
         parent_id = workspace.tasks[tid].parent_id
         assert parent_id != "", "任务{}不是裁剪训练任务".format(tid)
         parent_path = workspace.tasks[parent_id].path
-        sensitivities_path = osp.join(parent_path, 'prune',
-                                      'sensitivities.data')
-        eval_metric_loss = data['eval_metric_loss']
+        sensitivities_path = osp.join(parent_path, 'prune', 'model.sensi.data')
+        pruned_flops = data['pruned_flops']
         parent_best_model_path = osp.join(parent_path, 'output', 'best_model')
         params_conf_file = osp.join(path, 'params.pkl')
         with open(params_conf_file, 'rb') as f:
             params = pickle.load(f)
         params['train'].sensitivities_path = sensitivities_path
-        params['train'].eval_metric_loss = eval_metric_loss
+        params['train'].pruned_flops = pruned_flops
         params['train'].pretrain_weights = parent_best_model_path
         with open(params_conf_file, 'wb') as f:
             pickle.dump(params, f)

+ 2 - 2
paddlex_restful/restful/project/train/params.py

@@ -89,8 +89,8 @@ class Params(object):
     def set_sensitivities_path(self, sensitivities_path):
         self.sensitivities_path = sensitivities_path
 
-    def set_eval_metric_loss(self, eval_metric_loss):
-        self.eval_metric_loss = eval_metric_loss
+    def set_pruned_flops(self, pruned_flops):
+        self.pruned_flops = pruned_flops
 
     def set_image_shape(self, image_shape):
         self.image_shape = image_shape

+ 4 - 2
paddlex_restful/restful/utils.py

@@ -29,13 +29,15 @@ import logging
 import socket
 import logging.handlers
 import requests
-import json
 from json import JSONEncoder
 
 
 class CustomEncoder(JSONEncoder):
     def default(self, o):
-        return o.__dict__
+        try:
+            return o.__dict__
+        except AttributeError:
+            return o.tolist()
 
 
 class ShareData():