Browse Source

Merge pull request #436 from syyxsxx/restful

Restful
Jason 5 years ago
parent
commit
5dc922d8b1
43 changed files with 9008 additions and 1 deletions
  1. 1 0
      paddlex/__init__.py
  2. 29 0
      paddlex/command.py
  3. 15 0
      paddlex/restful/__init__.py
  4. 903 0
      paddlex/restful/app.py
  5. 13 0
      paddlex/restful/dataset/__init__.py
  6. 159 0
      paddlex/restful/dataset/cls_dataset.py
  7. 239 0
      paddlex/restful/dataset/dataset.py
  8. 86 0
      paddlex/restful/dataset/datasetbase.py
  9. 190 0
      paddlex/restful/dataset/det_dataset.py
  10. 314 0
      paddlex/restful/dataset/ins_seg_dataset.py
  11. 142 0
      paddlex/restful/dataset/operate.py
  12. 222 0
      paddlex/restful/dataset/seg_dataset.py
  13. 223 0
      paddlex/restful/dataset/utils.py
  14. 167 0
      paddlex/restful/demo.py
  15. 45 0
      paddlex/restful/dir.py
  16. 298 0
      paddlex/restful/model.py
  17. 13 0
      paddlex/restful/project/__init__.py
  18. 13 0
      paddlex/restful/project/evaluate/__init__.py
  19. 126 0
      paddlex/restful/project/evaluate/classification.py
  20. 783 0
      paddlex/restful/project/evaluate/detection.py
  21. 181 0
      paddlex/restful/project/evaluate/draw_pred_result.py
  22. 117 0
      paddlex/restful/project/evaluate/segmentation.py
  23. 914 0
      paddlex/restful/project/operate.py
  24. 143 0
      paddlex/restful/project/project.py
  25. 13 0
      paddlex/restful/project/prune/__init__.py
  26. 30 0
      paddlex/restful/project/prune/classification.py
  27. 46 0
      paddlex/restful/project/prune/detection.py
  28. 32 0
      paddlex/restful/project/prune/segmentation.py
  29. 797 0
      paddlex/restful/project/task.py
  30. 13 0
      paddlex/restful/project/train/__init__.py
  31. 134 0
      paddlex/restful/project/train/classification.py
  32. 218 0
      paddlex/restful/project/train/detection.py
  33. 438 0
      paddlex/restful/project/train/params.py
  34. 264 0
      paddlex/restful/project/train/params_v2.py
  35. 171 0
      paddlex/restful/project/train/segmentation.py
  36. 244 0
      paddlex/restful/project/visualize.py
  37. 88 0
      paddlex/restful/system.py
  38. 753 0
      paddlex/restful/utils.py
  39. 83 0
      paddlex/restful/workspace.proto
  40. 325 0
      paddlex/restful/workspace.py
  41. 21 0
      paddlex/restful/workspace_pb2.py
  42. 1 0
      requirements.txt
  43. 1 1
      setup.py

+ 1 - 0
paddlex/__init__.py

@@ -47,6 +47,7 @@ from . import slim
 from . import converter
 from . import tools
 from . import deploy
+from . import restful
 
 try:
     import pycocotools

+ 29 - 0
paddlex/command.py

@@ -120,6 +120,25 @@ def arg_parser():
         "-tv",
         default=None,
         help="define the value of test dataset(E.g 0.1)")
+    parser.add_argument(
+        "--start_restful",
+        "-sr",
+        action="store_true",
+        default=False,
+        help="start paddlex restful server")
+    parser.add_argument(
+        "--port",
+        "--pt",
+        type=_text_type,
+        default=None,
+        help="set the port of restful server")
+    parser.add_argument(
+        "--workspace_dir",
+        "--wd",
+        type=_text_type,
+        default=None,
+        help="set the workspace dir of restful server")
+
     return parser
 
 
@@ -226,6 +245,16 @@ def main():
         pdx.tools.split.dataset_split(dataset_dir, dataset_format, val_value,
                                       test_value, save_dir)
 
+    if args.start_restful:
+
+        assert args.port is not None, "--port should be defined while start restful server"
+        assert args.workspace_dir, "--workspace_dir should be define while start restful server"
+
+        port = args.port
+        workspace_dir = args.workspace_dir
+
+        pdx.restful.app.run(port, workspace_dir)
+
 
 if __name__ == "__main__":
     main()

+ 15 - 0
paddlex/restful/__init__.py

@@ -0,0 +1,15 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .app import run

+ 903 - 0
paddlex/restful/app.py

@@ -0,0 +1,903 @@
+from flask import Flask, request, render_template, send_from_directory, jsonify, session, send_file
+from werkzeug.utils import secure_filename
+from flask_cors import CORS
+import argparse
+from os import path as osp
+import os
+import time
+import json
+import sys
+import multiprocessing as mp
+from . import workspace_pb2 as w
+from .utils import CustomEncoder, ShareData, is_pic, get_logger
+import numpy as np
+
+app = Flask(__name__)
+CORS(app, supports_credentials=True)
+SESSION_TYPE = 'filesystem'
+app.config.from_object(__name__)
+SD = ShareData()
+
+
+def init(dirname, logger):
+    #初始化工作空间
+    from .workspace import init_workspace
+    from .system import get_system_info
+    SD.workspace = w.Workspace(path=dirname)
+    init_workspace(SD.workspace, dirname, logger)
+    SD.workspace_dir = dirname
+    get_system_info(SD.machine_info)
+
+
+@app.errorhandler(Exception)
+def handle_exception(e):
+    ret = {"status": -1, 'message': repr(e)}
+    return ret
+
+
+@app.route('/workspace', methods=['GET', 'PUT'])
+def workspace():
+    """
+    methods=='GET':获取工作目录中项目、数据集、任务的属性
+        Args:
+            struct(str):结构类型,可以是'dataset', 'project'或'task',
+            id(str):结构类型对应的id
+            attr_list(list):需要获取的属性的列表
+        Return:
+            attr(dict):key为属性,value为属性的值,
+            status
+    methods=='PUT':修改工作目录中项目、数据集、任务的属性
+        Args:
+            struct(str):结构类型,可以是'dataset', 'project'或'task',
+            id(str):结构类型对应的id
+            attr_dict(dict):key:需要修改的属性,value:需要修改属性的值
+        Return:
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if data:
+            from .workspace import get_attr
+            ret = get_attr(data, SD.workspace)
+            return ret
+        return {'status': 1, 'dirname': SD.workspace_dir}
+    if request.method == 'PUT':
+        from .workspace import set_attr
+        ret = set_attr(data, SD.workspace)
+        return ret
+
+
+@app.route('/dataset', methods=['GET', 'POST', 'PUT', 'DELETE'])
+def dataset():
+    """
+    methods=='GET':获取所有数据集或者单个数据集的信息
+        Args:
+            did(str, optional):数据集id(可选),如果存在就返回数据集id对应数据集的信息
+        Ruturn:
+            status
+            if 'did' in Args:
+                id(str):数据集id,
+                dataset_status(int):数据集状态(DatasetStatus)枚举变量的值
+                message(str):数据集状态信息
+                attr(dict):数据集属性
+            else:
+                datasets(list):所有数据集属性的列表
+
+    methods=='POST':创建一个新的数据集
+        Args:
+            name(str):数据集名字
+            desc(str):数据集描述
+            dataset_type(str):数据集类型,可以是['classification', 'detection', 'segmentation','instance_segmentation','remote_segmentation']
+        Return:
+            did(str):数据集id
+            status
+
+    methods=='PUT':异步,向数据集导入数据,支持分类、检测、语义分割、实例分割、摇杆分割数据集类型
+        Args:
+            did(str):数据集id
+            path(str):数据集路径
+        Return:
+            status
+
+    methods=='DELETE':删除已有的某个数据集
+        Args:
+            did(str):数据集id
+        Return:
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if 'did' in data:
+            from .dataset.dataset import get_dataset_status
+            ret = get_dataset_status(data, SD.workspace)
+            return ret
+        from .dataset.dataset import list_datasets
+        ret = list_datasets(SD.workspace)
+        return ret
+    if request.method == 'POST':
+        from .dataset.dataset import create_dataset
+        ret = create_dataset(data, SD.workspace)
+        return ret
+    if request.method == 'PUT':
+        from .dataset.dataset import import_dataset
+        ret = import_dataset(data, SD.workspace, SD.monitored_processes,
+                             SD.load_demo_proc_dict)
+        return ret
+
+    if request.method == 'DELETE':
+        from .dataset.dataset import delete_dataset
+        ret = delete_dataset(data, SD.workspace)
+        return ret
+
+
+@app.route('/dataset/details', methods=['GET'])
+def dataset_details():
+    """
+    methods=='GET':获取某个数据集的详细信息
+        Args:
+            did(str):数据集id
+        Return:
+            details(dict):数据集详细信息,
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .dataset.dataset import get_dataset_details
+        ret = get_dataset_details(data, SD.workspace)
+        return ret
+
+
+@app.route('/dataset/split', methods=['PUT'])
+def dataset_split():
+    """
+    Args:
+        did(str):数据集id
+        val_split(float): 验证集比例
+        test_split(float): 测试集比例
+    Return:
+        status
+    """
+    data = request.get_json()
+    if request.method == 'PUT':
+        from .dataset.dataset import split_dataset
+        ret = split_dataset(data, SD.workspace)
+        return ret
+
+
+@app.route('/dataset/image', methods=['GET'])
+def dataset_img_base64():
+    """
+    Args:
+        GET: 获取图片base64数据,参数:'path' 图片绝对路径
+    """
+    data = request.get_json()
+    if request.method == 'GET':
+        from .dataset.dataset import img_base64
+        ret = img_base64(data)
+        return ret
+
+
+@app.route('/dataset/file', methods=['GET'])
+def get_image_file():
+    """
+    Args:
+        GET: 获取文件数据,参数:'path' 文件绝对路径
+    """
+    data = request.get_json()
+    if request.method == 'GET':
+        ret = data['path']
+        return send_file(ret)
+
+
+@app.route('/dataset/npy', methods=['GET'])
+def get_npyfile():
+    """
+    Args:
+        GET: 获取文件数据,参数:'path' npy文件绝对路径
+    """
+    data = request.get_json()
+    if request.method == 'GET':
+        npy = np.load(data['path'], allow_pickle=True).tolist()
+        npy['gt_bbox'] = npy['gt_bbox'].tolist()
+        return npy
+
+
+@app.route('/file', methods=['GET'])
+def get_file():
+    """
+    Args:
+        path'(str):文件在服务端的路径
+	Return:
+		#数据为图片
+		img_data(str): base64图片数据
+		status
+		#数据为xml文件
+		ret:数据流
+		#数据为log文件
+		ret:json数据
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        path = data['path']
+        if not os.path.exists(path):
+            return {'status': -1}
+        if is_pic(path):
+            from .dataset.dataset import img_base64
+            ret = img_base64(data)
+            return ret
+        file_type = path[(path.rfind('.') + 1):]
+        if file_type in ['xml', 'npy', 'log']:
+            return send_file(path)
+        else:
+            pass
+
+
+@app.route('/project', methods=['GET', 'POST', 'DELETE'])
+def project():
+    """
+    methods=='GET':获取指定项目id的信息
+        Args:
+            'id'(str, optional):项目id,可选,如果存在就返回项目id对应项目的信息
+        Return:
+            status,
+            if 'id' in Args:
+                attr(dict):项目属性
+            else:
+                projects(list):所有项目属性
+
+    methods=='POST':创建一个项目
+        Args:
+            name(str): 项目名
+            desc(str):项目描述
+            project_type(str):项目类型
+        Return:
+            pid(str):项目id
+            status
+
+    methods=='DELETE':删除一个项目,以及项目相关的task
+        Args:
+            pid(str):项目id
+        Return:
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .project.project import list_projects
+        from .project.project import get_project
+        if 'id' in data:
+            ret = get_project(data, SD.workspace)
+            return ret
+        ret = list_projects(SD.workspace)
+        return ret
+    if request.method == 'POST':
+        from .project.project import create_project
+        ret = create_project(data, SD.workspace)
+        return ret
+    if request.method == 'DELETE':
+        from .project.project import delete_project
+        ret = delete_project(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task', methods=['GET', 'POST', 'DELETE'])
+def task():
+    """
+    methods=='GET':#获取某个任务的信息或者所有任务的信息
+        Args:
+            tid(str, optional):任务id,可选,若存在即返回id对应任务的信息
+            resume(str, optional):获取是否可以恢复训练的状态,可选,需在存在tid的情况下才生效
+            pid(str, optional):项目id,可选,若存在即返回该项目id下所有任务信息
+        Return:
+            status
+            if 'tid' in Args:
+                task_status(int):任务状态(TaskStatus)枚举变量的值
+                message(str):任务状态信息
+                resumable(bool):仅Args中存在resume时返回,任务训练是否可以恢复
+                max_saved_epochs(int):仅Args中存在resume时返回,当前训练模型保存的最大epoch
+            else:
+                tasks(list):所有任务属性
+
+    methods=='POST':#创建任务(训练或者裁剪)
+        Args:
+            pid(str):项目id
+            train(dict):训练参数
+            desc(str, optional):任务描述,可选
+            parent_id(str, optional):可选,若存在即表示新建的任务为裁剪任务,parent_id的值为裁剪任务对应的训练任务id
+        Return:
+            tid(str):任务id
+            status
+
+    methods=='DELETE':#删除任务
+        Args:
+            tid(str):任务id
+        Return:
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if data:
+            if 'pid' not in data:
+                from .project.task import get_task_status
+                ret = get_task_status(data, SD.workspace)
+                return ret
+        from .project.task import list_tasks
+        ret = list_tasks(data, SD.workspace)
+        return ret
+    if request.method == 'POST':
+        from .project.task import create_task
+        ret = create_task(data, SD.workspace)
+        return ret
+    if request.method == 'DELETE':
+        from .project.task import delete_task
+        ret = delete_task(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task/params', methods=['GET', 'POST'])
+def task_params():
+    """
+    methods=='GET':#获取任务id对应的参数,或者获取项目默认参数
+        Args:
+            tid(str, optional):获取任务对应的参数
+            pid(str,optional):获取项目对应的默认参数
+            model_type(str,optional):pid存在下有效,对应项目下获取指定模型的默认参数
+            gpu_list(list,optional):pid存在下有效,默认值为[0],使用指定的gpu并获取相应的默认参数
+        Return:
+            train(dict):训练或者裁剪的参数
+            status
+
+    methods=='POST':#设置任务参数,将前端用户设置训练参数dict保存在后端的pkl文件中
+        Args:
+            tid(str):任务id
+            train(dict):训练参数
+        Return:
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if 'tid' in data:
+            from .project.task import get_task_params
+            ret = get_task_params(data, SD.workspace)
+            ret['train'] = CustomEncoder().encode(ret['train'])
+            ret['train'] = json.loads(ret['train'])
+            return ret
+        if 'pid' in data:
+            from .project.task import get_default_params
+            ret = get_default_params(data, SD.workspace, SD.machine_info)
+            return ret
+    if request.method == 'POST':
+        from .project.task import set_task_params
+        ret = set_task_params(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task/metrics', methods=['GET'])
+def task_metrics():
+    """
+    methods=='GET':#获取日志数据
+        Args:
+            tid(str):任务id
+            type(str):可以获取日志的类型,[train,eval,sensitivities,prune],包括训练,评估,敏感度与模型裁剪率关系图,裁剪的日志
+        Return:
+            status
+            if type == 'train':
+                train_log(dict): 训练日志
+            elif type == 'eval':
+                eval_metrics(dict): 评估结果
+            elif type == 'sensitivities':
+                sensitivities_loss_img(dict): 敏感度与模型裁剪率关系图
+            elif type == 'prune':
+                prune_log(dict):裁剪日志
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if data['type'] == 'train':
+            from .project.task import get_train_metrics
+            ret = get_train_metrics(data, SD.workspace)
+            return ret
+        if data['type'] == 'eval':
+            from .project.task import get_eval_metrics
+            ret = get_eval_metrics(data, SD.workspace)
+            return ret
+        if data['type'] == 'eval_all':
+            from .project.task import get_eval_all_metrics
+            ret = get_eval_all_metrics(data, SD.workspace)
+            return ret
+        if data['type'] == 'sensitivities':
+            from .project.task import get_sensitivities_loss_img
+            ret = get_sensitivities_loss_img(data, SD.workspace)
+            return ret
+        if data['type'] == 'prune':
+            from .project.task import get_prune_metrics
+            ret = get_prune_metrics(data, SD.workspace)
+            return ret
+
+
+@app.route('/project/task/train', methods=['POST', 'PUT'])
+def task_train():
+    """
+    methods=='POST':#异步,启动训练或者裁剪任务
+        Args:
+            tid(str):任务id
+            eval_metric_loss(int,optional):可选,裁剪任务时可用,裁剪任务所需的评估loss
+        Return:
+            status
+
+    methods=='PUT':#改变任务训练的状态,即终止训练或者恢复训练
+        Args:
+            tid(str):任务id
+            act(str):[stop,resume]暂停或者恢复
+            epoch(int):(resume下可以设置)恢复训练的起始轮数
+        Return:
+            status
+    """
+    data = request.get_json()
+    if request.method == 'POST':
+        from .project.task import start_train_task
+        ret = start_train_task(data, SD.workspace, SD.monitored_processes)
+        return ret
+    if request.method == 'PUT':
+        if data['act'] == 'resume':
+            from .project.task import resume_train_task
+            ret = resume_train_task(data, SD.workspace, SD.monitored_processes)
+            return ret
+        if data['act'] == 'stop':
+            from .project.task import stop_train_task
+            ret = stop_train_task(data, SD.workspace)
+            return ret
+
+
+@app.route('/project/task/train/file', methods=['GET'])
+def log_file():
+    data = request.get_json()
+    if request.method == 'GET':
+        path = data['path']
+        if not os.path.exists(path):
+            return {'status': -1}
+        logs = open(path, encoding='utf-8').readlines()
+        if len(logs) < 50:
+            return {'status': 1, 'log': logs}
+        else:
+            logs = logs[-50:]
+            return {'status': 1, 'log': logs}
+
+
+@app.route('/project/task/prune', methods=['GET', 'POST', 'PUT'])
+def task_prune():
+    """
+    methods=='GET':#获取裁剪任务的状态
+        Args:
+            tid(str):任务id
+        Return:
+            prune_status(int): 裁剪任务状态(PruneStatus)枚举变量的值
+            status
+
+    methods=='POST':#异步,创建一个裁剪分析,对于启动裁剪任务前需要先启动裁剪分析
+        Args:
+            tid(str):任务id
+        Return:
+            status
+
+    methods=='PUT':#改变裁剪分析任务的状态
+        Args:
+            tid(str):任务id
+            act(str):[stop],目前仅支持停止一个裁剪分析任务
+        Return
+            status
+    """
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .project.task import get_prune_status
+        ret = get_prune_status(data, SD.workspace)
+        return ret
+    if request.method == 'POST':
+        from .project.task import start_prune_analysis
+        ret = start_prune_analysis(data, SD.workspace, SD.monitored_processes)
+        return ret
+    if request.method == 'PUT':
+        if data['act'] == 'stop':
+            from .project.task import stop_prune_analysis
+            ret = stop_prune_analysis(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task/evaluate', methods=['GET', 'POST'])
+def task_evaluate():
+    '''
+    methods=='GET':#获取模型评估的结果
+        Args:
+            tid(str):任务id
+        Return:
+            evaluate_status(int): 任务状态(TaskStatus)枚举变量的值
+            message(str):描述评估任务的信息
+            result(dict):如果评估成功,返回评估结果的dict,否则为None
+            status
+
+    methods=='POST':#异步,创建一个评估任务
+        Args:
+            tid(str):任务id
+            epoch(int,optional):需要评估的epoch,如果为None则会评估训练时指标最好的epoch
+            topk(int,optional):分类任务topk指标,如果为None默认输入为5
+            score_thresh(float):检测任务类别的score threshhold值,如果为None默认输入为0.5
+            overlap_thresh(float):实例分割任务IOU threshhold值,如果为None默认输入为0.3
+        Return:
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .project.task import get_evaluate_result
+        ret = get_evaluate_result(data, SD.workspace)
+        ret['evaluate_status'] = ret['evaluate_status'].value
+        if 'Confusion_Matrix' in ret['result']:
+            ret['result']['Confusion_Matrix'] = ret['result'][
+                'Confusion_Matrix'].tolist()
+        ret['result'] = CustomEncoder().encode(ret['result'])
+        return ret
+    if request.method == 'POST':
+        from .project.task import evaluate_model
+        ret = evaluate_model(data, SD.workspace, SD.monitored_processes)
+        return ret
+
+
+@app.route('/project/task/evaluate/file', methods=['GET'])
+def task_evaluate_file():
+    data = request.get_json()
+    if request.method == 'GET':
+        ret = data['path']
+        return send_file(ret)
+
+
+@app.route('/project/task/predict', methods=['GET', 'POST', 'PUT'])
+def task_predict():
+    '''
+    methods=='GET':#获取预测状态
+        Args:
+            tid(str):任务id
+        Return:
+            predict_status(int): 预测任务状态(PredictStatus)枚举变量的值
+            message(str): 预测信息
+            status
+
+    methods=='POST':#创建预测任务,目前仅支持单张图片的预测
+        Args:
+            tid(str):任务id
+            image_data(str):base64编码的image数据
+            score_thresh(float,optional):可选,检测任务时有效,检测类别的score threashold值默认是0.5
+            epoch(int,float,optional):可选,选择需要做预测的ephoch,默认为评估指标最好的那一个epoch
+        Return:
+            path(str):服务器上保存预测结果图片的路径
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .project.task import get_predict_status
+        ret = get_predict_status(data, SD.workspace)
+        return ret
+    if request.method == 'POST':
+        from .project.task import predict_test_pics
+        ret = predict_test_pics(data, SD.workspace, SD.monitored_processes)
+        if 'img_list' in data:
+            del ret['path']
+            return ret
+        return ret
+    if request.method == 'PUT':
+        from .project.task import stop_predict_task
+        ret = stop_predict_task(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task/export', methods=['GET', 'POST', 'PUT'])
+def task_export():
+    '''
+    methods=='GET':#获取导出模型的状态
+        Args:
+            tid(str):任务id
+            quant(str,optional)可选,[log,result],导出量模型导出状态,若值为log则返回量化的日志;若值为result则返回量化的结果
+        Return:
+            status
+            if quant == 'log':
+                quant_log(dict):量化日志
+            if quant == 'result'
+                quant_result(dict):量化结果
+            if quant not in Args:
+                export_status(int):模型导出状态(PredictStatus)枚举变量的值
+                message(str):模型导出提示信息
+
+    methods=='POST':#导出inference模型或者导出lite模型
+        Args:
+            tid(str):任务id
+            type(str):保存模型的类别[infer,lite],支持inference模型导出和lite的模型导出
+            save_dir(str):保存模型的路径
+            quant(bool,optional)可选,type为infer有效,是否导出量化后的模型
+            model_path(str,optional)可选,type为lite时有效,inference模型的地址
+        Return:
+            status
+            if type == 'infer':
+                save_dir:模型保存路径
+            if type == 'lite':
+                message:模型保存信息
+
+    methods=='PUT':#停止导出模型
+        Args:
+            tid(str):任务id
+        Return:
+            export_status(int):模型导出状态(PredictStatus)枚举变量的值
+            message(str):停止模型导出提示信息
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if 'quant' in data:
+            if data['quant'] == 'log':
+                from .project.task import get_quant_progress
+                ret = get_quant_progress(data, SD.workspace)
+                return ret
+            if data['quant'] == 'result':
+                from .project.task import get_quant_result
+                ret = get_quant_result(data, SD.workspace)
+                return ret
+        from .project.task import get_export_status
+        ret = get_export_status(data, SD.workspace)
+        ret['export_status'] = ret['export_status'].value
+        return ret
+    if request.method == 'POST':
+        if data['type'] == 'infer':
+            from .project.task import export_infer_model
+            ret = export_infer_model(data, SD.workspace,
+                                     SD.monitored_processes)
+            return ret
+        if data['type'] == 'lite':
+            from .project.task import export_lite_model
+            ret = export_lite_model(data, SD.workspace)
+            return ret
+    if request.method == 'PUT':
+        from .project.task import stop_export_task
+        stop_export_task(data, SD.workspace)
+        return ret
+
+
+@app.route('/project/task/vdl', methods=['GET'])
+def task_vdl():
+    '''
+    methods=='GET':#打开某个任务的可视化分析工具(VisualDL)
+        Args:
+            tid(str):任务id
+        Return:
+            url(str):vdl地址
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .project.task import open_vdl
+        ret = open_vdl(data, SD.workspace, SD.current_port,
+                       SD.monitored_processes, SD.running_boards)
+        return ret
+
+
+@app.route('/system', methods=['GET', 'DELETE'])
+def system():
+    '''
+    methods=='GET':#获取系统GPU、CPU信息
+        Args:
+            type(str):[machine_info,gpu_memory_size]选择需要获取的系统信息
+        Return:
+            status
+            if type=='machine_info'
+                info(dict):服务端信息
+            if type=='gpu_memory_size'
+                gpu_mem_infos(list):GPU内存信息
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if data['type'] == 'machine_info':
+            '''if 'path' not in data:
+                data['path'] = None
+            from .system import get_machine_info
+            ret = get_machine_info(data, SD.machine_info)'''
+            from .system import get_system_info
+            ret = get_system_info(SD.machine_info)
+            return ret
+
+        if data['type'] == 'gpu_memory_size':
+            #from .system import get_gpu_memory_size
+            from .system import get_gpu_memory_info
+            ret = get_gpu_memory_info(SD.machine_info)
+            return ret
+    if request.method == 'DELETE':
+        from .system import exit_system
+        ret = exit_system(SD.monitored_processes)
+        return ret
+
+
+@app.route('/demo', methods=['GET', 'POST', 'PUT'])
+def demo():
+    '''
+    methods=='GET':#获取demo下载进度
+        Args:
+            prj_type(int):项目类型ProjectType枚举变量的int值
+        Return:
+            status
+            attr(dict):demo下载信息
+
+    methods=='POST':#下载或创建demo工程
+        Args:
+            type(str):{download,load}下载或者创建样例
+            prj_type(int):项目类型ProjectType枚举变量的int值
+        Return:
+            status
+            if type=='load':
+                did:数据集id
+                pid:项目id
+
+    methods=='PUT':#停止下载或创建demo工程
+        Args:
+            prj_type(int):项目类型ProjectType枚举变量的int值
+        Return:
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        from .demo import get_download_demo_progress
+        ret = get_download_demo_progress(data, SD.workspace)
+        return ret
+    if request.method == 'POST':
+        if data['type'] == 'download':
+            from .demo import download_demo_dataset
+            ret = download_demo_dataset(data, SD.workspace,
+                                        SD.load_demo_proc_dict)
+            return ret
+        if data['type'] == 'load':
+            from .demo import load_demo_project
+            ret = load_demo_project(data, SD.workspace, SD.monitored_processes,
+                                    SD.load_demo_proj_data_dict,
+                                    SD.load_demo_proc_dict)
+            return ret
+    if request.method == 'PUT':
+        from .demo import stop_import_demo
+        ret = stop_import_demo(data, SD.workspace, SD.load_demo_proc_dict,
+                               SD.load_demo_proj_data_dict)
+        return ret
+
+
+@app.route('/model', methods=['GET', 'POST', 'DELETE'])
+def model():
+    '''
+    methods=='GET':#获取一个或者所有模型的信息
+        Args:
+            mid(str,optional)可选,若存在则返回某个模型的信息
+            type(str,optional)可选,[pretrained,exported].若存在则返回对应类型下所有的模型信息
+        Return:
+            status
+            if mid in Args:
+                dataset_attr(dict):数据集属性
+                task_params(dict):模型训练参数
+                eval_result(dict):模型评估结果
+            if type in Args and type == 'pretrained':
+                pretrained_models(list):所有预训练模型信息
+            if type in Args and type == 'exported':
+                exported_models(list):所有inference模型的信息
+
+    methods=='POST':#创建一个模型
+        Args:
+            pid(str):项目id
+            tid(str):任务id
+            name(str):模型名字
+            type(str):创建模型的类型,[pretrained,exported],pretrained代表创建预训练模型、exported代表创建inference或者lite模型
+            source_path(str):仅type为pretrained时有效,训练好的模型的路径
+            path(str):仅type为exported时有效,inference或者lite模型的路径
+            exported_type(int):0为inference模型,1为lite模型
+            eval_results(dict,optional):可选,仅type为pretrained时有效,模型评估的指标
+        Return:
+            status
+            if type == 'pretrained':
+                pmid(str):预训练模型id
+            if type == 'exported':
+                emid(str):inference模型id
+
+    methods=='DELETE':删除一个模型
+        Args:
+            type(str):删除模型的类型,[pretrained,exported],pretrained代表创建预训练模型、exported代表创建inference或者lite模型
+            if type='pretrained':
+                pmid:预训练模型id
+            if type='exported':
+                emid:inference或者lite模型id
+        Return:
+            status
+    '''
+    data = request.get_json()
+    if data is None:
+        data = request.args
+    if request.method == 'GET':
+        if 'type' in data:
+            if data['type'] == 'pretrained':
+                from .model import list_pretrained_models
+                ret = list_pretrained_models(SD.workspace)
+                return ret
+            if data['type'] == 'exported':
+                from .model import list_exported_models
+                ret = list_exported_models(SD.workspace)
+                return ret
+        from .model import get_model_details
+        ret = get_model_details(data, SD.workspace)
+        ret['eval_result']['Confusion_Matrix'] = ret['eval_result'][
+            'Confusion_Matrix'].tolist()
+        ret['eval_result'] = CustomEncoder().encode(ret['eval_result'])
+        ret['task_params'] = CustomEncoder().encode(ret['task_params'])
+        return ret
+    if request.method == 'POST':
+        if data['type'] == 'pretrained':
+            if 'eval_results' in data:
+                data['eval_results']['Confusion_Matrix'] = np.array(data[
+                    'eval_results']['Confusion_Matrix'])
+            from .model import create_pretrained_model
+            ret = create_pretrained_model(data, SD.workspace,
+                                          SD.monitored_processes)
+            return ret
+        if data['type'] == 'exported':
+            from .model import create_exported_model
+            ret = create_exported_model(data, SD.workspace)
+            return ret
+    if request.method == 'DELETE':
+        if data['type'] == 'pretrained':
+            from .model import delete_pretrained_model
+            ret = delete_pretrained_model(data, workspace)
+            return ret
+        if data['type'] == 'exported':
+            from .model import delete_exported_model
+            ret = delete_exported_model(data, workspace)
+            return ret
+
+
+@app.route('/model/file', methods=['GET'])
+def model_file():
+    data = request.get_json()
+    if request.method == 'GET':
+        ret = data['path']
+        return send_file(ret)
+
+
+def run(port, workspace_dir):
+    if workspace_dir is None:
+        user_home = os.path.expanduser('~')
+        dirname = osp.join(user_home, "paddlex_workspace")
+    else:
+        dirname = workspace_dir
+    if not osp.exists(dirname):
+        os.makedirs(dirname)
+    else:
+        if not osp.isdir(dirname):
+            os.remove(dirname)
+            os.makedirs(dirname)
+    logger = get_logger(osp.join(dirname, "mcessages.log"))
+    init(dirname, logger)
+    app.run(host='0.0.0.0', port=port, threaded=True)

+ 13 - 0
paddlex/restful/dataset/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 159 - 0
paddlex/restful/dataset/cls_dataset.py

@@ -0,0 +1,159 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+from ..utils import list_files
+from .utils import is_pic, get_encoding, check_list_txt
+from .datasetbase import DatasetBase
+
+
+class ClsDataset(DatasetBase):
+    def __init__(self, dataset_id, path):
+        super().__init__(dataset_id, path)
+
+    def check_dataset(self, source_path):
+        self.all_files = list_files(source_path)
+        # 对分类数据集进行统计分析
+        self.file_info = dict()
+        self.label_info = dict()
+        # 校验已切分的数据集
+        if osp.exists(osp.join(source_path, 'train_list.txt')):
+            return self.check_splited_dataset(source_path)
+
+        for f in self.all_files:
+            if not is_pic(f):
+                continue
+            items = osp.split(f)
+            if len(items) == 2:
+                if " " in items[0]:
+                    raise ValueError("类别-{}名称有误,分类数据集中类别名称不应包含空格".format(items[
+                        0]))
+                if items[0] not in self.label_info:
+                    self.label_info[items[0]] = list()
+                self.label_info[items[0]].append(f)
+                self.file_info[f] = items[0]
+        if len(self.label_info) < 2:
+            raise ValueError("分类数据集中至少需要包含两种图像类别")
+        self.labels = sorted(self.label_info.keys())
+        for label in self.labels:
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def check_splited_dataset(self, source_path):
+        labels_txt = osp.join(source_path, "labels.txt")
+        train_list_txt = osp.join(source_path, "train_list.txt")
+        val_list_txt = osp.join(source_path, "val_list.txt")
+        test_list_txt = osp.join(source_path, "test_list.txt")
+        for txt_file in [labels_txt, train_list_txt, val_list_txt]:
+            if not osp.exists(txt_file):
+                raise Exception(
+                    "已切分的数据集下应该包含labels.txt, train_list.txt, val_list.txt文件")
+        check_list_txt([train_list_txt, val_list_txt, test_list_txt])
+
+        self.labels = open(
+            labels_txt, 'r',
+            encoding=get_encoding(labels_txt)).read().strip().split('\n')
+
+        for txt_file in [train_list_txt, val_list_txt, test_list_txt]:
+            if not osp.exists(txt_file):
+                continue
+            with open(txt_file, "r") as f:
+                for line in f:
+                    items = line.strip().split()
+
+                    if not osp.exists(osp.join(source_path, items[0])):
+                        raise Exception("数据目录{}中不存在图片文件{}".format(
+                            osp.split(txt_file)[-1], items[0]))
+                    dir_name = osp.split(osp.split(items[0])[0])[-1]
+                    if dir_name != self.labels[int(items[1])]:
+                        raise Exception("labels.txt中label顺序不准确")
+                    img_file = osp.split(items[0])[-1]
+                    if not is_pic(img_file) or img_file.startswith('.'):
+                        raise ValueError("文件{}不是图片格式".format(img_file))
+                    self.file_info[items[0]] = self.labels[int(items[1])]
+
+                    if txt_file == train_list_txt:
+                        self.train_files.append(items[0])
+                        if self.labels[int(items[
+                                1])] in self.class_train_file_list:
+                            self.class_train_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+                        else:
+                            self.class_train_file_list[self.labels[int(items[
+                                1])]] = list()
+                            self.class_train_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+                    elif txt_file == val_list_txt:
+                        self.val_files.append(items[0])
+                        if self.labels[int(items[
+                                1])] in self.class_val_file_list:
+                            self.class_val_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+                        else:
+                            self.class_val_file_list[self.labels[int(items[
+                                1])]] = list()
+                            self.class_val_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+                    elif txt_file == test_list_txt:
+                        self.test_files.append(items[0])
+                        if self.labels[int(items[
+                                1])] in self.class_test_file_list:
+                            self.class_test_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+                        else:
+                            self.class_test_file_list[self.labels[int(items[
+                                1])]] = list()
+                            self.class_test_file_list[self.labels[int(items[
+                                1])]].append(items[0])
+
+        for img_file, label in self.file_info.items():
+            if label not in self.label_info:
+                self.label_info[label] = list()
+            self.label_info[label].append(img_file)
+
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def split(self, val_split, test_split):
+        super().split(val_split, test_split)
+        with open(
+                osp.join(self.path, 'train_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.train_files:
+                label = self.file_info[x]
+                label_idx = self.labels.index(label)
+                f.write('{} {}\n'.format(x, label_idx))
+        with open(
+                osp.join(self.path, 'val_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.val_files:
+                label = self.file_info[x]
+                label_idx = self.labels.index(label)
+                f.write('{} {}\n'.format(x, label_idx))
+        with open(
+                osp.join(self.path, 'test_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.test_files:
+                label = self.file_info[x]
+                label_idx = self.labels.index(label)
+                f.write('{} {}\n'.format(x, label_idx))
+        with open(
+                osp.join(self.path, 'labels.txt'), mode='w',
+                encoding='utf-8') as f:
+            for l in self.labels:
+                f.write('{}\n'.format(l))
+        self.dump_statis_info()

+ 239 - 0
paddlex/restful/dataset/dataset.py

@@ -0,0 +1,239 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ..utils import (set_folder_status, get_folder_status, DatasetStatus,
+                     TaskStatus, is_available, DownloadStatus,
+                     PretrainedModelStatus, ProjectType)
+
+from threading import Thread
+import random
+from .utils import copy_directory
+import traceback
+import shutil
+import psutil
+import pickle
+import os
+import os.path as osp
+import time
+import json
+import base64
+from .. import workspace_pb2 as w
+
+
+def create_dataset(data, workspace):
+    """
+    创建dataset
+    """
+    create_time = time.time()
+    time_array = time.localtime(create_time)
+    create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    id = workspace.max_dataset_id + 1
+    if id < 10000:
+        did = 'D%04d' % id
+    else:
+        did = 'D{}'.format(id)
+    assert not did in workspace.datasets, "【数据集创建】ID'{}'已经被占用.".format(did)
+    path = osp.join(workspace.path, 'datasets', did)
+    if osp.exists(path):
+        if not osp.isdir(path):
+            os.remove(path)
+        else:
+            shutil.rmtree(path)
+    os.makedirs(path)
+    set_folder_status(path, DatasetStatus.XEMPTY)
+    workspace.max_dataset_id = id
+    ds = w.Dataset(
+        id=did,
+        name=data['name'],
+        desc=data['desc'],
+        type=data['dataset_type'],
+        create_time=create_time,
+        path=path)
+    workspace.datasets[did].CopyFrom(ds)
+    return {'status': 1, 'did': did}
+
+
+def import_dataset(data, workspace, monitored_processes, load_demo_proc_dict):
+    """导入数据集到工作目录,包括数据检查和拷贝
+    Args:
+        data为dict, key包括
+        'did':数据集id,'path': 原数据集目录路径,
+        'demo'(可选): 该数据集为demo数据集
+    """
+    dataset_id = data['did']
+    source_path = data['path']
+    assert dataset_id in workspace.datasets, "数据集ID'{}'不存在.".format(dataset_id)
+    dataset_type = workspace.datasets[dataset_id].type
+    dataset_path = workspace.datasets[dataset_id].path
+    valid_dataset_type = [
+        'classification', 'detection', 'segmentation', 'instance_segmentation',
+        'remote_segmentation'
+    ]
+    assert dataset_type in valid_dataset_type, "无法识别的数据类型{}".format(
+        dataset_type)
+
+    from .operate import import_dataset
+    process = import_dataset(dataset_id, dataset_type, dataset_path,
+                             source_path)
+    monitored_processes.put(process.pid)
+    if 'demo' in data:
+        prj_type = getattr(ProjectType, dataset_type)
+        if prj_type not in load_demo_proc_dict:
+            load_demo_proc_dict[prj_type] = []
+        load_demo_proc_dict[prj_type].append(process)
+    return {'status': 1}
+
+
+def delete_dataset(data, workspace):
+    """删除dataset。
+
+    Args:
+        data为dict,key包括
+        'did'数据集id
+    """
+    dataset_id = data['did']
+    assert dataset_id in workspace.datasets, "数据集ID'{}'不存在.".format(dataset_id)
+    counter = 0
+    for key in workspace.projects:
+        if workspace.projects[key].did == dataset_id:
+            counter += 1
+    assert counter == 0, "无法删除数据集,当前仍被{}个项目中使用中,请先删除相关项目".format(counter)
+    path = workspace.datasets[dataset_id].path
+    if osp.exists(path):
+        shutil.rmtree(path)
+    del workspace.datasets[dataset_id]
+    return {'status': 1}
+
+
+def get_dataset_status(data, workspace):
+    """获取数据集当前状态
+
+    Args:
+        data为dict, key包括
+        'did':数据集id
+    """
+    from .operate import get_dataset_status
+    dataset_id = data['did']
+    assert dataset_id in workspace.datasets, "数据集ID'{}'不存在.".format(dataset_id)
+    dataset_type = workspace.datasets[dataset_id].type
+    dataset_path = workspace.datasets[dataset_id].path
+    dataset_name = workspace.datasets[dataset_id].name
+    dataset_desc = workspace.datasets[dataset_id].desc
+    dataset_create_time = workspace.datasets[dataset_id].create_time
+    status, message = get_dataset_status(dataset_id, dataset_type,
+                                         dataset_path)
+    dataset_pids = list()
+    for key in workspace.projects:
+        if dataset_id == workspace.projects[key].did:
+            dataset_pids.append(workspace.projects[key].id)
+
+    attr = {
+        "type": dataset_type,
+        "id": dataset_id,
+        "name": dataset_name,
+        "path": dataset_path,
+        "desc": dataset_desc,
+        "create_time": dataset_create_time,
+        "pids": dataset_pids
+    }
+    return {
+        'status': 1,
+        'id': dataset_id,
+        'dataset_status': status.value,
+        'message': message,
+        'attr': attr
+    }
+
+
+def list_datasets(workspace):
+    """
+    列出数据集列表,可根据request中的参数进行筛选
+    """
+    from .operate import get_dataset_status
+    dataset_list = list()
+    for key in workspace.datasets:
+        dataset_type = workspace.datasets[key].type
+        dataset_id = workspace.datasets[key].id
+        dataset_name = workspace.datasets[key].name
+        dataset_path = workspace.datasets[key].path
+        dataset_desc = workspace.datasets[key].desc
+        dataset_create_time = workspace.datasets[key].create_time
+        status, message = get_dataset_status(dataset_id, dataset_type,
+                                             dataset_path)
+        attr = {
+            "type": dataset_type,
+            "id": dataset_id,
+            "name": dataset_name,
+            "path": dataset_path,
+            "desc": dataset_desc,
+            "create_time": dataset_create_time,
+            'dataset_status': status.value,
+            'message': message
+        }
+        dataset_list.append({"id": dataset_id, "attr": attr})
+    return {'status': 1, "datasets": dataset_list}
+
+
+def get_dataset_details(data, workspace):
+    """获取数据集详情
+
+    Args:
+        data为dict, key包括
+        'did':数据集id
+    Return:
+        details(dict): 'file_info': 全量数据集文件与标签映射表,'label_info': 标签与全量数据集文件映射表,
+        'labels': 标签列表,'train_files': 训练集文件列表, 'val_files': 验证集文件列表,
+        'test_files': 测试集文件列表
+    """
+    from .operate import get_dataset_details
+    dataset_id = data['did']
+    assert dataset_id in workspace.datasets, "数据集ID'{}'不存在.".format(dataset_id)
+    dataset_path = workspace.datasets[dataset_id].path
+    details = get_dataset_details(dataset_path)
+    return {'status': 1, 'details': details}
+
+
+def split_dataset(data, workspace):
+    """将数据集切分为训练集、验证集和测试集
+
+    Args:
+        data为dict, key包括
+        'did':数据集id, 'val_split': 验证集比例, 'test_split': 测试集比例
+    """
+    from .operate import split_dataset
+    from .operate import get_dataset_details
+    dataset_id = data['did']
+    assert dataset_id in workspace.datasets, "数据集ID'{}'不存在.".format(dataset_id)
+    dataset_type = workspace.datasets[dataset_id].type
+    dataset_path = workspace.datasets[dataset_id].path
+    val_split = data['val_split']
+    test_split = data['test_split']
+    split_dataset(dataset_id, dataset_type, dataset_path, val_split,
+                  test_split)
+    return {'status': 1}
+
+
+def img_base64(data):
+    """将数据集切分为训练集、验证集和测试集
+
+    Args:
+        data为dict, key包括
+        'path':图片绝对路径
+    """
+    path = data['path']
+    print(path)
+    with open(path, 'rb') as f:
+        base64_data = base64.b64encode(f.read())
+        base64_str = str(base64_data, 'utf-8')
+    return {'status': 1, 'img_data': base64_str}

+ 86 - 0
paddlex/restful/dataset/datasetbase.py

@@ -0,0 +1,86 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pickle
+import os.path as osp
+import random
+from .utils import copy_directory
+
+
+class DatasetBase(object):
+    def __init__(self, dataset_id, path):
+        self.id = dataset_id
+        self.path = path
+        self.all_files = list()
+        self.file_info = dict()
+        self.label_info = dict()
+        self.labels = list()
+        self.train_files = list()
+        self.val_files = list()
+        self.test_files = list()
+        self.class_train_file_list = dict()
+        self.class_val_file_list = dict()
+        self.class_test_file_list = dict()
+
+    def copy_dataset(self, source_path, files):
+        # 将原数据集拷贝至目标路径
+        copy_directory(source_path, self.path, files)
+
+    def dump_statis_info(self):
+        # info['fields']指定了需要dump的信息
+        info = dict()
+        info['fields'] = [
+            'file_info', 'label_info', 'labels', 'train_files', 'val_files',
+            'test_files', 'class_train_file_list', 'class_val_file_list',
+            'class_test_file_list'
+        ]
+        for field in info['fields']:
+            if hasattr(self, field):
+                info[field] = getattr(self, field)
+        with open(osp.join(self.path, 'statis.pkl'), 'wb') as f:
+            pickle.dump(info, f)
+
+    def load_statis_info(self):
+        with open(osp.join(self.path, 'statis.pkl'), 'rb') as f:
+            info = pickle.load(f)
+        for field in info['fields']:
+            if field in info:
+                setattr(self, field, info[field])
+
+    def split(self, val_split, test_split):
+        all_files = list(self.file_info.keys())
+        random.shuffle(all_files)
+        val_num = int(len(all_files) * val_split)
+        test_num = int(len(all_files) * test_split)
+        train_num = len(all_files) - val_num - test_num
+        assert train_num > 0, "训练集样本数量需大于0"
+        assert val_num > 0, "验证集样本数量需大于0"
+        self.train_files = all_files[:train_num]
+        self.val_files = all_files[train_num:train_num + val_num]
+        self.test_files = all_files[train_num + val_num:]
+        self.train_set = set(self.train_files)
+        self.val_set = set(self.val_files)
+        self.test_set = set(self.test_files)
+
+        for label, file_list in self.label_info.items():
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+            for f in file_list:
+                if f in self.test_set:
+                    self.class_test_file_list[label].append(f)
+                if f in self.val_set:
+                    self.class_val_file_list[label].append(f)
+                if f in self.train_set:
+                    self.class_train_file_list[label].append(f)

+ 190 - 0
paddlex/restful/dataset/det_dataset.py

@@ -0,0 +1,190 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+from ..utils import list_files
+from .utils import is_pic, replace_ext, get_encoding, check_list_txt
+from .datasetbase import DatasetBase
+import xml.etree.ElementTree as ET
+
+
+class DetDataset(DatasetBase):
+    def __init__(self, dataset_id, path):
+        super().__init__(dataset_id, path)
+
+    def check_dataset(self, source_path):
+        if not osp.isdir(osp.join(source_path, 'Annotations')):
+            raise ValueError("标注文件应该放在{}目录下".format(
+                osp.join(source_path, 'Annotations')))
+        if not osp.isdir(osp.join(source_path, 'JPEGImages')):
+            raise ValueError("图片文件应该放在{}目录下".format(
+                osp.join(source_path, 'JPEGImages')))
+
+        self.all_files = list_files(source_path)
+        # 对检测数据集进行统计分析
+        self.file_info = dict()
+        self.label_info = dict()
+
+        if osp.exists(osp.join(source_path, 'train_list.txt')):
+            return self.check_splited_dataset(source_path)
+
+        for f in self.all_files:
+            if not is_pic(f):
+                continue
+            items = osp.split(f)
+            if len(items) == 2 and items[0] == "JPEGImages":
+                anno_name = replace_ext(items[1], "xml")
+                full_anno_path = osp.join(
+                    (osp.join(source_path, 'Annotations')), anno_name)
+                if osp.exists(full_anno_path):
+                    self.file_info[f] = osp.join('Annotations', anno_name)
+
+                # 解析XML文件,获取类别信息
+                try:
+                    tree = ET.parse(full_anno_path)
+                except:
+                    raise Exception("文件{}不是一个良构的xml文件".format(anno_name))
+                objs = tree.findall('object')
+                for i, obj in enumerate(objs):
+                    cname = obj.find('name').text
+                    if cname not in self.label_info:
+                        self.label_info[cname] = list()
+                    if f not in self.label_info[cname]:
+                        self.label_info[cname].append(f)
+
+        self.labels = sorted(self.label_info.keys())
+        for label in self.labels:
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def check_splited_dataset(self, source_path):
+        labels_txt = osp.join(source_path, "labels.txt")
+        train_list_txt = osp.join(source_path, "train_list.txt")
+        val_list_txt = osp.join(source_path, "val_list.txt")
+        test_list_txt = osp.join(source_path, "test_list.txt")
+        for txt_file in [labels_txt, train_list_txt, val_list_txt]:
+            if not osp.exists(txt_file):
+                raise Exception(
+                    "已切分的数据集下应该包含labels.txt, train_list.txt, val_list.txt文件")
+        check_list_txt([train_list_txt, val_list_txt, test_list_txt])
+
+        self.labels = open(
+            labels_txt, 'r',
+            encoding=get_encoding(labels_txt)).read().strip().split('\n')
+
+        for txt_file in [train_list_txt, val_list_txt, test_list_txt]:
+            if not osp.exists(txt_file):
+                continue
+            with open(txt_file, "r") as f:
+                for line in f:
+                    items = line.strip().split()
+                    img_file, xml_file = [items[0], items[1]]
+
+                    if not osp.isfile(osp.join(source_path, xml_file)):
+                        raise ValueError("数据目录{}中不存在标注文件{}".format(
+                            osp.split(txt_file)[-1], xml_file))
+                    if not osp.isfile(osp.join(source_path, img_file)):
+                        raise ValueError("数据目录{}中不存在图片文件{}".format(
+                            osp.split(txt_file)[-1], img_file))
+                    if not xml_file.split('.')[-1] == 'xml':
+                        raise ValueError("标注文件{}不是xml文件".format(xml_file))
+                    img_file_name = osp.split(img_file)[-1]
+                    if not is_pic(img_file_name) or img_file_name.startswith(
+                            '.'):
+                        raise ValueError("文件{}不是图片格式".format(img_file))
+
+                    self.file_info[img_file] = xml_file
+
+                    if txt_file == train_list_txt:
+                        self.train_files.append(img_file)
+                    elif txt_file == val_list_txt:
+                        self.val_files.append(img_file)
+                    elif txt_file == test_list_txt:
+                        self.test_files.append(img_file)
+
+                    try:
+                        tree = ET.parse(osp.join(source_path, xml_file))
+                    except:
+                        raise Exception("文件{}不是一个良构的xml文件".format(xml_file))
+                    objs = tree.findall('object')
+                    for i, obj in enumerate(objs):
+                        cname = obj.find('name').text
+                        if cname in self.labels:
+                            if cname not in self.label_info:
+                                self.label_info[cname] = list()
+                            if img_file not in self.label_info[cname]:
+                                self.label_info[cname].append(img_file)
+                                if txt_file == train_list_txt:
+                                    if cname in self.class_train_file_list:
+                                        self.class_train_file_list[
+                                            cname].append(img_file)
+                                    else:
+                                        self.class_train_file_list[
+                                            cname] = list()
+                                        self.class_train_file_list[
+                                            cname].append(img_file)
+                                elif txt_file == val_list_txt:
+                                    if cname in self.class_val_file_list:
+                                        self.class_val_file_list[cname].append(
+                                            img_file)
+                                    else:
+                                        self.class_val_file_list[cname] = list(
+                                        )
+                                        self.class_val_file_list[cname].append(
+                                            img_file)
+                                elif txt_file == test_list_txt:
+                                    if cname in self.class_test_file_list:
+                                        self.class_test_file_list[
+                                            cname].append(img_file)
+                                    else:
+                                        self.class_test_file_list[
+                                            cname] = list()
+                                        self.class_test_file_list[
+                                            cname].append(img_file)
+                        else:
+                            raise Exception("文件{}与labels.txt文件信息不对应".format(
+                                xml_file))
+
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def split(self, val_split, test_split):
+        super().split(val_split, test_split)
+        with open(
+                osp.join(self.path, 'train_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.train_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        with open(
+                osp.join(self.path, 'val_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.val_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        with open(
+                osp.join(self.path, 'test_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.test_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        with open(
+                osp.join(self.path, 'labels.txt'), mode='w',
+                encoding='utf-8') as f:
+            for l in self.labels:
+                f.write('{}\n'.format(l))
+        self.dump_statis_info()

+ 314 - 0
paddlex/restful/dataset/ins_seg_dataset.py

@@ -0,0 +1,314 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+import random
+from ..utils import list_files
+from .utils import is_pic, replace_ext, MyEncoder, read_coco_ann, get_npy_from_coco_json
+from .datasetbase import DatasetBase
+import numpy as np
+import json
+from pycocotools.coco import COCO
+
+
+class InsSegDataset(DatasetBase):
+    def __init__(self, dataset_id, path):
+        super().__init__(dataset_id, path)
+        self.annotation_dict = None
+
+    def check_dataset(self, source_path):
+        if not osp.isdir(osp.join(source_path, 'JPEGImages')):
+            raise ValueError("图片文件应该放在{}目录下".format(
+                osp.join(source_path, 'JPEGImages')))
+
+        self.all_files = list_files(source_path)
+        # 对检测数据集进行统计分析
+        self.file_info = dict()
+        self.label_info = dict()
+        # 若数据集已切分
+        if osp.exists(osp.join(source_path, 'train.json')):
+            return self.check_splited_dataset(source_path)
+
+        if not osp.exists(osp.join(source_path, 'annotations.json')):
+            raise ValueError("标注文件annotations.json应该放在{}目录下".format(
+                source_path))
+
+        filename_set = set()
+        anno_set = set()
+        for f in self.all_files:
+            items = osp.split(f)
+            if len(items) == 2 and items[0] == "JPEGImages":
+                if not is_pic(f) or f.startswith('.'):
+                    continue
+                filename_set.add(items[1])
+        # 解析包含标注信息的json文件
+        try:
+            coco = COCO(osp.join(source_path, 'annotations.json'))
+            img_ids = coco.getImgIds()
+            cat_ids = coco.getCatIds()
+            anno_ids = coco.getAnnIds()
+
+            catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
+            cid2cname = dict({
+                clsid: coco.loadCats(catid)[0]['name']
+                for catid, clsid in catid2clsid.items()
+            })
+            for img_id in img_ids:
+                img_anno = coco.loadImgs(img_id)[0]
+                img_name = osp.split(img_anno['file_name'])[-1]
+                anno_set.add(img_name)
+                anno_dict = read_coco_ann(img_id, coco, cid2cname, catid2clsid)
+                img_path = osp.join("JPEGImages", img_name)
+                anno_path = osp.join("Annotations", img_name)
+                anno = replace_ext(anno_path, "npy")
+                self.file_info[img_path] = anno
+
+                img_class = list(set(anno_dict["gt_class"]))
+                for category_name in img_class:
+                    if not category_name in self.label_info:
+                        self.label_info[category_name] = [img_path]
+                    else:
+                        self.label_info[category_name].append(img_path)
+
+            for label in sorted(self.label_info.keys()):
+                self.labels.append(label)
+        except:
+            raise Exception("标注文件存在错误")
+
+        if len(anno_set) > len(filename_set):
+            sub_list = list(anno_set - filename_set)
+            raise Exception("标注文件中{}等{}个信息无对应图片".format(sub_list[0],
+                                                        len(sub_list)))
+
+        # 生成每个图片对应的标注信息npy文件
+        npy_path = osp.join(self.path, "Annotations")
+        get_npy_from_coco_json(coco, npy_path, self.file_info)
+
+        for label in self.labels:
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def check_splited_dataset(self, source_path):
+        train_files_json = osp.join(source_path, "train.json")
+        val_files_json = osp.join(source_path, "val.json")
+        test_files_json = osp.join(source_path, "test.json")
+
+        for json_file in [train_files_json, val_files_json]:
+            if not osp.exists(json_file):
+                raise Exception("已切分的数据集下应该包含train.json, val.json文件")
+
+        filename_set = set()
+        anno_set = set()
+        # 获取全部图片名称
+        for f in self.all_files:
+            items = osp.split(f)
+            if len(items) == 2 and items[0] == "JPEGImages":
+                if not is_pic(f) or f.startswith('.'):
+                    continue
+                filename_set.add(items[1])
+
+        img_id_index = 0
+        anno_id_index = 0
+        new_img_list = list()
+        new_cat_list = list()
+        new_anno_list = list()
+        for json_file in [train_files_json, val_files_json, test_files_json]:
+            if not osp.exists(json_file):
+                continue
+            coco = COCO(json_file)
+            img_ids = coco.getImgIds()
+            cat_ids = coco.getCatIds()
+            anno_ids = coco.getAnnIds()
+
+            catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
+            clsid2catid = dict({i: catid for i, catid in enumerate(cat_ids)})
+            cid2cname = dict({
+                clsid: coco.loadCats(catid)[0]['name']
+                for catid, clsid in catid2clsid.items()
+            })
+
+            # 由原train.json中的category生成新的category信息
+            if json_file == train_files_json:
+                cname2catid = dict({
+                    coco.loadCats(catid)[0]['name']: clsid2catid[clsid]
+                    for catid, clsid in catid2clsid.items()
+                })
+                new_cat_list = coco.loadCats(cat_ids)
+            # 获取json中全部标注图片的名字
+            for img_id in img_ids:
+                img_anno = coco.loadImgs(img_id)[0]
+                im_fname = img_anno['file_name']
+                anno_set.add(im_fname)
+
+                if json_file == train_files_json:
+                    self.train_files.append(osp.join("JPEGImages", im_fname))
+                elif json_file == val_files_json:
+                    self.val_files.append(osp.join("JPEGImages", im_fname))
+                elif json_file == test_files_json:
+                    self.test_files.append(osp.join("JPEGImages", im_fname))
+                # 获取每张图片的对应标注信息,并记录为npy格式
+                anno_dict = read_coco_ann(img_id, coco, cid2cname, catid2clsid)
+                img_path = osp.join("JPEGImages", im_fname)
+                anno_path = osp.join("Annotations", im_fname)
+                anno = replace_ext(anno_path, "npy")
+                self.file_info[img_path] = anno
+
+                # 生成label_info
+                img_class = list(set(anno_dict["gt_class"]))
+                for category_name in img_class:
+                    if not category_name in self.label_info:
+                        self.label_info[category_name] = [img_path]
+                    else:
+                        self.label_info[category_name].append(img_path)
+                    if json_file == train_files_json:
+                        if category_name in self.class_train_file_list:
+                            self.class_train_file_list[category_name].append(
+                                img_path)
+                        else:
+                            self.class_train_file_list[category_name] = list()
+                            self.class_train_file_list[category_name].append(
+                                img_path)
+                    elif json_file == val_files_json:
+                        if category_name in self.class_val_file_list:
+                            self.class_val_file_list[category_name].append(
+                                img_path)
+                        else:
+                            self.class_val_file_list[category_name] = list()
+                            self.class_val_file_list[category_name].append(
+                                img_path)
+                    elif json_file == test_files_json:
+                        if category_name in self.class_test_file_list:
+                            self.class_test_file_list[category_name].append(
+                                img_path)
+                        else:
+                            self.class_test_file_list[category_name] = list()
+                            self.class_test_file_list[category_name].append(
+                                img_path)
+
+                # 生成新的图片信息
+                new_img = img_anno
+                new_img["id"] = img_id_index
+                img_id_index += 1
+                new_img_list.append(new_img)
+                # 生成新的标注信息
+                ins_anno_ids = coco.getAnnIds(imgIds=img_id, iscrowd=0)
+                for ins_anno_id in ins_anno_ids:
+                    anno = coco.loadAnns(ins_anno_id)[0]
+                    new_anno = anno
+                    new_anno["image_id"] = new_img["id"]
+                    new_anno["id"] = anno_id_index
+                    anno_id_index += 1
+                    cat = coco.loadCats(anno["category_id"])[0]
+                    new_anno_list.append(new_anno)
+
+        if len(anno_set) > len(filename_set):
+            sub_list = list(anno_set - filename_set)
+            raise Exception("标注文件中{}等{}个信息无对应图片".format(sub_list[0],
+                                                        len(sub_list)))
+
+        for label in sorted(self.label_info.keys()):
+            self.labels.append(label)
+
+        self.annotation_dict = {
+            "images": new_img_list,
+            "categories": new_cat_list,
+            "annotations": new_anno_list
+        }
+
+        # 若原数据集已切分,无annotations.json文件
+        if not osp.exists(osp.join(self.path, "annotations.json")):
+            json_file = open(osp.join(self.path, "annotations.json"), 'w+')
+            json.dump(self.annotation_dict, json_file, cls=MyEncoder)
+            json_file.close()
+
+        # 生成每个图片对应的标注信息npy文件
+        coco = COCO(osp.join(self.path, "annotations.json"))
+        npy_path = osp.join(self.path, "Annotations")
+        get_npy_from_coco_json(coco, npy_path, self.file_info)
+
+        self.dump_statis_info()
+
+    def split(self, val_split, test_split):
+        all_files = list(self.file_info.keys())
+        val_num = int(len(all_files) * val_split)
+        test_num = int(len(all_files) * test_split)
+        train_num = len(all_files) - val_num - test_num
+        assert train_num > 0, "训练集样本数量需大于0"
+        assert val_num > 0, "验证集样本数量需大于0"
+        self.train_files = list()
+        self.val_files = list()
+        self.test_files = list()
+
+        coco = COCO(osp.join(self.path, 'annotations.json'))
+        img_ids = coco.getImgIds()
+        cat_ids = coco.getCatIds()
+        anno_ids = coco.getAnnIds()
+        random.shuffle(img_ids)
+
+        train_files_ids = img_ids[:train_num]
+        val_files_ids = img_ids[train_num:train_num + val_num]
+        test_files_ids = img_ids[train_num + val_num:]
+
+        for img_id_list in [train_files_ids, val_files_ids, test_files_ids]:
+            img_anno_ids = coco.getAnnIds(imgIds=img_id_list, iscrowd=0)
+            imgs = coco.loadImgs(img_id_list)
+            instances = coco.loadAnns(img_anno_ids)
+            categories = coco.loadCats(cat_ids)
+            img_dict = {
+                "annotations": instances,
+                "images": imgs,
+                "categories": categories
+            }
+
+            if img_id_list == train_files_ids:
+                for img in imgs:
+                    self.train_files.append(
+                        osp.join("JPEGImages", img["file_name"]))
+                json_file = open(osp.join(self.path, 'train.json'), 'w+')
+                json.dump(img_dict, json_file, cls=MyEncoder)
+            elif img_id_list == val_files_ids:
+                for img in imgs:
+                    self.val_files.append(
+                        osp.join("JPEGImages", img["file_name"]))
+                json_file = open(osp.join(self.path, 'val.json'), 'w+')
+                json.dump(img_dict, json_file, cls=MyEncoder)
+            elif img_id_list == test_files_ids:
+                for img in imgs:
+                    self.test_files.append(
+                        osp.join("JPEGImages", img["file_name"]))
+                json_file = open(osp.join(self.path, 'test.json'), 'w+')
+                json.dump(img_dict, json_file, cls=MyEncoder)
+
+            self.train_set = set(self.train_files)
+            self.val_set = set(self.val_files)
+            self.test_set = set(self.test_files)
+
+            for label, file_list in self.label_info.items():
+                self.class_train_file_list[label] = list()
+                self.class_val_file_list[label] = list()
+                self.class_test_file_list[label] = list()
+                for f in file_list:
+                    if f in self.test_set:
+                        self.class_test_file_list[label].append(f)
+                    if f in self.val_set:
+                        self.class_val_file_list[label].append(f)
+                    if f in self.train_set:
+                        self.class_train_file_list[label].append(f)
+
+        self.dump_statis_info()

+ 142 - 0
paddlex/restful/dataset/operate.py

@@ -0,0 +1,142 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import pickle
+import traceback
+import os.path as osp
+import multiprocessing as mp
+from .cls_dataset import ClsDataset
+from .det_dataset import DetDataset
+from .seg_dataset import SegDataset
+from .ins_seg_dataset import InsSegDataset
+from ..utils import set_folder_status, get_folder_status, DatasetStatus, DownloadStatus, download, list_files
+
+dataset_url_list = [
+    'https://bj.bcebos.com/paddlex/demos/vegetables_cls.tar.gz',
+    'https://bj.bcebos.com/paddlex/demos/insect_det.tar.gz',
+    'https://bj.bcebos.com/paddlex/demos/optic_disc_seg.tar.gz',
+    'https://bj.bcebos.com/paddlex/demos/xiaoduxiong_ins_det.tar.gz',
+    'https://bj.bcebos.com/paddlex/demos/remote_sensing_seg.tar.gz'
+]
+
+
+def _check_and_copy(dataset, dataset_path, source_path):
+    try:
+        dataset.check_dataset(source_path)
+    except Exception as e:
+        error_info = traceback.format_exc()
+        set_folder_status(dataset_path, DatasetStatus.XCHECKFAIL, error_info)
+        return
+    set_folder_status(dataset_path, DatasetStatus.XCOPYING, os.getpid())
+    try:
+        dataset.copy_dataset(source_path, dataset.all_files)
+    except Exception as e:
+        error_info = traceback.format_exc()
+        set_folder_status(dataset_path, DatasetStatus.XCOPYFAIL, error_info)
+        return
+    # 若上传已切分好的数据集
+    if len(dataset.train_files) != 0:
+        set_folder_status(dataset_path, DatasetStatus.XSPLITED)
+
+
+def import_dataset(dataset_id, dataset_type, dataset_path, source_path):
+    set_folder_status(dataset_path, DatasetStatus.XCHECKING)
+    if dataset_type == 'classification':
+        ds = ClsDataset(dataset_id, dataset_path)
+    elif dataset_type == 'detection':
+        ds = DetDataset(dataset_id, dataset_path)
+    elif dataset_type == 'segmentation':
+        ds = SegDataset(dataset_id, dataset_path)
+    elif dataset_type == 'instance_segmentation':
+        ds = InsSegDataset(dataset_id, dataset_path)
+    p = mp.Process(
+        target=_check_and_copy, args=(ds, dataset_path, source_path))
+    p.start()
+    return p
+
+
+def _download_proc(url, target_path, dataset_type):
+    # 下载数据集压缩包
+    from paddlex.utils import decompress
+    target_path = osp.join(target_path, dataset_type)
+    fname = download(url, target_path)
+    # 解压
+    decompress(fname)
+    set_folder_status(target_path, DownloadStatus.XDDECOMPRESSED)
+
+
+def download_demo_dataset(prj_type, target_path):
+    url = dataset_url_list[prj_type.value]
+    dataset_type = prj_type.name
+    p = mp.Process(
+        target=_download_proc, args=(url, target_path, dataset_type))
+    p.start()
+    return p
+
+
+def get_dataset_status(dataset_id, dataset_type, dataset_path):
+    status, message = get_folder_status(dataset_path, True)
+    if status is None:
+        status = DatasetStatus.XEMPTY
+    if status == DatasetStatus.XCOPYING:
+        items = message.strip().split()
+        pid = None
+        if len(items) < 2:
+            percent = 0.0
+        else:
+            pid = int(items[0])
+            if int(items[1]) == 0:
+                percent = 1.0
+            else:
+                copyed_files_num = len(list_files(dataset_path)) - 1
+                percent = copyed_files_num * 1.0 / int(items[1])
+        message = {'pid': pid, 'percent': percent}
+    if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
+        if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
+            p = import_dataset(dataset_id, dataset_type, dataset_path,
+                               dataset_path)
+            status = DatasetStatus.XCHECKING
+    return status, message
+
+
+def split_dataset(dataset_id, dataset_type, dataset_path, val_split,
+                  test_split):
+    status, message = get_folder_status(dataset_path, True)
+    if status != DatasetStatus.XCOPYDONE and status != DatasetStatus.XSPLITED:
+        raise Exception("数据集还未导入完成,请等数据集导入成功后再进行切分")
+    if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
+        raise Exception("数据集需重新校验,请刷新数据集后再进行切分")
+
+    if dataset_type == 'classification':
+        ds = ClsDataset(dataset_id, dataset_path)
+    elif dataset_type == 'detection':
+        ds = DetDataset(dataset_id, dataset_path)
+    elif dataset_type == 'segmentation':
+        ds = SegDataset(dataset_id, dataset_path)
+    elif dataset_type == 'instance_segmentation':
+        ds = InsSegDataset(dataset_id, dataset_path)
+
+    ds.load_statis_info()
+    ds.split(val_split, test_split)
+    set_folder_status(dataset_path, DatasetStatus.XSPLITED)
+
+
+def get_dataset_details(dataset_path):
+    status, message = get_folder_status(dataset_path, True)
+    if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
+        with open(osp.join(dataset_path, 'statis.pkl'), 'rb') as f:
+            details = pickle.load(f)
+        return details
+    return None

+ 222 - 0
paddlex/restful/dataset/seg_dataset.py

@@ -0,0 +1,222 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+import cv2
+from ..utils import list_files
+from .utils import is_pic, replace_ext, get_encoding, check_list_txt, read_seg_ann
+from .datasetbase import DatasetBase
+
+
+class SegDataset(DatasetBase):
+    def __init__(self, dataset_id, path):
+        super().__init__(dataset_id, path)
+
+    def check_dataset(self, source_path):
+        if not osp.isdir(osp.join(source_path, 'Annotations')):
+            raise ValueError("标注文件应该放在{}目录下".format(
+                osp.join(source_path, 'Annotations')))
+        if not osp.isdir(osp.join(source_path, 'JPEGImages')):
+            raise ValueError("图片文件应该放在{}目录下".format(
+                osp.join(source_path, 'JPEGImages')))
+
+        labels_txt = osp.join(source_path, 'labels.txt')
+        if osp.exists(labels_txt):
+            with open(labels_txt, encoding=get_encoding(labels_txt)) as fid:
+                lines = fid.readlines()
+                for line in lines:
+                    self.labels.append(line.strip())
+
+        self.all_files = list_files(source_path)
+        # 对语义分割数据集进行统计分析
+        self.file_info = dict()
+        self.label_info = dict()
+
+        if osp.exists(osp.join(source_path, 'train_list.txt')):
+            return self.check_splited_dataset(source_path)
+
+        for f in self.all_files:
+            if not is_pic(f):
+                continue
+            items = osp.split(f)
+            if len(items) == 2 and items[0] == "JPEGImages":
+                anno_name = replace_ext(items[1], "png")
+                full_anno_path = osp.join(
+                    (osp.join(source_path, 'Annotations')), anno_name)
+                if osp.exists(full_anno_path):
+                    self.file_info[f] = osp.join('Annotations', anno_name)
+
+                # 解析PNG标注文件,获取类别信息
+                labels, ann_img_shape = read_seg_ann(full_anno_path)
+                img_shape = cv2.imread(osp.join(source_path, f)).shape
+                if img_shape[0] != ann_img_shape[0] or img_shape[
+                        1] != ann_img_shape[1]:
+                    raise ValueError("文件{}与标注图片尺寸不一致".format(items[1]))
+                for i in labels:
+                    if str(i) not in self.label_info:
+                        self.label_info[str(i)] = list()
+                    self.label_info[str(i)].append(f)
+
+        # 如果类标签的最大值大于类别数,统计相应的类别为零
+        max_label = max([int(i) for i in self.label_info]) + 1
+        for i in range(max_label):
+            if str(i) not in self.label_info:
+                self.label_info[str(i)] = list()
+
+        if len(self.labels) == 0:
+            self.labels = [int(i) for i in self.label_info]
+            self.labels.sort()
+            self.labels = [str(i) for i in self.labels]
+        else:
+            keys = list(self.label_info.keys())
+            try:
+                for key in keys:
+                    label = self.labels[int(key)]
+                    self.label_info[label] = self.label_info.pop(key)
+            except:
+                raise ValueError("标注信息与实际类别不一致")
+
+        for label in self.labels:
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def check_splited_dataset(self, source_path):
+        labels_txt = osp.join(source_path, "labels.txt")
+        train_list_txt = osp.join(source_path, "train_list.txt")
+        val_list_txt = osp.join(source_path, "val_list.txt")
+        test_list_txt = osp.join(source_path, "test_list.txt")
+        for txt_file in [train_list_txt, val_list_txt]:
+            if not osp.exists(txt_file):
+                raise Exception("已切分的数据集下应该包含train_list.txt, val_list.txt文件")
+        check_list_txt([train_list_txt, val_list_txt, test_list_txt])
+
+        if osp.exists(labels_txt):
+            self.labels = open(
+                labels_txt, 'r',
+                encoding=get_encoding(labels_txt)).read().strip().split('\n')
+
+        for txt_file in [train_list_txt, val_list_txt, test_list_txt]:
+            if not osp.exists(txt_file):
+                continue
+            with open(txt_file, "r") as f:
+                for line in f:
+                    items = line.strip().split()
+                    img_file, png_file = [items[0], items[1]]
+
+                    if not osp.isfile(osp.join(source_path, png_file)):
+                        raise ValueError("数据目录{}中不存在标注文件{}".format(
+                            osp.split(txt_file)[-1], png_file))
+                    if not osp.isfile(osp.join(source_path, img_file)):
+                        raise ValueError("数据目录{}中不存在图片文件{}".format(
+                            osp.split(txt_file)[-1], img_file))
+                    if not png_file.split('.')[-1] == 'png':
+                        raise ValueError("标注文件{}不是png文件".format(png_file))
+                    img_file_name = osp.split(img_file)[-1]
+                    if not is_pic(img_file_name) or img_file_name.startswith(
+                            '.'):
+                        raise ValueError("文件{}不是图片格式".format(img_file_name))
+
+                    self.file_info[img_file] = png_file
+
+                    if txt_file == train_list_txt:
+                        self.train_files.append(img_file)
+                    elif txt_file == val_list_txt:
+                        self.val_files.append(img_file)
+                    elif txt_file == test_list_txt:
+                        self.test_files.append(img_file)
+
+                    # 解析PNG标注文件
+                    labels, ann_img_shape = read_seg_ann(
+                        osp.join(source_path, png_file))
+                    img_shape = cv2.imread(osp.join(source_path,
+                                                    img_file)).shape
+                    if img_shape[0] != ann_img_shape[0] or img_shape[
+                            1] != ann_img_shape[1]:
+                        raise ValueError("文件{}与标注图片尺寸不一致".format(
+                            img_file_name))
+                    for i in labels:
+                        if str(i) not in self.label_info:
+                            self.label_info[str(i)] = list()
+                        self.label_info[str(i)].append(img_file)
+
+        # 如果类标签的最大值大于类别数,统计相应的类别为零
+        max_label = max([int(i) for i in self.label_info]) + 1
+        for i in range(max_label):
+            if str(i) not in self.label_info:
+                self.label_info[str(i)] = list()
+
+        if len(self.labels) == 0:
+            self.labels = [int(i) for i in self.label_info]
+            self.labels.sort()
+            self.labels = [str(i) for i in self.labels]
+        else:
+            keys = list(self.label_info.keys())
+            try:
+                for key in keys:
+                    label = self.labels[int(key)]
+                    self.label_info[label] = self.label_info.pop(key)
+            except:
+                raise ValueError("标注信息与实际类别不一致")
+
+        self.train_set = set(self.train_files)
+        self.val_set = set(self.val_files)
+        self.test_set = set(self.test_files)
+
+        for label, file_list in self.label_info.items():
+            self.class_train_file_list[label] = list()
+            self.class_val_file_list[label] = list()
+            self.class_test_file_list[label] = list()
+            for f in file_list:
+                if f in self.test_set:
+                    self.class_test_file_list[label].append(f)
+                if f in self.val_set:
+                    self.class_val_file_list[label].append(f)
+                if f in self.train_set:
+                    self.class_train_file_list[label].append(f)
+
+        # 将数据集分析信息dump到本地
+        self.dump_statis_info()
+
+    def split(self, val_split, test_split):
+        super().split(val_split, test_split)
+        with open(
+                osp.join(self.path, 'train_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.train_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        with open(
+                osp.join(self.path, 'val_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.val_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        with open(
+                osp.join(self.path, 'test_list.txt'), mode='w',
+                encoding='utf-8') as f:
+            for x in self.test_files:
+                label = self.file_info[x]
+                f.write('{} {}\n'.format(x, label))
+        if not osp.exists(osp.join(self.path, 'labels.txt')):
+            with open(
+                    osp.join(self.path, 'labels.txt'), mode='w',
+                    encoding='utf-8') as f:
+                max_label = max([int(i) for i in self.labels]) + 1
+                for i in range(max_label):
+                    f.write('{}\n'.format(str(i)))
+        self.dump_statis_info()

+ 223 - 0
paddlex/restful/dataset/utils.py

@@ -0,0 +1,223 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+import shutil
+from enum import Enum
+import traceback
+import chardet
+from PIL import Image
+import numpy as np
+import json
+from ..utils import set_folder_status, get_folder_status, DatasetStatus
+
+
+def copy_directory(src, dst, files=None):
+    """从src目录copy文件至dst目录,
+       注意:拷贝前会先清空dst中的所有文件
+
+    Args:
+        src: 源目录路径
+        dst: 目标目录路径
+        files: 需要拷贝的文件列表(src的相对路径)
+    """
+    set_folder_status(dst, DatasetStatus.XCOPYING, os.getpid())
+    if files is None:
+        files = list_files(src)
+    try:
+        message = '{} {}'.format(os.getpid(), len(files))
+        set_folder_status(dst, DatasetStatus.XCOPYING, message)
+        if not osp.samefile(src, dst):
+            for i, f in enumerate(files):
+                items = osp.split(f)
+                if len(items) > 2:
+                    continue
+                if len(items) == 2:
+                    if not osp.isdir(osp.join(dst, items[0])):
+                        if osp.exists(osp.join(dst, items[0])):
+                            os.remove(osp.join(dst, items[0]))
+                        os.makedirs(osp.join(dst, items[0]))
+                shutil.copy(osp.join(src, f), osp.join(dst, f))
+        set_folder_status(dst, DatasetStatus.XCOPYDONE)
+    except Exception as e:
+        error_info = traceback.format_exc()
+        set_folder_status(dst, DatasetStatus.XCOPYFAIL, error_info)
+
+
+def is_pic(filename):
+    """ 判断文件是否为图片格式
+
+    Args:
+        filename: 文件路径
+    """
+    suffixes = {'JPEG', 'jpeg', 'JPG', 'jpg', 'BMP', 'bmp', 'PNG', 'png'}
+    suffix = filename.strip().split('.')[-1]
+    if suffix not in suffixes:
+        return False
+    return True
+
+
+def replace_ext(filename, new_ext):
+    """ 替换文件后缀
+
+    Args:
+        filename: 文件路径
+        new_ext: 需要替换的新的后缀
+    """
+    items = filename.split(".")
+    items[-1] = new_ext
+    new_filename = ".".join(items)
+    return new_filename
+
+
+def get_encoding(filename):
+    """ 获取文件编码方式
+
+    Args:
+        filename: 文件路径
+    """
+    f = open(filename, 'rb')
+    data = f.read()
+    file_encoding = chardet.detect(data).get('encoding')
+    return file_encoding
+
+
+def pil_imread(file_path):
+    """ 获取分割标注图片信息
+
+    Args:
+        filename: 文件路径
+    """
+    im = Image.open(file_path)
+    return np.asarray(im)
+
+
+def check_list_txt(list_txts):
+    """ 检查切分信息文件的格式
+
+    Args:
+        list_txts: 包含切分信息文件路径的list
+    """
+    for list_txt in list_txts:
+        if not osp.exists(list_txt):
+            continue
+        with open(list_txt) as f:
+            for line in f:
+                items = line.strip().split()
+                if len(items) != 2:
+                    raise Exception('{} 格式错误. 列表应包含两列,由空格分离。'.format(list_txt))
+
+
+def read_seg_ann(pngfile):
+    """ 解析语义分割的标注png图片
+
+    Args:
+        pngfile: 包含标注信息的png图片路径
+    """
+    grt = pil_imread(pngfile)
+    labels = list(np.unique(grt))
+    if 255 in labels:
+        labels.remove(255)
+    return labels, grt.shape
+
+
+def read_coco_ann(img_id, coco, cid2cname, catid2clsid):
+    img_anno = coco.loadImgs(img_id)[0]
+    im_w = float(img_anno['width'])
+    im_h = float(img_anno['height'])
+
+    ins_anno_ids = coco.getAnnIds(imgIds=img_id, iscrowd=0)
+    instances = coco.loadAnns(ins_anno_ids)
+
+    bboxes = []
+    for inst in instances:
+        x, y, box_w, box_h = inst['bbox']
+        x1 = max(0, x)
+        y1 = max(0, y)
+        x2 = min(im_w - 1, x1 + max(0, box_w - 1))
+        y2 = min(im_h - 1, y1 + max(0, box_h - 1))
+        if inst['area'] > 0 and x2 >= x1 and y2 >= y1:
+            inst['clean_bbox'] = [x1, y1, x2, y2]
+            bboxes.append(inst)
+        else:
+            raise Exception("标注文件存在错误")
+    num_bbox = len(bboxes)
+
+    gt_bbox = np.zeros((num_bbox, 4), dtype=np.float32)
+    gt_class = [""] * num_bbox
+    gt_poly = [None] * num_bbox
+
+    for i, box in enumerate(bboxes):
+        catid = box['category_id']
+        gt_class[i] = cid2cname[catid2clsid[catid]]
+        gt_bbox[i, :] = box['clean_bbox']
+        # is_crowd[i][0] = box['iscrowd']
+        if 'segmentation' in box:
+            gt_poly[i] = box['segmentation']
+
+    anno_dict = {
+        'h': im_h,
+        'w': im_w,
+        'gt_class': gt_class,
+        'gt_bbox': gt_bbox,
+        'gt_poly': gt_poly,
+    }
+    return anno_dict
+
+
+def get_npy_from_coco_json(coco, npy_path, files):
+    """ 从实例分割标注的json文件中,获取每张图片的信息,并存为npy格式
+
+    Args:
+        coco: 从json文件中解析出的标注信息
+        npy_path: npy文件保存的地址
+        files: 需要生成npy文件的目录
+    """
+    img_ids = coco.getImgIds()
+    cat_ids = coco.getCatIds()
+    anno_ids = coco.getAnnIds()
+    catid2clsid = dict({catid: i for i, catid in enumerate(cat_ids)})
+    cid2cname = dict({
+        clsid: coco.loadCats(catid)[0]['name']
+        for catid, clsid in catid2clsid.items()
+    })
+    iname2id = dict()
+    for img_id in img_ids:
+        img_name = osp.split(coco.loadImgs(img_id)[0]["file_name"])[-1]
+        iname2id[img_name] = img_id
+
+    if not osp.exists(npy_path):
+        os.makedirs(npy_path)
+
+    for img in files:
+        img_id = iname2id[osp.split(img)[-1]]
+        anno_dict = read_coco_ann(img_id, coco, cid2cname, catid2clsid)
+
+        img_name = osp.split(img)[-1]
+        npy_name = replace_ext(img_name, "npy")
+        np.save(osp.join(npy_path, npy_name), anno_dict)
+
+
+class MyEncoder(json.JSONEncoder):
+    # 调整json文件存储形式
+    def default(self, obj):
+        if isinstance(obj, np.integer):
+            return int(obj)
+        elif isinstance(obj, np.floating):
+            return float(obj)
+        elif isinstance(obj, np.ndarray):
+            return obj.tolist()
+        else:
+            return super(MyEncoder, self).default(obj)

+ 167 - 0
paddlex/restful/demo.py

@@ -0,0 +1,167 @@
+import os
+import json
+from os import path as osp
+from .utils import DownloadStatus, DatasetStatus, ProjectType, get_folder_status
+from .project.train.params import PARAMS_CLASS_LIST
+from .utils import CustomEncoder
+
+
+def download_demo_dataset(data, workspace, load_demo_proc_dict):
+    """下载样例工程
+
+        Args:
+            data为dict, key包括
+            'prj_type' 样例类型(ProjectType)
+        """
+    prj_type = ProjectType(data['prj_type'])
+    assert prj_type.value >= 0 and prj_type.value <= 4, "不支持此样例类型的导入(type:{})".format(
+        prj_type)
+    target_path = osp.join(workspace.path, "demo_datasets")
+    if not osp.exists(target_path):
+        os.makedirs(target_path)
+    from .dataset.operate import download_demo_dataset
+    proc = download_demo_dataset(prj_type, target_path)
+    if prj_type not in load_demo_proc_dict:
+        load_demo_proc_dict[prj_type] = []
+    load_demo_proc_dict[prj_type].append(proc)
+    return {'status': 1}
+
+
+def load_demo_project(data, workspace, monitored_processes,
+                      load_demo_proj_data_dict, load_demo_proc_dict):
+    """导入样例工程
+
+    Args:
+        data为dict, key包括
+        'prj_type' 样例类型(ProjectType)
+    """
+    prj_type = ProjectType(data['prj_type'])
+    assert prj_type.value >= 0 and prj_type.value <= 4, "不支持此样例类型的导入(type:{})".format(
+        prj_type)
+
+    target_path = osp.join(workspace.path, "demo_datasets")
+    assert osp.exists(target_path), "样例数据集暂未下载,无法导入样例工程"
+    target_path = osp.join(target_path, prj_type.name)
+    assert osp.exists(target_path), "样例{}数据集暂未下载,无法导入样例工程".format(
+        prj_type.name)
+
+    status = get_folder_status(target_path)
+    assert status == DownloadStatus.XDDECOMPRESSED, "样例{}数据集暂未解压,无法导入样例工程".format(
+        prj_type.name)
+
+    from .dataset.operate import dataset_url_list
+    url = dataset_url_list[prj_type.value]
+    fname = osp.split(url)[-1]
+    for suffix in ['tar', 'tgz', 'zip']:
+        pos = fname.find(suffix)
+        if pos >= 2:
+            fname = fname[0:pos - 1]
+            break
+    source_dataset_path = osp.join(target_path, fname)
+    params_path = osp.join(target_path, fname, fname + "_params.json")
+    params = {}
+    with open(params_path, "r", encoding="utf-8") as f:
+        params = json.load(f)
+
+    dataset_params = params['dataset_info']
+    proj_params = params['project_info']
+    train_params = params['train_params']
+
+    # 判断数据集、项目名称是否已存在
+    dataset_name = dataset_params['name']
+    project_name = proj_params['name']
+    for id in workspace.datasets:
+        if dataset_name == workspace.datasets[id].name:
+            return {'status': 1, 'loading_status': 'dataset already exists'}
+
+    for id in workspace.projects:
+        if project_name == workspace.projects[id].name:
+            return {'status': 1, 'loading_status': 'project already exists'}
+
+    # 创建数据集
+    from .dataset.dataset import create_dataset
+    results = create_dataset(dataset_params, workspace)
+    dataset_id = results['did']
+
+    # 导入数据集
+    from .dataset.dataset import import_dataset
+    data = {'did': dataset_id, 'path': source_dataset_path}
+    import_dataset(data, workspace, monitored_processes, load_demo_proc_dict)
+
+    # 创建项目
+    from .project.project import create_project
+    results = create_project(proj_params, workspace)
+
+    pid = results['pid']
+    # 绑定数据集
+    from .workspace import set_attr
+    attr_dict = {'did': dataset_id}
+    params = {'struct': 'project', 'id': pid, 'attr_dict': attr_dict}
+    set_attr(params, workspace)
+    # 创建任务
+    task_params = PARAMS_CLASS_LIST[prj_type.value]()
+    for k, v in train_params.items():
+        if hasattr(task_params, k):
+            setattr(task_params, k, v)
+    task_params = CustomEncoder().encode(task_params)
+    from .project.task import create_task
+    params = {'pid': pid, 'train': task_params}
+    create_task(params, workspace)
+    load_demo_proj_data_dict[prj_type] = (pid, dataset_id)
+    return {'status': 1, 'did': dataset_id, 'pid': pid}
+
+
+def get_download_demo_progress(data, workspace):
+    """查询样例工程的下载进度
+
+    Args:
+        data为dict, key包括
+        'prj_type' 样例类型(ProjectType)
+    """
+    prj_type = ProjectType(data['prj_type'])
+    target_path = osp.join(workspace.path, "demo_datasets", prj_type.name)
+    status, message = get_folder_status(target_path, True)
+    if status == DownloadStatus.XDDOWNLOADING:
+        from .dataset.operate import dataset_url_list
+        url = dataset_url_list[prj_type.value]
+        fname = osp.split(url)[-1] + "_tmp"
+        fullname = osp.join(target_path, fname)
+        total_size = int(message)
+        download_size = osp.getsize(fullname)
+        message = download_size * 100 / total_size
+    if status is not None:
+        attr = {'status': status.value, 'progress': message}
+    else:
+        attr = {'status': status, 'progress': message}
+    return {'status': 1, 'attr': attr}
+
+
+def stop_import_demo(data, workspace, load_demo_proc_dict,
+                     load_demo_proj_data_dict):
+    """停止样例工程的导入进度
+
+    Args:
+        request(comm.Request): 其中request.params为dict, key包括
+        'prj_type' 样例类型(ProjectType)
+    """
+    prj_type = ProjectType(data['prj_type'])
+    for proc in load_demo_proc_dict[prj_type]:
+        if proc.is_alive():
+            proc.terminate()
+    # 只删除未完成导入的样例项目
+    if prj_type in load_demo_proj_data_dict:
+        pid, did = load_demo_proj_data_dict[prj_type]
+        params = {'did': did}
+        from .dataset.dataset import get_dataset_status
+        results = get_dataset_status(params, workspace)
+        dataset_status = DatasetStatus(results['dataset_status'])
+        if dataset_status not in [
+                DatasetStatus.XCOPYDONE, DatasetStatus.XSPLITED
+        ]:
+            params = {'pid': pid}
+            from .project.project import delete_project
+            delete_project(params, workspace)
+            from .dataset.dataset import delete_dataset
+            params = {'did': did}
+            delete_dataset(params, workspace)
+    return {'status': 1}

+ 45 - 0
paddlex/restful/dir.py

@@ -0,0 +1,45 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+
+
+def gen_user_home():
+    if "PADDLE_HOME" in os.environ:
+        home_path = os.environ["PADDLE_HOME"]
+        if os.path.exists(home_path) and os.path.isdir(home_path):
+            return home_path
+    return os.path.expanduser('~')
+
+
+def gen_paddlex_home():
+    path = osp.join(gen_user_home(), ".paddlex_server")
+    if not osp.exists(path):
+        os.makedirs(path)
+    return path
+
+
+USER_HOME = gen_user_home()
+PADDLEX_HOME = gen_paddlex_home()
+WORKSPACE_HOME = osp.join(USER_HOME, "paddlex_workspace")
+LOG_HOME = osp.join(PADDLEX_HOME, "logs")
+SINGLE_LOCK_HOME = osp.join(PADDLEX_HOME, "single_lock")
+CACHE_HOME = osp.join(PADDLEX_HOME, "cache")
+
+for home in [WORKSPACE_HOME, LOG_HOME, SINGLE_LOCK_HOME, CACHE_HOME]:
+    if not osp.exists(home):
+        os.makedirs(home)

+ 298 - 0
paddlex/restful/model.py

@@ -0,0 +1,298 @@
+import time
+import os
+import shutil
+import pickle
+from os import path as osp
+from .utils import set_folder_status, TaskStatus, copy_pretrained_model, PretrainedModelStatus
+from . import workspace_pb2 as w
+
+
+def list_pretrained_models(workspace):
+    """列出预训练模型列表
+    """
+    pretrained_model_list = list()
+    for id in workspace.pretrained_models:
+        pretrained_model = workspace.pretrained_models[id]
+        model_id = pretrained_model.id
+        model_name = pretrained_model.name
+        model_model = pretrained_model.model
+        model_type = pretrained_model.type
+        model_pid = pretrained_model.pid
+        model_tid = pretrained_model.tid
+        model_create_time = pretrained_model.create_time
+        model_path = pretrained_model.path
+        attr = {
+            'id': model_id,
+            'name': model_name,
+            'model': model_model,
+            'type': model_type,
+            'pid': model_pid,
+            'tid': model_tid,
+            'create_time': model_create_time,
+            'path': model_path
+        }
+        pretrained_model_list.append(attr)
+
+    return {'status': 1, "pretrained_models": pretrained_model_list}
+
+
+def create_pretrained_model(data, workspace, monitored_processes):
+    """根据request创建预训练模型。
+
+    Args:
+        data为dict,key包括
+        'pid'所属项目id, 'tid'所属任务id,'name'预训练模型名称,
+        'source_path' 原模型路径, 'eval_results'(可选) 评估结果数据
+    """
+    time_array = time.localtime(time.time())
+    create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    id = workspace.max_pretrained_model_id + 1
+    workspace.max_pretrained_model_id = id
+    if id < 10000:
+        id = 'PM%04d' % id
+    else:
+        id = 'PM{}'.format(id)
+    pid = data['pid']
+    tid = data['tid']
+    name = data['name']
+    source_path = data['source_path']
+    assert pid in workspace.projects, "【预训练模型创建】项目ID'{}'不存在.".format(pid)
+    assert tid in workspace.tasks, "【预训练模型创建】任务ID'{}'不存在.".format(tid)
+    assert not id in workspace.pretrained_models, "【预训练模型创建】预训练模型'{}'已经被占用.".format(
+        id)
+    assert osp.exists(source_path), "原模型路径不存在: {}".format(source_path)
+    path = osp.join(workspace.path, 'pretrain', id)
+    if not osp.exists(path):
+        os.makedirs(path)
+    set_folder_status(path, PretrainedModelStatus.XPINIT)
+    params = {'tid': tid}
+    from .project.task import get_task_params
+    ret = get_task_params(params, workspace)
+    train_params = ret['train']
+    model_structure = train_params.model
+    if hasattr(train_params, "backbone"):
+        model_structure = "{}-{}".format(model_structure,
+                                         train_params.backbone)
+    if hasattr(train_params, "with_fpn"):
+        if train_params.with_fpn:
+            model_structure = "{}-{}".format(model_structure, "WITH_FPN")
+
+    pm = w.PretrainedModel(
+        id=id,
+        name=name,
+        model=model_structure,
+        type=workspace.projects[pid].type,
+        pid=pid,
+        tid=tid,
+        create_time=create_time,
+        path=path)
+    workspace.pretrained_models[id].CopyFrom(pm)
+    # 保存评估结果
+    if 'eval_results' in data:
+        with open(osp.join(source_path, "eval_res.pkl"), "wb") as f:
+            pickle.dump(data['eval_results'], f)
+    # 拷贝训练参数文件
+    task_path = workspace.tasks[tid].path
+    task_params_path = osp.join(task_path, 'params.pkl')
+    if osp.exists(task_params_path):
+        shutil.copy(task_params_path, path)
+    # 拷贝数据集信息文件
+    did = workspace.projects[pid].did
+    dataset_path = workspace.datasets[did].path
+    dataset_info_path = osp.join(dataset_path, "statis.pkl")
+    if osp.exists(dataset_info_path):
+        # 写入部分数据集信息
+        with open(dataset_info_path, "rb") as f:
+            dataset_info_dict = pickle.load(f)
+        dataset_info_dict['name'] = workspace.datasets[did].name
+        dataset_info_dict['desc'] = workspace.datasets[did].desc
+        with open(dataset_info_path, "wb") as f:
+            pickle.dump(dataset_info_dict, f)
+        shutil.copy(dataset_info_path, path)
+
+    # copy from source_path to path
+    proc = copy_pretrained_model(source_path, path)
+    monitored_processes.put(proc.pid)
+    return {'status': 1, 'pmid': id}
+
+
+def delete_pretrained_model(data, workspace):
+    """删除pretrained_model。
+
+    Args:
+        data为dict,
+        key包括'pmid'预训练模型id
+    """
+    pmid = data['pmid']
+    assert pmid in workspace.pretrained_models, "预训练模型ID'{}'不存在.".format(pmid)
+    if osp.exists(workspace.pretrained_models[pmid].path):
+        shutil.rmtree(workspace.pretrained_models[pmid].path)
+    del workspace.pretrained_models[pmid]
+    return {'status': 1}
+
+
+def create_exported_model(data, workspace):
+    """根据request创建已发布模型。
+    Args:
+        data为dict,key包括
+        'pid'所属项目id, 'tid'所属任务id,'name'已发布模型名称,
+        'path' 模型路径, 'exported_type' 已发布模型类型,
+    """
+    time_array = time.localtime(time.time())
+    create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    emid = workspace.max_exported_model_id + 1
+    workspace.max_exported_model_id = emid
+    if emid < 10000:
+        emid = 'EM%04d' % emid
+    else:
+        emid = 'EM{}'.format(emid)
+    pid = data['pid']
+    tid = data['tid']
+    name = data['name']
+    path = data['path']
+    exported_type = data['exported_type']
+    assert pid in workspace.projects, "【已发布模型创建】项目ID'{}'不存在.".format(pid)
+    assert tid in workspace.tasks, "【已发布模型创建】任务ID'{}'不存在.".format(tid)
+    assert emid not in workspace.exported_models, "【已发布模型创建】已发布模型'{}'已经被占用.".format(
+        emid)
+    #assert osp.exists(path), "已发布模型路径不存在: {}".format(path)
+    if not osp.exists(path):
+        os.makedirs(path)
+    task_path = workspace.tasks[tid].path
+    # 拷贝评估结果
+    eval_res_path = osp.join(task_path, 'eval_res.pkl')
+    if osp.exists(eval_res_path):
+        shutil.copy(eval_res_path, path)
+    # 拷贝训练参数文件
+    task_params_path = osp.join(task_path, 'params.pkl')
+    if osp.exists(task_params_path):
+        shutil.copy(task_params_path, path)
+    # 拷贝数据集信息文件
+    did = workspace.projects[pid].did
+    dataset_path = workspace.datasets[did].path
+    dataset_info_path = osp.join(dataset_path, "statis.pkl")
+    if osp.exists(dataset_info_path):
+        # 写入部分数据集信息
+        with open(dataset_info_path, "rb") as f:
+            dataset_info_dict = pickle.load(f)
+        dataset_info_dict['name'] = workspace.datasets[did].name
+        dataset_info_dict['desc'] = workspace.datasets[did].desc
+        with open(dataset_info_path, "wb") as f:
+            pickle.dump(dataset_info_dict, f)
+        shutil.copy(dataset_info_path, path)
+    from .project.task import get_task_params
+    params = {'tid': tid}
+    ret = get_task_params(params, workspace)
+    train_params = ret['train']
+    model_structure = train_params.model
+    if hasattr(train_params, "backbone"):
+        model_structure = "{}-{}".format(model_structure,
+                                         train_params.backbone)
+    if hasattr(train_params, "with_fpn"):
+        if train_params.with_fpn:
+            model_structure = "{}-{}".format(model_structure, "WITH_FPN")
+
+    em = w.ExportedModel(
+        id=emid,
+        name=name,
+        model=model_structure,
+        type=workspace.projects[pid].type,
+        pid=pid,
+        tid=tid,
+        create_time=create_time,
+        path=path,
+        exported_type=exported_type)
+
+    workspace.exported_models[emid].CopyFrom(em)
+    return {'status': 1, 'emid': emid}
+
+
+def list_exported_models(workspace):
+    """列出预训练模型列表,可根据request中的参数进行筛选
+
+    Args:
+    """
+    exported_model_list = list()
+    for id in workspace.exported_models:
+        exported_model = workspace.exported_models[id]
+        model_id = exported_model.id
+        model_name = exported_model.name
+        model_model = exported_model.model
+        model_type = exported_model.type
+        model_pid = exported_model.pid
+        model_tid = exported_model.tid
+        model_create_time = exported_model.create_time
+        model_path = exported_model.path
+        model_exported_type = exported_model.exported_type
+        attr = {
+            'id': model_id,
+            'name': model_name,
+            'model': model_model,
+            'type': model_type,
+            'pid': model_pid,
+            'tid': model_tid,
+            'create_time': model_create_time,
+            'path': model_path,
+            'exported_type': model_exported_type
+        }
+        if model_tid in workspace.tasks:
+            from .project.task import get_export_status
+            params = {'tid': model_tid}
+            results = get_export_status(params, workspace)
+            if results['export_status'] == TaskStatus.XEXPORTED:
+                exported_model_list.append(attr)
+        else:
+            exported_model_list.append(attr)
+    return {'status': 1, "exported_models": exported_model_list}
+
+
+def delete_exported_model(data, workspace):
+    """删除exported_model。
+
+    Args:
+        data为dict,
+        key包括'emid'已发布模型id
+    """
+    emid = data['emid']
+    assert emid in workspace.exported_models, "已发布模型模型ID'{}'不存在.".format(emid)
+    if osp.exists(workspace.exported_models[emid].path):
+        shutil.rmtree(workspace.exported_models[emid].path)
+    del workspace.exported_models[emid]
+    return {'status': 1}
+
+
+def get_model_details(data, workspace):
+    """获取模型详情。
+
+    Args:
+        data为dict,
+        key包括'mid'模型id
+    """
+    mid = data['mid']
+    if mid in workspace.pretrained_models:
+        model_path = workspace.pretrained_models[mid].path
+    elif mid in workspace.exported_models:
+        model_path = workspace.exported_models[mid].path
+    else:
+        raise "模型{}不存在".format(mid)
+    dataset_file = osp.join(model_path, 'statis.pkl')
+    dataset_info = pickle.load(open(dataset_file, 'rb'))
+    dataset_attr = {
+        'name': dataset_info['name'],
+        'desc': dataset_info['desc'],
+        'labels': dataset_info['labels'],
+        'train_num': len(dataset_info['train_files']),
+        'val_num': len(dataset_info['val_files']),
+        'test_num': len(dataset_info['test_files'])
+    }
+    task_params_file = osp.join(model_path, 'params.pkl')
+    task_params = pickle.load(open(task_params_file, 'rb'))
+    eval_result_file = osp.join(model_path, 'eval_res.pkl')
+    eval_result = pickle.load(open(eval_result_file, 'rb'))
+    #model_file = {'task_attr': task_params_file, 'eval_result': eval_result_file}
+    return {
+        'status': 1,
+        'dataset_attr': dataset_attr,
+        'task_params': task_params,
+        'eval_result': eval_result
+    }

+ 13 - 0
paddlex/restful/project/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 13 - 0
paddlex/restful/project/evaluate/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 126 - 0
paddlex/restful/project/evaluate/classification.py

@@ -0,0 +1,126 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import yaml
+import os.path as osp
+import numpy as np
+from sklearn.metrics import confusion_matrix, roc_curve, auc
+
+
+class Evaluator(object):
+    def __init__(self, model_path, topk=5):
+        with open(osp.join(model_path, "model.yml")) as f:
+            model_info = yaml.load(f.read(), Loader=yaml.Loader)
+        with open(osp.join(model_path, 'eval_details.json'), 'r') as f:
+            eval_details = json.load(f)
+        self.topk = topk
+
+        self.labels = model_info['_Attributes']['labels']
+        self.true_labels = np.array(eval_details['true_labels'])
+        self.pred_scores = np.array(eval_details['pred_scores'])
+        label_ids_list = list(range(len(self.labels)))
+        self.no_appear_label_ids = set(label_ids_list) - set(
+            self.true_labels.tolist())
+
+    def cal_confusion_matrix(self):
+        '''计算混淆矩阵。
+        '''
+        pred_labels = np.argsort(self.pred_scores)[:, -1:].flatten()
+        cm = confusion_matrix(
+            self.true_labels.tolist(),
+            pred_labels.tolist(),
+            labels=list(range(len(self.labels))))
+        return cm
+
+    def cal_precision_recall_F1(self):
+        '''计算precision、recall、F1。
+        '''
+        out = {}
+        out_avg = {}
+        out_avg['precision'] = 0.0
+        out_avg['recall'] = 0.0
+        out_avg['F1'] = 0.0
+        pred_labels = np.argsort(self.pred_scores)[:, -1:].flatten()
+        for label_id in range(len(self.labels)):
+            out[self.labels[label_id]] = {}
+            if label_id in self.no_appear_label_ids:
+                out[self.labels[label_id]]['precision'] = -1.0
+                out[self.labels[label_id]]['recall'] = -1.0
+                out[self.labels[label_id]]['F1'] = -1.0
+                continue
+            pred_index = np.where(pred_labels == label_id)[0].tolist()
+            tp = np.sum(
+                self.true_labels[pred_index] == pred_labels[pred_index])
+            tp_fp = len(pred_index)
+            tp_fn = len(np.where(self.true_labels == label_id)[0].tolist())
+            out[self.labels[label_id]]['precision'] = tp * 1.0 / tp_fp
+            out[self.labels[label_id]]['recall'] = tp * 1.0 / tp_fn
+            out[self.labels[label_id]]['F1'] = 2 * tp * 1.0 / (tp_fp + tp_fn)
+            ratio = tp_fn * 1.0 / self.true_labels.shape[0]
+            out_avg['precision'] += out[self.labels[label_id]][
+                'precision'] * ratio
+            out_avg['recall'] += out[self.labels[label_id]]['recall'] * ratio
+            out_avg['F1'] += out[self.labels[label_id]]['F1'] * ratio
+        return out, out_avg
+
+    def cal_auc(self):
+        '''计算AUC。
+        '''
+        out = {}
+        for label_id in range(len(self.labels)):
+            part_pred_scores = self.pred_scores[:, label_id:label_id + 1]
+            part_pred_scores = part_pred_scores.flatten()
+            fpr, tpr, thresholds = roc_curve(
+                self.true_labels, part_pred_scores, pos_label=label_id)
+            label_auc = auc(fpr, tpr)
+            if label_id in self.no_appear_label_ids:
+                out[self.labels[label_id]] = -1.0
+                continue
+            out[self.labels[label_id]] = label_auc
+        return out
+
+    def cal_accuracy(self):
+        '''计算Accuracy。
+        '''
+        out = {}
+        k = min(self.topk, len(self.labels))
+        pred_top1_label = np.argsort(self.pred_scores)[:, -1]
+        pred_topk_label = np.argsort(self.pred_scores)[:, -k:]
+        acc1 = sum(pred_top1_label == self.true_labels) / len(self.true_labels)
+        acck = sum([
+            np.isin(x, y) for x, y in zip(self.true_labels, pred_topk_label)
+        ]) / len(self.true_labels)
+        out['acc1'] = acc1
+        out['acck'] = acck
+        out['k'] = k
+        return out
+
+    def generate_report(self):
+        '''生成评估报告。
+        '''
+        report = dict()
+        report['Confusion_Matrix'] = self.cal_confusion_matrix()
+        report['PRF1_average'] = {}
+        report['PRF1'], report['PRF1_average'][
+            'over_all'] = self.cal_precision_recall_F1()
+        auc = self.cal_auc()
+        for k, v in auc.items():
+            report['PRF1'][k]['auc'] = v
+        acc = self.cal_accuracy()
+        report["Acc1"] = acc["acc1"]
+        report["Acck"] = acc["acck"]
+        report["topk"] = acc["k"]
+        report['label_list'] = self.labels
+        return report

+ 783 - 0
paddlex/restful/project/evaluate/detection.py

@@ -0,0 +1,783 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import yaml
+import copy
+import os.path as osp
+import numpy as np
+
+backup_linspace = np.linspace
+
+
+def fixed_linspace(start,
+                   stop,
+                   num=50,
+                   endpoint=True,
+                   retstep=False,
+                   dtype=None,
+                   axis=0):
+    '''解决numpy > 1.17.2时pycocotools中linspace的报错问题。
+    '''
+    num = int(num)
+    return backup_linspace(start, stop, num, endpoint, retstep, dtype, axis)
+
+
+def jaccard_overlap(pred, gt):
+    '''计算两个框之间的IoU。
+    '''
+
+    def bbox_area(bbox):
+        width = bbox[2] - bbox[0] + 1
+        height = bbox[3] - bbox[1] + 1
+        return width * height
+    if pred[0] >= gt[2] or pred[2] <= gt[0] or \
+        pred[1] >= gt[3] or pred[3] <= gt[1]:
+        return 0.
+    inter_xmin = max(pred[0], gt[0])
+    inter_ymin = max(pred[1], gt[1])
+    inter_xmax = min(pred[2], gt[2])
+    inter_ymax = min(pred[3], gt[3])
+    inter_size = bbox_area([inter_xmin, inter_ymin, inter_xmax, inter_ymax])
+    pred_size = bbox_area(pred)
+    gt_size = bbox_area(gt)
+    overlap = float(inter_size) / (pred_size + gt_size - inter_size)
+    return overlap
+
+
+def loadRes(coco_obj, anns):
+    '''导入结果文件并返回pycocotools中的COCO对象。
+    '''
+    from pycocotools.coco import COCO
+    import pycocotools.mask as maskUtils
+    import time
+    res = COCO()
+    res.dataset['images'] = [img for img in coco_obj.dataset['images']]
+
+    tic = time.time()
+    assert type(anns) == list, 'results in not an array of objects'
+    annsImgIds = [ann['image_id'] for ann in anns]
+    assert set(annsImgIds) == (set(annsImgIds) & set(coco_obj.getImgIds())), \
+           'Results do not correspond to current coco set'
+    if 'bbox' in anns[0] and not anns[0]['bbox'] == []:
+        res.dataset['categories'] = copy.deepcopy(coco_obj.dataset[
+            'categories'])
+        for id, ann in enumerate(anns):
+            bb = ann['bbox']
+            x1, x2, y1, y2 = [bb[0], bb[0] + bb[2], bb[1], bb[1] + bb[3]]
+            if not 'segmentation' in ann:
+                ann['segmentation'] = [[x1, y1, x1, y2, x2, y2, x2, y1]]
+            ann['area'] = bb[2] * bb[3]
+            ann['id'] = id + 1
+            ann['iscrowd'] = 0
+    elif 'segmentation' in anns[0]:
+        res.dataset['categories'] = copy.deepcopy(coco_obj.dataset[
+            'categories'])
+        for id, ann in enumerate(anns):
+            ann['area'] = maskUtils.area(ann['segmentation'])
+            if not 'bbox' in ann:
+                ann['bbox'] = maskUtils.toBbox(ann['segmentation'])
+            ann['id'] = id + 1
+            ann['iscrowd'] = 0
+    res.dataset['annotations'] = anns
+    res.createIndex()
+    return res
+
+
+class DetectionMAP(object):
+    def __init__(self,
+                 num_classes,
+                 overlap_thresh=0.5,
+                 map_type='11point',
+                 is_bbox_normalized=False,
+                 evaluate_difficult=False):
+        self.num_classes = num_classes
+        self.overlap_thresh = overlap_thresh
+        assert map_type in ['11point', 'integral'], \
+                "map_type currently only support '11point' "\
+                "and 'integral'"
+        self.map_type = map_type
+        self.is_bbox_normalized = is_bbox_normalized
+        self.evaluate_difficult = evaluate_difficult
+        self.reset()
+
+    def update(self, bbox, gt_box, gt_label, difficult=None):
+        '''用预测值和真值更新指标。
+        '''
+        if difficult is None:
+            difficult = np.zeros_like(gt_label)
+
+        for gtl, diff in zip(gt_label, difficult):
+            if self.evaluate_difficult or int(diff) == 0:
+                self.class_gt_counts[int(np.array(gtl))] += 1
+
+        visited = [False] * len(gt_label)
+        for b in bbox:
+            label, score, xmin, ymin, xmax, ymax = b.tolist()
+            pred = [xmin, ymin, xmax, ymax]
+            max_idx = -1
+            max_overlap = -1.0
+            for i, gl in enumerate(gt_label):
+                if int(gl) == int(label):
+                    overlap = jaccard_overlap(pred, gt_box[i])
+                    if overlap > max_overlap:
+                        max_overlap = overlap
+                        max_idx = i
+
+            if max_overlap > self.overlap_thresh:
+                if self.evaluate_difficult or \
+                        int(np.array(difficult[max_idx])) == 0:
+                    if not visited[max_idx]:
+                        self.class_score_poss[int(label)].append([score, 1.0])
+                        visited[max_idx] = True
+                    else:
+                        self.class_score_poss[int(label)].append([score, 0.0])
+            else:
+                self.class_score_poss[int(label)].append([score, 0.0])
+
+    def reset(self):
+        '''初始化指标。
+        '''
+        self.class_score_poss = [[] for _ in range(self.num_classes)]
+        self.class_gt_counts = [0] * self.num_classes
+        self.mAP = None
+        self.APs = [None] * self.num_classes
+
+    def accumulate(self):
+        '''汇总指标并由此计算mAP。
+        '''
+        mAP = 0.
+        valid_cnt = 0
+        for id, (
+                score_pos, count
+        ) in enumerate(zip(self.class_score_poss, self.class_gt_counts)):
+            if count == 0: continue
+            if len(score_pos) == 0:
+                valid_cnt += 1
+                continue
+
+            accum_tp_list, accum_fp_list = \
+                    self._get_tp_fp_accum(score_pos)
+            precision = []
+            recall = []
+            for ac_tp, ac_fp in zip(accum_tp_list, accum_fp_list):
+                precision.append(float(ac_tp) / (ac_tp + ac_fp))
+                recall.append(float(ac_tp) / count)
+
+            if self.map_type == '11point':
+                max_precisions = [0.] * 11
+                start_idx = len(precision) - 1
+                for j in range(10, -1, -1):
+                    for i in range(start_idx, -1, -1):
+                        if recall[i] < float(j) / 10.:
+                            start_idx = i
+                            if j > 0:
+                                max_precisions[j - 1] = max_precisions[j]
+                                break
+                        else:
+                            if max_precisions[j] < precision[i]:
+                                max_precisions[j] = precision[i]
+                mAP += sum(max_precisions) / 11.
+                self.APs[id] = sum(max_precisions) / 11.
+                valid_cnt += 1
+            elif self.map_type == 'integral':
+                import math
+                ap = 0.
+                prev_recall = 0.
+                for i in range(len(precision)):
+                    recall_gap = math.fabs(recall[i] - prev_recall)
+                    if recall_gap > 1e-6:
+                        ap += precision[i] * recall_gap
+                        prev_recall = recall[i]
+                mAP += ap
+                self.APs[id] = sum(max_precisions) / 11.
+                valid_cnt += 1
+            else:
+                raise Exception("Unspported mAP type {}".format(self.map_type))
+
+        self.mAP = mAP / float(valid_cnt) if valid_cnt > 0 else mAP
+
+    def get_map(self):
+        '''获取mAP。
+        '''
+        if self.mAP is None:
+            raise Exception("mAP is not calculated.")
+        return self.mAP
+
+    def _get_tp_fp_accum(self, score_pos_list):
+        '''计算真阳/假阳。
+        '''
+        sorted_list = sorted(score_pos_list, key=lambda s: s[0], reverse=True)
+        accum_tp = 0
+        accum_fp = 0
+        accum_tp_list = []
+        accum_fp_list = []
+        for (score, pos) in sorted_list:
+            accum_tp += int(pos)
+            accum_tp_list.append(accum_tp)
+            accum_fp += 1 - int(pos)
+            accum_fp_list.append(accum_fp)
+        return accum_tp_list, accum_fp_list
+
+
+class DetConfusionMatrix(object):
+    def __init__(self,
+                 num_classes,
+                 overlap_thresh=0.5,
+                 evaluate_difficult=False,
+                 score_threshold=0.3):
+        self.overlap_thresh = overlap_thresh
+        self.evaluate_difficult = evaluate_difficult
+        self.confusion_matrix = np.zeros(shape=(num_classes, num_classes))
+        self.score_threshold = score_threshold
+        self.total_tp = [0] * num_classes
+        self.total_gt = [0] * num_classes
+        self.total_pred = [0] * num_classes
+
+    def update(self, bbox, gt_box, gt_label, difficult=None):
+        '''更新混淆矩阵。
+        '''
+        if difficult is None:
+            difficult = np.zeros_like(gt_label)
+
+        dtind = np.argsort([-d[1] for d in bbox], kind='mergesort')
+        bbox = [bbox[i] for i in dtind]
+        det_bbox = []
+        det_label = []
+        G = len(gt_box)
+        D = len(bbox)
+        gtm = np.full((G, ), -1)
+        dtm = np.full((D, ), -1)
+        for j, b in enumerate(bbox):
+            label, score, xmin, ymin, xmax, ymax = b.tolist()
+            if float(score) < self.score_threshold:
+                continue
+            det_label.append(int(label) - 1)
+            self.total_pred[int(label) - 1] += 1
+            det_bbox.append([xmin, ymin, xmax, ymax])
+        for i, gl in enumerate(gt_label):
+            self.total_gt[int(gl) - 1] += 1
+
+        for j, pred in enumerate(det_bbox):
+            m = -1
+            for i, gt in enumerate(gt_box):
+                overlap = jaccard_overlap(pred, gt)
+                if overlap >= self.overlap_thresh:
+                    m = i
+            if m == -1:
+                continue
+            gtm[m] = j
+            dtm[j] = m
+        for i, gl in enumerate(gt_label):
+            if gtm[i] == -1:
+                self.confusion_matrix[int(gl) - 1][self.confusion_matrix.shape[
+                    1] - 1] += 1
+        for i, b in enumerate(det_bbox):
+            if dtm[i] > -1:
+                gl = int(gt_label[dtm[i]]) - 1
+                self.confusion_matrix[gl][int(det_label[i])] += 1
+            if dtm[i] == -1:
+                self.confusion_matrix[self.confusion_matrix.shape[0] - 1][int(
+                    det_label[i])] += 1
+
+        gtm = np.full((G, ), -1)
+        dtm = np.full((D, ), -1)
+        for j, pred in enumerate(det_bbox):
+            m = -1
+            max_overlap = -1
+            for i, gt in enumerate(gt_box):
+                if int(gt_label[i]) - 1 == int(det_label[j]):
+                    overlap = jaccard_overlap(pred, gt)
+                    if overlap > max_overlap:
+                        max_overlap = overlap
+                        m = i
+            if max_overlap < self.overlap_thresh:
+                continue
+            if difficult[m]:
+                continue
+            if m == -1 or gtm[m] > -1:
+                continue
+            gtm[m] = j
+            dtm[j] = m
+            self.total_tp[int(gt_label[m]) - 1] += 1
+
+    def get_confusion_matrix(self):
+        return self.confusion_matrix
+
+
+class InsSegConfusionMatrix(object):
+    def __init__(self,
+                 num_classes,
+                 overlap_thresh=0.5,
+                 evaluate_difficult=False,
+                 score_threshold=0.3):
+        self.overlap_thresh = overlap_thresh
+        self.evaluate_difficult = evaluate_difficult
+        self.confusion_matrix = np.zeros(shape=(num_classes, num_classes))
+        self.score_threshold = score_threshold
+        self.total_tp = [0] * num_classes
+        self.total_gt = [0] * num_classes
+        self.total_pred = [0] * num_classes
+
+    def update(self, mask, gt_mask, gt_label, is_crowd=None):
+        '''更新混淆矩阵。
+        '''
+        dtind = np.argsort([-d[1] for d in mask], kind='mergesort')
+        mask = [mask[i] for i in dtind]
+        det_mask = []
+        det_label = []
+        for j, b in enumerate(mask):
+            label, score, d_b = b
+            if float(score) < self.score_threshold:
+                continue
+            self.total_pred[int(label) - 1] += 1
+            det_label.append(label - 1)
+            det_mask.append(d_b)
+        for i, gl in enumerate(gt_label):
+            self.total_gt[int(gl) - 1] += 1
+
+        g = [gt for gt in gt_mask]
+        d = [dt for dt in det_mask]
+        import pycocotools.mask as maskUtils
+        ious = maskUtils.iou(d, g, is_crowd)
+        G = len(gt_mask)
+        D = len(det_mask)
+        gtm = np.full((G, ), -1)
+        dtm = np.full((D, ), -1)
+        gtIg = np.array(is_crowd)
+        dtIg = np.zeros((D, ))
+        for dind, d in enumerate(det_mask):
+            m = -1
+            for gind, g in enumerate(gt_mask):
+                if ious[dind, gind] >= self.overlap_thresh:
+                    m = gind
+            if m == -1:
+                continue
+            dtIg[dind] = gtIg[m]
+            dtm[dind] = m
+            gtm[m] = dind
+        for i, gl in enumerate(gt_label):
+            if gtm[i] == -1 and gtIg[i] == 0:
+                self.confusion_matrix[int(gl) - 1][self.confusion_matrix.shape[
+                    1] - 1] += 1
+        for i, b in enumerate(det_mask):
+            if dtm[i] > -1 and dtIg[i] == 0:
+                gl = int(gt_label[dtm[i]]) - 1
+                self.confusion_matrix[gl][int(det_label[i])] += 1
+            if dtm[i] == -1 and dtIg[i] == 0:
+                self.confusion_matrix[self.confusion_matrix.shape[0] - 1][int(
+                    det_label[i])] += 1
+
+        gtm = np.full((G, ), -1)
+        dtm = np.full((D, ), -1)
+        for dind, d in enumerate(det_mask):
+            m = -1
+            max_overlap = -1
+            for gind, g in enumerate(gt_mask):
+                if int(gt_label[gind]) - 1 == int(det_label[dind]):
+                    if ious[dind, gind] > max_overlap:
+                        max_overlap = ious[dind, gind]
+                        m = gind
+
+            if max_overlap < self.overlap_thresh:
+                continue
+            if m == -1 or gtm[m] > -1:
+                continue
+            dtm[dind] = m
+            gtm[m] = dind
+            self.total_tp[int(gt_label[m]) - 1] += 1
+
+    def get_confusion_matrix(self):
+        return self.confusion_matrix
+
+
+class DetEvaluator(object):
+    def __init__(self, model_path, overlap_thresh=0.5, score_threshold=0.3):
+        self.model_path = model_path
+        self.overlap_thresh = overlap_thresh
+        self.score_threshold = score_threshold
+
+    def _prepare_data(self):
+        with open(osp.join(self.model_path, 'eval_details.json'), 'r') as f:
+            eval_details = json.load(f)
+        self.bbox = eval_details['bbox']
+        self.mask = None
+        if 'mask' in eval_details:
+            self.mask = eval_details['mask']
+        gt_dataset = eval_details['gt']
+
+        from pycocotools.coco import COCO
+        from pycocotools.cocoeval import COCOeval
+        self.coco = COCO()
+        self.coco.dataset = gt_dataset
+        self.coco.createIndex()
+        img_ids = self.coco.getImgIds()
+        cat_ids = self.coco.getCatIds()
+        self.catid2clsid = dict(
+            {catid: i + 1
+             for i, catid in enumerate(cat_ids)})
+        self.cname2cid = dict({
+            self.coco.loadCats(catid)[0]['name']: clsid
+            for catid, clsid in self.catid2clsid.items()
+        })
+        self.cid2cname = dict(
+            {cid: cname
+             for cname, cid in self.cname2cid.items()})
+        self.cid2cname[0] = 'back_ground'
+
+        self.gt = dict()
+        for img_id in img_ids:
+            img_anno = self.coco.loadImgs(img_id)[0]
+            im_fname = img_anno['file_name']
+            im_w = float(img_anno['width'])
+            im_h = float(img_anno['height'])
+
+            ins_anno_ids = self.coco.getAnnIds(imgIds=img_id, iscrowd=False)
+            instances = self.coco.loadAnns(ins_anno_ids)
+
+            bboxes = []
+            for inst in instances:
+                x, y, box_w, box_h = inst['bbox']
+                x1 = max(0, x)
+                y1 = max(0, y)
+                x2 = min(im_w - 1, x1 + max(0, box_w - 1))
+                y2 = min(im_h - 1, y1 + max(0, box_h - 1))
+                if inst['area'] > 0 and x2 >= x1 and y2 >= y1:
+                    inst['clean_bbox'] = [x1, y1, x2, y2]
+                    bboxes.append(inst)
+                else:
+                    pass
+            num_bbox = len(bboxes)
+
+            gt_bbox = np.zeros((num_bbox, 4), dtype=np.float32)
+            gt_class = np.zeros((num_bbox, 1), dtype=np.int32)
+            gt_score = np.ones((num_bbox, 1), dtype=np.float32)
+            is_crowd = np.zeros((num_bbox), dtype=np.int32)
+            difficult = np.zeros((num_bbox, 1), dtype=np.int32)
+            gt_poly = [None] * num_bbox
+
+            for i, box in enumerate(bboxes):
+                catid = box['category_id']
+                gt_class[i][0] = self.catid2clsid[catid]
+                gt_bbox[i, :] = box['clean_bbox']
+                is_crowd[i] = box['iscrowd']
+                if 'segmentation' in box:
+                    gt_poly[i] = self.coco.annToRLE(box)
+                if 'difficult' in box:
+                    difficult[i][0] = box['difficult']
+
+            coco_rec = {
+                'is_crowd': is_crowd,
+                'gt_class': gt_class,
+                'gt_bbox': gt_bbox,
+                'gt_score': gt_score,
+                'gt_poly': gt_poly,
+                'difficult': difficult
+            }
+            self.gt[img_id] = coco_rec
+        self.gtimgids = list(self.gt.keys())
+        self.detimgids = [ann['image_id'] for ann in self.bbox]
+        self.det = dict()
+        if len(self.bbox) > 0:
+            if 'bbox' in self.bbox[0] and not self.bbox[0]['bbox'] == []:
+                for id, ann in enumerate(self.bbox):
+                    im_id = ann['image_id']
+                    bb = ann['bbox']
+                    x1, x2, y1, y2 = [
+                        bb[0], bb[0] + bb[2] - 1, bb[1], bb[1] + bb[3] - 1
+                    ]
+                    score = ann['score']
+                    category_id = self.catid2clsid[ann['category_id']]
+                    if int(im_id) not in self.det:
+                        self.det[int(im_id)] = [[
+                            category_id, score, x1, y1, x2, y2
+                        ]]
+                    else:
+                        self.det[int(im_id)].extend(
+                            [[category_id, score, x1, y1, x2, y2]])
+
+        if self.mask is not None:
+            self.maskimgids = [ann['image_id'] for ann in self.mask]
+            self.segm = dict()
+            if len(self.mask) > 0:
+                if 'segmentation' in self.mask[0]:
+                    for id, ann in enumerate(self.mask):
+                        im_id = ann['image_id']
+                        score = ann['score']
+                        segmentation = self.coco.annToRLE(ann)
+                        category_id = self.catid2clsid[ann['category_id']]
+                        if int(im_id) not in self.segm:
+                            self.segm[int(im_id)] = [[
+                                category_id, score, segmentation
+                            ]]
+                        else:
+                            self.segm[int(im_id)].extend(
+                                [[category_id, score, segmentation]])
+
+    def cal_confusion_matrix(self):
+        '''计算混淆矩阵。
+        '''
+        self._prepare_data()
+        confusion_matrix = DetConfusionMatrix(
+            num_classes=len(self.cid2cname.keys()),
+            overlap_thresh=self.overlap_thresh,
+            score_threshold=self.score_threshold)
+        for im_id in self.gtimgids:
+            if im_id not in set(self.detimgids):
+                bbox = []
+            else:
+                bbox = np.array(self.det[im_id])
+            gt_box = self.gt[im_id]['gt_bbox']
+            gt_label = self.gt[im_id]['gt_class']
+            difficult = self.gt[im_id]['difficult']
+            confusion_matrix.update(bbox, gt_box, gt_label, difficult)
+        self.confusion_matrix = confusion_matrix.get_confusion_matrix()
+
+        self.precision_recall = dict()
+        for id in range(len(self.cid2cname.keys()) - 1):
+            if confusion_matrix.total_gt[id] == 0:
+                recall = -1
+            else:
+                recall = confusion_matrix.total_tp[
+                    id] / confusion_matrix.total_gt[id]
+            if confusion_matrix.total_pred[id] == 0:
+                precision = -1
+            else:
+                precision = confusion_matrix.total_tp[
+                    id] / confusion_matrix.total_pred[id]
+            self.precision_recall[self.cid2cname[id + 1]] = {
+                "precision": precision,
+                "recall": recall
+            }
+        return self.confusion_matrix
+
+    def cal_precision_recall(self):
+        '''计算precision、recall。
+        '''
+        return self.precision_recall
+
+    def cal_map(self):
+        '''计算mAP。
+        '''
+        detection_map = DetectionMAP(
+            num_classes=len(self.cid2cname.keys()),
+            overlap_thresh=self.overlap_thresh)
+        for im_id in self.gtimgids:
+            if im_id not in set(self.detimgids):
+                bbox = []
+            else:
+                bbox = np.array(self.det[im_id])
+            gt_box = self.gt[im_id]['gt_bbox']
+            gt_label = self.gt[im_id]['gt_class']
+            difficult = self.gt[im_id]['difficult']
+            detection_map.update(bbox, gt_box, gt_label, difficult)
+        detection_map.accumulate()
+        self.map = detection_map.get_map()
+        self.APs = detection_map.APs
+        return self.map
+
+    def cal_ap(self):
+        '''计算各类AP。
+        '''
+        self.aps = dict()
+        for id, ap in enumerate(self.APs):
+            if id == 0:
+                continue
+            self.aps[self.cid2cname[id]] = ap
+        return self.aps
+
+    def generate_report(self):
+        '''生成评估报告。
+        '''
+        report = dict()
+        report['Confusion_Matrix'] = copy.deepcopy(self.cal_confusion_matrix())
+        report['mAP'] = copy.deepcopy(self.cal_map())
+        report['PRAP'] = copy.deepcopy(self.cal_precision_recall())
+        report['label_list'] = copy.deepcopy(list(self.cname2cid.keys()))
+        report['label_list'].append('back_ground')
+        per_ap = copy.deepcopy(self.cal_ap())
+        for k, v in per_ap.items():
+            report['PRAP'][k]["AP"] = v
+        return report
+
+
+class InsSegEvaluator(DetEvaluator):
+    def __init__(self, model_path, overlap_thresh=0.5, score_threshold=0.3):
+        super(DetEvaluator, self).__init__()
+        self.model_path = model_path
+        self.overlap_thresh = overlap_thresh
+        self.score_threshold = score_threshold
+
+    def cal_confusion_matrix_mask(self):
+        '''计算Mask的混淆矩阵。
+        '''
+        confusion_matrix = InsSegConfusionMatrix(
+            num_classes=len(self.cid2cname.keys()),
+            overlap_thresh=self.overlap_thresh,
+            score_threshold=self.score_threshold)
+        for im_id in self.gtimgids:
+            if im_id not in set(self.maskimgids):
+                segm = []
+            else:
+                segm = self.segm[im_id]
+            gt_segm = self.gt[im_id]['gt_poly']
+            gt_label = self.gt[im_id]['gt_class']
+            is_crowd = self.gt[im_id]['is_crowd']
+            confusion_matrix.update(segm, gt_segm, gt_label, is_crowd)
+        self.confusion_matrix_mask = confusion_matrix.get_confusion_matrix()
+
+        self.precision_recall_mask = dict()
+        for id in range(len(self.cid2cname.keys()) - 1):
+            if confusion_matrix.total_gt[id] == 0:
+                recall = -1
+            else:
+                recall = confusion_matrix.total_tp[
+                    id] / confusion_matrix.total_gt[id]
+            if confusion_matrix.total_pred[id] == 0:
+                precision = -1
+            else:
+                precision = confusion_matrix.total_tp[
+                    id] / confusion_matrix.total_pred[id]
+            self.precision_recall_mask[self.cid2cname[id + 1]] = {
+                "precision": precision,
+                "recall": recall
+            }
+        return self.confusion_matrix_mask
+
+    def cal_precision_recall_mask(self):
+        '''计算Mask的precision、recall。
+        '''
+        return self.precision_recall_mask
+
+    def _summarize(self,
+                   coco_gt,
+                   ap=1,
+                   iouThr=None,
+                   areaRng='all',
+                   maxDets=100):
+        p = coco_gt.params
+        aind = [i for i, aRng in enumerate(p.areaRngLbl) if aRng == areaRng]
+        mind = [i for i, mDet in enumerate(p.maxDets) if mDet == maxDets]
+        if ap == 1:
+            s = coco_gt.eval['precision']
+            if iouThr is not None:
+                t = np.where(iouThr == p.iouThrs)[0]
+                s = s[t]
+            s = s[:, :, :, aind, mind]
+        else:
+            s = coco_gt.eval['recall']
+            if iouThr is not None:
+                t = np.where(iouThr == p.iouThrs)[0]
+                s = s[t]
+            s = s[:, :, aind, mind]
+        if len(s[s > -1]) == 0:
+            mean_s = -1
+        else:
+            mean_s = np.mean(s[s > -1])
+        return mean_s
+
+    def cal_map(self):
+        '''计算BBox的mAP。
+        '''
+        if len(self.bbox) > 0:
+            from pycocotools.cocoeval import COCOeval
+            coco_dt = loadRes(self.coco, self.bbox)
+            np.linspace = fixed_linspace
+            coco_eval = COCOeval(self.coco, coco_dt, 'bbox')
+            coco_eval.params.iouThrs = np.linspace(
+                self.overlap_thresh, self.overlap_thresh, 1, endpoint=True)
+            np.linspace = backup_linspace
+            coco_eval.evaluate()
+            coco_eval.accumulate()
+            self.map = self._summarize(coco_eval, iouThr=self.overlap_thresh)
+
+            precision = coco_eval.eval['precision'][0, :, :, 0, 2]
+            num_classes = len(coco_eval.params.catIds)
+            self.APs = [None] * num_classes
+            for i in range(num_classes):
+                per = precision[:, i]
+                per = per[per > -1]
+                self.APs[i] = np.sum(per) / 101 if per.shape[0] > 0 else None
+        else:
+            self.map = None
+            self.APs = [None] * len(self.catid2clsid)
+        return self.map
+
+    def cal_ap(self):
+        '''计算BBox的各类AP。
+        '''
+        self.aps = dict()
+        for id, ap in enumerate(self.APs):
+            self.aps[self.cid2cname[id + 1]] = ap
+        return self.aps
+
+    def cal_map_mask(self):
+        '''计算Mask的mAP。
+        '''
+        if len(self.mask) > 0:
+            from pycocotools.cocoeval import COCOeval
+            coco_dt = loadRes(self.coco, self.mask)
+            np.linspace = fixed_linspace
+            coco_eval = COCOeval(self.coco, coco_dt, 'segm')
+            coco_eval.params.iouThrs = np.linspace(
+                self.overlap_thresh, self.overlap_thresh, 1, endpoint=True)
+            np.linspace = backup_linspace
+            coco_eval.evaluate()
+            coco_eval.accumulate()
+            self.map_mask = self._summarize(
+                coco_eval, iouThr=self.overlap_thresh)
+
+            precision = coco_eval.eval['precision'][0, :, :, 0, 2]
+            num_classes = len(coco_eval.params.catIds)
+            self.mask_APs = [None] * num_classes
+            for i in range(num_classes):
+                per = precision[:, i]
+                per = per[per > -1]
+                self.mask_APs[i] = np.sum(per) / 101 if per.shape[
+                    0] > 0 else None
+        else:
+            self.map_mask = None
+            self.mask_APs = [None] * len(self.catid2clsid)
+        return self.map_mask
+
+    def cal_ap_mask(self):
+        '''计算Mask的各类AP。
+        '''
+        self.mask_aps = dict()
+        for id, ap in enumerate(self.mask_APs):
+            self.mask_aps[self.cid2cname[id + 1]] = ap
+        return self.mask_aps
+
+    def generate_report(self):
+        '''生成评估报告。
+        '''
+        report = dict()
+        report['BBox_Confusion_Matrix'] = copy.deepcopy(
+            self.cal_confusion_matrix())
+        report['BBox_mAP'] = copy.deepcopy(self.cal_map())
+        report['BBox_PRAP'] = copy.deepcopy(self.cal_precision_recall())
+        report['label_list'] = copy.deepcopy(list(self.cname2cid.keys()))
+        report['label_list'].append('back_ground')
+        per_ap = copy.deepcopy(self.cal_ap())
+        for k, v in per_ap.items():
+            report['BBox_PRAP'][k]['AP'] = v
+
+        report['Mask_Confusion_Matrix'] = copy.deepcopy(
+            self.cal_confusion_matrix_mask())
+        report['Mask_mAP'] = copy.deepcopy(self.cal_map_mask())
+        report['Mask_PRAP'] = copy.deepcopy(self.cal_precision_recall_mask())
+        per_ap_mask = copy.deepcopy(self.cal_ap_mask())
+        for k, v in per_ap_mask.items():
+            report['Mask_PRAP'][k]['AP'] = v
+        return report

+ 181 - 0
paddlex/restful/project/evaluate/draw_pred_result.py

@@ -0,0 +1,181 @@
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+import cv2
+import numpy as np
+from PIL import Image
+import matplotlib.pyplot as plt
+import matplotlib.patches as patches
+
+
+def visualize_segmented_result(save_path, image_groundtruth, groundtruth,
+                               image_predict, predict, legend):
+    tail = save_path.split(".")[-1]
+    save_path = (save_path[:-len(tail)] + "png")
+    import matplotlib.patches as mpatches
+    from matplotlib import use
+    use('Agg')
+    if image_groundtruth is not None:
+        image_groundtruth = image_groundtruth[..., ::-1]
+    image_predict = image_predict[..., ::-1]
+    if groundtruth is not None:
+        groundtruth = groundtruth[..., ::-1]
+    predict = predict[..., ::-1]
+    fig = plt.figure()
+    red_patches = []
+    for key, value in legend.items():
+        red_patch = mpatches.Patch(
+            color=[x / 255.0 for x in value[::-1]], label=key)
+        red_patches.append(red_patch)
+    plt.legend(
+        handles=red_patches, bbox_to_anchor=(1.05, 0), loc=3, borderaxespad=0)
+    plt.axis('off')
+
+    if image_groundtruth is not None and \
+            groundtruth is not None:
+        left, bottom, width, height = 0.02, 0.51, 0.38, 0.38
+        fig.add_axes([left, bottom, width, height])
+        plt.imshow(image_groundtruth)
+        plt.axis('off')
+        plt.title("Ground Truth", loc='left')
+        left, bottom, width, height = 0.52, 0.51, 0.38, 0.38
+        fig.add_axes([left, bottom, width, height])
+        plt.imshow(groundtruth)
+        plt.axis('off')
+        left, bottom, width, height = 0.01, 0.5, 0.9, 0.45
+        fig.add_axes([left, bottom, width, height])
+        currentAxis = plt.gca()
+        rect = patches.Rectangle(
+            (0.0, 0.0), 1.0, 1.0, linewidth=1, edgecolor='k', facecolor='none')
+        currentAxis.add_patch(rect)
+        plt.axis('off')
+
+        left, bottom, width, height = 0.02, 0.06, 0.38, 0.38
+        fig.add_axes([left, bottom, width, height])
+        plt.imshow(image_predict)
+        plt.axis('off')
+        plt.title("Prediction", loc='left')
+        left, bottom, width, height = 0.52, 0.06, 0.38, 0.38
+        fig.add_axes([left, bottom, width, height])
+        plt.imshow(predict)
+        plt.axis('off')
+        left, bottom, width, height = 0.01, 0.05, 0.9, 0.45
+        fig.add_axes([left, bottom, width, height])
+        currentAxis = plt.gca()
+        rect = patches.Rectangle(
+            (0.0, 0.0), 1.0, 1.0, linewidth=1, edgecolor='k', facecolor='none')
+        currentAxis.add_patch(rect)
+        plt.axis('off')
+    else:
+        plt.subplot(1, 2, 1)
+        plt.imshow(image_predict)
+        plt.axis('off')
+        plt.title("Combination ", y=-0.12)
+        plt.subplot(1, 2, 2)
+        plt.imshow(predict)
+        plt.axis('off')
+        plt.title("Prediction", y=-0.12)
+    plt.savefig(save_path, dpi=200, bbox_inches='tight')
+    plt.close()
+
+
+def visualize_detected_result(save_path, image_groundtruth, image_predict):
+    tail = save_path.split(".")[-1]
+    save_path = (save_path[:-len(tail)] + "png")
+    from matplotlib import use
+    use('Agg')
+    if image_groundtruth is not None:
+        plt.subplot(1, 2, 1)
+        plt.imshow(cv2.cvtColor(image_groundtruth, cv2.COLOR_BGR2RGB))
+        plt.axis('off')
+        plt.title("Ground Truth", y=-0.12)
+        plt.subplot(1, 2, 2)
+        plt.imshow(cv2.cvtColor(image_predict, cv2.COLOR_BGR2RGB))
+        plt.axis('off')
+        plt.title("Prediction", y=-0.12)
+    else:
+        plt.subplot(1, 1, 1)
+        plt.imshow(cv2.cvtColor(image_predict, cv2.COLOR_BGR2RGB))
+        plt.axis('off')
+        plt.title("Prediction", y=-0.12)
+    plt.tight_layout(pad=1.08)
+    plt.autoscale()
+    plt.savefig(save_path, dpi=600, bbox_inches='tight')
+    plt.close()
+
+
+def visualize_classified_result(save_path, image_predict, res_info):
+    from matplotlib import use
+    use('Agg')
+    if isinstance(image_predict, str):
+        img = Image.open(image_predict)
+        name_part = osp.split(image_predict)
+        filename = name_part[-1]
+        foldername = osp.split(name_part[-2])[-1]
+        tail = filename.split(".")[-1]
+        filename = (foldername + '_' + filename[:-len(tail)] + "png")
+    elif isinstance(image_predict, np.ndarray):
+        img = Image.fromarray(cv2.cvtColor(image_predict, cv2.COLOR_BGR2RGB))
+        filename = "predict_result.png"
+    if np.array(img).ndim == 3:
+        cmap = None
+    else:
+        cmap = 'gray'
+    plt.subplot(1, 2, 1)
+    plt.imshow(img, cmap=cmap)
+    plt.axis('off')
+    if "gt_label" in res_info:
+        plt.title(
+            "Test Image, Label: {}".format(res_info["gt_label"]), y=-0.15)
+    else:
+        plt.title("Test Image", y=-0.15)
+    plt.subplot(1, 2, 2)
+    topk = res_info["topk"]
+    start_height = (topk + 2) // 2 * 10 + 45
+    plt.text(
+        15, start_height, 'Probability of each class:', va='center', ha='left')
+    for i in range(topk):
+        if "gt_label" in res_info:
+            color = "red" if res_info["label"][i] == res_info[
+                "gt_label"] else "black"
+        else:
+            color = 'black'
+        if i == 0:
+            color = "green"
+        plt.text(
+            70,
+            start_height - (i + 1) * 10,
+            '    {}: {:.4f}'.format(res_info["label"][i],
+                                    res_info["score"][i]),
+            va='center',
+            ha='right',
+            color=color)
+    if "gt_label" in res_info:
+        plt.text(
+            15,
+            start_height - (topk + 1) * 10,
+            'True Label: {}'.format(res_info["gt_label"]),
+            va='center',
+            ha='left',
+            color="black")
+    plt.axis('off')
+    plt.axis([0, 100, 0, 100])
+    plt.gca().set_aspect('equal', adjustable='box')
+    plt.tight_layout(pad=0.08)
+    plt.savefig(osp.join(save_path, filename), dpi=200, bbox_inches='tight')
+    plt.close()

+ 117 - 0
paddlex/restful/project/evaluate/segmentation.py

@@ -0,0 +1,117 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import yaml
+import os.path as osp
+import numpy as np
+
+
+class Evaluator(object):
+    def __init__(self, model_path):
+        with open(osp.join(model_path, "model.yml")) as f:
+            model_info = yaml.load(f.read(), Loader=yaml.Loader)
+        self.labels = model_info['_Attributes']['labels']
+        with open(osp.join(model_path, 'eval_details.json'), 'r') as f:
+            eval_details = json.load(f)
+        self.confusion_matrix = np.array(eval_details['confusion_matrix'])
+        self.num_classes = len(self.confusion_matrix)
+
+    def cal_iou(self):
+        '''计算IoU。
+        '''
+        category_iou = []
+        mean_iou = 0
+        vji = np.sum(self.confusion_matrix, axis=1)
+        vij = np.sum(self.confusion_matrix, axis=0)
+
+        for c in range(self.num_classes):
+            total = vji[c] + vij[c] - self.confusion_matrix[c][c]
+            if total == 0:
+                iou = 0
+            else:
+                iou = float(self.confusion_matrix[c][c]) / total
+            mean_iou += iou
+            category_iou.append(iou)
+        mean_iou = float(mean_iou) / float(self.num_classes)
+        return np.array(category_iou), mean_iou
+
+    def cal_acc(self):
+        '''计算Acc。
+        '''
+        total = self.confusion_matrix.sum()
+        total_tp = 0
+        for c in range(self.num_classes):
+            total_tp += self.confusion_matrix[c][c]
+        if total == 0:
+            mean_acc = 0
+        else:
+            mean_acc = float(total_tp) / total
+
+        vij = np.sum(self.confusion_matrix, axis=0)
+        category_acc = []
+        for c in range(self.num_classes):
+            if vij[c] == 0:
+                acc = 0
+            else:
+                acc = self.confusion_matrix[c][c] / float(vij[c])
+            category_acc.append(acc)
+        return np.array(category_acc), mean_acc
+
+    def cal_confusion_matrix(self):
+        '''计算混淆矩阵。
+        '''
+        return self.confusion_matrix
+
+    def cal_precision_recall(self):
+        '''计算precision、recall.
+        '''
+        self.precision_recall = dict()
+        for i in range(len(self.labels)):
+            label_name = self.labels[i]
+            if np.isclose(np.sum(self.confusion_matrix[i, :]), 0, atol=1e-6):
+                recall = -1
+            else:
+                total_gt = np.sum(self.confusion_matrix[i, :]) + 1e-06
+                recall = self.confusion_matrix[i, i] / total_gt
+            if np.isclose(np.sum(self.confusion_matrix[:, i]), 0, atol=1e-6):
+                precision = -1
+            else:
+                total_pred = np.sum(self.confusion_matrix[:, i]) + 1e-06
+                precision = self.confusion_matrix[i, i] / total_pred
+            self.precision_recall[label_name] = {
+                'precision': precision,
+                'recall': recall
+            }
+        return self.precision_recall
+
+    def generate_report(self):
+        '''生成评估报告。
+        '''
+        category_iou, mean_iou = self.cal_iou()
+        category_acc, mean_acc = self.cal_acc()
+
+        category_iou_dict = {}
+        for i in range(len(category_iou)):
+            category_iou_dict[self.labels[i]] = category_iou[i]
+
+        report = dict()
+        report['Confusion_Matrix'] = self.cal_confusion_matrix()
+        report['Mean_IoU'] = mean_iou
+        report['Mean_Acc'] = mean_acc
+        report['PRIoU'] = self.cal_precision_recall()
+        for key in report['PRIoU']:
+            report['PRIoU'][key]["iou"] = category_iou_dict[key]
+        report['label_list'] = self.labels
+        return report

+ 914 - 0
paddlex/restful/project/operate.py

@@ -0,0 +1,914 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+import os
+import numpy as np
+from PIL import Image
+import sys
+import cv2
+import psutil
+import shutil
+import pickle
+import base64
+import multiprocessing as mp
+from ..utils import (pkill, set_folder_status, get_folder_status, TaskStatus,
+                     PredictStatus, PruneStatus)
+from .evaluate.draw_pred_result import visualize_classified_result, visualize_detected_result, visualize_segmented_result
+from .visualize import plot_det_label, plot_insseg_label, get_color_map_list
+
+
+def _call_paddle_prune(best_model_path, prune_analysis_path, params):
+    mode = 'w'
+    sys.stdout = open(
+        osp.join(prune_analysis_path, 'out.log'), mode, encoding='utf-8')
+    sys.stderr = open(
+        osp.join(prune_analysis_path, 'err.log'), mode, encoding='utf-8')
+    sensitivities_path = osp.join(prune_analysis_path, "sensitivities.data")
+    task_type = params['task_type']
+    dataset_path = params['dataset_path']
+    os.environ['CUDA_VISIBLE_DEVICES'] = params['train'].cuda_visible_devices
+    if task_type == "classification":
+        from .prune.classification import prune
+    elif task_type in ["detection", "instance_segmentation"]:
+        from .prune.detection import prune
+    elif task_type == "segmentation":
+        from .prune.segmentation import prune
+    batch_size = params['train'].batch_size
+    prune(best_model_path, dataset_path, sensitivities_path, batch_size)
+    import paddlex as pdx
+    from paddlex.cv.models.slim.visualize import visualize
+    model = pdx.load_model(best_model_path)
+    visualize(model, sensitivities_path, prune_analysis_path)
+    set_folder_status(prune_analysis_path, PruneStatus.XSPRUNEDONE)
+
+
+def _call_paddlex_train(task_path, params):
+    '''
+    Args:
+        params为dict,字段包括'pretrain_weights_download_save_dir': 预训练模型保存路径,
+        'task_type': 任务类型,'dataset_path': 数据集路径,'train':训练参数
+    '''
+
+    mode = 'w'
+    if params['train'].resume_checkpoint is not None:
+        mode = 'a'
+    sys.stdout = open(osp.join(task_path, 'out.log'), mode, encoding='utf-8')
+    sys.stderr = open(osp.join(task_path, 'err.log'), mode, encoding='utf-8')
+    sys.stdout.write("This log file path is {}\n".format(
+        osp.join(task_path, 'out.log')))
+    sys.stdout.write("注意:标志为WARNING/INFO类的仅为警告或提示类信息,非错误信息\n")
+    sys.stderr.write("This log file path is {}\n".format(
+        osp.join(task_path, 'err.log')))
+    sys.stderr.write("注意:标志为WARNING/INFO类的仅为警告或提示类信息,非错误信息\n")
+    os.environ['CUDA_VISIBLE_DEVICES'] = params['train'].cuda_visible_devices
+    import paddlex as pdx
+    pdx.gui_mode = True
+    pdx.log_level = 3
+    pdx.pretrain_dir = params['pretrain_weights_download_save_dir']
+    task_type = params['task_type']
+    dataset_path = params['dataset_path']
+    if task_type == "classification":
+        from .train.classification import train
+    elif task_type in ["detection", "instance_segmentation"]:
+        from .train.detection import train
+    elif task_type == "segmentation":
+        from .train.segmentation import train
+    train(task_path, dataset_path, params['train'])
+    set_folder_status(task_path, TaskStatus.XTRAINDONE)
+
+
+def _call_paddlex_evaluate_model(task_path,
+                                 model_path,
+                                 task_type,
+                                 epoch,
+                                 topk=5,
+                                 score_thresh=0.3,
+                                 overlap_thresh=0.5):
+    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    sys.stdout = open(
+        osp.join(evaluate_status_path, 'out.log'), 'w', encoding='utf-8')
+    sys.stderr = open(
+        osp.join(evaluate_status_path, 'err.log'), 'w', encoding='utf-8')
+    if task_type == "classification":
+        from .evaluate.classification import Evaluator
+        evaluator = Evaluator(model_path, topk=topk)
+    elif task_type == "detection":
+        from .evaluate.detection import DetEvaluator
+        evaluator = DetEvaluator(
+            model_path,
+            score_threshold=score_thresh,
+            overlap_thresh=overlap_thresh)
+    elif task_type == "instance_segmentation":
+        from .evaluate.detection import InsSegEvaluator
+        evaluator = InsSegEvaluator(
+            model_path,
+            score_threshold=score_thresh,
+            overlap_thresh=overlap_thresh)
+    elif task_type == "segmentation":
+        from .evaluate.segmentation import Evaluator
+        evaluator = Evaluator(model_path)
+    report = evaluator.generate_report()
+    report['epoch'] = epoch
+    pickle.dump(report, open(osp.join(task_path, "eval_res.pkl"), "wb"))
+    set_folder_status(evaluate_status_path, TaskStatus.XEVALUATED)
+    set_folder_status(task_path, TaskStatus.XEVALUATED)
+
+
+def _call_paddlex_predict(task_path,
+                          predict_status_path,
+                          params,
+                          img_list,
+                          img_data,
+                          save_dir,
+                          score_thresh,
+                          epoch=None):
+    total_num = open(
+        osp.join(predict_status_path, 'total_num'), 'w', encoding='utf-8')
+
+    def write_file_num(total_file_num):
+        total_num.write(str(total_file_num))
+        total_num.close()
+
+    sys.stdout = open(
+        osp.join(predict_status_path, 'out.log'), 'w', encoding='utf-8')
+    sys.stderr = open(
+        osp.join(predict_status_path, 'err.log'), 'w', encoding='utf-8')
+
+    import paddlex as pdx
+    pdx.log_level = 3
+    task_type = params['task_type']
+    dataset_path = params['dataset_path']
+    if epoch is None:
+        model_path = osp.join(task_path, 'output', 'best_model')
+    else:
+        model_path = osp.join(task_path, 'output', 'epoch_{}'.format(epoch))
+    model = pdx.load_model(model_path)
+    file_list = dict()
+    predicted_num = 0
+    if task_type == "classification":
+        if img_data is None:
+            if len(img_list) == 0 and osp.exists(
+                    osp.join(dataset_path, "test_list.txt")):
+                with open(osp.join(dataset_path, "test_list.txt")) as f:
+                    for line in f:
+                        items = line.strip().split()
+                        file_list[osp.join(dataset_path, items[0])] = items[1]
+            else:
+                for image in img_list:
+                    file_list[image] = None
+            total_file_num = len(file_list)
+            write_file_num(total_file_num)
+            for image, label_id in file_list.items():
+                pred_result = {}
+                if label_id is not None:
+                    pred_result["gt_label"] = model.labels[int(label_id)]
+                results = model.predict(img_file=image)
+                pred_result["label"] = []
+                pred_result["score"] = []
+                pred_result["topk"] = len(results)
+                for res in results:
+                    pred_result["label"].append(res['category'])
+                    pred_result["score"].append(res['score'])
+                visualize_classified_result(save_dir, image, pred_result)
+                predicted_num += 1
+        else:
+            img_data = base64.b64decode(img_data)
+            img_array = np.frombuffer(img_data, np.uint8)
+            img = cv2.imdecode(img_array, cv2.COLOR_RGB2BGR)
+            results = model.predict(img)
+            pred_result = {}
+            pred_result["label"] = []
+            pred_result["score"] = []
+            pred_result["topk"] = len(results)
+            for res in results:
+                pred_result["label"].append(res['category'])
+                pred_result["score"].append(res['score'])
+            visualize_classified_result(save_dir, img, pred_result)
+    elif task_type in ["detection", "instance_segmentation"]:
+        if img_data is None:
+            if task_type == "detection" and osp.exists(
+                    osp.join(dataset_path, "test_list.txt")):
+                if len(img_list) == 0 and osp.exists(
+                        osp.join(dataset_path, "test_list.txt")):
+                    with open(osp.join(dataset_path, "test_list.txt")) as f:
+                        for line in f:
+                            items = line.strip().split()
+                            file_list[osp.join(dataset_path, items[0])] = \
+                                osp.join(dataset_path, items[1])
+                else:
+                    for image in img_list:
+                        file_list[image] = None
+                total_file_num = len(file_list)
+                write_file_num(total_file_num)
+                for image, anno in file_list.items():
+                    results = model.predict(img_file=image)
+                    image_pred = pdx.det.visualize(
+                        image, results, threshold=score_thresh, save_dir=None)
+                    save_name = osp.join(save_dir, osp.split(image)[-1])
+                    image_gt = None
+                    if anno is not None:
+                        image_gt = plot_det_label(image, anno, model.labels)
+                    visualize_detected_result(save_name, image_gt, image_pred)
+                    predicted_num += 1
+            elif len(img_list) == 0 and osp.exists(
+                    osp.join(dataset_path, "test.json")):
+                from pycocotools.coco import COCO
+                anno_path = osp.join(dataset_path, "test.json")
+                coco = COCO(anno_path)
+                img_ids = coco.getImgIds()
+                total_file_num = len(img_ids)
+                write_file_num(total_file_num)
+                for img_id in img_ids:
+                    img_anno = coco.loadImgs(img_id)[0]
+                    file_name = img_anno['file_name']
+                    name = (osp.split(file_name)[-1]).split(".")[0]
+                    anno = osp.join(dataset_path, "Annotations", name + ".npy")
+                    img_file = osp.join(dataset_path, "JPEGImages", file_name)
+                    results = model.predict(img_file=img_file)
+                    image_pred = pdx.det.visualize(
+                        img_file,
+                        results,
+                        threshold=score_thresh,
+                        save_dir=None)
+                    save_name = osp.join(save_dir, osp.split(img_file)[-1])
+                    if task_type == "detection":
+                        image_gt = plot_det_label(img_file, anno, model.labels)
+                    else:
+                        image_gt = plot_insseg_label(img_file, anno,
+                                                     model.labels)
+                    visualize_detected_result(save_name, image_gt, image_pred)
+                    predicted_num += 1
+            else:
+                total_file_num = len(img_list)
+                write_file_num(total_file_num)
+                for image in img_list:
+                    results = model.predict(img_file=image)
+                    image_pred = pdx.det.visualize(
+                        image, results, threshold=score_thresh, save_dir=None)
+                    save_name = osp.join(save_dir, osp.split(image)[-1])
+                    visualize_detected_result(save_name, None, image_pred)
+                    predicted_num += 1
+        else:
+            img_data = base64.b64decode(img_data)
+            img_array = np.frombuffer(img_data, np.uint8)
+            img = cv2.imdecode(img_array, cv2.COLOR_RGB2BGR)
+            results = model.predict(img)
+            image_pred = pdx.det.visualize(
+                img, results, threshold=score_thresh, save_dir=None)
+            image_gt = None
+            save_name = osp.join(save_dir, 'predict_result.png')
+            visualize_detected_result(save_name, image_gt, image_pred)
+
+    elif task_type == "segmentation":
+        if img_data is None:
+            if len(img_list) == 0 and osp.exists(
+                    osp.join(dataset_path, "test_list.txt")):
+                with open(osp.join(dataset_path, "test_list.txt")) as f:
+                    for line in f:
+                        items = line.strip().split()
+                        file_list[osp.join(dataset_path, items[0])] = \
+                            osp.join(dataset_path, items[1])
+            else:
+                for image in img_list:
+                    file_list[image] = None
+            total_file_num = len(file_list)
+            write_file_num(total_file_num)
+            color_map = get_color_map_list(256)
+            legend = {}
+            for i in range(len(model.labels)):
+                legend[model.labels[i]] = color_map[i]
+            for image, anno in file_list.items():
+                results = model.predict(img_file=image)
+                image_pred = pdx.seg.visualize(image, results, save_dir=None)
+                pse_pred = pdx.seg.visualize(
+                    image, results, weight=0, save_dir=None)
+                image_ground = None
+                pse_label = None
+                if anno is not None:
+                    label = np.asarray(Image.open(anno)).astype('uint8')
+                    image_ground = pdx.seg.visualize(
+                        image, {'label_map': label}, save_dir=None)
+                    pse_label = pdx.seg.visualize(
+                        image, {'label_map': label}, weight=0, save_dir=None)
+                save_name = osp.join(save_dir, osp.split(image)[-1])
+                visualize_segmented_result(save_name, image_ground, pse_label,
+                                           image_pred, pse_pred, legend)
+                predicted_num += 1
+        else:
+            img_data = base64.b64decode(img_data)
+            img_array = np.frombuffer(img_data, np.uint8)
+            img = cv2.imdecode(img_array, cv2.COLOR_RGB2BGR)
+            color_map = get_color_map_list(256)
+            legend = {}
+            for i in range(len(model.labels)):
+                legend[model.labels[i]] = color_map[i]
+            results = model.predict(img)
+            image_pred = pdx.seg.visualize(image, results, save_dir=None)
+            pse_pred = pdx.seg.visualize(
+                image, results, weight=0, save_dir=None)
+            image_ground = None
+            pse_label = None
+            save_name = osp.join(save_dir, 'predict_result.png')
+            visualize_segmented_result(save_name, image_ground, pse_label,
+                                       image_pred, pse_pred, legend)
+    set_folder_status(predict_status_path, PredictStatus.XPREDONE)
+
+
+def _call_paddlex_export_infer(task_path, save_dir, export_status_path, epoch):
+    # 导出模型不使用GPU
+    sys.stdout = open(
+        osp.join(export_status_path, 'out.log'), 'w', encoding='utf-8')
+    sys.stderr = open(
+        osp.join(export_status_path, 'err.log'), 'w', encoding='utf-8')
+    import os
+    os.environ['CUDA_VISIBLE_DEVICES'] = ''
+    import paddlex as pdx
+    model_dir = "epoch_{}".format(epoch)
+    model_path = osp.join(task_path, 'output', model_dir)
+    model = pdx.load_model(model_path)
+    model.export_inference_model(save_dir)
+    set_folder_status(export_status_path, TaskStatus.XEXPORTED)
+    set_folder_status(task_path, TaskStatus.XEXPORTED)
+
+
+def _call_paddlex_export_quant(task_path, params, save_dir, export_status_path,
+                               epoch):
+    sys.stdout = open(
+        osp.join(export_status_path, 'out.log'), 'w', encoding='utf-8')
+    sys.stderr = open(
+        osp.join(export_status_path, 'err.log'), 'w', encoding='utf-8')
+    dataset_path = params['dataset_path']
+    task_type = params['task_type']
+    os.environ['CUDA_VISIBLE_DEVICES'] = params['train'].cuda_visible_devices
+    import paddlex as pdx
+    model_dir = "epoch_{}".format(epoch)
+    model_path = osp.join(task_path, 'output', model_dir)
+    model = pdx.load_model(model_path)
+    if task_type == "classification":
+        train_file_list = osp.join(dataset_path, 'train_list.txt')
+        val_file_list = osp.join(dataset_path, 'val_list.txt')
+        label_list = osp.join(dataset_path, 'labels.txt')
+        quant_dataset = pdx.datasets.ImageNet(
+            data_dir=dataset_path,
+            file_list=train_file_list,
+            label_list=label_list,
+            transforms=model.test_transforms)
+        eval_dataset = pdx.datasets.ImageNet(
+            data_dir=dataset_path,
+            file_list=val_file_list,
+            label_list=label_list,
+            transforms=model.eval_transforms)
+    elif task_type == "detection":
+        train_file_list = osp.join(dataset_path, 'train_list.txt')
+        val_file_list = osp.join(dataset_path, 'val_list.txt')
+        label_list = osp.join(dataset_path, 'labels.txt')
+        quant_dataset = pdx.datasets.VOCDetection(
+            data_dir=dataset_path,
+            file_list=train_file_list,
+            label_list=label_list,
+            transforms=model.test_transforms)
+        eval_dataset = pdx.datasets.VOCDetection(
+            data_dir=dataset_path,
+            file_list=val_file_list,
+            label_list=label_list,
+            transforms=model.eval_transforms)
+    elif task_type == "instance_segmentation":
+        train_json = osp.join(dataset_path, 'train.json')
+        val_json = osp.join(dataset_path, 'val.json')
+        quant_dataset = pdx.datasets.CocoDetection(
+            data_dir=osp.join(dataset_path, 'JPEGImages'),
+            ann_file=train_json,
+            transforms=model.test_transforms)
+        eval_dataset = pdx.datasets.CocoDetection(
+            data_dir=osp.join(dataset_path, 'JPEGImages'),
+            ann_file=val_json,
+            transforms=model.eval_transforms)
+    elif task_type == "segmentation":
+        train_file_list = osp.join(dataset_path, 'train_list.txt')
+        val_file_list = osp.join(dataset_path, 'val_list.txt')
+        label_list = osp.join(dataset_path, 'labels.txt')
+        quant_dataset = pdx.datasets.SegDataset(
+            data_dir=dataset_path,
+            file_list=train_file_list,
+            label_list=label_list,
+            transforms=model.test_transforms)
+        eval_dataset = pdx.datasets.SegDataset(
+            data_dir=dataset_path,
+            file_list=val_file_list,
+            label_list=label_list,
+            transforms=model.eval_transforms)
+    metric_before = model.evaluate(eval_dataset)
+    pdx.log_level = 3
+    pdx.slim.export_quant_model(
+        model, quant_dataset, batch_size=1, save_dir=save_dir, cache_dir=None)
+    model_quant = pdx.load_model(save_dir)
+    metric_after = model_quant.evaluate(eval_dataset)
+    metrics = {}
+    if task_type == "segmentation":
+        metrics['before'] = {'miou': metric_before['miou']}
+        metrics['after'] = {'miou': metric_after['miou']}
+    else:
+        metrics['before'] = metric_before
+        metrics['after'] = metric_after
+    import json
+    with open(
+            osp.join(export_status_path, 'quant_result.json'),
+            'w',
+            encoding='utf-8') as f:
+        json.dump(metrics, f)
+    set_folder_status(export_status_path, TaskStatus.XEXPORTED)
+    set_folder_status(task_path, TaskStatus.XEXPORTED)
+
+
+def _call_paddlelite_export_lite(model_path, save_dir=None, place="arm"):
+    import paddlelite.lite as lite
+    opt = lite.Opt()
+    model_file = os.path.join(model_path, '__model__')
+    params_file = os.path.join(model_path, '__params__')
+    if save_dir is None:
+        save_dir = osp.join(model_path, "lite_model")
+    if not osp.exists(save_dir):
+        os.makedirs(save_dir)
+    path = osp.join(save_dir, "model")
+    opt.run_optimize("", model_file, params_file, "naive_buffer", place, path)
+
+
+def safe_clean_folder(folder):
+    if osp.exists(folder):
+        try:
+            shutil.rmtree(folder)
+            os.makedirs(folder)
+        except Exception as e:
+            pass
+        if osp.exists(folder):
+            for root, dirs, files in os.walk(folder):
+                for name in files:
+                    try:
+                        os.remove(os.path.join(root, name))
+                    except Exception as e:
+                        pass
+        else:
+            os.makedirs(folder)
+    else:
+        os.makedirs(folder)
+    if not osp.exists(folder):
+        os.makedirs(folder)
+
+
+def get_task_max_saved_epochs(task_path):
+    saved_epoch_num = -1
+    output_path = osp.join(task_path, "output")
+    if osp.exists(output_path):
+        for file in os.listdir(output_path):
+            if file.startswith("epoch_"):
+                curr_epoch_num = int(file[6:])
+                if curr_epoch_num > saved_epoch_num:
+                    saved_epoch_num = curr_epoch_num
+    return saved_epoch_num
+
+
+def get_task_status(task_path):
+    status, message = get_folder_status(task_path, True)
+    task_id = os.path.split(task_path)[-1]
+    err_log = os.path.join(task_path, 'err.log')
+    if status in [TaskStatus.XTRAINING, TaskStatus.XPRUNETRAIN]:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = TaskStatus.XTRAINFAIL
+            message = "训练任务{}异常终止,请查阅错误日志具体确认原因{}。\n\n 如若通过日志无法确定原因,可尝试以下几种方法,\n" \
+            "1. 尝试重新启动训练,看是否能正常训练; \n" \
+            "2. 调低batch_size(需同时按比例调低学习率等参数)排除是否是显存或内存不足的原因导致;\n" \
+            "3. 前往GitHub提ISSUE,描述清楚问题会有工程师及时回复: https://github.com/PaddlePaddle/PaddleX/issues ; \n" \
+            "3. 加QQ群1045148026或邮件至paddlex@baidu.com在线咨询工程师".format(task_id, err_log)
+            set_folder_status(task_path, status, message)
+    return status, message
+
+
+def train_model(task_path):
+    """训练模型
+
+    Args:
+        task_path(str): 模型训练的参数保存在task_path下的'params.pkl'文件中
+    """
+    params_conf_file = osp.join(task_path, 'params.pkl')
+    assert osp.exists(
+        params_conf_file), "任务无法启动,路径{}下不存在参数配置文件params.pkl".format(task_path)
+    with open(params_conf_file, 'rb') as f:
+        params = pickle.load(f)
+    sensitivities_path = params['train'].sensitivities_path
+    p = mp.Process(target=_call_paddlex_train, args=(task_path, params))
+    p.start()
+    if sensitivities_path is None:
+        set_folder_status(task_path, TaskStatus.XTRAINING, p.pid)
+    else:
+        set_folder_status(task_path, TaskStatus.XPRUNETRAIN, p.pid)
+    return p
+
+
+def stop_train_model(task_path):
+    """停止正在训练的模型
+
+    Args:
+        task_path(str): 从task_path下的'XTRANING'文件中获取训练的进程id
+    """
+    status, message = get_task_status(task_path)
+    if status in [TaskStatus.XTRAINING, TaskStatus.XPRUNETRAIN]:
+        pid = int(message)
+        pkill(pid)
+        best_model_saved = True
+        if not osp.exists(osp.join(task_path, 'output', 'best_model')):
+            best_model_saved = False
+        set_folder_status(task_path, TaskStatus.XTRAINEXIT, best_model_saved)
+    else:
+        raise Exception("模型训练任务没在运行中")
+
+
+def prune_analysis_model(task_path):
+    """模型裁剪分析
+
+    Args:
+        task_path(str): 模型训练的参数保存在task_path
+        dataset_path(str) 模型裁剪中评估数据集的路径
+    """
+    best_model_path = osp.join(task_path, 'output', 'best_model')
+    assert osp.exists(best_model_path), "该任务暂未保存模型,无法进行模型裁剪分析"
+    prune_analysis_path = osp.join(task_path, 'prune')
+    if not osp.exists(prune_analysis_path):
+        os.makedirs(prune_analysis_path)
+
+    params_conf_file = osp.join(task_path, 'params.pkl')
+    assert osp.exists(
+        params_conf_file), "任务无法启动,路径{}下不存在参数配置文件params.pkl".format(task_path)
+    with open(params_conf_file, 'rb') as f:
+        params = pickle.load(f)
+    assert params['train'].model.lower() not in [
+        "ppyolo", "fasterrcnn", "maskrcnn", "fastscnn", "HRNet_W18"
+    ], "暂不支持PPYOLO、FasterRCNN、MaskRCNN、HRNet_W18、FastSCNN模型裁剪"
+    p = mp.Process(
+        target=_call_paddle_prune,
+        args=(best_model_path, prune_analysis_path, params))
+    p.start()
+    set_folder_status(prune_analysis_path, PruneStatus.XSPRUNEING, p.pid)
+    set_folder_status(task_path, TaskStatus.XPRUNEING, p.pid)
+    return p
+
+
+def get_prune_status(prune_path):
+    status, message = get_folder_status(prune_path, True)
+    if status in [PruneStatus.XSPRUNEING]:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = PruneStatus.XSPRUNEFAIL
+            message = "模型裁剪异常终止,可能原因如下:\n1.暂不支持FasterRCNN、MaskRCNN模型的模型裁剪\n2.模型裁剪过程中进程被异常结束,建议重新启动模型裁剪任务"
+            set_folder_status(prune_path, status, message)
+    return status, message
+
+
+def stop_prune_analysis(prune_path):
+    """停止正在裁剪分析的模型
+
+    Args:
+        prune_path(str): prune_path'XSSLMING'文件中获取训练的进程id
+    """
+    status, message = get_prune_status(prune_path)
+    if status == PruneStatus.XSPRUNEING:
+        pid = int(message)
+        pkill(pid)
+        set_folder_status(prune_path, PruneStatus.XSPRUNEEXIT)
+    else:
+        raise Exception("模型裁剪分析任务未在运行中")
+
+
+def evaluate_model(task_path,
+                   task_type,
+                   epoch=None,
+                   topk=5,
+                   score_thresh=0.3,
+                   overlap_thresh=0.5):
+    """评估最优模型
+
+    Args:
+        task_path(str): 模型训练相关结果的保存路径
+    """
+    output_path = osp.join(task_path, 'output')
+    if not osp.exists(osp.join(output_path, 'best_model')):
+        raise Exception("未在训练路径{}下发现保存的best_model,无法进行评估".format(output_path))
+    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    safe_clean_folder(evaluate_status_path)
+    if epoch is None:
+        model_path = osp.join(output_path, 'best_model')
+    else:
+        epoch_dir = "{}_{}".format('epoch', epoch)
+        model_path = osp.join(output_path, epoch_dir)
+    p = mp.Process(
+        target=_call_paddlex_evaluate_model,
+        args=(task_path, model_path, task_type, epoch, topk, score_thresh,
+              overlap_thresh))
+    p.start()
+    set_folder_status(evaluate_status_path, TaskStatus.XEVALUATING, p.pid)
+    return p
+
+
+def get_evaluate_status(task_path):
+    """获取导出状态
+    Args:
+        task_path(str): 训练任务文件夹
+    """
+    evaluate_status_path = osp.join(task_path, './logs/evaluate')
+    if not osp.exists(evaluate_status_path):
+        return None, "No evaluate fold in path {}".format(task_path)
+    status, message = get_folder_status(evaluate_status_path, True)
+    if status == TaskStatus.XEVALUATING:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = TaskStatus.XEVALUATEFAIL
+            message = "评估过程出现异常,请尝试重新评估!"
+            set_folder_status(evaluate_status_path, status, message)
+    if status not in [
+            TaskStatus.XEVALUATING, TaskStatus.XEVALUATED,
+            TaskStatus.XEVALUATEFAIL
+    ]:
+        raise ValueError("Wrong status in evaluate task {}".format(status))
+    return status, message
+
+
+def get_predict_status(task_path):
+    """获取预测任务状态
+
+    Args:
+        task_path(str): 从predict_path下的'XPRESTART'文件中获取训练的进程id
+    """
+    from ..utils import list_files
+    predict_status_path = osp.join(task_path, "./logs/predict")
+    save_dir = osp.join(task_path, "visualized_test_results")
+    if not osp.exists(save_dir):
+        return None, "任务目录下没有visualized_test_results文件夹,{}".format(
+            task_path), 0, 0
+    status, message = get_folder_status(predict_status_path, True)
+    if status == PredictStatus.XPRESTART:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = PredictStatus.XPREFAIL
+            message = "图片预测过程出现异常,请尝试重新预测!"
+            set_folder_status(predict_status_path, status, message)
+    if status not in [
+            PredictStatus.XPRESTART, PredictStatus.XPREDONE,
+            PredictStatus.XPREFAIL
+    ]:
+        raise ValueError("预测任务状态异常,{}".format(status))
+    predict_num = len(list_files(save_dir))
+    if predict_num > 0:
+        if predict_num == 1:
+            total_num = 1
+        else:
+            total_num = int(
+                open(
+                    osp.join(predict_status_path, "total_num"),
+                    encoding='utf-8').readline().strip())
+    else:
+        predict_num = 0
+        total_num = 0
+    return status, message, predict_num, total_num
+
+
+def predict_test_pics(task_path,
+                      img_list=[],
+                      img_data=None,
+                      save_dir=None,
+                      score_thresh=0.5,
+                      epoch=None):
+    """模型预测
+
+    Args:
+        task_path(str): 模型训练的参数保存在task_path下的'params.pkl'文件中
+    """
+    params_conf_file = osp.join(task_path, 'params.pkl')
+    assert osp.exists(
+        params_conf_file), "任务无法启动,路径{}下不存在参数配置文件params.pkl".format(task_path)
+    with open(params_conf_file, 'rb') as f:
+        params = pickle.load(f)
+    predict_status_path = osp.join(task_path, "./logs/predict")
+    safe_clean_folder(predict_status_path)
+    save_dir = osp.join(task_path, 'visualized_test_results')
+    safe_clean_folder(save_dir)
+    p = mp.Process(
+        target=_call_paddlex_predict,
+        args=(task_path, predict_status_path, params, img_list, img_data,
+              save_dir, score_thresh, epoch))
+    p.start()
+    set_folder_status(predict_status_path, PredictStatus.XPRESTART, p.pid)
+    return p, save_dir
+
+
+def stop_predict_task(task_path):
+    """停止预测任务
+
+    Args:
+        task_path(str): 从predict_path下的'XPRESTART'文件中获取训练的进程id
+    """
+    from ..utils import list_files
+    predict_status_path = osp.join(task_path, "./logs/predict")
+    save_dir = osp.join(task_path, "visualized_test_results")
+    if not osp.exists(save_dir):
+        return None, "任务目录下没有visualized_test_results文件夹,{}".format(
+            task_path), 0, 0
+    status, message = get_folder_status(predict_status_path, True)
+    if status == PredictStatus.XPRESTART:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = PredictStatus.XPREFAIL
+            message = "图片预测过程出现异常,请尝试重新预测!"
+            set_folder_status(predict_status_path, status, message)
+        else:
+            pkill(pid)
+            status = PredictStatus.XPREFAIL
+            message = "图片预测进程已停止!"
+            set_folder_status(predict_status_path, status, message)
+    if status not in [
+            PredictStatus.XPRESTART, PredictStatus.XPREDONE,
+            PredictStatus.XPREFAIL
+    ]:
+        raise ValueError("预测任务状态异常,{}".format(status))
+    predict_num = len(list_files(save_dir))
+    if predict_num > 0:
+        total_num = int(
+            open(
+                osp.join(predict_status_path, "total_num"), encoding='utf-8')
+            .readline().strip())
+    else:
+        predict_num = 0
+        total_num = 0
+    return status, message, predict_num, total_num
+
+
+def get_export_status(task_path):
+    """获取导出状态
+
+    Args:
+        task_path(str): 从task_path下的'export/XEXPORTING'文件中获取训练的进程id
+    Return:
+        导出的状态和其他消息.
+    """
+    export_status_path = osp.join(task_path, './logs/export')
+    if not osp.exists(export_status_path):
+        return None, "{}任务目录下没有export文件夹".format(task_path)
+    status, message = get_folder_status(export_status_path, True)
+    if status == TaskStatus.XEXPORTING:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = TaskStatus.XEXPORTFAIL
+            message = "导出过程出现异常,请尝试重新评估!"
+            set_folder_status(export_status_path, status, message)
+    if status not in [
+            TaskStatus.XEXPORTING, TaskStatus.XEXPORTED, TaskStatus.XEXPORTFAIL
+    ]:
+        # raise ValueError("获取到的导出状态异常,{}。".format(status))
+        return None, "获取到的导出状态异常,{}。".format(status)
+    return status, message
+
+
+def export_quant_model(task_path, save_dir, epoch):
+    """导出量化模型
+
+    Args:
+        task_path(str): 模型训练的路径
+        save_dir(str): 导出后的模型保存路径
+    """
+    output_path = osp.join(task_path, 'output')
+    if not osp.exists(osp.join(output_path, 'best_model')):
+        raise Exception("未在训练路径{}下发现保存的best_model,导出失败".format(output_path))
+    export_status_path = osp.join(task_path, './logs/export')
+    safe_clean_folder(export_status_path)
+
+    params_conf_file = osp.join(task_path, 'params.pkl')
+    assert osp.exists(
+        params_conf_file), "任务无法启动,路径{}下不存在参数配置文件params.pkl".format(task_path)
+    with open(params_conf_file, 'rb') as f:
+        params = pickle.load(f)
+    p = mp.Process(
+        target=_call_paddlex_export_quant,
+        args=(task_path, params, save_dir, export_status_path, epoch))
+    p.start()
+    set_folder_status(export_status_path, TaskStatus.XEXPORTING, p.pid)
+    set_folder_status(task_path, TaskStatus.XEXPORTING, p.pid)
+    return p
+
+
+def export_noquant_model(task_path, save_dir, epoch):
+    """导出inference模型
+
+    Args:
+        task_path(str): 模型训练的路径
+        save_dir(str): 导出后的模型保存路径
+    """
+    output_path = osp.join(task_path, 'output')
+    if not osp.exists(osp.join(output_path, 'best_model')):
+        raise Exception("未在训练路径{}下发现保存的best_model,导出失败".format(output_path))
+    export_status_path = osp.join(task_path, './logs/export')
+    safe_clean_folder(export_status_path)
+    p = mp.Process(
+        target=_call_paddlex_export_infer,
+        args=(task_path, save_dir, export_status_path, epoch))
+    p.start()
+    set_folder_status(export_status_path, TaskStatus.XEXPORTING, p.pid)
+    set_folder_status(task_path, TaskStatus.XEXPORTING, p.pid)
+    return p
+
+
+def opt_lite_model(model_path, save_dir=None, place='arm'):
+    p = mp.Process(
+        target=_call_paddlelite_export_lite,
+        args=(model_path, save_dir, place))
+    p.start()
+    p.join()
+
+
+def stop_export_task(task_path):
+    """停止导出
+
+    Args:
+        task_path(str): 从task_path下的'export/XEXPORTING'文件中获取训练的进程id
+    Return:
+        the export status and message.
+    """
+    export_status_path = osp.join(task_path, './logs/export')
+    if not osp.exists(export_status_path):
+        return None, "{}任务目录下没有export文件夹".format(task_path)
+    status, message = get_folder_status(export_status_path, True)
+    if status == TaskStatus.XEXPORTING:
+        pid = int(message)
+        is_dead = False
+        if not psutil.pid_exists(pid):
+            is_dead = True
+        else:
+            p = psutil.Process(pid)
+            if p.status() == 'zombie':
+                is_dead = True
+        if is_dead:
+            status = TaskStatus.XEXPORTFAIL
+            message = "导出过程出现异常,请尝试重新评估!"
+            set_folder_status(export_status_path, status, message)
+        else:
+            pkill(pid)
+            status = TaskStatus.XEXPORTFAIL
+            message = "已停止导出进程!"
+            set_folder_status(export_status_path, status, message)
+    if status not in [
+            TaskStatus.XEXPORTING, TaskStatus.XEXPORTED, TaskStatus.XEXPORTFAIL
+    ]:
+        raise ValueError("获取到的导出状态异常,{}。".format(status))
+    return status, message

+ 143 - 0
paddlex/restful/project/project.py

@@ -0,0 +1,143 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import time
+import os
+import os.path as osp
+from .. import workspace_pb2 as w
+import shutil
+
+
+def create_project(data, workspace):
+    """根据request创建project。
+
+    Args:
+        data为dict,key包括
+        'name'项目名, 'desc'项目描述,'project_type'项目类型和'path'
+        项目路径,`path`(可选),不设置或为None,表示使用默认的生成路径
+    """
+    create_time = time.time()
+    time_array = time.localtime(create_time)
+    create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    id = workspace.max_project_id + 1
+    workspace.max_project_id = id
+    if id < 10000:
+        id = 'P%04d' % id
+    else:
+        id = 'P{}'.format(id)
+    assert not id in workspace.projects, "【项目创建】ID'{}'已经被占用.".format(id)
+    if 'path' not in data:
+        path = osp.join(workspace.path, 'projects', id)
+    if not osp.exists(path):
+        os.makedirs(path)
+    pj = w.Project(
+        id=id,
+        name=data['name'],
+        desc=data['desc'],
+        type=data['project_type'],
+        create_time=create_time,
+        path=path)
+    workspace.projects[id].CopyFrom(pj)
+
+    with open(os.path.join(path, 'info.pb'), 'wb') as f:
+        f.write(pj.SerializeToString())
+
+    return {'status': 1, 'pid': id}
+
+
+def delete_project(data, workspace):
+    """删除project和与project相关的task
+
+    Args:
+        data为dict,key包括
+        'pid'项目id
+    """
+    proj_id = data['pid']
+    assert proj_id in workspace.projects, "项目ID'{}'不存在.".format(proj_id)
+
+    tids = list()
+    for key in workspace.tasks:
+        tids.append(key)
+    for tid in tids:
+        if workspace.tasks[tid].pid == proj_id:
+            from .task import delete_task
+            data['tid'] = tid
+            delete_task(data, workspace)
+    if osp.exists(workspace.projects[proj_id].path):
+        shutil.rmtree(workspace.projects[proj_id].path)
+    del workspace.projects[proj_id]
+    return {'status': 1}
+
+
+def list_projects(workspace):
+    '''列出项目列表
+    Args:
+    '''
+    project_list = list()
+    for key in workspace.projects:
+        project_id = workspace.projects[key].id
+        project_name = workspace.projects[key].name
+        project_desc = workspace.projects[key].desc
+        project_type = workspace.projects[key].type
+        project_did = workspace.projects[key].did
+        project_path = workspace.projects[key].path
+        project_create_time = workspace.projects[key].create_time
+        attr = {
+            "id": project_id,
+            "name": project_name,
+            "desc": project_desc,
+            "type": project_type,
+            "did": project_did,
+            "path": project_path,
+            "create_time": project_create_time
+        }
+        project_list.append({"id": project_id, "attr": attr})
+    return {'status': 1, 'projects': project_list}
+
+
+def get_project(data, workspace):
+    '''获取项目信息
+    Args:
+        data为dict,
+        'id': 项目id
+    '''
+    pid = data["id"]
+    attr = {}
+    assert pid in workspace.projects, "项目ID'{}'不存在".format(pid)
+    if pid in workspace.projects:
+        project_id = workspace.projects[pid].id
+        project_name = workspace.projects[pid].name
+        project_desc = workspace.projects[pid].desc
+        project_type = workspace.projects[pid].type
+        project_did = workspace.projects[pid].did
+        project_path = workspace.projects[pid].path
+        project_create_time = workspace.projects[pid].create_time
+        project_tasks = {}
+        from .operate import get_task_status
+        for tid in workspace.tasks:
+            if workspace.tasks[tid].pid == pid:
+                path = workspace.tasks[tid].path
+                status, message = get_task_status(path)
+                project_tasks[tid] = {'status': status.value}
+        attr = {
+            "id": project_id,
+            "name": project_name,
+            "desc": project_desc,
+            "type": project_type,
+            "did": project_did,
+            "path": project_path,
+            "create_time": project_create_time,
+            "tasks": project_tasks
+        }
+    return {'status': 1, 'attr': attr}

+ 13 - 0
paddlex/restful/project/prune/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 30 - 0
paddlex/restful/project/prune/classification.py

@@ -0,0 +1,30 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+
+
+def prune(best_model_path, dataset_path, sensitivities_path, batch_size):
+    import paddlex as pdx
+    print(best_model_path)
+    model = pdx.load_model(best_model_path)
+
+    eval_dataset = pdx.datasets.ImageNet(
+        data_dir=dataset_path,
+        file_list=osp.join(dataset_path, 'val_list.txt'),
+        label_list=osp.join(dataset_path, 'labels.txt'),
+        transforms=model.eval_transforms)
+    pdx.slim.cal_params_sensitivities(model, sensitivities_path, eval_dataset,
+                                      batch_size)

+ 46 - 0
paddlex/restful/project/prune/detection.py

@@ -0,0 +1,46 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+
+
+def prune(best_model_path, dataset_path, sensitivities_path, batch_size):
+    import paddlex as pdx
+    model = pdx.load_model(best_model_path)
+
+    # build coco dataset
+    if osp.exists(osp.join(dataset_path, 'JPEGImages')) and \
+        osp.exists(osp.join(dataset_path, 'train.json')) and \
+        osp.exists(osp.join(dataset_path, 'val.json')):
+        data_dir = osp.join(dataset_path, 'JPEGImages')
+        eval_ann_file = osp.join(dataset_path, 'val.json')
+        eval_dataset = pdx.datasets.CocoDetection(
+            data_dir=data_dir,
+            ann_file=eval_ann_file,
+            transforms=model.eval_transforms)
+    # build voc
+    elif osp.exists(osp.join(dataset_path, 'train_list.txt')) and \
+        osp.exists(osp.join(dataset_path, 'val_list.txt')) and \
+        osp.exists(osp.join(dataset_path, 'labels.txt')):
+        eval_file_list = osp.join(dataset_path, 'val_list.txt')
+        label_list = osp.join(dataset_path, 'labels.txt')
+        eval_dataset = pdx.datasets.VOCDetection(
+            data_dir=dataset_path,
+            file_list=eval_file_list,
+            label_list=label_list,
+            transforms=model.eval_transforms)
+
+    pdx.slim.cal_params_sensitivities(model, sensitivities_path, eval_dataset,
+                                      batch_size)

+ 32 - 0
paddlex/restful/project/prune/segmentation.py

@@ -0,0 +1,32 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import os.path as osp
+
+
+def prune(best_model_path, dataset_path, sensitivities_path, batch_size):
+    import paddlex as pdx
+    model = pdx.load_model(best_model_path)
+
+    eval_file_list = osp.join(dataset_path, 'val_list.txt')
+    label_list = osp.join(dataset_path, 'labels.txt')
+    eval_dataset = pdx.datasets.SegDataset(
+        data_dir=dataset_path,
+        file_list=eval_file_list,
+        label_list=label_list,
+        transforms=model.eval_transforms)
+
+    pdx.slim.cal_params_sensitivities(model, sensitivities_path, eval_dataset,
+                                      batch_size)

+ 797 - 0
paddlex/restful/project/task.py

@@ -0,0 +1,797 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .. import workspace_pb2 as w
+import os
+import os.path as osp
+import shutil
+import time
+import pickle
+import json
+import multiprocessing as mp
+from ..utils import set_folder_status, TaskStatus, get_folder_status, is_available, get_ip
+from .train.params import ClsParams, DetParams, SegParams
+
+
+def create_task(data, workspace):
+    """根据request创建task。
+
+    Args:
+        data为dict,key包括
+        'pid'所属项目id, 'train'训练参数。训练参数和数据增强参数以pickle的形式保存
+        在任务目录下的params.pkl文件中。 'parent_id'(可选)该裁剪训练任务的父任务,
+        'desc'(可选)任务描述。
+    """
+    create_time = time.time()
+    time_array = time.localtime(create_time)
+    create_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+    id = workspace.max_task_id + 1
+    workspace.max_task_id = id
+    if id < 10000:
+        id = 'T%04d' % id
+    else:
+        id = 'T{}'.format(id)
+    pid = data['pid']
+    assert pid in workspace.projects, "【任务创建】项目ID'{}'不存在.".format(pid)
+    assert not id in workspace.tasks, "【任务创建】任务ID'{}'已经被占用.".format(id)
+    did = workspace.projects[pid].did
+    assert did in workspace.datasets, "【任务创建】数据集ID'{}'不存在".format(did)
+    path = osp.join(workspace.projects[pid].path, id)
+    if not osp.exists(path):
+        os.makedirs(path)
+    set_folder_status(path, TaskStatus.XINIT)
+    data['task_type'] = workspace.projects[pid].type
+    data['dataset_path'] = workspace.datasets[did].path
+    data['pretrain_weights_download_save_dir'] = osp.join(workspace.path,
+                                                          'pretrain')
+    #获取参数
+    if 'train' in data:
+        params_json = json.loads(data['train'])
+        if (data['task_type'] == 'classification'):
+            params_init = ClsParams()
+        if (data['task_type'] == 'detection' or
+                data['task_type'] == 'instance_segmentation'):
+            params_init = DetParams()
+        if (data['task_type'] == 'segmentation' or
+                data['task_type'] == 'remote_segmentation'):
+            params_init = SegParams()
+        params_init.load_from_dict(params_json)
+        data['train'] = params_init
+    parent_id = ''
+    if 'parent_id' in data:
+        data['tid'] = data['parent_id']
+        parent_id = data['parent_id']
+        assert data['parent_id'] in workspace.tasks, "【任务创建】裁剪任务创建失败".format(
+            data['parent_id'])
+        r = get_task_params(data, workspace)
+        train_params = r['train']
+        data['train'] = train_params
+    desc = ""
+    if 'desc' in data:
+        desc = data['desc']
+    with open(osp.join(path, 'params.pkl'), 'wb') as f:
+        pickle.dump(data, f)
+    task = w.Task(
+        id=id,
+        pid=pid,
+        path=path,
+        create_time=create_time,
+        parent_id=parent_id,
+        desc=desc)
+    workspace.tasks[id].CopyFrom(task)
+
+    with open(os.path.join(path, 'info.pb'), 'wb') as f:
+        f.write(task.SerializeToString())
+
+    return {'status': 1, 'tid': id}
+
+
+def delete_task(data, workspace):
+    """删除task。
+
+    Args:
+        data为dict,key包括
+        'tid'任务id
+    """
+    task_id = data['tid']
+    assert task_id in workspace.tasks, "任务ID'{}'不存在.".format(task_id)
+    if osp.exists(workspace.tasks[task_id].path):
+        shutil.rmtree(workspace.tasks[task_id].path)
+    del workspace.tasks[task_id]
+    return {'status': 1}
+
+
+def get_task_params(data, workspace):
+    """根据request获取task的参数。
+
+    Args:
+        data为dict,key包括
+        'tid'任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "【任务创建】任务ID'{}'不存在.".format(tid)
+    path = workspace.tasks[tid].path
+    with open(osp.join(path, 'params.pkl'), 'rb') as f:
+        task_params = pickle.load(f)
+    return {'status': 1, 'train': task_params['train']}
+
+
+def list_tasks(data, workspace):
+    '''列出任务列表,可request的参数进行筛选
+    Args:
+        data为dict, 包括
+        'pid'(可选)所属项目id
+    '''
+    task_list = list()
+    for key in workspace.tasks:
+        task_id = workspace.tasks[key].id
+        task_name = workspace.tasks[key].name
+        task_desc = workspace.tasks[key].desc
+        task_pid = workspace.tasks[key].pid
+        task_path = workspace.tasks[key].path
+        task_create_time = workspace.tasks[key].create_time
+        from .operate import get_task_status
+        path = workspace.tasks[task_id].path
+        status, message = get_task_status(path)
+        if data is not None:
+            if "pid" in data:
+                if data["pid"] != task_pid:
+                    continue
+        attr = {
+            "id": task_id,
+            "name": task_name,
+            "desc": task_desc,
+            "pid": task_pid,
+            "path": task_path,
+            "create_time": task_create_time,
+            "status": status.value
+        }
+        task_list.append(attr)
+    return {'status': 1, 'tasks': task_list}
+
+
+def set_task_params(data, workspace):
+    """根据request设置task的参数。只有在task是TaskStatus.XINIT状态时才有效
+
+    Args:
+        data为dict,key包括
+        'tid'任务id, 'train'训练参数. 训练
+        参数和数据增强参数以pickle的形式保存在任务目录下的params.pkl文件
+        中。
+    """
+    tid = data['tid']
+    train = data['train']
+    assert tid in workspace.tasks, "【任务创建】任务ID'{}'不存在.".format(tid)
+    path = workspace.tasks[tid].path
+    status = get_folder_status(path)
+    assert status == TaskStatus.XINIT, "该任务不在初始化阶段,设置参数失败"
+    with open(osp.join(path, 'params.pkl'), 'rb') as f:
+        task_params = pickle.load(f)
+    train_json = json.loads(train)
+    task_params['train'].load_from_dict(train_json)
+    with open(osp.join(path, 'params.pkl'), 'wb') as f:
+        pickle.dump(task_params, f)
+    return {'status': 1}
+
+
+def get_default_params(data, workspace, machine_info):
+    from .train.params_v2 import get_params
+    from ..dataset.dataset import get_dataset_details
+    pid = data['pid']
+    assert pid in workspace.projects, "项目ID{}不存在.".format(pid)
+    project_type = workspace.projects[pid].type
+    did = workspace.projects[pid].did
+
+    result = get_dataset_details({'did': did}, workspace)
+    if result['status'] == 1:
+        details = result['details']
+    else:
+        raise Exception("Fail to get dataset details!")
+    train_num = len(details['train_files'])
+    class_num = len(details['labels'])
+    if machine_info['gpu_num'] == 0:
+        gpu_num = 0
+        per_gpu_memory = 0
+        gpu_list = None
+    else:
+        if gpu_list in data:
+            gpu_list = data['gpu_list']
+            gpu_num = len(gpu_list)
+            per_gpu_memory = None
+            for gpu_id in gpu_list:
+                if per_gpu_memory is None:
+                    per_gpu_memory = machine_info['gpu_free_mem'][gpu_id]
+                elif machine_info['gpu_free_mem'][gpu_id] < per_gpu_memory:
+                    per_gpu_memory = machine_info['gpu_free_mem'][gpu_id]
+        else:
+            gpu_num = 1
+            per_gpu_memory = machine_info['gpu_free_mem'][0]
+            gpu_list = [0]
+    params = get_params(data, project_type, train_num, class_num, gpu_num,
+                        per_gpu_memory, gpu_list)
+    return {"status": 1, "train": params}
+
+
+def get_task_params(data, workspace):
+    """根据request获取task的参数。
+
+    Args:
+        data为dict,key包括
+        'tid'任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "【任务创建】任务ID'{}'不存在.".format(tid)
+    path = workspace.tasks[tid].path
+    with open(osp.join(path, 'params.pkl'), 'rb') as f:
+        task_params = pickle.load(f)
+    return {'status': 1, 'train': task_params['train']}
+
+
+def get_task_status(data, workspace):
+    """ 获取任务状态
+
+    Args:
+        data为dict, key包括
+        'tid'任务id, 'resume'(可选):获取是否可以恢复训练的状态
+    """
+    from .operate import get_task_status, get_task_max_saved_epochs
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    status, message = get_task_status(path)
+    if 'resume' in data:
+        max_saved_epochs = get_task_max_saved_epochs(path)
+        params = {'tid': tid}
+        results = get_task_params(params, workspace)
+        total_epochs = results['train'].num_epochs
+        resumable = max_saved_epochs > 0 and max_saved_epochs < total_epochs
+        return {
+            'status': 1,
+            'task_status': status.value,
+            'message': message,
+            'resumable': resumable,
+            'max_saved_epochs': max_saved_epochs
+        }
+
+    return {'status': 1, 'task_status': status.value, 'message': message}
+
+
+def get_train_metrics(data, workspace):
+    """ 获取任务日志
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    Return:
+        train_log(dict): 'eta':剩余时间,'train_metrics': 训练指标,'eval_metircs': 评估指标,
+        'download_status': 下载模型状态,'eval_done': 是否已保存模型,'train_error': 训练错误原因
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    from ..utils import TrainLogReader
+    task_path = workspace.tasks[tid].path
+    log_file = osp.join(task_path, 'out.log')
+    train_log = TrainLogReader(log_file)
+    train_log.update()
+    train_log = train_log.__dict__
+    return {'status': 1, 'train_log': train_log}
+
+
+def get_eval_metrics(data, workspace):
+    """ 获取任务日志
+
+    Args:
+        data为dict, key包括
+        'tid'父任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    best_model_path = osp.join(workspace.tasks[tid].path, "output",
+                               "best_model", "model.yml")
+    import yaml
+    f = open(best_model_path, "r", encoding="utf-8")
+    eval_metrics = yaml.load(f)['_Attributes']['eval_metrics']
+    f.close()
+    return {'status': 1, 'eval_metric': eval_metrics}
+
+
+def get_eval_all_metrics(data, workspace):
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    output_dir = osp.join(workspace.tasks[tid].path, "output")
+    epoch_result_dict = dict()
+    best_epoch = -1
+    best_result = -1
+    import yaml
+    for file_dir in os.listdir(output_dir):
+        if file_dir.startswith("epoch"):
+            epoch_dir = osp.join(output_dir, file_dir)
+            if osp.exists(osp.join(epoch_dir, ".success")):
+                epoch_index = int(file_dir.split('_')[-1])
+                yml_file_path = osp.join(epoch_dir, "model.yml")
+                f = open(yml_file_path, 'r', encoding='utf-8')
+                yml_file = yaml.load(f.read())
+                result = yml_file["_Attributes"]["eval_metrics"]
+                key = list(result.keys())[0]
+                value = result[key]
+                if value > best_result:
+                    best_result = value
+                    best_epoch = epoch_index
+                elif value == best_result:
+                    if epoch_index < best_epoch:
+                        best_epoch = epoch_index
+                epoch_result_dict[epoch_index] = value
+    return {
+        'status': 1,
+        'key': key,
+        'epoch_result_dict': epoch_result_dict,
+        'best_epoch': best_epoch,
+        'best_result': best_result
+    }
+
+
+def get_sensitivities_loss_img(data, workspace):
+    """ 获取敏感度与模型裁剪率关系图
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    task_path = workspace.tasks[tid].path
+    pkl_path = osp.join(task_path, 'prune', 'sensitivities_xy.pkl')
+    import pickle
+    f = open(pkl_path, 'rb')
+    sensitivities_xy = pickle.load(f)
+    return {'status': 1, 'sensitivities_loss_img': sensitivities_xy}
+
+
+def start_train_task(data, workspace, monitored_processes):
+    """启动训练任务。
+
+    Args:
+        data为dict,key包括
+        'tid'任务id, 'eval_metric_loss'(可选)裁剪任务所需的评估loss
+    """
+    from .operate import train_model
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    if 'eval_metric_loss' in data and \
+        data['eval_metric_loss'] is not None:
+        # 裁剪任务
+        parent_id = workspace.tasks[tid].parent_id
+        assert parent_id != "", "任务{}不是裁剪训练任务".format(tid)
+        parent_path = workspace.tasks[parent_id].path
+        sensitivities_path = osp.join(parent_path, 'prune',
+                                      'sensitivities.data')
+        eval_metric_loss = data['eval_metric_loss']
+        parent_best_model_path = osp.join(parent_path, 'output', 'best_model')
+        params_conf_file = osp.join(path, 'params.pkl')
+        with open(params_conf_file, 'rb') as f:
+            params = pickle.load(f)
+        params['train'].sensitivities_path = sensitivities_path
+        params['train'].eval_metric_loss = eval_metric_loss
+        params['train'].pretrain_weights = parent_best_model_path
+        with open(params_conf_file, 'wb') as f:
+            pickle.dump(params, f)
+    p = train_model(path)
+    monitored_processes.put(p.pid)
+    return {'status': 1}
+
+
+def resume_train_task(data, workspace, monitored_processes):
+    """恢复训练任务
+
+    Args:
+        data为dict, key包括
+        'tid'任务id,'epoch'恢复训练的起始轮数
+    """
+    from .operate import train_model
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    epoch_path = "epoch_" + str(data['epoch'])
+    resume_checkpoint_path = osp.join(path, "output", epoch_path)
+
+    params_conf_file = osp.join(path, 'params.pkl')
+    with open(params_conf_file, 'rb') as f:
+        params = pickle.load(f)
+    params['train'].resume_checkpoint = resume_checkpoint_path
+    with open(params_conf_file, 'wb') as f:
+        pickle.dump(params, f)
+
+    p = train_model(path)
+    monitored_processes.put(p.pid)
+    return {'status': 1}
+
+
+def stop_train_task(data, workspace):
+    """停止训练任务
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    """
+    from .operate import stop_train_model
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    stop_train_model(path)
+    return {'status': 1}
+
+
+def start_prune_analysis(data, workspace, monitored_processes):
+    """开始模型裁剪分析
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    task_path = workspace.tasks[tid].path
+    from .operate import prune_analysis_model
+    p = prune_analysis_model(task_path)
+    monitored_processes.put(p.pid)
+    return {'status': 1}
+
+
+def get_prune_metrics(data, workspace):
+    """ 获取模型裁剪分析日志
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    Return:
+        prune_log(dict): 'eta':剩余时间,'iters': 模型裁剪总轮数,'current': 当前轮数,
+        'progress': 模型裁剪进度
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    from ..utils import PruneLogReader
+    task_path = workspace.tasks[tid].path
+    log_file = osp.join(task_path, 'prune', 'out.log')
+    # assert osp.exists(log_file), "模型裁剪分析任务还未开始,请稍等"
+    if not osp.exists(log_file):
+        return {'status': 1, 'prune_log': None}
+    prune_log = PruneLogReader(log_file)
+    prune_log.update()
+    prune_log = prune_log.__dict__
+    return {'status': 1, 'prune_log': prune_log}
+
+
+def get_prune_status(data, workspace):
+    """ 获取模型裁剪状态
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    """
+    from .operate import get_prune_status
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    prune_path = osp.join(path, "prune")
+    status, message = get_prune_status(prune_path)
+    if status is not None:
+        status = status.value
+    return {'status': 1, 'prune_status': status, 'message': message}
+
+
+def stop_prune_analysis(data, workspace):
+    """停止模型裁剪分析
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    from .operate import stop_prune_analysis
+    prune_path = osp.join(workspace.tasks[tid].path, 'prune')
+    stop_prune_analysis(prune_path)
+    return {'status': 1}
+
+
+def evaluate_model(data, workspace, monitored_processes):
+    """ 模型评估
+
+    Args:
+        data为dict, key包括
+        'tid'任务id, topk, score_thresh, overlap_thresh这些评估所需参数
+    Return:
+        None
+    """
+    from .operate import evaluate_model
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    pid = workspace.tasks[tid].pid
+    assert pid in workspace.projects, "项目ID'{}'不存在".format(pid)
+    path = workspace.tasks[tid].path
+    type = workspace.projects[pid].type
+    p = evaluate_model(path, type, data['epoch'], data['topk'],
+                       data['score_thresh'], data['overlap_thresh'])
+    monitored_processes.put(p.pid)
+    return {'status': 1}
+
+
+def get_evaluate_result(data, workspace):
+    """ 获评估结果
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    Return:
+        包含评估指标的dict
+    """
+    from .operate import get_evaluate_status
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    task_path = workspace.tasks[tid].path
+    status, message = get_evaluate_status(task_path)
+    if status == TaskStatus.XEVALUATED:
+        result_file = osp.join(task_path, 'eval_res.pkl')
+        if os.path.exists(result_file):
+            result = pickle.load(open(result_file, "rb"))
+            return {
+                'status': 1,
+                'evaluate_status': status,
+                'message': "{}评估完成".format(tid),
+                'path': result_file,
+                'result': result
+            }
+        else:
+            return {
+                'status': -1,
+                'evaluate_status': status,
+                'message': "评估结果丢失,建议重新评估!",
+                'result': None
+            }
+    if status == TaskStatus.XEVALUATEFAIL:
+        return {
+            'status': -1,
+            'evaluate_status': status,
+            'message': "评估失败,请重新评估!",
+            'result': None
+        }
+    return {
+        'status': 1,
+        'evaluate_status': status,
+        'message': "{}正在评估中,请稍后!".format(tid),
+        'result': None
+    }
+
+
+def get_predict_status(data, workspace):
+    from .operate import get_predict_status
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    status, message, predict_num, total_num = get_predict_status(path)
+    return {
+        'status': 1,
+        'predict_status': status.value,
+        'message': message,
+        'predict_num': predict_num,
+        'total_num': total_num
+    }
+
+
+def predict_test_pics(data, workspace, monitored_processes):
+    from .operate import predict_test_pics
+    tid = data['tid']
+
+    if 'img_list' in data:
+        img_list = data['img_list']
+    else:
+        img_list = list()
+    if 'image_data' in data:
+        image_data = data['image_data']
+    else:
+        image_data = None
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    save_dir = data['save_dir'] if 'save_dir' in data else None
+    epoch = data['epoch'] if 'epoch' in data else None
+    score_thresh = data['score_thresh'] if 'score_thresh' in data else 0.5
+    p, save_dir = predict_test_pics(
+        path,
+        save_dir=save_dir,
+        img_list=img_list,
+        img_data=image_data,
+        score_thresh=score_thresh,
+        epoch=epoch)
+    monitored_processes.put(p.pid)
+    if 'image_data' in data:
+        path = osp.join(save_dir, 'predict_result.png')
+    else:
+        path = None
+    return {'status': 1, 'path': path}
+
+
+def stop_predict_task(data, workspace):
+    from .operate import stop_predict_task
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    status, message, predict_num, total_num = stop_predict_task(path)
+    return {
+        'status': 1,
+        'predict_status': status.value,
+        'message': message,
+        'predict_num': predict_num,
+        'total_num': total_num
+    }
+
+
+def get_quant_progress(data, workspace):
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    from ..utils import QuantLogReader
+    export_path = osp.join(workspace.tasks[tid].path, "./logs/export")
+    log_file = osp.join(export_path, 'out.log')
+    quant_log = QuantLogReader(log_file)
+    quant_log.update()
+    quant_log = quant_log.__dict__
+    return {'status': 1, 'quant_log': quant_log}
+
+
+def get_quant_result(data, workspace):
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    export_path = osp.join(workspace.tasks[tid].path, "./logs/export")
+    result_json = osp.join(export_path, 'quant_result.json')
+    result = {}
+    import json
+    if osp.exists(result_json):
+        with open(result_json, 'r') as f:
+            result = json.load(f)
+    return {'status': 1, 'quant_result': result}
+
+
+def get_export_status(data, workspace):
+    """ 获取导出状态
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    Return:
+        目前导出状态.
+    """
+    from .operate import get_export_status
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    task_path = workspace.tasks[tid].path
+    status, message = get_export_status(task_path)
+    if status == TaskStatus.XEXPORTED:
+        return {
+            'status': 1,
+            'export_status': status,
+            'message': "恭喜您,{}任务模型导出成功!".format(tid)
+        }
+    if status == TaskStatus.XEXPORTFAIL:
+        return {
+            'status': -1,
+            'export_status': status,
+            'message': "{}任务模型导出失败,请重试!".format(tid)
+        }
+    return {
+        'status': 1,
+        'export_status': status,
+        'message': "{}任务模型导出中,请稍等!".format(tid)
+    }
+
+
+def export_infer_model(data, workspace, monitored_processes):
+    """导出部署模型
+
+    Args:
+        data为dict,key包括
+        'tid'任务id, 'save_dir'导出模型保存路径
+    """
+    from .operate import export_noquant_model, export_quant_model
+    tid = data['tid']
+    save_dir = data['save_dir']
+    epoch = data['epoch']
+    quant = data['quant']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    path = workspace.tasks[tid].path
+    if quant:
+        p = export_quant_model(path, save_dir, epoch)
+    else:
+        p = export_noquant_model(path, save_dir, epoch)
+    monitored_processes.put(p.pid)
+    return {'status': 1, 'save_dir': save_dir}
+
+
+def export_lite_model(data, workspace):
+    """ 导出lite模型
+
+    Args:
+        data为dict, key包括
+        'tid'任务id, 'save_dir'导出模型保存路径
+    """
+    from .operate import opt_lite_model
+    model_path = data['model_path']
+    save_dir = data['save_dir']
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    opt_lite_model(model_path, save_dir)
+    if not osp.exists(osp.join(save_dir, "model.nb")):
+        if osp.exists(save_dir):
+            shutil.rmtree(save_dir)
+        return {'status': -1, 'message': "导出为lite模型失败,Ubuntu下请保证 gcc版本 >= 5.0"}
+    return {'status': 1, 'message': "完成"}
+
+
+def stop_export_task(data, workspace):
+    """ 停止导出任务
+
+    Args:
+        data为dict, key包括
+        'tid'任务id
+    Return:
+        目前导出的状态.
+    """
+    from .operate import stop_export_task
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    task_path = workspace.tasks[tid].path
+    status, message = stop_export_task(task_path)
+    return {'status': 1, 'export_status': status.value, 'message': message}
+
+
+def _open_vdl(logdir, current_port):
+    from visualdl.server import app
+    app.run(logdir=logdir, host='0.0.0.0', port=current_port)
+
+
+def open_vdl(data, workspace, current_port, monitored_processes,
+             running_boards):
+    """打开vdl页面
+
+    Args:
+        data为dict,
+        'tid' 任务id
+    """
+    tid = data['tid']
+    assert tid in workspace.tasks, "任务ID'{}'不存在".format(tid)
+    ip = get_ip()
+    if tid in running_boards:
+        url = ip + ":{}".format(running_boards[tid][0])
+        return {'status': 1, 'url': url}
+    task_path = workspace.tasks[tid].path
+    logdir = osp.join(task_path, 'output', 'vdl_log')
+    assert osp.exists(logdir), "该任务还未正常产生日志文件"
+    port_available = is_available(ip, current_port)
+    while not port_available:
+        current_port += 1
+        port_available = is_available(ip, current_port)
+        assert current_port <= 8500, "找不到可用的端口"
+    p = mp.Process(target=_open_vdl, args=(logdir, current_port))
+    p.start()
+    monitored_processes.put(p.pid)
+    url = ip + ":{}".format(current_port)
+    running_boards[tid] = [current_port, p.pid]
+    current_port += 1
+    total_time = 0
+    while True:
+        if not is_available(ip, current_port - 1):
+            break
+        print(current_port)
+        time.sleep(0.5)
+        total_time += 0.5
+        assert total_time <= 8, "VisualDL服务启动超时,请重新尝试打开"
+
+    return {'status': 1, 'url': url}

+ 13 - 0
paddlex/restful/project/train/__init__.py

@@ -0,0 +1,13 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

+ 134 - 0
paddlex/restful/project/train/classification.py

@@ -0,0 +1,134 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+
+
+def build_transforms(params):
+    from paddlex.cls import transforms
+    crop_size = params.image_shape[0]
+    train_transforms = transforms.Compose([
+        transforms.RandomCrop(
+            crop_size=crop_size,
+            lower_scale=0.88,
+            lower_ratio=3. / 4,
+            upper_ratio=4. / 3),
+        transforms.RandomHorizontalFlip(prob=params.horizontal_flip_prob),
+        transforms.RandomVerticalFlip(prob=params.vertical_flip_prob),
+        transforms.RandomDistort(
+            brightness_range=params.brightness_range,
+            brightness_prob=params.brightness_prob,
+            contrast_range=params.contrast_range,
+            contrast_prob=params.contrast_prob,
+            saturation_range=params.saturation_range,
+            saturation_prob=params.saturation_prob,
+            hue_range=params.hue_range,
+            hue_prob=params.hue_prob), transforms.RandomRotate(
+                rotate_range=params.rotate_range,
+                prob=params.rotate_prob), transforms.Normalize(
+                    mean=params.image_mean, std=params.image_std)
+    ])
+    eval_transforms = transforms.Compose([
+        transforms.ResizeByShort(short_size=int(crop_size * 1.143)),
+        transforms.CenterCrop(crop_size=crop_size), transforms.Normalize(
+            mean=params.image_mean, std=params.image_std)
+    ])
+    return train_transforms, eval_transforms
+
+
+def build_datasets(dataset_path, train_transforms, eval_transforms):
+    import paddlex as pdx
+    train_file_list = osp.join(dataset_path, 'train_list.txt')
+    eval_file_list = osp.join(dataset_path, 'val_list.txt')
+    label_list = osp.join(dataset_path, 'labels.txt')
+    train_dataset = pdx.datasets.ImageNet(
+        data_dir=dataset_path,
+        file_list=train_file_list,
+        label_list=label_list,
+        transforms=train_transforms,
+        shuffle=True)
+    eval_dataset = pdx.datasets.ImageNet(
+        data_dir=dataset_path,
+        file_list=eval_file_list,
+        label_list=label_list,
+        transforms=eval_transforms)
+    return train_dataset, eval_dataset
+
+
+def build_optimizer(step_each_epoch, params):
+    import paddle.fluid as fluid
+    from paddle.fluid.regularizer import L2Decay
+    learning_rate = params.learning_rate
+    num_epochs = params.num_epochs
+    if params.lr_policy == 'Cosine':
+        learning_rate = fluid.layers.cosine_decay(
+            learning_rate=learning_rate,
+            step_each_epoch=step_each_epoch,
+            epochs=num_epochs)
+    elif params.lr_policy == 'Linear':
+        learning_rate = fluid.layers.polynomial_decay(
+            learning_rate=learning_rate,
+            decay_steps=step_each_epoch * num_epochs,
+            end_learning_rate=0.0,
+            power=1.0)
+    elif params.lr_policy == 'Piecewise':
+        lr_decay_epochs = params.lr_decay_epochs
+        values = [
+            learning_rate * (0.1**i) for i in range(len(lr_decay_epochs) + 1)
+        ]
+        boundaries = [b * step_each_epoch for b in lr_decay_epochs]
+        learning_rate = fluid.layers.piecewise_decay(
+            boundaries=boundaries, values=values)
+    optimizer = fluid.optimizer.Momentum(
+        learning_rate=learning_rate,
+        momentum=0.9,
+        regularization=L2Decay(1e-04))
+    return optimizer
+
+
+def train(task_path, dataset_path, params):
+    import paddlex as pdx
+    pdx.log_level = 3
+    train_transforms, eval_transforms = build_transforms(params)
+    train_dataset, eval_dataset = build_datasets(
+        dataset_path=dataset_path,
+        train_transforms=train_transforms,
+        eval_transforms=eval_transforms)
+
+    step_each_epoch = train_dataset.num_samples // params.batch_size
+    save_interval_epochs = params.save_interval_epochs
+    save_dir = osp.join(task_path, 'output')
+    pretrain_weights = params.pretrain_weights
+
+    optimizer = build_optimizer(step_each_epoch, params)
+    classifier = getattr(pdx.cv.models, params.model)
+    sensitivities_path = params.sensitivities_path
+    eval_metric_loss = params.eval_metric_loss
+    if eval_metric_loss is None:
+        eval_metric_loss = 0.05
+    model = classifier(num_classes=len(train_dataset.labels))
+    model.train(
+        num_epochs=params.num_epochs,
+        train_dataset=train_dataset,
+        train_batch_size=params.batch_size,
+        eval_dataset=eval_dataset,
+        save_interval_epochs=save_interval_epochs,
+        log_interval_steps=2,
+        save_dir=save_dir,
+        pretrain_weights=pretrain_weights,
+        optimizer=optimizer,
+        use_vdl=True,
+        sensitivities_file=sensitivities_path,
+        eval_metric_loss=eval_metric_loss,
+        resume_checkpoint=params.resume_checkpoint)

+ 218 - 0
paddlex/restful/project/train/detection.py

@@ -0,0 +1,218 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+import math
+import pickle
+import os
+
+
+def build_yolo_transforms(params):
+    from paddlex.det import transforms
+    target_size = params.image_shape[0]
+    use_mixup = params.use_mixup
+    dt_list = []
+    if use_mixup:
+        dt_list.append(
+            transforms.MixupImage(
+                alpha=params.mixup_alpha,
+                beta=params.mixup_beta,
+                mixup_epoch=int(params.num_epochs * 25. / 27)))
+    dt_list.extend([
+        transforms.RandomDistort(
+            brightness_range=params.brightness_range,
+            brightness_prob=params.brightness_prob,
+            contrast_range=params.contrast_range,
+            contrast_prob=params.contrast_prob,
+            saturation_range=params.saturation_range,
+            saturation_prob=params.saturation_prob,
+            hue_range=params.hue_range,
+            hue_prob=params.hue_prob), transforms.RandomExpand(
+                prob=params.expand_prob,
+                fill_value=[float(int(x * 255)) for x in params.image_mean])
+    ])
+    crop_image = params.crop_image
+    if crop_image:
+        dt_list.append(transforms.RandomCrop())
+    dt_list.extend([
+        transforms.Resize(
+            target_size=target_size, interp='RANDOM'),
+        transforms.RandomHorizontalFlip(prob=params.horizontal_flip_prob),
+        transforms.Normalize(
+            mean=params.image_mean, std=params.image_std)
+    ])
+    train_transforms = transforms.Compose(dt_list)
+    eval_transforms = transforms.Compose([
+        transforms.Resize(
+            target_size=target_size, interp='CUBIC'),
+        transforms.Normalize(
+            mean=params.image_mean, std=params.image_std),
+    ])
+    return train_transforms, eval_transforms
+
+
+def build_rcnn_transforms(params):
+    from paddlex.det import transforms
+    short_size = min(params.image_shape)
+    max_size = max(params.image_shape)
+    train_transforms = transforms.Compose([
+        transforms.RandomDistort(
+            brightness_range=params.brightness_range,
+            brightness_prob=params.brightness_prob,
+            contrast_range=params.contrast_range,
+            contrast_prob=params.contrast_prob,
+            saturation_range=params.saturation_range,
+            saturation_prob=params.saturation_prob,
+            hue_range=params.hue_range,
+            hue_prob=params.hue_prob),
+        transforms.RandomHorizontalFlip(prob=params.horizontal_flip_prob),
+        transforms.Normalize(
+            mean=params.image_mean, std=params.image_std),
+        transforms.ResizeByShort(
+            short_size=short_size, max_size=max_size),
+        transforms.Padding(coarsest_stride=32 if params.with_fpn else 1),
+    ])
+    eval_transforms = transforms.Compose([
+        transforms.Normalize(), transforms.ResizeByShort(
+            short_size=short_size, max_size=max_size),
+        transforms.Padding(coarsest_stride=32 if params.with_fpn else 1)
+    ])
+    return train_transforms, eval_transforms
+
+
+def build_voc_datasets(dataset_path, train_transforms, eval_transforms):
+    import paddlex as pdx
+    train_file_list = osp.join(dataset_path, 'train_list.txt')
+    eval_file_list = osp.join(dataset_path, 'val_list.txt')
+    label_list = osp.join(dataset_path, 'labels.txt')
+    train_dataset = pdx.datasets.VOCDetection(
+        data_dir=dataset_path,
+        file_list=train_file_list,
+        label_list=label_list,
+        transforms=train_transforms,
+        shuffle=True)
+    eval_dataset = pdx.datasets.VOCDetection(
+        data_dir=dataset_path,
+        file_list=eval_file_list,
+        label_list=label_list,
+        transforms=eval_transforms)
+    return train_dataset, eval_dataset
+
+
+def build_coco_datasets(dataset_path, train_transforms, eval_transforms):
+    import paddlex as pdx
+    data_dir = osp.join(dataset_path, 'JPEGImages')
+    train_ann_file = osp.join(dataset_path, 'train.json')
+    eval_ann_file = osp.join(dataset_path, 'val.json')
+    train_dataset = pdx.datasets.CocoDetection(
+        data_dir=data_dir,
+        ann_file=train_ann_file,
+        transforms=train_transforms,
+        shuffle=True)
+    eval_dataset = pdx.datasets.CocoDetection(
+        data_dir=data_dir, ann_file=eval_ann_file, transforms=eval_transforms)
+    return train_dataset, eval_dataset
+
+
+def build_optimizer(step_each_epoch, params):
+    import paddle.fluid as fluid
+    from paddle.fluid.regularizer import L2Decay
+    learning_rate = params.learning_rate
+    num_epochs = params.num_epochs
+    lr_decay_epochs = params.lr_decay_epochs
+    warmup_steps = params.warmup_steps
+    warmup_start_lr = params.warmup_start_lr
+
+    boundaries = [b * step_each_epoch for b in lr_decay_epochs]
+    values = [
+        learning_rate * (0.1**i) for i in range(len(lr_decay_epochs) + 1)
+    ]
+    lr = fluid.layers.piecewise_decay(boundaries=boundaries, values=values)
+    lr = fluid.layers.linear_lr_warmup(
+        learning_rate=lr,
+        warmup_steps=warmup_steps,
+        start_lr=warmup_start_lr,
+        end_lr=learning_rate)
+    factor = 1e-04 if params.model in ['FasterRCNN', 'MaskRCNN'] else 5e-04
+    optimizer = fluid.optimizer.Momentum(
+        learning_rate=lr, momentum=0.9, regularization=L2Decay(factor))
+    return optimizer
+
+
+def train(task_path, dataset_path, params):
+    import paddlex as pdx
+    pdx.log_level = 3
+    if params.model in ['YOLOv3', 'PPYOLO']:
+        train_transforms, eval_transforms = build_yolo_transforms(params)
+    elif params.model in ['FasterRCNN', 'MaskRCNN']:
+        train_transforms, eval_transforms = build_rcnn_transforms(params)
+    if osp.exists(osp.join(dataset_path, 'JPEGImages')) and \
+        osp.exists(osp.join(dataset_path, 'train.json')) and \
+        osp.exists(osp.join(dataset_path, 'val.json')):
+        train_dataset, eval_dataset = build_coco_datasets(
+            dataset_path=dataset_path,
+            train_transforms=train_transforms,
+            eval_transforms=eval_transforms)
+    elif osp.exists(osp.join(dataset_path, 'train_list.txt')) and \
+        osp.exists(osp.join(dataset_path, 'val_list.txt')) and \
+        osp.exists(osp.join(dataset_path, 'labels.txt')):
+        train_dataset, eval_dataset = build_voc_datasets(
+            dataset_path=dataset_path,
+            train_transforms=train_transforms,
+            eval_transforms=eval_transforms)
+    step_each_epoch = train_dataset.num_samples // params.batch_size
+    train_batch_size = params.batch_size
+    save_interval_epochs = params.save_interval_epochs
+    save_dir = osp.join(task_path, 'output')
+    pretrain_weights = params.pretrain_weights
+
+    optimizer = build_optimizer(step_each_epoch, params)
+    detector = getattr(pdx.cv.models, params.model)
+    num_classes = len(train_dataset.labels) if params.model in ['YOLOv3', 'PPYOLO'] else \
+        len(train_dataset.labels) + 1
+    sensitivities_path = params.sensitivities_path
+    eval_metric_loss = params.eval_metric_loss
+    if eval_metric_loss is None:
+        eval_metric_loss = 0.05
+    model = detector(num_classes=num_classes, backbone=params.backbone)
+    if params.model in ['YOLOv3', 'PPYOLO']:
+        model.train_random_shapes = params.random_shape_sizes
+    if params.model == 'YOLOv3':
+        model.train(
+            num_epochs=params.num_epochs,
+            train_dataset=train_dataset,
+            train_batch_size=train_batch_size,
+            eval_dataset=eval_dataset,
+            save_interval_epochs=save_interval_epochs,
+            log_interval_steps=2,
+            save_dir=save_dir,
+            pretrain_weights=pretrain_weights,
+            optimizer=optimizer,
+            use_vdl=True,
+            sensitivities_file=sensitivities_path,
+            eval_metric_loss=eval_metric_loss,
+            resume_checkpoint=params.resume_checkpoint)
+    else:
+        model.train(
+            num_epochs=params.num_epochs,
+            train_dataset=train_dataset,
+            train_batch_size=train_batch_size,
+            eval_dataset=eval_dataset,
+            save_interval_epochs=save_interval_epochs,
+            log_interval_steps=2,
+            save_dir=save_dir,
+            pretrain_weights=pretrain_weights,
+            optimizer=optimizer,
+            use_vdl=True,
+            resume_checkpoint=params.resume_checkpoint)

+ 438 - 0
paddlex/restful/project/train/params.py

@@ -0,0 +1,438 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import platform
+import os
+
+
+class Params(object):
+    def __init__(self):
+        self.init_train_params()
+        self.init_transform_params()
+
+    def init_train_params(self):
+        self.cuda_visible_devices = ''
+        self.batch_size = 2
+        self.save_interval_epochs = 1
+        self.pretrain_weights = 'IMAGENET'
+        self.model = 'MobileNetV2'
+        self.num_epochs = 4
+        self.learning_rate = 0.000125
+        self.lr_decay_epochs = [2, 3]
+        self.train_num = 0
+        self.resume_checkpoint = None
+        self.sensitivities_path = None
+        self.eval_metric_loss = None
+
+    def init_transform_params(self):
+        self.image_shape = [224, 224]
+        self.image_mean = [0.485, 0.456, 0.406]
+        self.image_std = [0.229, 0.224, 0.225]
+        self.horizontal_flip_prob = 0.5
+        self.brightness_range = 0.9
+        self.brightness_prob = 0.5
+        self.contrast_range = 0.9
+        self.contrast_prob = 0.5
+        self.saturation_range = 0.9
+        self.saturation_prob = 0.5
+        self.hue_range = 18
+        self.hue_prob = 0.5
+        self.horizontal_flip = True
+        self.brightness = True
+        self.contrast = True
+        self.saturation = True
+        self.hue = True
+
+    def load_from_dict(self, params_dict):
+        for attr in params_dict:
+            if hasattr(self, attr):
+                method = getattr(self, "set_" + attr)
+                method(params_dict[attr])
+
+    def set_cuda_visible_devices(self, cuda_visible_devices):
+        self.cuda_visible_devices = cuda_visible_devices
+
+    def set_batch_size(self, batch_size):
+        self.batch_size = batch_size
+
+    def set_save_interval_epochs(self, save_interval_epochs):
+        self.save_interval_epochs = save_interval_epochs
+
+    def set_pretrain_weights(self, pretrain_weights):
+        self.pretrain_weights = pretrain_weights
+
+    def set_model(self, model):
+        self.model = model
+
+    def set_num_epochs(self, num_epochs):
+        self.num_epochs = num_epochs
+
+    def set_learning_rate(self, learning_rate):
+        self.learning_rate = learning_rate
+
+    def set_lr_decay_epochs(self, lr_decay_epochs):
+        self.lr_decay_epochs = lr_decay_epochs
+
+    def set_resume_checkpoint(self, resume_checkpoint):
+        self.resume_checkpoint = resume_checkpoint
+
+    def set_sensitivities_path(self, sensitivities_path):
+        self.sensitivities_path = sensitivities_path
+
+    def set_eval_metric_loss(self, eval_metric_loss):
+        self.eval_metric_loss = eval_metric_loss
+
+    def set_image_shape(self, image_shape):
+        self.image_shape = image_shape
+
+    def set_image_mean(self, image_mean):
+        self.image_mean = image_mean
+
+    def set_image_std(self, image_std):
+        self.image_std = image_std
+
+    def set_horizontal_flip(self, horizontal_flip):
+        self.horizontal_flip = horizontal_flip
+        if not horizontal_flip:
+            self.horizontal_flip_prob = 0.0
+
+    def set_horizontal_flip_prob(self, horizontal_flip_prob):
+        if self.horizontal_flip:
+            self.horizontal_flip_prob = horizontal_flip_prob
+
+    def set_brightness_range(self, brightness_range):
+        self.brightness_range = brightness_range
+
+    def set_brightness_prob(self, brightness_prob):
+        if self.brightness:
+            self.brightness_prob = brightness_prob
+
+    def set_brightness(self, brightness):
+        self.brightness = brightness
+        if not brightness:
+            self.brightness_prob = 0.0
+
+    def set_contrast(self, contrast):
+        self.contrast = contrast
+        if not contrast:
+            self.contrast_prob = 0.0
+
+    def set_contrast_prob(self, contrast_prob):
+        if self.contrast:
+            self.contrast_prob = contrast_prob
+
+    def set_contrast_range(self, contrast_range):
+        self.contrast_range = contrast_range
+
+    def set_saturation(self, saturation):
+        self.saturation = saturation
+        if not saturation:
+            self.saturation_prob = 0.0
+
+    def set_saturation_prob(self, saturation_prob):
+        if self.saturation:
+            self.saturation_prob = saturation_prob
+
+    def set_saturation_range(self, saturation_range):
+        self.saturation_range = saturation_range
+
+    def set_hue(self, hue):
+        self.hue = hue
+        if not hue:
+            self.hue_prob = 0.0
+
+    def set_hue_prob(self, hue_prob):
+        if self.hue_prob:
+            self.hue_prob = hue_prob
+
+    def set_hue_range(self, hue_range):
+        self.hue_range = hue_range
+
+    def set_train_num(self, train_num):
+        self.train_num = train_num
+
+
+class ClsParams(Params):
+    def __init__(self):
+        super(ClsParams, self).__init__()
+        self.lr_policy = 'Piecewise'
+        self.vertical_flip_prob = 0.0
+        self.vertical_flip = True
+        self.rotate_prob = 0.0
+        self.rotate_range = 30
+        self.rotate = True
+
+    def set_lr_policy(self, lr_policy):
+        self.lr_policy = lr_policy
+
+    def set_vertical_flip(self, vertical_flip):
+        self.vertical_flip = vertical_flip
+        if not self.vertical_flip:
+            self.vertical_flip_prob = 0.0
+
+    def set_vertical_flip_prob(self, vertical_flip_prob):
+        if self.vertical_flip:
+            self.vertical_flip_prob = vertical_flip_prob
+
+    def set_rotate(self, rotate):
+        self.rotate = rotate
+        if not rotate:
+            self.rotate_prob = 0.0
+
+    def set_rotate_prob(self, rotate_prob):
+        if self.rotate:
+            self.rotate_prob = rotate_prob
+
+    def set_rotate_range(self, rotate_range):
+        self.rotate_range = rotate_range
+
+
+class DetParams(Params):
+    def __init__(self):
+        super(DetParams, self).__init__()
+        self.warmup_steps = 10
+        self.warmup_start_lr = 0.
+        self.use_mixup = True
+        self.mixup_alpha = 1.5
+        self.mixup_beta = 1.5
+        self.expand_prob = 0.5
+        self.expand_image = True
+        self.crop_image = True
+        self.backbone = 'ResNet18'
+        self.model = 'FasterRCNN'
+        self.with_fpn = True
+        self.random_shape = True
+        self.random_shape_sizes = [
+            320, 352, 384, 416, 448, 480, 512, 544, 576, 608
+        ]
+
+    def set_warmup_steps(self, warmup_steps):
+        self.warmup_steps = warmup_steps
+
+    def set_warmup_start_lr(self, warmup_start_lr):
+        self.warmup_start_lr = warmup_start_lr
+
+    def set_use_mixup(self, use_mixup):
+        self.use_mixup = use_mixup
+
+    def set_mixup_alpha(self, mixup_alpha):
+        self.mixup_alpha = mixup_alpha
+
+    def set_mixup_beta(self, mixup_beta):
+        self.mixup_beta = mixup_beta
+
+    def set_expand_image(self, expand_image):
+        self.expand_image = expand_image
+        if not expand_image:
+            self.expand_prob = 0.0
+
+    def set_expand_prob(self, expand_prob):
+        if self.expand_image:
+            self.expand_prob = expand_prob
+
+    def set_crop_image(self, crop_image):
+        self.crop_image = crop_image
+
+    def set_backbone(self, backbone):
+        self.backbone = backbone
+
+    def set_with_fpn(self, with_fpn):
+        self.with_fpn = with_fpn
+
+    def set_random_shape(self, random_shape):
+        self.random_shape = random_shape
+
+    def set_random_shape_sizes(self, random_shape_sizes):
+        self.random_shape_sizes = random_shape_sizes
+
+
+class SegParams(Params):
+    def __init__(self):
+        super(SegParams, self).__init__()
+        self.loss_type = [True, True]
+        self.lr_policy = 'Piecewise'
+        self.optimizer = 'Adam'
+        self.backbone = 'MobileNetV2_x1.0'
+        self.blur = True
+        self.blur_prob = 0.
+        self.rotate = False
+        self.max_rotation = 15
+        self.scale_aspect = False
+        self.min_ratio = 0.5
+        self.aspect_ratio = 0.33
+        self.vertical_flip_prob = 0.0
+        self.vertical_flip = True
+        self.model = 'UNet'
+
+    def set_loss_type(self, loss_type):
+        self.loss_type = loss_type
+
+    def set_lr_policy(self, lr_policy):
+        self.lr_policy = lr_policy
+
+    def set_optimizer(self, optimizer):
+        self.optimizer = optimizer
+
+    def set_backbone(self, backbone):
+        self.backbone = backbone
+
+    def set_blur(self, blur):
+        self.blur = blur
+        if not blur:
+            self.blur_prob = 0.
+
+    def set_blur_prob(self, blur_prob):
+        if self.blur:
+            self.blur_prob = blur_prob
+
+    def set_rotate(self, rotate):
+        self.rotate = rotate
+
+    def set_max_rotation(self, max_rotation):
+        self.max_rotation = max_rotation
+
+    def set_scale_aspect(self, scale_aspect):
+        self.scale_aspect = scale_aspect
+
+    def set_min_ratio(self, min_ratio):
+        self.min_ratio = min_ratio
+
+    def set_aspect_ratio(self, aspect_ratio):
+        self.aspect_ratio = aspect_ratio
+
+    def set_vertical_flip(self, vertical_flip):
+        self.vertical_flip = vertical_flip
+        if not vertical_flip:
+            self.vertical_flip_prob = 0.0
+
+    def set_vertical_flip_prob(self, vertical_flip_prob):
+        if vertical_flip_prob:
+            self.vertical_flip_prob = vertical_flip_prob
+
+
+PARAMS_CLASS_LIST = [ClsParams, DetParams, SegParams, DetParams, SegParams]
+
+
+def recommend_parameters(params, train_nums, class_nums, memory_size_per_gpu):
+    model_type = params['model']
+    gpu_list = params['cuda_visible_devices']
+    if 'cpu_num' in params:
+        cpu_num = params['cpu_num']
+    else:
+        cpu_num = int(os.environ.get('CPU_NUM', 1))
+        if cpu_num > 8:
+            os.environ['CPU_NUM'] = '8'
+    if not params['use_gpu']:
+        gpu_nums = 0
+    else:
+        gpu_nums = len(gpu_list.split(','))
+
+    # set batch_size
+    if gpu_nums == 0 or platform.platform().startswith("Darwin"):
+        if model_type.startswith('MobileNet'):
+            batch_size = 8 * cpu_num
+        elif model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
+            or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
+            or model_type.startswith('ShuffleNet'):
+            batch_size = 4 * cpu_num
+        elif model_type.startswith('YOLOv3') or model_type.startswith(
+                'PPYOLO'):
+            batch_size = 2 * cpu_num
+        elif model_type.startswith('FasterRCNN') or model_type.startswith(
+                'MaskRCNN'):
+            batch_size = 1 * cpu_num
+        elif model_type.startswith('DeepLab') or model_type.startswith('UNet') \
+            or model_type.startswith('HRNet_W18') or model_type.startswith('FastSCNN'):
+            batch_size = 2 * cpu_num
+    else:
+        if model_type.startswith('MobileNet'):
+            batch_size = (memory_size_per_gpu - 513) // 57 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 125)
+        elif model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
+            or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
+            or model_type.startswith('ShuffleNet'):
+            batch_size = (memory_size_per_gpu - 739) // 211 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 16)
+        elif model_type.startswith('YOLOv3'):
+            batch_size = (memory_size_per_gpu - 1555) // 943 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 8)
+        elif model_type.startswith('PPYOLO'):
+            batch_size = (memory_size_per_gpu - 1691) // 1025 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 8)
+        elif model_type.startswith('FasterRCNN'):
+            batch_size = (memory_size_per_gpu - 1755) // 915 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 2)
+        elif model_type.startswith('MaskRCNN'):
+            batch_size = (memory_size_per_gpu - 2702) // 1188 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 2)
+        elif model_type.startswith('DeepLab'):
+            batch_size = (memory_size_per_gpu - 1469) // 1605 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('UNet'):
+            batch_size = (memory_size_per_gpu - 1275) // 1256 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('HRNet_W18'):
+            batch_size = (memory_size_per_gpu - 800) // 682 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('FastSCNN'):
+            batch_size = (memory_size_per_gpu - 636) // 144 * gpu_nums
+            batch_size = min(batch_size, gpu_nums * 4)
+
+    if batch_size > train_nums // 2:
+        batch_size = train_nums // 2
+        gpu_list = '{}'.format(gpu_list.split(',')[0]) if gpu_nums > 0 else ''
+    if batch_size <= 0:
+        batch_size = 1
+
+    # set learning_rate
+    if model_type.startswith('MobileNet'):
+        lr = (batch_size / 500.0) * 0.1
+    elif model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
+        or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
+        or model_type.startswith('ShuffleNet'):
+        lr = (batch_size / 256.0) * 0.1
+    elif model_type.startswith('YOLOv3') or model_type.startswith('PPYOLO'):
+        lr = 0.001 * batch_size / 64
+        num_steps_each_epoch = train_nums // batch_size
+        min_warmup_step = max(3 * num_steps_each_epoch, 50 * class_nums)
+        if gpu_nums == 0:
+            gpu_nums = 1
+        warmup_step = min(min_warmup_step, int(400 * class_nums / gpu_nums))
+    elif model_type.startswith('FasterRCNN') or model_type.startswith(
+            'MaskRCNN'):
+        lr = 0.02 * batch_size / 16
+        num_steps_each_epoch = train_nums // batch_size
+        min_warmup_step = max(num_steps_each_epoch, 50)
+        if gpu_nums == 0:
+            gpu_nums = 1
+        warmup_step = min(min_warmup_step, int(4000 / gpu_nums))
+    elif model_type.startswith('DeepLab') or model_type.startswith('UNet') \
+        or model_type.startswith('HRNet_W18') or model_type.startswith('FastSCNN'):
+        lr = 0.01 * batch_size / 2
+        loss_type = [False, False]
+
+    params['batch_size'] = batch_size
+    params['learning_rate'] = lr
+    params['cuda_visible_devices'] = gpu_list
+    if model_type in ['YOLOv3', 'PPYOLO', 'FasterRCNN', 'MaskRCNN']:
+        num_epochs = params['num_epochs']
+        lr_decay_epochs = params['lr_decay_epochs']
+        if warmup_step >= lr_decay_epochs[0] * num_steps_each_epoch:
+            for i in range(len(lr_decay_epochs)):
+                lr_decay_epochs[i] += warmup_step // num_steps_each_epoch
+            num_epochs += warmup_step // num_steps_each_epoch
+        params['num_epochs'] = num_epochs
+        params['lr_decay_epochs'] = lr_decay_epochs
+        params['warmup_steps'] = warmup_step
+    if 'loss_type' in params:
+        params['loss_type'] = loss_type

+ 264 - 0
paddlex/restful/project/train/params_v2.py

@@ -0,0 +1,264 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+def get_base_params(model_type, per_gpu_memory, num_train_samples, num_gpu,
+                    gpu_list, num_classes):
+    params = dict()
+    params["model"] = model_type
+    params["cpu_num"] = 1
+    if num_gpu == 0:
+        batch_size = 4
+        params['cuda_visible_devices'] = ''
+    else:
+        params['cuda_visible_devices'] = str(gpu_list).strip("[]")
+        if model_type.startswith('MobileNet'):
+            batch_size = (per_gpu_memory - 513) // 57 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 125)
+        elif model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
+            or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
+            or model_type.startswith('ShuffleNet'):
+            batch_size = (per_gpu_memory - 739) // 211 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 16)
+        elif model_type.startswith('YOLOv3'):
+            batch_size = (per_gpu_memory - 1555) // 943 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 8)
+        elif model_type.startswith('PPYOLO'):
+            batch_size = (per_gpu_memory - 1691) // 1025 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 8)
+        elif model_type.startswith('FasterRCNN'):
+            batch_size = (per_gpu_memory - 1755) // 915 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 2)
+        elif model_type.startswith('MaskRCNN'):
+            batch_size = (per_gpu_memory - 2702) // 1188 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 2)
+        elif model_type.startswith('DeepLab'):
+            batch_size = (per_gpu_memory - 1469) // 1605 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('UNet'):
+            batch_size = (per_gpu_memory - 1275) // 1256 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('HRNet_W18'):
+            batch_size = (per_gpu_memory - 800) // 682 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 4)
+        elif model_type.startswith('FastSCNN'):
+            batch_size = (per_gpu_memory - 636) // 144 * num_gpu
+            batch_size = min(batch_size, gpu_nums * 4)
+    if batch_size > num_train_samples // 2:
+        batch_size = num_train_samples // 2
+    if batch_size < 1:
+        batch_size = 1
+
+    brightness_range = 0.5
+    contrast_range = 0.5
+    saturation_range = 0.5
+    saturation = False
+    hue = False
+    if model_type.startswith('DenseNet') or model_type.startswith('ResNet') \
+        or model_type.startswith('Xception') or model_type.startswith('DarkNet') \
+        or model_type.startswith('ShuffleNet') or model_type.startswith('MobileNet'):
+        if model_type.startswith('MobileNet'):
+            lr = (batch_size / 500.0) * 0.1
+        else:
+            lr = (batch_size / 256.0) * 0.1
+        shape = [224, 224]
+        save_interval_epochs = 5
+        num_epochs = 120
+        lr_decay_epochs = [30, 60, 90]
+        image_mean = [0.485, 0.456, 0.406]
+        image_std = [0.229, 0.224, 0.225]
+        brightness_range = 0.9
+        contrast_range = 0.9
+        saturation_range = 0.9
+        brightness = True
+        contrast = True
+    elif model_type.startswith('YOLOv3') or model_type.startswith('PPYOLO'):
+        shape = [608, 608]
+        save_interval_epochs = 30
+        num_epochs = 270
+        lr_decay_epochs = [210, 240]
+        lr = 0.001 * batch_size / 64
+        image_mean = [0.485, 0.456, 0.406]
+        image_std = [0.229, 0.224, 0.225]
+        brightness = True
+        contrast = True
+        saturation = True
+        hue = True
+        num_steps_each_epoch = num_train_samples // batch_size
+        min_warmup_step = max(3 * num_steps_each_epoch, 50 * num_classes)
+        if num_gpu == 0:
+            num_gpu = 1
+        warmup_step = min(min_warmup_step, int(400 * num_classes / num_gpu))
+    elif model_type.startswith('FasterRCNN') or model_type.startswith(
+            'MaskRCNN'):
+        shape = [800, 1333]
+        save_interval_epochs = 1
+        num_epochs = 12
+        lr_decay_epochs = [8, 11]
+        image_mean = [0.485, 0.456, 0.406]
+        image_std = [0.229, 0.224, 0.225]
+        brightness = False
+        contrast = False
+        lr = 0.02 * batch_size / 16
+        num_steps_each_epoch = num_train_samples // batch_size
+        min_warmup_step = max(num_steps_each_epoch, 50)
+        if num_gpu == 0:
+            num_gpu = 1
+        warmup_step = min(min_warmup_step, int(4000 / num_gpu))
+    elif model_type.startswith('DeepLab') or model_type.startswith('UNet') \
+        or model_type.startswith('HRNet_W18') or model_type.startswith('FastSCNN'):
+        shape = [512, 512]
+        save_interval_epochs = 10
+        num_epochs = 100
+        lr_decay_epochs = [10, 20]
+        image_mean = [0.5, 0.5, 0.5]
+        image_std = [0.5, 0.5, 0.5]
+        brightness = False
+        contrast = False
+        lr = 0.01 * batch_size / 2
+
+    params['batch_size'] = batch_size
+    params['learning_rate'] = lr
+    params["image_shape"] = shape
+    params['save_interval_epochs'] = save_interval_epochs
+    params['num_epochs'] = num_epochs
+    params['lr_decay_epochs'] = lr_decay_epochs
+    params['resume_checkpoint'] = None
+    params["sensitivities_path"] = None
+    params["image_mean"] = image_mean
+    params["image_std"] = image_std
+    params["horizontal_flip_prob"] = 0.5
+    params['brightness'] = brightness
+    params["brightness_range"] = brightness_range
+    params["brightness_prob"] = 0.5
+    params['contrast'] = contrast
+    params['contrast_range'] = contrast_range
+    params['contrast_prob'] = 0.5
+    params['saturation'] = saturation
+    params['saturation_range'] = saturation_range
+    params['saturation_prob'] = 0.5
+    params['hue'] = hue
+    params['hue_range'] = 18
+    params['hue_prob'] = 0.5
+    params['horizontal_flip'] = True
+
+    if model_type in ['YOLOv3', 'PPYOLO', 'FasterRCNN', 'MaskRCNN']:
+        num_epochs = params['num_epochs']
+        lr_decay_epochs = params['lr_decay_epochs']
+        if warmup_step >= lr_decay_epochs[0] * num_steps_each_epoch:
+            for i in range(len(lr_decay_epochs)):
+                lr_decay_epochs[i] += warmup_step // num_steps_each_epoch
+            num_epochs += warmup_step // num_steps_each_epoch
+        params['num_epochs'] = num_epochs
+        params['lr_decay_epochs'] = lr_decay_epochs
+        params['warmup_steps'] = warmup_step
+
+    return params
+
+
+def get_classification_params(params):
+    params["pretrain_weights"] = 'IMAGENET'
+    params["lr_policy"] = "Piecewise"
+    params['vertical_flip_prob'] = 0.5
+    params['vertical_flip'] = True
+    params['rotate'] = True
+    params['rotate_prob'] = 0.5
+    params['rotate_range'] = 30
+    return params
+
+
+def get_detection_params(params):
+    params['with_fpn'] = True
+    params["pretrain_weights"] = 'IMAGENET'
+    if params['model'].startswith('YOLOv3') or params['model'].startswith(
+            'PPYOLO'):
+        if params['model'].startswith('YOLOv3'):
+            params['backbone'] = 'DarkNet53'
+        elif params['model'].startswith('PPYOLO'):
+            params['backbone'] = 'ResNet50_vd_ssld'
+        params['use_mixup'] = True
+        params['mixup_alpha'] = 1.5
+        params['mixup_beta'] = 1.5
+        params['expand_prob'] = 0.5
+        params['expand_image'] = True
+        params['crop_image'] = True
+        params['random_shape'] = True
+        params['random_shape_sizes'] = [
+            320, 352, 384, 416, 448, 480, 512, 544, 576, 608
+        ]
+    elif params['model'].startswith('FasterRCNN') or params[
+            'model'].startswith('MaskRCNN'):
+        params['backbone'] = 'ResNet50'
+
+    return params
+
+
+def get_segmentation_params(params):
+    if params['model'].startswith('DeepLab'):
+        params['backbone'] = 'Xception65'
+        params["pretrain_weights"] = 'IMAGENET'
+    elif params['model'].startswith('UNet'):
+        params["pretrain_weights"] = 'COCO'
+    elif params['model'].startswith('HRNet_W18') or params['model'].startswith(
+            'FastSCNN'):
+        params["pretrain_weights"] = 'CITYSCAPES'
+    params['loss_type'] = [False, False]
+    params['lr_policy'] = 'Polynomial'
+    params['optimizer'] = 'SGD'
+    params['blur'] = False
+    params['blur_prob'] = 0.1
+    params['rotate'] = False
+    params['max_rotation'] = 15
+    params['scale_aspect'] = False
+    params['min_ratio'] = 0.5
+    params['aspect_ratio'] = 0.33
+    params['vertical_flip'] = False
+    params['vertical_flip_prob'] = 0.5
+    return params
+
+
+def get_params(data, project_type, num_train_samples, num_classes, num_gpu,
+               per_gpu_memory, gpu_list):
+    if project_type == "classification":
+        if 'model_type' in data:
+            model_type = data['model_type']
+        else:
+            model_type = "MobileNetV2"
+        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
+                                 num_gpu, gpu_list, num_classes)
+        return get_classification_params(params)
+    if project_type == "detection":
+        if 'model_type' in data:
+            model_type = data['model_type']
+        else:
+            model_type = "YOLOv3"
+        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
+                                 num_gpu, gpu_list, num_classes)
+        return get_detection_params(params)
+    if project_type == "instance_segmentation":
+        if 'model_type' in data:
+            model_type = data['model_type']
+        else:
+            model_type = "MaskRCNN"
+        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
+                                 num_gpu, gpu_list, num_classes)
+        return get_detection_params(params)
+    if project_type == 'segmentation' or project_type == "remote_segmentation":
+        if 'model_type' in data:
+            model_type = data['model_type']
+        else:
+            model_type = "DeepLabv3+"
+        params = get_base_params(model_type, per_gpu_memory, num_train_samples,
+                                 num_gpu, gpu_list, num_classes)
+        return get_segmentation_params(params)

+ 171 - 0
paddlex/restful/project/train/segmentation.py

@@ -0,0 +1,171 @@
+# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os.path as osp
+
+
+def build_transforms(params):
+    from paddlex.seg import transforms
+    seg_list = []
+    min_value = max(params.image_shape) * 4 // 5
+    max_value = max(params.image_shape) * 6 // 5
+    seg_list.extend([
+        transforms.ResizeRangeScaling(
+            min_value=min_value, max_value=max_value),
+        transforms.RandomBlur(prob=params.blur_prob)
+    ])
+    if params.rotate:
+        seg_list.append(
+            transforms.RandomRotate(rotate_range=params.max_rotation))
+    if params.scale_aspect:
+        seg_list.append(
+            transforms.RandomScaleAspect(
+                min_scale=params.min_ratio, aspect_ratio=params.aspect_ratio))
+    seg_list.extend([
+        transforms.RandomDistort(
+            brightness_range=params.brightness_range,
+            brightness_prob=params.brightness_prob,
+            contrast_range=params.contrast_range,
+            contrast_prob=params.contrast_prob,
+            saturation_range=params.saturation_range,
+            saturation_prob=params.saturation_prob,
+            hue_range=params.hue_range,
+            hue_prob=params.hue_prob),
+        transforms.RandomVerticalFlip(prob=params.vertical_flip_prob),
+        transforms.RandomHorizontalFlip(prob=params.horizontal_flip_prob),
+        transforms.RandomPaddingCrop(crop_size=max(params.image_shape)),
+        transforms.Normalize(
+            mean=params.image_mean, std=params.image_std)
+    ])
+
+    train_transforms = transforms.Compose(seg_list)
+    eval_transforms = transforms.Compose([
+        transforms.ResizeByLong(long_size=max(params.image_shape)),
+        transforms.Padding(target_size=max(params.image_shape)),
+        transforms.Normalize(
+            mean=params.image_mean, std=params.image_std)
+    ])
+    return train_transforms, eval_transforms
+
+
+def build_datasets(dataset_path, train_transforms, eval_transforms):
+    import paddlex as pdx
+    train_file_list = osp.join(dataset_path, 'train_list.txt')
+    eval_file_list = osp.join(dataset_path, 'val_list.txt')
+    label_list = osp.join(dataset_path, 'labels.txt')
+    train_dataset = pdx.datasets.SegDataset(
+        data_dir=dataset_path,
+        file_list=train_file_list,
+        label_list=label_list,
+        transforms=train_transforms,
+        shuffle=True)
+    eval_dataset = pdx.datasets.SegDataset(
+        data_dir=dataset_path,
+        file_list=eval_file_list,
+        label_list=label_list,
+        transforms=eval_transforms)
+    return train_dataset, eval_dataset
+
+
+def build_optimizer(step_each_epoch, params):
+    import paddle.fluid as fluid
+    if params.lr_policy == 'Piecewise':
+        gamma = 0.1
+        bd = [step_each_epoch * e for e in params.lr_decay_epochs]
+        lr = [params.learning_rate * (gamma**i) for i in range(len(bd) + 1)]
+        decayed_lr = fluid.layers.piecewise_decay(boundaries=bd, values=lr)
+    elif params.lr_policy == 'Polynomial':
+        decay_step = params.num_epochs * step_each_epoch
+        decayed_lr = fluid.layers.polynomial_decay(
+            params.learning_rate, decay_step, end_learning_rate=0, power=0.9)
+    elif params.lr_policy == 'Cosine':
+        decayed_lr = fluid.layers.cosine_decay(
+            params.learning_rate, step_each_epoch, params.num_epochs)
+    else:
+        raise Exception(
+            'lr_policy only support Polynomial or Piecewise, but you set {}'.
+            format(params.lr_policy))
+
+    if params.optimizer.lower() == 'sgd':
+        momentum = 0.9
+        regularize_coef = 1e-4
+        optimizer = fluid.optimizer.Momentum(
+            learning_rate=decayed_lr,
+            momentum=momentum,
+            regularization=fluid.regularizer.L2Decay(
+                regularization_coeff=regularize_coef), )
+    elif params.optimizer.lower() == 'adam':
+        momentum = 0.9
+        momentum2 = 0.999
+        regularize_coef = 1e-4
+        optimizer = fluid.optimizer.Adam(
+            learning_rate=decayed_lr,
+            beta1=momentum,
+            beta2=momentum2,
+            regularization=fluid.regularizer.L2Decay(
+                regularization_coeff=regularize_coef), )
+
+    return optimizer
+
+
+def train(task_path, dataset_path, params):
+    import paddlex as pdx
+    pdx.log_level = 3
+    train_transforms, eval_transforms = build_transforms(params)
+    train_dataset, eval_dataset = build_datasets(
+        dataset_path=dataset_path,
+        train_transforms=train_transforms,
+        eval_transforms=eval_transforms)
+
+    step_each_epoch = train_dataset.num_samples // params.batch_size
+    save_interval_epochs = params.save_interval_epochs
+    save_dir = osp.join(task_path, 'output')
+    pretrain_weights = params.pretrain_weights
+
+    optimizer = build_optimizer(step_each_epoch, params)
+    segmenter = getattr(pdx.cv.models, 'HRNet'
+                        if params.model.startswith('HRNet') else params.model)
+    use_dice_loss, use_bce_loss = params.loss_type
+    backbone = params.backbone
+    sensitivities_path = params.sensitivities_path
+    eval_metric_loss = params.eval_metric_loss
+    if eval_metric_loss is None:
+        eval_metric_loss = 0.05
+    if params.model in ['UNet', 'HRNet_W18', 'FastSCNN']:
+        model = segmenter(
+            num_classes=len(train_dataset.labels),
+            use_bce_loss=use_bce_loss,
+            use_dice_loss=use_dice_loss)
+    elif params.model == 'DeepLabv3p':
+        model = segmenter(
+            num_classes=len(train_dataset.labels),
+            backbone=backbone,
+            use_bce_loss=use_bce_loss,
+            use_dice_loss=use_dice_loss)
+        if backbone == 'MobileNetV3_large_x1_0_ssld':
+            model.pooling_crop_size = params.image_shape
+    model.train(
+        num_epochs=params.num_epochs,
+        train_dataset=train_dataset,
+        train_batch_size=params.batch_size,
+        eval_dataset=eval_dataset,
+        save_interval_epochs=save_interval_epochs,
+        log_interval_steps=2,
+        save_dir=save_dir,
+        pretrain_weights=pretrain_weights,
+        optimizer=optimizer,
+        use_vdl=True,
+        sensitivities_file=sensitivities_path,
+        eval_metric_loss=eval_metric_loss,
+        resume_checkpoint=params.resume_checkpoint)

+ 244 - 0
paddlex/restful/project/visualize.py

@@ -0,0 +1,244 @@
+# copytrue (c) 2020 PaddlePaddle Authors. All Rights Reserve.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#! /usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import numpy as np
+import cv2
+import math
+import xml.etree.ElementTree as ET
+from PIL import Image
+
+
+def resize_img(img):
+    """ 调整图片尺寸
+
+    Args:
+        img: 图片信息
+    """
+    h, w = img.shape[:2]
+    min_size = 580
+
+    if w >= h and w > min_size:
+        new_w = min_size
+        new_h = new_w * h / w
+    elif h >= w and h > min_size:
+        new_h = min_size
+        new_w = new_h * w / h
+    else:
+        new_h = h
+        new_w = w
+    new_img = cv2.resize(
+        img, (int(new_w), int(new_h)), interpolation=cv2.INTER_CUBIC)
+
+    scale_value = new_w / w
+    return new_img, scale_value
+
+
+def plot_det_label(image, anno, labels):
+    """ 目标检测类型生成标注图
+
+    Args:
+        image: 图片路径
+        anno: 图片标注
+        labels: 图片所属数据集的类别信息
+    """
+    catid2color = {}
+    img = cv2.imread(image)
+    img, scale_value = resize_img(img)
+    tree = ET.parse(anno)
+    objs = tree.findall('object')
+    color_map = get_color_map_list(len(labels) + 1)
+    for i, obj in enumerate(objs):
+        cname = obj.find('name').text
+        catid = labels.index(cname)
+        if cname not in labels:
+            continue
+        xmin = int(float(obj.find('bndbox').find('xmin').text) * scale_value)
+        ymin = int(float(obj.find('bndbox').find('ymin').text) * scale_value)
+        xmax = int(float(obj.find('bndbox').find('xmax').text) * scale_value)
+        ymax = int(float(obj.find('bndbox').find('ymax').text) * scale_value)
+
+        if catid not in catid2color:
+            catid2color[catid] = color_map[catid + 1]
+        color = tuple(catid2color[catid])
+        img = draw_rectangle_and_cname(img, xmin, ymin, xmax, ymax, cname,
+                                       color)
+    return img
+
+
+def plot_seg_label(anno):
+    """ 语义分割类型生成标注图
+
+    Args:
+        anno: 图片标注
+    """
+    label = pil_imread(anno)
+    pse_label = gray2pseudo(label)
+    return pse_label
+
+
+def plot_insseg_label(image, anno, labels, alpha=0.7):
+    """ 实例分割类型生成标注图
+
+    Args:
+        image: 图片路径
+        anno: 图片标注
+        labels: 图片所属数据集的类别信息
+    """
+    anno = np.load(anno, allow_pickle=True).tolist()
+    catid2color = dict()
+    img = cv2.imread(image)
+    img, scale_value = resize_img(img)
+    color_map = get_color_map_list(len(labels) + 1)
+    img_h = anno['h']
+    img_w = anno['w']
+    gt_class = anno['gt_class']
+    gt_bbox = anno['gt_bbox']
+    gt_poly = anno['gt_poly']
+    num_bbox = gt_bbox.shape[0]
+    num_mask = len(gt_poly)
+    # 描绘mask信息
+    img_array = np.array(img).astype('float32')
+    for i in range(num_mask):
+        cname = gt_class[i]
+        catid = labels.index(cname)
+        if cname not in labels:
+            continue
+        if catid not in catid2color:
+            catid2color[catid] = color_map[catid + 1]
+        color = np.array(catid2color[catid]).astype('float32')
+
+        import pycocotools.mask as mask_util
+        for x in range(len(gt_poly[i])):
+            for y in range(len(gt_poly[i][x])):
+                gt_poly[i][x][y] = int(float(gt_poly[i][x][y]) * scale_value)
+        poly = gt_poly[i]
+        rles = mask_util.frPyObjects(poly,
+                                     int(float(img_h) * scale_value),
+                                     int(float(img_w) * scale_value))
+        rle = mask_util.merge(rles)
+        mask = mask_util.decode(rle) * 255
+        idx = np.nonzero(mask)
+        img_array[idx[0], idx[1], :] *= 1.0 - alpha
+        img_array[idx[0], idx[1], :] += alpha * color
+    img = img_array.astype('uint8')
+
+    for i in range(num_bbox):
+        cname = gt_class[i]
+        catid = labels.index(cname)
+        if cname not in labels:
+            continue
+        if catid not in catid2color:
+            catid2color[catid] = color_map[catid]
+        color = tuple(catid2color[catid])
+        xmin, ymin, xmax, ymax = gt_bbox[i]
+
+        img = draw_rectangle_and_cname(img,
+                                       int(float(xmin) * scale_value),
+                                       int(float(ymin) * scale_value),
+                                       int(float(xmax) * scale_value),
+                                       int(float(ymax) * scale_value), cname,
+                                       color)
+
+    return img
+
+
+def draw_rectangle_and_cname(img, xmin, ymin, xmax, ymax, cname, color):
+    """ 根据提供的标注信息,给图片描绘框体和类别显示
+
+    Args:
+        img: 图片路径
+        xmin: 检测框最小的x坐标
+        ymin: 检测框最小的y坐标
+        xmax: 检测框最大的x坐标
+        ymax: 检测框最大的y坐标
+        cname: 类别信息
+        color: 类别与颜色的对应信息
+    """
+    # 描绘检测框
+    line_width = math.ceil(2 * max(img.shape[0:2]) / 600)
+    cv2.rectangle(
+        img,
+        pt1=(xmin, ymin),
+        pt2=(xmax, ymax),
+        color=color,
+        thickness=line_width)
+
+    # 计算并描绘类别信息
+    text_thickness = math.ceil(2 * max(img.shape[0:2]) / 1200)
+    fontscale = math.ceil(0.5 * max(img.shape[0:2]) / 600)
+    tw, th = cv2.getTextSize(
+        cname, 0, fontScale=fontscale, thickness=text_thickness)[0]
+    cv2.rectangle(
+        img,
+        pt1=(xmin + 1, ymin - th),
+        pt2=(xmin + int(0.7 * tw) + 1, ymin),
+        color=color,
+        thickness=-1)
+    cv2.putText(
+        img,
+        cname, (int(xmin) + 3, int(ymin) - 5),
+        0,
+        0.6 * fontscale, (255, 255, 255),
+        lineType=cv2.LINE_AA,
+        thickness=text_thickness)
+    return img
+
+
+def pil_imread(file_path):
+    """ 将图片读成np格式数据
+
+    Args:
+        file_path: 图片路径
+    """
+    img = Image.open(file_path)
+    return np.asarray(img)
+
+
+def get_color_map_list(num_classes):
+    """ 为类别信息生成对应的颜色列表
+
+    Args:
+        num_classes: 类别数量
+    """
+    color_map = num_classes * [0, 0, 0]
+    for i in range(0, num_classes):
+        j = 0
+        lab = i
+        while lab:
+            color_map[i * 3] |= (((lab >> 0) & 1) << (7 - j))
+            color_map[i * 3 + 1] |= (((lab >> 1) & 1) << (7 - j))
+            color_map[i * 3 + 2] |= (((lab >> 2) & 1) << (7 - j))
+            j += 1
+            lab >>= 3
+    color_map = [color_map[i:i + 3] for i in range(0, len(color_map), 3)]
+    return color_map
+
+
+def gray2pseudo(gray_image):
+    """ 将分割的结果映射到图片
+
+    Args:
+        gray_image: 灰度图
+    """
+    color_map = get_color_map_list(256)
+    color_map = np.array(color_map).astype("uint8")
+    # 用OpenCV进行色彩映射
+    c1 = cv2.LUT(gray_image, color_map[:, 0])
+    c2 = cv2.LUT(gray_image, color_map[:, 1])
+    c3 = cv2.LUT(gray_image, color_map[:, 2])
+    pseudo_img = np.dstack((c1, c2, c3))
+    return pseudo_img

+ 88 - 0
paddlex/restful/system.py

@@ -0,0 +1,88 @@
+import sys
+import os
+import psutil
+import platform
+
+
+def pkill(pid):
+    try:
+        parent = psutil.Process(pid)
+        for child in parent.children(recursive=True):
+            child.kill()
+        parent.kill()
+    except:
+        print("Try to kill process {} failed.".format(pid))
+
+
+def get_system_info(machine_info={}):
+    if machine_info:
+        return {'status': 1, 'info': machine_info}
+    from .utils import get_gpu_info
+    gpu_info, message = get_gpu_info()
+    cpu_num = os.environ.get('CPU_NUM', 1)
+    sysstr = platform.system()
+    machine_info['message'] = message
+    machine_info['cpu_num'] = cpu_num
+    machine_info['gpu_num'] = gpu_info['gpu_num']
+    machine_info['sysstr'] = sysstr
+    if gpu_info['gpu_num'] > 0:
+        machine_info['driver_version'] = gpu_info['driver_version']
+        machine_info['gpu_free_mem'] = gpu_info['mem_free']
+    return {'status': 1, 'info': machine_info}
+
+
+def get_gpu_memory_info(machine_info):
+    gpu_mem_infos = list()
+    if machine_info['gpu_num'] == 0:
+        pass
+    else:
+        from .utils import get_gpu_info
+        gpu_info, message = get_gpu_info()
+        for i in range(gpu_info['gpu_num']):
+            attr = {
+                'free': gpu_info['mem_free'][i],
+                'used': gpu_info['mem_used'][i],
+                'total': gpu_info['mem_total'][i]
+            }
+            gpu_mem_infos.append(attr)
+    return {'status': 1, 'gpu_mem_infos': gpu_mem_infos}
+
+
+def get_machine_info(data, machine_info):
+    path = None
+    if "path" in data:
+        path = data['path']
+    if path in machine_info:
+        return {'status': 1, 'info': machine_info}
+    from .utils import get_machine_info
+    info = get_machine_info(path)
+    machine_info = info
+    return {'status': 1, 'info': machine_info}
+
+
+def get_gpu_memory_size(data):
+    """获取显存大小
+
+    Args:
+        request(comm.Request): 其中request.params为dict, key包括
+        'path' 显卡驱动动态链接库路径
+    """
+    from .utils import PyNvml
+    p = PyNvml()
+    p.nvml_init(data['path'])
+    count = p.nvml_device_get_count()
+    gpu_mem_infos = []
+    for i in range(count):
+        handler = p.nvml_device_get_handle_by_index(i)
+        mem = p.nvml_device_get_memory_info(handler)
+        attr = {'free': mem.free, 'used': mem.used, 'total': mem.total}
+        gpu_mem_infos.append(attr)
+    return {'status': 1, 'gpu_mem_infos': gpu_mem_infos}
+
+
+def exit_system(monitored_processes):
+    while not monitored_processes.empty():
+        pid = monitored_processes.get(timeout=0.5)
+        print("Try to kill process {}".format(pid))
+        pkill(pid)
+    return {'status': 1}

+ 753 - 0
paddlex/restful/utils.py

@@ -0,0 +1,753 @@
+import psutil
+import shutil
+import os
+import os.path as osp
+from enum import Enum
+import multiprocessing as mp
+from queue import Queue
+import time
+import threading
+from ctypes import CDLL, c_char, c_uint, c_ulonglong
+from _ctypes import byref, Structure, POINTER
+import platform
+import string
+import logging
+import socket
+import logging.handlers
+import requests
+import json
+from json import JSONEncoder
+
+
+class CustomEncoder(JSONEncoder):
+    def default(self, o):
+        return o.__dict__
+
+
+class ShareData():
+    workspace = None
+    workspace_dir = ""
+    has_gpu = True
+    monitored_processes = mp.Queue(4096)
+    current_port = 8000
+    running_boards = {}
+    machine_info = dict()
+    load_demo_proc_dict = {}
+    load_demo_proj_data_dict = {}
+
+
+DatasetStatus = Enum(
+    'DatasetStatus', ('XEMPTY', 'XCHECKING', 'XCHECKFAIL', 'XCOPYING',
+                      'XCOPYDONE', 'XCOPYFAIL', 'XSPLITED'),
+    start=0)
+
+TaskStatus = Enum(
+    'TaskStatus', ('XUNINIT', 'XINIT', 'XDOWNLOADING', 'XTRAINING',
+                   'XTRAINDONE', 'XEVALUATED', 'XEXPORTING', 'XEXPORTED',
+                   'XTRAINEXIT', 'XDOWNLOADFAIL', 'XTRAINFAIL', 'XEVALUATING',
+                   'XEVALUATEFAIL', 'XEXPORTFAIL', 'XPRUNEING', 'XPRUNETRAIN'),
+    start=0)
+
+ProjectType = Enum(
+    'ProjectType', ('classification', 'detection', 'segmentation',
+                    'instance_segmentation', 'remote_segmentation'),
+    start=0)
+
+DownloadStatus = Enum(
+    'DownloadStatus',
+    ('XDDOWNLOADING', 'XDDOWNLOADFAIL', 'XDDOWNLOADDONE', 'XDDECOMPRESSED'),
+    start=0)
+
+PredictStatus = Enum(
+    'PredictStatus', ('XPRESTART', 'XPREDONE', 'XPREFAIL'), start=0)
+
+PruneStatus = Enum(
+    'PruneStatus', ('XSPRUNESTART', 'XSPRUNEING', 'XSPRUNEDONE', 'XSPRUNEFAIL',
+                    'XSPRUNEEXIT'),
+    start=0)
+
+PretrainedModelStatus = Enum(
+    'PretrainedModelStatus',
+    ('XPINIT', 'XPSAVING', 'XPSAVEFAIL', 'XPSAVEDONE'),
+    start=0)
+
+ExportedModelType = Enum(
+    'ExportedModelType', ('XQUANTMOBILE', 'XPRUNEMOBILE', 'XTRAINMOBILE',
+                          'XQUANTSERVER', 'XPRUNESERVER', 'XTRAINSERVER'),
+    start=0)
+
+process_pool = Queue(1000)
+
+
+def get_ip():
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+        s.connect(('8.8.8.8', 80))
+        ip = s.getsockname()[0]
+    finally:
+        s.close()
+    return ip
+
+
+def get_logger(filename):
+    flask_logger = logging.getLogger()
+    flask_logger.setLevel(level=logging.INFO)
+    fmt = '%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s:%(message)s'
+    format_str = logging.Formatter(fmt)
+    ch = logging.StreamHandler()
+    ch.setLevel(level=logging.INFO)
+    ch.setFormatter(format_str)
+    th = logging.handlers.TimedRotatingFileHandler(
+        filename=filename, when='D', backupCount=5, encoding='utf-8')
+    th.setFormatter(format_str)
+    flask_logger.addHandler(th)
+    flask_logger.addHandler(ch)
+    return flask_logger
+
+
+def start_process(target, args):
+    global process_pool
+    p = mp.Process(target=target, args=args)
+    p.start()
+    process_pool.put(p)
+
+
+def pkill(pid):
+    """结束进程pid,和与其相关的子进程
+
+    Args:
+        pid(int): 进程id
+    """
+    try:
+        parent = psutil.Process(pid)
+        for child in parent.children(recursive=True):
+            child.kill()
+        parent.kill()
+    except:
+        print("Try to kill process {} failed.".format(pid))
+
+
+def set_folder_status(dirname, status, message=""):
+    """设置目录状态
+
+    Args:
+        dirname(str): 目录路径
+        status(DatasetStatus): 状态
+        message(str): 需要写到状态文件里的信息
+    """
+    if not osp.isdir(dirname):
+        raise Exception("目录路径{}不存在".format(dirname))
+    tmp_file = osp.join(dirname, status.name + '.tmp')
+    with open(tmp_file, 'w', encoding='utf-8') as f:
+        f.write("{}\n".format(message))
+    shutil.move(tmp_file, osp.join(dirname, status.name))
+    for status_type in [
+            DatasetStatus, TaskStatus, PredictStatus, PruneStatus,
+            DownloadStatus, PretrainedModelStatus
+    ]:
+        for s in status_type:
+            if s == status:
+                continue
+            if osp.exists(osp.join(dirname, s.name)):
+                os.remove(osp.join(dirname, s.name))
+
+
+def get_folder_status(dirname, with_message=False):
+    """获取目录状态
+
+    Args:
+        dirname(str): 目录路径
+        with_message(bool): 是否需要返回状态文件内的信息
+    """
+    status = None
+    closest_time = 0
+    message = ''
+    for status_type in [
+            DatasetStatus, TaskStatus, PredictStatus, PruneStatus,
+            DownloadStatus, PretrainedModelStatus
+    ]:
+        for s in status_type:
+            if osp.exists(osp.join(dirname, s.name)):
+                modify_time = os.stat(osp.join(dirname, s.name)).st_mtime
+                if modify_time > closest_time:
+                    closest_time = modify_time
+                    status = getattr(status_type, s.name)
+                    if with_message:
+                        encoding = 'utf-8'
+                        try:
+                            f = open(
+                                osp.join(dirname, s.name),
+                                'r',
+                                encoding=encoding)
+                            message = f.read()
+                            f.close()
+                        except:
+                            try:
+                                import chardet
+                                f = open(filename, 'rb')
+                                data = f.read()
+                                f.close()
+                                encoding = chardet.detect(data).get('encoding')
+                                f = open(
+                                    osp.join(dirname, s.name),
+                                    'r',
+                                    encoding=encoding)
+                                message = f.read()
+                                f.close()
+                            except:
+                                pass
+    if with_message:
+        return status, message
+    return status
+
+
+def _machine_check_proc(queue, path):
+    info = dict()
+    p = PyNvml()
+    gpu_num = 0
+    try:
+        # import paddle.fluid.core as core
+        # gpu_num = core.get_cuda_device_count()
+        p.nvml_init(path)
+        gpu_num = p.nvml_device_get_count()
+        driver_version = bytes.decode(p.nvml_system_get_driver_version())
+    except:
+        driver_version = "N/A"
+    info['gpu_num'] = gpu_num
+    info['gpu_free_mem'] = list()
+    try:
+        for i in range(gpu_num):
+            handle = p.nvml_device_get_handle_by_index(i)
+            meminfo = p.nvml_device_get_memory_info(handle)
+            free_mem = meminfo.free / 1024 / 1024
+            info['gpu_free_mem'].append(free_mem)
+    except:
+        pass
+
+    info['cpu_num'] = os.environ.get('CPU_NUM', 1)
+    info['driver_version'] = driver_version
+    info['path'] = p.nvml_lib_path
+    queue.put(info, timeout=2)
+
+
+def get_machine_info(path=None):
+    queue = mp.Queue(1)
+    p = mp.Process(target=_machine_check_proc, args=(queue, path))
+    p.start()
+    p.join()
+    return queue.get(timeout=2)
+
+
+def download(url, target_path):
+    if not osp.exists(target_path):
+        os.makedirs(target_path)
+    fname = osp.split(url)[-1]
+    fullname = osp.join(target_path, fname)
+    retry_cnt = 0
+    DOWNLOAD_RETRY_LIMIT = 3
+    while not (osp.exists(fullname)):
+        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
+            retry_cnt += 1
+        else:
+            # 设置下载失败
+            msg = "Download from {} failed. Retry limit reached".format(url)
+            set_folder_status(target_path, DownloadStatus.XDDOWNLOADFAIL, msg)
+            raise RuntimeError(msg)
+        req = requests.get(url, stream=True)
+        if req.status_code != 200:
+            msg = "Downloading from {} failed with code {}!".format(
+                url, req.status_code)
+            set_folder_status(target_path, DownloadStatus.XDDOWNLOADFAIL, msg)
+            raise RuntimeError(msg)
+
+        # For protecting download interupted, download to
+        # tmp_fullname firstly, move tmp_fullname to fullname
+        # after download finished
+        tmp_fullname = fullname + "_tmp"
+        total_size = req.headers.get('content-length')
+        set_folder_status(target_path, DownloadStatus.XDDOWNLOADING,
+                          total_size)
+
+        with open(tmp_fullname, 'wb') as f:
+            if total_size:
+                download_size = 0
+                for chunk in req.iter_content(chunk_size=1024):
+                    f.write(chunk)
+                    download_size += 1024
+            else:
+                for chunk in req.iter_content(chunk_size=1024):
+                    if chunk:
+                        f.write(chunk)
+        shutil.move(tmp_fullname, fullname)
+    set_folder_status(target_path, DownloadStatus.XDDOWNLOADDONE)
+    return fullname
+
+
+def is_pic(filename):
+    suffixes = {'JPEG', 'jpeg', 'JPG', 'jpg', 'BMP', 'bmp', 'PNG', 'png'}
+    suffix = filename.strip().split('.')[-1]
+    if suffix not in suffixes:
+        return False
+    return True
+
+
+def is_available(ip, port):
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    try:
+        s.connect((ip, int(port)))
+        s.shutdown(2)
+        return False
+    except:
+        return True
+
+
+def list_files(dirname):
+    """ 列出目录下所有文件(包括所属的一级子目录下文件)
+
+    Args:
+        dirname: 目录路径
+    """
+
+    def filter_file(f):
+        if f.startswith('.'):
+            return True
+        if hasattr(PretrainedModelStatus, f):
+            return True
+        return False
+
+    all_files = list()
+    dirs = list()
+    for f in os.listdir(dirname):
+        if filter_file(f):
+            continue
+        if osp.isdir(osp.join(dirname, f)):
+            dirs.append(f)
+        else:
+            all_files.append(f)
+    for d in dirs:
+        for f in os.listdir(osp.join(dirname, d)):
+            if filter_file(f):
+                continue
+            if osp.isdir(osp.join(dirname, d, f)):
+                continue
+            all_files.append(osp.join(d, f))
+    return all_files
+
+
+def copy_model_directory(src, dst, files=None, filter_files=[]):
+    """从src目录copy文件至dst目录,
+           注意:拷贝前会先清空dst中的所有文件
+
+        Args:
+            src: 源目录路径
+            dst: 目标目录路径
+            files: 需要拷贝的文件列表(src的相对路径)
+        """
+    set_folder_status(dst, PretrainedModelStatus.XPSAVING, os.getpid())
+    if files is None:
+        files = list_files(src)
+    try:
+        message = '{} {}'.format(os.getpid(), len(files))
+        set_folder_status(dst, PretrainedModelStatus.XPSAVING, message)
+        if not osp.samefile(src, dst):
+            for i, f in enumerate(files):
+                items = osp.split(f)
+                if len(items) > 2:
+                    continue
+                if len(items) == 2:
+                    if not osp.isdir(osp.join(dst, items[0])):
+                        if osp.exists(osp.join(dst, items[0])):
+                            os.remove(osp.join(dst, items[0]))
+                        os.makedirs(osp.join(dst, items[0]))
+                if f not in filter_files:
+                    shutil.copy(osp.join(src, f), osp.join(dst, f))
+        set_folder_status(dst, PretrainedModelStatus.XPSAVEDONE)
+    except Exception as e:
+        import traceback
+        error_info = traceback.format_exc()
+        set_folder_status(dst, PretrainedModelStatus.XPSAVEFAIL, error_info)
+
+
+def copy_pretrained_model(src, dst):
+    p = mp.Process(
+        target=copy_model_directory, args=(src, dst, None, ['model.pdopt']))
+    p.start()
+    return p
+
+
+def _get_gpu_info(queue):
+    gpu_info = dict()
+    mem_free = list()
+    mem_used = list()
+    mem_total = list()
+    import pycuda.driver as drv
+    from pycuda.tools import clear_context_caches
+    drv.init()
+    driver_version = drv.get_driver_version()
+    gpu_num = drv.Device.count()
+    for gpu_id in range(gpu_num):
+        dev = drv.Device(gpu_id)
+        try:
+            context = dev.make_context()
+            free, total = drv.mem_get_info()
+            context.pop()
+            free = free // 1024 // 1024
+            total = total // 1024 // 1024
+            used = total - free
+        except:
+            free = 0
+            total = 0
+            used = 0
+        mem_free.append(free)
+        mem_used.append(used)
+        mem_total.append(total)
+    gpu_info['mem_free'] = mem_free
+    gpu_info['mem_used'] = mem_used
+    gpu_info['mem_total'] = mem_total
+    gpu_info['driver_version'] = driver_version
+    gpu_info['gpu_num'] = gpu_num
+    queue.put(gpu_info)
+
+
+def get_gpu_info():
+    try:
+        import pycuda
+    except:
+        gpu_info = dict()
+        message = "未检测到GPU \n 若存在GPU请确保安装pycuda \n 若未安装pycuda请使用'pip install pycuda'来安装"
+        gpu_info['gpu_num'] = 0
+        return gpu_info, message
+    queue = mp.Queue(1)
+    p = mp.Process(target=_get_gpu_info, args=(queue, ))
+    p.start()
+    p.join()
+    gpu_info = queue.get(timeout=2)
+    if gpu_info['gpu_num'] == 0:
+        message = "未检测到GPU"
+    else:
+        message = "检测到GPU"
+
+    return gpu_info, message
+
+
+class TrainLogReader(object):
+    def __init__(self, log_file):
+        self.log_file = log_file
+        self.eta = None
+        self.train_metrics = None
+        self.eval_metrics = None
+        self.download_status = None
+        self.eval_done = False
+        self.train_error = None
+        self.train_stage = None
+        self.running_duration = None
+
+    def update(self):
+        if not osp.exists(self.log_file):
+            return
+        if self.train_stage == "Train Error":
+            return
+        if self.download_status == "Failed":
+            return
+        if self.train_stage == "Train Complete":
+            return
+        logs = open(self.log_file, encoding='utf-8').read().strip().split('\n')
+        self.eta = None
+        self.train_metrics = None
+        self.eval_metrics = None
+        if self.download_status != "Done":
+            self.download_status = None
+
+        start_time_timestamp = osp.getctime(self.log_file)
+        for line in logs[::1]:
+            try:
+                start_time_str = " ".join(line.split()[0:2])
+                start_time_array = time.strptime(start_time_str,
+                                                 "%Y-%m-%d %H:%M:%S")
+                start_time_timestamp = time.mktime(start_time_array)
+                break
+            except Exception as e:
+                pass
+        for line in logs[::-1]:
+            if line.count('Train Complete!'):
+                self.train_stage = "Train Complete"
+            if line.count('Training stop with error!'):
+                self.train_error = line
+            if self.train_metrics is not None \
+                    and self.eval_metrics is not None and self.eval_done and self.eta is not None:
+                break
+            items = line.strip().split()
+            if line.count('Model saved in'):
+                self.eval_done = True
+            if line.count('download completed'):
+                self.download_status = 'Done'
+                break
+            if line.count('download failed'):
+                self.download_status = 'Failed'
+                break
+            if self.download_status != 'Done':
+                if line.count('[DEBUG]\tDownloading'
+                              ) and self.download_status is None:
+                    self.download_status = dict()
+                    if not line.endswith('KB/s'):
+                        continue
+                    speed = items[-1].strip('KB/s').split('=')[-1]
+                    download = items[-2].strip('M, ').split('=')[-1]
+                    total = items[-3].strip('M, ').split('=')[-1]
+                    self.download_status['speed'] = speed
+                    self.download_status['download'] = float(download)
+                    self.download_status['total'] = float(total)
+            if self.eta is None:
+                if line.count('eta') > 0 and (line[-3] == ':' or
+                                              line[-4] == ':'):
+                    eta = items[-1].strip().split('=')[1]
+                    h, m, s = [int(x) for x in eta.split(':')]
+                    self.eta = h * 3600 + m * 60 + s
+            if self.train_metrics is None:
+                if line.count('[INFO]\t[TRAIN]') > 0 and line.count(
+                        'Step') > 0:
+                    if not items[-1].startswith('eta'):
+                        continue
+                    self.train_metrics = dict()
+                    metrics = items[4:]
+                    for metric in metrics:
+                        try:
+                            name, value = metric.strip(', ').split('=')
+                            value = value.split('/')[0]
+                            if value.count('.') > 0:
+                                value = float(value)
+                            elif value == 'nan':
+                                value = 'nan'
+                            else:
+                                value = int(value)
+                            self.train_metrics[name] = value
+                        except:
+                            pass
+            if self.eval_metrics is None:
+                if line.count('[INFO]\t[EVAL]') > 0 and line.count(
+                        'Finished') > 0:
+                    if not line.strip().endswith(' .'):
+                        continue
+                    self.eval_metrics = dict()
+                    metrics = items[5:]
+                    for metric in metrics:
+                        try:
+                            name, value = metric.strip(', ').split('=')
+                            value = value.split('/')[0]
+                            if value.count('.') > 0:
+                                value = float(value)
+                            else:
+                                value = int(value)
+                            self.eval_metrics[name] = value
+                        except:
+                            pass
+
+        end_time_timestamp = osp.getmtime(self.log_file)
+        t_diff = time.gmtime(end_time_timestamp - start_time_timestamp)
+        self.running_duration = "{}小时{}分{}秒".format(
+            t_diff.tm_hour, t_diff.tm_min, t_diff.tm_sec)
+
+
+class PruneLogReader(object):
+    def init_attr(self):
+        self.eta = None
+        self.iters = None
+        self.current = None
+        self.progress = None
+
+    def __init__(self, log_file):
+        self.log_file = log_file
+        self.init_attr()
+
+    def update(self):
+        if not osp.exists(self.log_file):
+            return
+        logs = open(self.log_file, encoding='utf-8').read().strip().split('\n')
+        self.init_attr()
+        for line in logs[::-1]:
+            metric_loaded = True
+            for k, v in self.__dict__.items():
+                if v is None:
+                    metric_loaded = False
+                    break
+            if metric_loaded:
+                break
+            if line.count("Total evaluate iters") > 0:
+                items = line.split(',')
+                for item in items:
+                    kv_list = item.strip().split()[-1].split('=')
+                    kv_list = [v.strip() for v in kv_list]
+                    setattr(self, kv_list[0], kv_list[1])
+
+
+class QuantLogReader:
+    def __init__(self, log_file):
+        self.log_file = log_file
+        self.stage = None
+        self.running_duration = None
+
+    def update(self):
+        if not osp.exists(self.log_file):
+            return
+        logs = open(self.log_file, encoding='utf-8').read().strip().split('\n')
+        for line in logs[::-1]:
+            items = line.strip().split(' ')
+            if line.count('[Run batch data]'):
+                info = items[-3][:-1].split('=')[1]
+                batch_id = float(info.split('/')[0])
+                batch_all = float(info.split('/')[1])
+                self.running_duration = \
+                    batch_id / batch_all * (10.0 / 30.0)
+                self.stage = 'Batch'
+                break
+            elif line.count('[Calculate weight]'):
+                info = items[-3][:-1].split('=')[1]
+                weight_id = float(info.split('/')[0])
+                weight_all = float(info.split('/')[1])
+                self.running_duration = \
+                    weight_id / weight_all * (3.0 / 30.0) + (10.0 / 30.0)
+                self.stage = 'Weight'
+                break
+            elif line.count('[Calculate activation]'):
+                info = items[-3][:-1].split('=')[1]
+                activation_id = float(info.split('/')[0])
+                activation_all = float(info.split('/')[1])
+                self.running_duration = \
+                    activation_id / activation_all * (16.0 / 30.0) + (13.0 / 30.0)
+                self.stage = 'Activation'
+                break
+            elif line.count('Finish quant!'):
+                self.stage = 'Finish'
+                break
+
+
+class PyNvml(object):
+    """ Nvidia GPU驱动检测类,可检测当前GPU驱动版本"""
+
+    class PrintableStructure(Structure):
+        _fmt_ = {}
+
+        def __str__(self):
+            result = []
+            for x in self._fields_:
+                key = x[0]
+                value = getattr(self, key)
+                fmt = "%s"
+                if key in self._fmt_:
+                    fmt = self._fmt_[key]
+                elif "<default>" in self._fmt_:
+                    fmt = self._fmt_["<default>"]
+                result.append(("%s: " + fmt) % (key, value))
+            return self.__class__.__name__ + "(" + string.join(result,
+                                                               ", ") + ")"
+
+    class c_nvmlMemory_t(PrintableStructure):
+        _fields_ = [
+            ('total', c_ulonglong),
+            ('free', c_ulonglong),
+            ('used', c_ulonglong),
+        ]
+        _fmt_ = {'<default>': "%d B"}
+
+    ## Device structures
+    class struct_c_nvmlDevice_t(Structure):
+        pass  # opaque handle
+
+    c_nvmlDevice_t = POINTER(struct_c_nvmlDevice_t)
+
+    def __init__(self):
+        self.nvml_lib = None
+        self.nvml_lib_refcount = 0
+        self.lib_load_lock = threading.Lock()
+        self.nvml_lib_path = None
+
+    def nvml_init(self, nvml_lib_path=None):
+        self.lib_load_lock.acquire()
+        sysstr = platform.system()
+        if nvml_lib_path is None or nvml_lib_path.strip() == "":
+            if sysstr == "Windows":
+                nvml_lib_path = osp.join(
+                    os.getenv("ProgramFiles", "C:/Program Files"),
+                    "NVIDIA Corporation/NVSMI")
+                if not osp.exists(osp.join(nvml_lib_path, "nvml.dll")):
+                    nvml_lib_path = "C:\\Windows\\System32"
+
+            elif sysstr == "Linux":
+                p1 = "/usr/lib/x86_64-linux-gnu"
+                p2 = "/usr/lib/i386-linux-gnu"
+                if osp.exists(osp.join(p1, "libnvidia-ml.so.1")):
+                    nvml_lib_path = p1
+                elif osp.exists(osp.join(p2, "libnvidia-ml.so.1")):
+                    nvml_lib_path = p2
+                else:
+                    nvml_lib_path = ""
+            else:
+                nvml_lib_path = "N/A"
+        nvml_lib_dir = nvml_lib_path
+        if sysstr == "Windows":
+            nvml_lib_path = osp.join(nvml_lib_dir, "nvml.dll")
+        else:
+            nvml_lib_path = osp.join(nvml_lib_dir, "libnvidia-ml.so.1")
+        self.nvml_lib_path = nvml_lib_path
+        try:
+            self.nvml_lib = CDLL(nvml_lib_path)
+            fn = self._get_fn_ptr("nvmlInit_v2")
+            fn()
+            if sysstr == "Windows":
+                driver_version = bytes.decode(
+                    self.nvml_system_get_driver_version())
+                if driver_version.strip() == "":
+                    nvml_lib_path = osp.join(nvml_lib_dir, "nvml9.dll")
+                    self.nvml_lib = CDLL(nvml_lib_path)
+                    fn = self._get_fn_ptr("nvmlInit_v2")
+                    fn()
+
+        except Exception as e:
+            raise e
+        finally:
+            self.lib_load_lock.release()
+        self.lib_load_lock.acquire()
+        self.nvml_lib_refcount += 1
+        self.lib_load_lock.release()
+
+    def create_string_buffer(self, init, size=None):
+        if isinstance(init, bytes):
+            if size is None:
+                size = len(init) + 1
+            buftype = c_char * size
+            buf = buftype()
+            buf.value = init
+            return buf
+        elif isinstance(init, int):
+            buftype = c_char * init
+            buf = buftype()
+            return buf
+        raise TypeError(init)
+
+    def _get_fn_ptr(self, name):
+        return getattr(self.nvml_lib, name)
+
+    def nvml_system_get_driver_version(self):
+        c_version = self.create_string_buffer(81)
+        fn = self._get_fn_ptr("nvmlSystemGetDriverVersion")
+        ret = fn(c_version, c_uint(81))
+        return c_version.value
+
+    def nvml_device_get_count(self):
+        c_count = c_uint()
+        fn = self._get_fn_ptr("nvmlDeviceGetCount_v2")
+        ret = fn(byref(c_count))
+        return c_count.value
+
+    def nvml_device_get_handle_by_index(self, index):
+        c_index = c_uint(index)
+        device = PyNvml.c_nvmlDevice_t()
+        fn = self._get_fn_ptr("nvmlDeviceGetHandleByIndex_v2")
+        ret = fn(c_index, byref(device))
+        return device
+
+    def nvml_device_get_memory_info(self, handle):
+        c_memory = PyNvml.c_nvmlMemory_t()
+        fn = self._get_fn_ptr("nvmlDeviceGetMemoryInfo")
+        ret = fn(handle, byref(c_memory))
+        return c_memory

+ 83 - 0
paddlex/restful/workspace.proto

@@ -0,0 +1,83 @@
+syntax = "proto3";
+package suitebackend;
+
+message Dataset {
+    string id = 1;
+    string name = 2;
+    string desc = 3;
+    // 'classification': 分类数据
+    // 'segmentation': 分割数据
+    // 'detection_voc': 检测数据(仅用于检测)
+    // 'detection_coco': 检测数据(用于检测,分割,实例分割)
+    string type = 4;
+    string path = 5;
+    string create_time = 6;
+}
+
+message Project {
+    string id = 1;
+    string name = 2;
+    string desc = 3;
+    // 'classification'
+    // 'segmentation'
+    // 'segmentation'
+    // 'instance_segmentation'
+    string type = 4;
+    string did = 5;
+    string path = 6;
+    string create_time = 7;
+}
+
+message Task {
+    string id = 1;
+    string name = 2;
+    string desc = 3;
+    string pid = 4;
+    string path = 5;
+    string create_time = 6;
+    string parent_id = 7;
+}
+
+message PretrainedModel {
+    string id = 1;
+    string name = 2;
+    string model = 3;
+    string type = 4;
+    // 所属项目id
+    string pid = 5;
+    string tid = 6;
+    string create_time = 7;
+    string path = 8;
+}
+
+message ExportedModel {
+    string id = 1;
+    string name = 2;
+    string model = 3;
+    string type = 4;
+    // 所属项目id
+    string pid = 5;
+    string tid = 6;
+    string create_time = 7;
+    string path = 8;
+    int32 exported_type = 9;
+}
+
+message Workspace {
+    string version = 1;
+    string path = 2;
+    map<string, Dataset> datasets = 3;
+    map<string, Project> projects = 4;
+    map<string, Task> tasks = 5;
+    int32 max_dataset_id = 6;
+    int32 max_project_id = 7;
+    int32 max_task_id = 8;
+    string current_time = 9;
+
+    int32 max_pretrained_model_id = 10;
+    map<string, PretrainedModel> pretrained_models = 11;
+
+    int32 max_exported_model_id = 12;
+    map<string, ExportedModel> exported_models = 13;
+}
+

+ 325 - 0
paddlex/restful/workspace.py

@@ -0,0 +1,325 @@
+from . import workspace_pb2 as w
+from .utils import get_logger
+from .dir import *
+import os
+import os.path as osp
+from threading import Thread
+import traceback
+import platform
+import configparser
+import time
+import shutil
+import copy
+
+
+class Workspace():
+    def __init__(self, workspace, dirname, logger):
+        self.workspace = workspace
+        #self.machine_info = {}
+        # app init
+        self.init_app_resource(dirname)
+        # 当前workspace版本
+        self.current_version = "0.2.0"
+        self.logger = logger
+        # 设置PaddleX的预训练模型下载存储路径
+        # 设置路径后不会重复下载相同模型
+        self.load_workspace()
+        self.stop_running = False
+        self.sync_thread = self.sync_with_local(interval=2)
+        #检查硬件环境
+        #self.check_hardware_env()
+
+    def init_app_resource(self, dirname):
+        self.m_cfgfile = configparser.ConfigParser()
+        app_conf_file_name = "PaddleX".lower() + ".cfg"
+        paddlex_cfg_file = os.path.join(PADDLEX_HOME, app_conf_file_name)
+        try:
+            self.m_cfgfile.read(paddlex_cfg_file)
+        except Exception as e:
+            print("[ERROR] Fail to read {}".format(paddlex_cfg_file))
+        if not self.m_cfgfile.has_option("USERCFG", "workspacedir"):
+            self.m_cfgfile.add_section("USERCFG")
+            self.m_cfgfile.set("USERCFG", "workspacedir", "")
+        self.m_cfgfile["USERCFG"]["workspacedir"] = dirname
+
+    def load_workspace(self):
+        path = self.workspace.path
+        newest_file = osp.join(self.workspace.path, 'workspace.newest.pb')
+        bak_file = osp.join(self.workspace.path, 'workspace.bak.pb')
+        flag_file = osp.join(self.workspace.path, '.pb.success')
+        self.workspace.version = self.current_version
+        try:
+            if osp.exists(flag_file):
+                with open(newest_file, 'rb') as f:
+                    self.workspace.ParseFromString(f.read())
+            elif osp.exists(bak_file):
+                with open(bak_file, 'rb') as f:
+                    self.workspace.ParseFromString(f.read())
+            else:
+                print("it is a new workspace")
+        except Exception as e:
+            print(traceback.format_exc())
+        self.workspace.path = path
+        if self.workspace.version < "0.2.0":
+            self.update_workspace()
+        self.recover_workspace()
+
+    def update_workspace(self):
+        if len(self.workspace.projects) == 0 and len(
+                self.workspace.datasets) == 0:
+            self.workspace.version == '0.2.0'
+            return
+
+        for key in self.workspace.datasets:
+            ds = self.workspace.datasets[key]
+            try:
+                info_file = os.path.join(ds.path, 'info.pb')
+                with open(info_file, 'wb') as f:
+                    f.write(ds.SerializeToString())
+            except Exception as e:
+                self.logger.info(traceback.format_exc())
+
+        for key in self.workspace.projects:
+            pj = self.workspace.projects[key]
+            try:
+                info_file = os.path.join(pj.path, 'info.pb')
+                with open(info_file, 'wb') as f:
+                    f.write(pj.SerializeToString())
+            except Exception as e:
+                self.logger.info(traceback.format_exc())
+
+        for key in self.workspace.tasks:
+            task = self.workspace.tasks[key]
+            try:
+                info_file = os.path.join(task.path, 'info.pb')
+                with open(info_file, 'wb') as f:
+                    f.write(task.SerializeToString())
+            except Exception as e:
+                self.logger.info(traceback.format_exc())
+        self.workspace.version == '0.2.0'
+
+    def recover_workspace(self):
+        if len(self.workspace.projects) > 0 or len(
+                self.workspace.datasets) > 0:
+            return
+        projects_dir = os.path.join(self.workspace.path, 'projects')
+        datasets_dir = os.path.join(self.workspace.path, 'datasets')
+        if not os.path.exists(projects_dir):
+            os.makedirs(projects_dir)
+        if not os.path.exists(datasets_dir):
+            os.makedirs(datasets_dir)
+
+        max_project_id = 0
+        max_dataset_id = 0
+        max_task_id = 0
+        for pd in os.listdir(projects_dir):
+            try:
+                if pd[0] != 'P':
+                    continue
+                if int(pd[1:]) > max_project_id:
+                    max_project_id = int(pd[1:])
+            except:
+                continue
+            info_pb_file = os.path.join(projects_dir, pd, 'info.pb')
+            if not os.path.exists(info_pb_file):
+                continue
+            try:
+                pj = w.Project()
+                with open(info_pb_file, 'rb') as f:
+                    pj.ParseFromString(f.read())
+                self.workspace.projects[pd].CopyFrom(pj)
+            except Exception as e:
+                self.logger.info(traceback.format_exc())
+
+            for td in os.listdir(os.path.join(projects_dir, pd)):
+                try:
+                    if td[0] != 'T':
+                        continue
+                    if int(td[1:]) > max_task_id:
+                        max_task_id = int(td[1:])
+                except:
+                    continue
+                info_pb_file = os.path.join(projects_dir, pd, td, 'info.pb')
+                if not os.path.exists(info_pb_file):
+                    continue
+                try:
+                    task = w.Task()
+                    with open(info_pb_file, 'rb') as f:
+                        task.ParseFromString(f.read())
+                    self.workspace.tasks[td].CopyFrom(task)
+                except Exception as e:
+                    self.logger.info(traceback.format_exc())
+
+        for dd in os.listdir(datasets_dir):
+            try:
+                if dd[0] != 'D':
+                    continue
+                if int(dd[1:]) > max_dataset_id:
+                    max_dataset_id = int(dd[1:])
+            except:
+                continue
+            info_pb_file = os.path.join(datasets_dir, dd, 'info.pb')
+            if not os.path.exists(info_pb_file):
+                continue
+            try:
+                ds = w.Dataset()
+                with open(info_pb_file, 'rb') as f:
+                    ds.ParseFromString(f.read())
+                self.workspace.datasets[dd].CopyFrom(ds)
+            except Exception as e:
+                self.logger.info(traceback.format_exc())
+
+        self.workspace.max_dataset_id = max_dataset_id
+        self.workspace.max_project_id = max_project_id
+        self.workspace.max_task_id = max_task_id
+
+    # 每间隔interval秒,将workspace同步到本地文件
+    def sync_with_local(self, interval=2):
+        def sync_func(s, interval_seconds=2):
+            newest_file = osp.join(self.workspace.path, 'workspace.newest.pb')
+            stable_file = osp.join(self.workspace.path, 'workspace.stable.pb')
+            bak_file = osp.join(self.workspace.path, 'workspace.bak.pb')
+            flag_file = osp.join(self.workspace.path, '.pb.success')
+            while True:
+                current_time = time.time()
+                time_array = time.localtime(current_time)
+                current_time = time.strftime("%Y-%m-%d %H:%M:%S", time_array)
+                self.workspace.current_time = current_time
+
+                if osp.exists(flag_file):
+                    os.remove(flag_file)
+                f = open(newest_file, mode='wb')
+                f.write(s.workspace.SerializeToString())
+                f.close()
+                open(flag_file, 'w').close()
+                if osp.exists(stable_file):
+                    shutil.copyfile(stable_file, bak_file)
+                shutil.copyfile(newest_file, stable_file)
+                if s.stop_running:
+                    break
+                time.sleep(interval_seconds)
+
+        t = Thread(target=sync_func, args=(self, interval))
+        t.start()
+        return t
+
+    def check_hardware_env(self):
+        # 判断是否有gpu,cpu值是否已经设置
+        hasGpu = True
+        try:
+            '''data = {'path' : path}
+            from .system import get_machine_info
+            info = get_machine_info(data, self.machine_info)['info']
+            if info is None:
+                return
+            if (info['gpu_num'] == 0 and self.sysstr == "Windows"):
+                data['path'] = os.path.abspath(os.path.dirname(__file__))
+                info = get_machine_info(data, self.machine_info)['info']'''
+            from .system import get_system_info
+            info = get_system_info()['info']
+            hasGpu = (info['gpu_num'] > 0)
+            self.machine_info = info
+            #driver_ver = info['driver_version']
+            # driver_ver_list = driver_ver.split(".")
+            # major_ver, minor_ver = driver_ver_list[0:2]
+            # if sysstr == "Windows":
+            #     if int(major_ver) < 411 or \
+            #             (int(major_ver) == 411 and int(minor_ver) < 31):
+            #         raise Exception("The GPU dirver version should be larger than 411.31")
+            #
+            # elif sysstr == "Linux":
+            #     if int(major_ver) < 410 or \
+            #             (int(major_ver) == 410 and int(minor_ver) < 48):
+            #         raise Exception("The GPU dirver version should be larger than 410.48")
+
+        except Exception as e:
+            hasGpu = False
+
+        self.m_HasGpu = hasGpu
+        self.save_app_cfg_file()
+
+    def save_app_cfg_file(self):
+        #更新程序配置信息
+        app_conf_file_name = 'PaddleX'.lower() + ".cfg"
+
+        with open(os.path.join(PADDLEX_HOME, app_conf_file_name),
+                  'w+') as file:
+            self.m_cfgfile.write(file)
+
+
+def init_workspace(workspace, dirname, logger):
+    wp = Workspace(workspace, dirname, logger)
+    #if not machine_info:
+    #machine_info.update(wp.machine_info)
+    return {'status': 1}
+
+
+def set_attr(data, workspace):
+    """对workspace中项目,数据,任务变量进行修改赋值
+
+    Args:
+        data为dict,key包括
+        'struct'结构类型,可以是'dataset', 'project'或'task';
+        'id'查询id, 其余的key:value则分别为待修改的变量名和相应的修改值。
+    """
+    struct = data['struct']
+    id = data['id']
+    assert struct in ['dataset', 'project', 'task'
+                      ], "struct只能为dataset, project或task"
+    if struct == 'dataset':
+        assert id in workspace.datasets, "数据集ID'{}'不存在".format(id)
+        modify_struct = workspace.datasets[id]
+    elif struct == 'project':
+        assert id in workspace.projects, "项目ID'{}'不存在".format(id)
+        modify_struct = workspace.projects[id]
+    elif struct == 'task':
+        assert id in workspace.tasks, "任务ID'{}'不存在".format(id)
+        modify_struct = workspace.tasks[id]
+    '''for k, v in data.items():
+        if k in ['id', 'struct']:
+            continue
+        assert hasattr(modify_struct,
+                        k), "{}不存在成员变量'{}'".format(type(modify_struct), k)
+        setattr(modify_struct, k, v)'''
+    for k, v in data['attr_dict'].items():
+        assert hasattr(modify_struct,
+                       k), "{}不存在成员变量'{}'".format(type(modify_struct), k)
+        setattr(modify_struct, k, v)
+    with open(os.path.join(modify_struct.path, 'info.pb'), 'wb') as f:
+        f.write(modify_struct.SerializeToString())
+
+    return {'status': 1}
+
+
+def get_attr(data, workspace):
+    """取出workspace中项目,数据,任务变量值
+
+    Args:
+        data为dict,key包括
+        'struct'结构类型,可以是'dataset', 'project'或'task';
+        'id'查询id, 'attr_list'需要获取的属性值列表
+    """
+    struct = data['struct']
+    id = data['id']
+    assert struct in ['dataset', 'project', 'task'
+                      ], "struct只能为dataset, project或task"
+    if struct == 'dataset':
+        assert id in workspace.datasets, "数据集ID'{}'不存在".format(id)
+        modify_struct = workspace.datasets[id]
+    elif struct == 'project':
+        assert id in workspace.projects, "项目ID'{}'不存在".format(id)
+        modify_struct = workspace.projects[id]
+    elif struct == 'task':
+        assert id in workspace.tasks, "任务ID'{}'不存在".format(id)
+        modify_struct = workspace.tasks[id]
+
+    attr = {}
+    for k in data['attr_list']:
+        if k in ['id', 'struct']:
+            continue
+        assert hasattr(modify_struct,
+                       k), "{}不存在成员变量'{}'".format(type(modify_struct), k)
+        v = getattr(modify_struct, k)
+        attr[k] = v
+
+    return {'status': 1, 'attr': attr}

File diff suppressed because it is too large
+ 21 - 0
paddlex/restful/workspace_pb2.py


+ 1 - 0
requirements.txt

@@ -9,3 +9,4 @@ shapely
 paddle2onnx
 paddlepaddle-gpu
 opencv-python
+psutil

+ 1 - 1
setup.py

@@ -31,7 +31,7 @@ setuptools.setup(
     install_requires=[
         "pycocotools;platform_system!='Windows'", 'pyyaml', 'colorama', 'tqdm',
         'paddleslim==1.1.1', 'visualdl>=2.0.0', 'paddlehub>=1.8.2',
-        'shapely>=1.7.0', "opencv-python"
+        'shapely>=1.7.0', 'opencv-python', 'flask_cors', 'sklearn', 'psutil'
     ],
     classifiers=[
         "Programming Language :: Python :: 3",

Some files were not shown because too many files changed in this diff