operate.py 5.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142
  1. # copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os
  15. import pickle
  16. import traceback
  17. import os.path as osp
  18. import multiprocessing as mp
  19. from .cls_dataset import ClsDataset
  20. from .det_dataset import DetDataset
  21. from .seg_dataset import SegDataset
  22. from .ins_seg_dataset import InsSegDataset
  23. from ..utils import set_folder_status, get_folder_status, DatasetStatus, DownloadStatus, download, list_files
  24. dataset_url_list = [
  25. 'https://bj.bcebos.com/paddlex/demos/vegetables_cls.tar.gz',
  26. 'https://bj.bcebos.com/paddlex/demos/insect_det.tar.gz',
  27. 'https://bj.bcebos.com/paddlex/demos/optic_disc_seg.tar.gz',
  28. 'https://bj.bcebos.com/paddlex/demos/xiaoduxiong_ins_det.tar.gz',
  29. 'https://bj.bcebos.com/paddlex/demos/remote_sensing_seg.tar.gz'
  30. ]
  31. def _check_and_copy(dataset, dataset_path, source_path):
  32. try:
  33. dataset.check_dataset(source_path)
  34. except Exception as e:
  35. error_info = traceback.format_exc()
  36. set_folder_status(dataset_path, DatasetStatus.XCHECKFAIL, error_info)
  37. return
  38. set_folder_status(dataset_path, DatasetStatus.XCOPYING, os.getpid())
  39. try:
  40. dataset.copy_dataset(source_path, dataset.all_files)
  41. except Exception as e:
  42. error_info = traceback.format_exc()
  43. set_folder_status(dataset_path, DatasetStatus.XCOPYFAIL, error_info)
  44. return
  45. # 若上传已切分好的数据集
  46. if len(dataset.train_files) != 0:
  47. set_folder_status(dataset_path, DatasetStatus.XSPLITED)
  48. def import_dataset(dataset_id, dataset_type, dataset_path, source_path):
  49. set_folder_status(dataset_path, DatasetStatus.XCHECKING)
  50. if dataset_type == 'classification':
  51. ds = ClsDataset(dataset_id, dataset_path)
  52. elif dataset_type == 'detection':
  53. ds = DetDataset(dataset_id, dataset_path)
  54. elif dataset_type == 'segmentation':
  55. ds = SegDataset(dataset_id, dataset_path)
  56. elif dataset_type == 'instance_segmentation':
  57. ds = InsSegDataset(dataset_id, dataset_path)
  58. p = mp.Process(
  59. target=_check_and_copy, args=(ds, dataset_path, source_path))
  60. p.start()
  61. return p
  62. def _download_proc(url, target_path, dataset_type):
  63. # 下载数据集压缩包
  64. from paddlex.utils import decompress
  65. target_path = osp.join(target_path, dataset_type)
  66. fname = download(url, target_path)
  67. # 解压
  68. decompress(fname)
  69. set_folder_status(target_path, DownloadStatus.XDDECOMPRESSED)
  70. def download_demo_dataset(prj_type, target_path):
  71. url = dataset_url_list[prj_type.value]
  72. dataset_type = prj_type.name
  73. p = mp.Process(
  74. target=_download_proc, args=(url, target_path, dataset_type))
  75. p.start()
  76. return p
  77. def get_dataset_status(dataset_id, dataset_type, dataset_path):
  78. status, message = get_folder_status(dataset_path, True)
  79. if status is None:
  80. status = DatasetStatus.XEMPTY
  81. if status == DatasetStatus.XCOPYING:
  82. items = message.strip().split()
  83. pid = None
  84. if len(items) < 2:
  85. percent = 0.0
  86. else:
  87. pid = int(items[0])
  88. if int(items[1]) == 0:
  89. percent = 1.0
  90. else:
  91. copyed_files_num = len(list_files(dataset_path)) - 1
  92. percent = copyed_files_num * 1.0 / int(items[1])
  93. message = {'pid': pid, 'percent': percent}
  94. if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
  95. if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
  96. p = import_dataset(dataset_id, dataset_type, dataset_path,
  97. dataset_path)
  98. status = DatasetStatus.XCHECKING
  99. return status, message
  100. def split_dataset(dataset_id, dataset_type, dataset_path, val_split,
  101. test_split):
  102. status, message = get_folder_status(dataset_path, True)
  103. if status != DatasetStatus.XCOPYDONE and status != DatasetStatus.XSPLITED:
  104. raise Exception("数据集还未导入完成,请等数据集导入成功后再进行切分")
  105. if not osp.exists(osp.join(dataset_path, 'statis.pkl')):
  106. raise Exception("数据集需重新校验,请刷新数据集后再进行切分")
  107. if dataset_type == 'classification':
  108. ds = ClsDataset(dataset_id, dataset_path)
  109. elif dataset_type == 'detection':
  110. ds = DetDataset(dataset_id, dataset_path)
  111. elif dataset_type == 'segmentation':
  112. ds = SegDataset(dataset_id, dataset_path)
  113. elif dataset_type == 'instance_segmentation':
  114. ds = InsSegDataset(dataset_id, dataset_path)
  115. ds.load_statis_info()
  116. ds.split(val_split, test_split)
  117. set_folder_status(dataset_path, DatasetStatus.XSPLITED)
  118. def get_dataset_details(dataset_path):
  119. status, message = get_folder_status(dataset_path, True)
  120. if status == DatasetStatus.XCOPYDONE or status == DatasetStatus.XSPLITED:
  121. with open(osp.join(dataset_path, 'statis.pkl'), 'rb') as f:
  122. details = pickle.load(f)
  123. return details
  124. return None