imagenet.py 3.7 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import os.path as osp
  15. import copy
  16. from paddle.io import Dataset
  17. from paddlex.utils import logging, get_num_workers, get_encoding, path_normalization, is_pic
  18. class ImageNet(Dataset):
  19. """读取ImageNet格式的分类数据集,并对样本进行相应的处理。
  20. Args:
  21. data_dir (str): 数据集所在的目录路径。
  22. file_list (str): 描述数据集图片文件和类别id的文件路径(文本内每行路径为相对data_dir的相对路)。
  23. label_list (str): 描述数据集包含的类别信息文件路径。
  24. transforms (paddlex.transforms): 数据集中每个样本的预处理/增强算子。
  25. num_workers (int|str): 数据集中样本在预处理过程中的线程或进程数。默认为'auto'。当设为'auto'时,根据
  26. 系统的实际CPU核数设置`num_workers`: 如果CPU核数的一半大于8,则`num_workers`为8,否则为CPU核
  27. 数的一半。
  28. shuffle (bool): 是否需要对数据集中样本打乱顺序。默认为False。
  29. """
  30. def __init__(self,
  31. data_dir,
  32. file_list,
  33. label_list,
  34. transforms=None,
  35. num_workers='auto',
  36. shuffle=False):
  37. super(ImageNet, self).__init__()
  38. self.transforms = copy.deepcopy(transforms)
  39. # TODO batch padding
  40. self.batch_transforms = None
  41. self.num_workers = get_num_workers(num_workers)
  42. self.shuffle = shuffle
  43. self.file_list = list()
  44. self.labels = list()
  45. with open(label_list, encoding=get_encoding(label_list)) as f:
  46. for line in f:
  47. item = line.strip()
  48. self.labels.append(item)
  49. logging.info("Starting to read file list from dataset...")
  50. with open(file_list, encoding=get_encoding(file_list)) as f:
  51. for line in f:
  52. items = line.strip().split()
  53. if len(items) > 2:
  54. raise Exception(
  55. "A space is defined as the delimiter to separate the image and label path, " \
  56. "so the space cannot be in the image or label path, but the line[{}] of " \
  57. " file_list[{}] has a space in the image or label path.".format(line, file_list))
  58. items[0] = path_normalization(items[0])
  59. if not is_pic(items[0]):
  60. continue
  61. full_path = osp.join(data_dir, items[0])
  62. if not osp.exists(full_path):
  63. raise IOError('The image file {} does not exist!'.format(
  64. full_path))
  65. self.file_list.append({
  66. 'image': full_path,
  67. 'label': int(items[1])
  68. })
  69. self.num_samples = len(self.file_list)
  70. logging.info("{} samples in file {}".format(
  71. len(self.file_list), file_list))
  72. def __getitem__(self, idx):
  73. sample = copy.deepcopy(self.file_list[idx])
  74. outputs = self.transforms(sample)
  75. return outputs
  76. def __len__(self):
  77. return len(self.file_list)