common_dataset.py 2.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from __future__ import print_function
  15. import numpy as np
  16. from paddle.io import Dataset
  17. import cv2
  18. from paddlex.ppcls.data import preprocess
  19. from paddlex.ppcls.data.preprocess import transform
  20. from paddlex.ppcls.utils import logger
  21. def create_operators(params):
  22. """
  23. create operators based on the config
  24. Args:
  25. params(list): a dict list, used to create some operators
  26. """
  27. assert isinstance(params, list), ('operator config should be a list')
  28. ops = []
  29. for operator in params:
  30. assert isinstance(operator,
  31. dict) and len(operator) == 1, "yaml format error"
  32. op_name = list(operator)[0]
  33. param = {} if operator[op_name] is None else operator[op_name]
  34. op = getattr(preprocess, op_name)(**param)
  35. ops.append(op)
  36. return ops
  37. class CommonDataset(Dataset):
  38. def __init__(
  39. self,
  40. image_root,
  41. cls_label_path,
  42. transform_ops=None, ):
  43. self._img_root = image_root
  44. self._cls_path = cls_label_path
  45. if transform_ops:
  46. self._transform_ops = create_operators(transform_ops)
  47. self.images = []
  48. self.labels = []
  49. self._load_anno()
  50. def _load_anno(self):
  51. pass
  52. def __getitem__(self, idx):
  53. try:
  54. with open(self.images[idx], 'rb') as f:
  55. img = f.read()
  56. if self._transform_ops:
  57. img = transform(img, self._transform_ops)
  58. img = img.transpose((2, 0, 1))
  59. return (img, self.labels[idx])
  60. except Exception as ex:
  61. logger.error("Exception occured when parse line: {} with msg: {}".
  62. format(self.images[idx], ex))
  63. rnd_idx = np.random.randint(self.__len__())
  64. return self.__getitem__(rnd_idx)
  65. def __len__(self):
  66. return len(self.images)
  67. @property
  68. def class_num(self):
  69. return len(set(self.labels))