__init__.py 5.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144
  1. # Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. import inspect
  15. import copy
  16. import paddle
  17. import numpy as np
  18. from paddle.io import DistributedBatchSampler, BatchSampler, DataLoader
  19. from paddlex.ppcls.utils import logger
  20. from paddlex.ppcls.data import dataloader
  21. # dataset
  22. from paddlex.ppcls.data.dataloader.imagenet_dataset import ImageNetDataset
  23. from paddlex.ppcls.data.dataloader.multilabel_dataset import MultiLabelDataset
  24. from paddlex.ppcls.data.dataloader.common_dataset import create_operators
  25. from paddlex.ppcls.data.dataloader.vehicle_dataset import CompCars, VeriWild
  26. from paddlex.ppcls.data.dataloader.logo_dataset import LogoDataset
  27. from paddlex.ppcls.data.dataloader.icartoon_dataset import ICartoonDataset
  28. from paddlex.ppcls.data.dataloader.mix_dataset import MixDataset
  29. # sampler
  30. from paddlex.ppcls.data.dataloader.DistributedRandomIdentitySampler import DistributedRandomIdentitySampler
  31. from paddlex.ppcls.data.dataloader.pk_sampler import PKSampler
  32. from paddlex.ppcls.data.dataloader.mix_sampler import MixSampler
  33. from paddlex.ppcls.data import preprocess
  34. from paddlex.ppcls.data.preprocess import transform
  35. def create_operators(params, class_num=None):
  36. """
  37. create operators based on the config
  38. Args:
  39. params(list): a dict list, used to create some operators
  40. """
  41. assert isinstance(params, list), ('operator config should be a list')
  42. ops = []
  43. for operator in params:
  44. assert isinstance(operator,
  45. dict) and len(operator) == 1, "yaml format error"
  46. op_name = list(operator)[0]
  47. param = {} if operator[op_name] is None else operator[op_name]
  48. op_func = getattr(preprocess, op_name)
  49. if "class_num" in inspect.getfullargspec(op_func).args:
  50. param.update({"class_num": class_num})
  51. op = op_func(**param)
  52. ops.append(op)
  53. return ops
  54. def build_dataloader(config, mode, device, use_dali=False, seed=None):
  55. assert mode in [
  56. 'Train', 'Eval', 'Test', 'Gallery', 'Query'
  57. ], "Dataset mode should be Train, Eval, Test, Gallery, Query"
  58. # build dataset
  59. if use_dali:
  60. from paddlex.ppcls.data.dataloader.dali import dali_dataloader
  61. return dali_dataloader(config, mode, paddle.device.get_device(), seed)
  62. class_num = config.get("class_num", None)
  63. config_dataset = config[mode]['dataset']
  64. config_dataset = copy.deepcopy(config_dataset)
  65. dataset_name = config_dataset.pop('name')
  66. if 'batch_transform_ops' in config_dataset:
  67. batch_transform = config_dataset.pop('batch_transform_ops')
  68. else:
  69. batch_transform = None
  70. dataset = eval(dataset_name)(**config_dataset)
  71. logger.debug("build dataset({}) success...".format(dataset))
  72. # build sampler
  73. config_sampler = config[mode]['sampler']
  74. if "name" not in config_sampler:
  75. batch_sampler = None
  76. batch_size = config_sampler["batch_size"]
  77. drop_last = config_sampler["drop_last"]
  78. shuffle = config_sampler["shuffle"]
  79. else:
  80. sampler_name = config_sampler.pop("name")
  81. batch_sampler = eval(sampler_name)(dataset, **config_sampler)
  82. logger.debug("build batch_sampler({}) success...".format(batch_sampler))
  83. # build batch operator
  84. def mix_collate_fn(batch):
  85. batch = transform(batch, batch_ops)
  86. # batch each field
  87. slots = []
  88. for items in batch:
  89. for i, item in enumerate(items):
  90. if len(slots) < len(items):
  91. slots.append([item])
  92. else:
  93. slots[i].append(item)
  94. return [np.stack(slot, axis=0) for slot in slots]
  95. if isinstance(batch_transform, list):
  96. batch_ops = create_operators(batch_transform, class_num)
  97. batch_collate_fn = mix_collate_fn
  98. else:
  99. batch_collate_fn = None
  100. # build dataloader
  101. config_loader = config[mode]['loader']
  102. num_workers = config_loader["num_workers"]
  103. use_shared_memory = config_loader["use_shared_memory"]
  104. if batch_sampler is None:
  105. data_loader = DataLoader(
  106. dataset=dataset,
  107. places=device,
  108. num_workers=num_workers,
  109. return_list=True,
  110. use_shared_memory=use_shared_memory,
  111. batch_size=batch_size,
  112. shuffle=shuffle,
  113. drop_last=drop_last,
  114. collate_fn=batch_collate_fn)
  115. else:
  116. data_loader = DataLoader(
  117. dataset=dataset,
  118. places=device,
  119. num_workers=num_workers,
  120. return_list=True,
  121. use_shared_memory=use_shared_memory,
  122. batch_sampler=batch_sampler,
  123. collate_fn=batch_collate_fn)
  124. logger.debug("build data_loader({}) success...".format(data_loader))
  125. return data_loader