det_transforms.py 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387
  1. # copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. try:
  15. from collections.abc import Sequence
  16. except Exception:
  17. from collections import Sequence
  18. import random
  19. import os.path as osp
  20. import numpy as np
  21. import cv2
  22. from PIL import Image, ImageEnhance
  23. from .imgaug_support import execute_imgaug
  24. from .ops import *
  25. from .box_utils import *
  26. import paddlex.utils.logging as logging
  27. class DetTransform:
  28. """检测数据处理基类
  29. """
  30. def __init__(self):
  31. pass
  32. class Compose(DetTransform):
  33. """根据数据预处理/增强列表对输入数据进行操作。
  34. 所有操作的输入图像流形状均是[H, W, C],其中H为图像高,W为图像宽,C为图像通道数。
  35. Args:
  36. transforms (list): 数据预处理/增强列表。
  37. Raises:
  38. TypeError: 形参数据类型不满足需求。
  39. ValueError: 数据长度不匹配。
  40. """
  41. def __init__(self, transforms):
  42. if not isinstance(transforms, list):
  43. raise TypeError('The transforms must be a list!')
  44. if len(transforms) < 1:
  45. raise ValueError('The length of transforms ' + \
  46. 'must be equal or larger than 1!')
  47. self.transforms = transforms
  48. self.use_mixup = False
  49. for t in self.transforms:
  50. if type(t).__name__ == 'MixupImage':
  51. self.use_mixup = True
  52. # 检查transforms里面的操作,目前支持PaddleX定义的或者是imgaug操作
  53. for op in self.transforms:
  54. if not isinstance(op, DetTransform):
  55. import imgaug.augmenters as iaa
  56. if not isinstance(op, iaa.Augmenter):
  57. raise Exception(
  58. "Elements in transforms should be defined in 'paddlex.det.transforms' or class of imgaug.augmenters.Augmenter, see docs here: https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/"
  59. )
  60. def __call__(self, im, im_info=None, label_info=None):
  61. """
  62. Args:
  63. im (str/np.ndarray): 图像路径/图像np.ndarray数据。
  64. im_info (dict): 存储与图像相关的信息,dict中的字段如下:
  65. - im_id (np.ndarray): 图像序列号,形状为(1,)。
  66. - image_shape (np.ndarray): 图像原始大小,形状为(2,),
  67. image_shape[0]为高,image_shape[1]为宽。
  68. - mixup (list): list为[im, im_info, label_info],分别对应
  69. 与当前图像进行mixup的图像np.ndarray数据、图像相关信息、标注框相关信息;
  70. 注意,当前epoch若无需进行mixup,则无该字段。
  71. label_info (dict): 存储与标注框相关的信息,dict中的字段如下:
  72. - gt_bbox (np.ndarray): 真实标注框坐标[x1, y1, x2, y2],形状为(n, 4),
  73. 其中n代表真实标注框的个数。
  74. - gt_class (np.ndarray): 每个真实标注框对应的类别序号,形状为(n, 1),
  75. 其中n代表真实标注框的个数。
  76. - gt_score (np.ndarray): 每个真实标注框对应的混合得分,形状为(n, 1),
  77. 其中n代表真实标注框的个数。
  78. - gt_poly (list): 每个真实标注框内的多边形分割区域,每个分割区域由点的x、y坐标组成,
  79. 长度为n,其中n代表真实标注框的个数。
  80. - is_crowd (np.ndarray): 每个真实标注框中是否是一组对象,形状为(n, 1),
  81. 其中n代表真实标注框的个数。
  82. - difficult (np.ndarray): 每个真实标注框中的对象是否为难识别对象,形状为(n, 1),
  83. 其中n代表真实标注框的个数。
  84. Returns:
  85. tuple: 根据网络所需字段所组成的tuple;
  86. 字段由transforms中的最后一个数据预处理操作决定。
  87. """
  88. def decode_image(im_file, im_info, label_info):
  89. if im_info is None:
  90. im_info = dict()
  91. if isinstance(im_file, np.ndarray):
  92. if len(im_file.shape) != 3:
  93. raise Exception(
  94. "im should be 3-dimensions, but now is {}-dimensions".
  95. format(len(im_file.shape)))
  96. im = im_file
  97. else:
  98. try:
  99. im = cv2.imread(im_file).astype('float32')
  100. except:
  101. raise TypeError('Can\'t read The image file {}!'.format(
  102. im_file))
  103. im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
  104. # make default im_info with [h, w, 1]
  105. im_info['im_resize_info'] = np.array(
  106. [im.shape[0], im.shape[1], 1.], dtype=np.float32)
  107. im_info['image_shape'] = np.array([im.shape[0],
  108. im.shape[1]]).astype('int32')
  109. if not self.use_mixup:
  110. if 'mixup' in im_info:
  111. del im_info['mixup']
  112. # decode mixup image
  113. if 'mixup' in im_info:
  114. im_info['mixup'] = \
  115. decode_image(im_info['mixup'][0],
  116. im_info['mixup'][1],
  117. im_info['mixup'][2])
  118. if label_info is None:
  119. return (im, im_info)
  120. else:
  121. return (im, im_info, label_info)
  122. outputs = decode_image(im, im_info, label_info)
  123. im = outputs[0]
  124. im_info = outputs[1]
  125. if len(outputs) == 3:
  126. label_info = outputs[2]
  127. for op in self.transforms:
  128. if im is None:
  129. return None
  130. if isinstance(op, DetTransform):
  131. outputs = op(im, im_info, label_info)
  132. im = outputs[0]
  133. else:
  134. im = execute_imgaug(op, im)
  135. if label_info is not None:
  136. outputs = (im, im_info, label_info)
  137. else:
  138. outputs = (im, im_info)
  139. return outputs
  140. def add_augmenters(self, augmenters):
  141. if not isinstance(augmenters, list):
  142. raise Exception(
  143. "augmenters should be list type in func add_augmenters()")
  144. transform_names = [type(x).__name__ for x in self.transforms]
  145. for aug in augmenters:
  146. if type(aug).__name__ in transform_names:
  147. logging.error(
  148. "{} is already in ComposedTransforms, need to remove it from add_augmenters().".
  149. format(type(aug).__name__))
  150. self.transforms = augmenters + self.transforms
  151. class ResizeByShort(DetTransform):
  152. """根据图像的短边调整图像大小(resize)。
  153. 1. 获取图像的长边和短边长度。
  154. 2. 根据短边与short_size的比例,计算长边的目标长度,
  155. 此时高、宽的resize比例为short_size/原图短边长度。
  156. 3. 如果max_size>0,调整resize比例:
  157. 如果长边的目标长度>max_size,则高、宽的resize比例为max_size/原图长边长度。
  158. 4. 根据调整大小的比例对图像进行resize。
  159. Args:
  160. target_size (int): 短边目标长度。默认为800。
  161. max_size (int): 长边目标长度的最大限制。默认为1333。
  162. Raises:
  163. TypeError: 形参数据类型不满足需求。
  164. """
  165. def __init__(self, short_size=800, max_size=1333):
  166. self.max_size = int(max_size)
  167. if not isinstance(short_size, int):
  168. raise TypeError(
  169. "Type of short_size is invalid. Must be Integer, now is {}".
  170. format(type(short_size)))
  171. self.short_size = short_size
  172. if not (isinstance(self.max_size, int)):
  173. raise TypeError("max_size: input type is invalid.")
  174. def __call__(self, im, im_info=None, label_info=None):
  175. """
  176. Args:
  177. im (numnp.ndarraypy): 图像np.ndarray数据。
  178. im_info (dict, 可选): 存储与图像相关的信息。
  179. label_info (dict, 可选): 存储与标注框相关的信息。
  180. Returns:
  181. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  182. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  183. 存储与标注框相关信息的字典。
  184. 其中,im_info更新字段为:
  185. - im_resize_info (np.ndarray): resize后的图像高、resize后的图像宽、resize后的图像相对原始图的缩放比例
  186. 三者组成的np.ndarray,形状为(3,)。
  187. Raises:
  188. TypeError: 形参数据类型不满足需求。
  189. ValueError: 数据长度不匹配。
  190. """
  191. if im_info is None:
  192. im_info = dict()
  193. if not isinstance(im, np.ndarray):
  194. raise TypeError("ResizeByShort: image type is not numpy.")
  195. if len(im.shape) != 3:
  196. raise ValueError('ResizeByShort: image is not 3-dimensional.')
  197. im_short_size = min(im.shape[0], im.shape[1])
  198. im_long_size = max(im.shape[0], im.shape[1])
  199. scale = float(self.short_size) / im_short_size
  200. if self.max_size > 0 and np.round(scale *
  201. im_long_size) > self.max_size:
  202. scale = float(self.max_size) / float(im_long_size)
  203. resized_width = int(round(im.shape[1] * scale))
  204. resized_height = int(round(im.shape[0] * scale))
  205. im_resize_info = [resized_height, resized_width, scale]
  206. im = cv2.resize(
  207. im, (resized_width, resized_height),
  208. interpolation=cv2.INTER_LINEAR)
  209. im_info['im_resize_info'] = np.array(im_resize_info).astype(np.float32)
  210. if label_info is None:
  211. return (im, im_info)
  212. else:
  213. return (im, im_info, label_info)
  214. class Padding(DetTransform):
  215. """1.将图像的长和宽padding至coarsest_stride的倍数。如输入图像为[300, 640],
  216. `coarest_stride`为32,则由于300不为32的倍数,因此在图像最右和最下使用0值
  217. 进行padding,最终输出图像为[320, 640]。
  218. 2.或者,将图像的长和宽padding到target_size指定的shape,如输入的图像为[300,640],
  219. a. `target_size` = 960,在图像最右和最下使用0值进行padding,最终输出
  220. 图像为[960, 960]。
  221. b. `target_size` = [640, 960],在图像最右和最下使用0值进行padding,最终
  222. 输出图像为[640, 960]。
  223. 1. 如果coarsest_stride为1,target_size为None则直接返回。
  224. 2. 获取图像的高H、宽W。
  225. 3. 计算填充后图像的高H_new、宽W_new。
  226. 4. 构建大小为(H_new, W_new, 3)像素值为0的np.ndarray,
  227. 并将原图的np.ndarray粘贴于左上角。
  228. Args:
  229. coarsest_stride (int): 填充后的图像长、宽为该参数的倍数,默认为1。
  230. target_size (int|list|tuple): 填充后的图像长、宽,默认为None,coarset_stride优先级更高。
  231. Raises:
  232. TypeError: 形参`target_size`数据类型不满足需求。
  233. ValueError: 形参`target_size`为(list|tuple)时,长度不满足需求。
  234. """
  235. def __init__(self, coarsest_stride=1, target_size=None):
  236. self.coarsest_stride = coarsest_stride
  237. if target_size is not None:
  238. if not isinstance(target_size, int):
  239. if not isinstance(target_size, tuple) and not isinstance(
  240. target_size, list):
  241. raise TypeError(
  242. "Padding: Type of target_size must in (int|list|tuple)."
  243. )
  244. elif len(target_size) != 2:
  245. raise ValueError(
  246. "Padding: Length of target_size must equal 2.")
  247. self.target_size = target_size
  248. def __call__(self, im, im_info=None, label_info=None):
  249. """
  250. Args:
  251. im (numnp.ndarraypy): 图像np.ndarray数据。
  252. im_info (dict, 可选): 存储与图像相关的信息。
  253. label_info (dict, 可选): 存储与标注框相关的信息。
  254. Returns:
  255. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  256. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  257. 存储与标注框相关信息的字典。
  258. Raises:
  259. TypeError: 形参数据类型不满足需求。
  260. ValueError: 数据长度不匹配。
  261. ValueError: coarsest_stride,target_size需有且只有一个被指定。
  262. ValueError: target_size小于原图的大小。
  263. """
  264. if im_info is None:
  265. im_info = dict()
  266. if not isinstance(im, np.ndarray):
  267. raise TypeError("Padding: image type is not numpy.")
  268. if len(im.shape) != 3:
  269. raise ValueError('Padding: image is not 3-dimensional.')
  270. im_h, im_w, im_c = im.shape[:]
  271. if isinstance(self.target_size, int):
  272. padding_im_h = self.target_size
  273. padding_im_w = self.target_size
  274. elif isinstance(self.target_size, list) or isinstance(self.target_size,
  275. tuple):
  276. padding_im_w = self.target_size[0]
  277. padding_im_h = self.target_size[1]
  278. elif self.coarsest_stride > 0:
  279. padding_im_h = int(
  280. np.ceil(im_h / self.coarsest_stride) * self.coarsest_stride)
  281. padding_im_w = int(
  282. np.ceil(im_w / self.coarsest_stride) * self.coarsest_stride)
  283. else:
  284. raise ValueError(
  285. "coarsest_stridei(>1) or target_size(list|int) need setting in Padding transform"
  286. )
  287. pad_height = padding_im_h - im_h
  288. pad_width = padding_im_w - im_w
  289. if pad_height < 0 or pad_width < 0:
  290. raise ValueError(
  291. 'the size of image should be less than target_size, but the size of image ({}, {}), is larger than target_size ({}, {})'
  292. .format(im_w, im_h, padding_im_w, padding_im_h))
  293. padding_im = np.zeros(
  294. (padding_im_h, padding_im_w, im_c), dtype=np.float32)
  295. padding_im[:im_h, :im_w, :] = im
  296. if label_info is None:
  297. return (padding_im, im_info)
  298. else:
  299. return (padding_im, im_info, label_info)
  300. class Resize(DetTransform):
  301. """调整图像大小(resize)。
  302. - 当目标大小(target_size)类型为int时,根据插值方式,
  303. 将图像resize为[target_size, target_size]。
  304. - 当目标大小(target_size)类型为list或tuple时,根据插值方式,
  305. 将图像resize为target_size。
  306. 注意:当插值方式为“RANDOM”时,则随机选取一种插值方式进行resize。
  307. Args:
  308. target_size (int/list/tuple): 短边目标长度。默认为608。
  309. interp (str): resize的插值方式,与opencv的插值方式对应,取值范围为
  310. ['NEAREST', 'LINEAR', 'CUBIC', 'AREA', 'LANCZOS4', 'RANDOM']。默认为"LINEAR"。
  311. Raises:
  312. TypeError: 形参数据类型不满足需求。
  313. ValueError: 插值方式不在['NEAREST', 'LINEAR', 'CUBIC',
  314. 'AREA', 'LANCZOS4', 'RANDOM']中。
  315. """
  316. # The interpolation mode
  317. interp_dict = {
  318. 'NEAREST': cv2.INTER_NEAREST,
  319. 'LINEAR': cv2.INTER_LINEAR,
  320. 'CUBIC': cv2.INTER_CUBIC,
  321. 'AREA': cv2.INTER_AREA,
  322. 'LANCZOS4': cv2.INTER_LANCZOS4
  323. }
  324. def __init__(self, target_size=608, interp='LINEAR'):
  325. self.interp = interp
  326. if not (interp == "RANDOM" or interp in self.interp_dict):
  327. raise ValueError("interp should be one of {}".format(
  328. self.interp_dict.keys()))
  329. if isinstance(target_size, list) or isinstance(target_size, tuple):
  330. if len(target_size) != 2:
  331. raise TypeError(
  332. 'when target is list or tuple, it should include 2 elements, but it is {}'
  333. .format(target_size))
  334. elif not isinstance(target_size, int):
  335. raise TypeError(
  336. "Type of target_size is invalid. Must be Integer or List or tuple, now is {}"
  337. .format(type(target_size)))
  338. self.target_size = target_size
  339. def __call__(self, im, im_info=None, label_info=None):
  340. """
  341. Args:
  342. im (np.ndarray): 图像np.ndarray数据。
  343. im_info (dict, 可选): 存储与图像相关的信息。
  344. label_info (dict, 可选): 存储与标注框相关的信息。
  345. Returns:
  346. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  347. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  348. 存储与标注框相关信息的字典。
  349. Raises:
  350. TypeError: 形参数据类型不满足需求。
  351. ValueError: 数据长度不匹配。
  352. """
  353. if im_info is None:
  354. im_info = dict()
  355. if not isinstance(im, np.ndarray):
  356. raise TypeError("Resize: image type is not numpy.")
  357. if len(im.shape) != 3:
  358. raise ValueError('Resize: image is not 3-dimensional.')
  359. if self.interp == "RANDOM":
  360. interp = random.choice(list(self.interp_dict.keys()))
  361. else:
  362. interp = self.interp
  363. im = resize(im, self.target_size, self.interp_dict[interp])
  364. if label_info is None:
  365. return (im, im_info)
  366. else:
  367. return (im, im_info, label_info)
  368. class RandomHorizontalFlip(DetTransform):
  369. """随机翻转图像、标注框、分割信息,模型训练时的数据增强操作。
  370. 1. 随机采样一个0-1之间的小数,当小数小于水平翻转概率时,
  371. 执行2-4步操作,否则直接返回。
  372. 2. 水平翻转图像。
  373. 3. 计算翻转后的真实标注框的坐标,更新label_info中的gt_bbox信息。
  374. 4. 计算翻转后的真实分割区域的坐标,更新label_info中的gt_poly信息。
  375. Args:
  376. prob (float): 随机水平翻转的概率。默认为0.5。
  377. Raises:
  378. TypeError: 形参数据类型不满足需求。
  379. """
  380. def __init__(self, prob=0.5):
  381. self.prob = prob
  382. if not isinstance(self.prob, float):
  383. raise TypeError("RandomHorizontalFlip: input type is invalid.")
  384. def __call__(self, im, im_info=None, label_info=None):
  385. """
  386. Args:
  387. im (np.ndarray): 图像np.ndarray数据。
  388. im_info (dict, 可选): 存储与图像相关的信息。
  389. label_info (dict, 可选): 存储与标注框相关的信息。
  390. Returns:
  391. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  392. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  393. 存储与标注框相关信息的字典。
  394. 其中,im_info更新字段为:
  395. - gt_bbox (np.ndarray): 水平翻转后的标注框坐标[x1, y1, x2, y2],形状为(n, 4),
  396. 其中n代表真实标注框的个数。
  397. - gt_poly (list): 水平翻转后的多边形分割区域的x、y坐标,长度为n,
  398. 其中n代表真实标注框的个数。
  399. Raises:
  400. TypeError: 形参数据类型不满足需求。
  401. ValueError: 数据长度不匹配。
  402. """
  403. if not isinstance(im, np.ndarray):
  404. raise TypeError(
  405. "RandomHorizontalFlip: image is not a numpy array.")
  406. if len(im.shape) != 3:
  407. raise ValueError(
  408. "RandomHorizontalFlip: image is not 3-dimensional.")
  409. if im_info is None or label_info is None:
  410. raise TypeError(
  411. 'Cannot do RandomHorizontalFlip! ' +
  412. 'Becasuse the im_info and label_info can not be None!')
  413. if 'gt_bbox' not in label_info:
  414. raise TypeError('Cannot do RandomHorizontalFlip! ' + \
  415. 'Becasuse gt_bbox is not in label_info!')
  416. image_shape = im_info['image_shape']
  417. gt_bbox = label_info['gt_bbox']
  418. height = image_shape[0]
  419. width = image_shape[1]
  420. if np.random.uniform(0, 1) < self.prob:
  421. im = horizontal_flip(im)
  422. if gt_bbox.shape[0] == 0:
  423. if label_info is None:
  424. return (im, im_info)
  425. else:
  426. return (im, im_info, label_info)
  427. label_info['gt_bbox'] = box_horizontal_flip(gt_bbox, width)
  428. if 'gt_poly' in label_info and \
  429. len(label_info['gt_poly']) != 0:
  430. label_info['gt_poly'] = segms_horizontal_flip(
  431. label_info['gt_poly'], height, width)
  432. if label_info is None:
  433. return (im, im_info)
  434. else:
  435. return (im, im_info, label_info)
  436. class Normalize(DetTransform):
  437. """对图像进行标准化。
  438. 1. 归一化图像到到区间[0.0, 1.0]。
  439. 2. 对图像进行减均值除以标准差操作。
  440. Args:
  441. mean (list): 图像数据集的均值。默认为[0.485, 0.456, 0.406]。
  442. std (list): 图像数据集的标准差。默认为[0.229, 0.224, 0.225]。
  443. Raises:
  444. TypeError: 形参数据类型不满足需求。
  445. """
  446. def __init__(self, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]):
  447. self.mean = mean
  448. self.std = std
  449. if not (isinstance(self.mean, list) and isinstance(self.std, list)):
  450. raise TypeError("NormalizeImage: input type is invalid.")
  451. from functools import reduce
  452. if reduce(lambda x, y: x * y, self.std) == 0:
  453. raise TypeError('NormalizeImage: std is invalid!')
  454. def __call__(self, im, im_info=None, label_info=None):
  455. """
  456. Args:
  457. im (numnp.ndarraypy): 图像np.ndarray数据。
  458. im_info (dict, 可选): 存储与图像相关的信息。
  459. label_info (dict, 可选): 存储与标注框相关的信息。
  460. Returns:
  461. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  462. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  463. 存储与标注框相关信息的字典。
  464. """
  465. mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
  466. std = np.array(self.std)[np.newaxis, np.newaxis, :]
  467. im = normalize(im, mean, std)
  468. if label_info is None:
  469. return (im, im_info)
  470. else:
  471. return (im, im_info, label_info)
  472. class RandomDistort(DetTransform):
  473. """以一定的概率对图像进行随机像素内容变换,模型训练时的数据增强操作
  474. 1. 对变换的操作顺序进行随机化操作。
  475. 2. 按照1中的顺序以一定的概率在范围[-range, range]对图像进行随机像素内容变换。
  476. Args:
  477. brightness_range (float): 明亮度因子的范围。默认为0.5。
  478. brightness_prob (float): 随机调整明亮度的概率。默认为0.5。
  479. contrast_range (float): 对比度因子的范围。默认为0.5。
  480. contrast_prob (float): 随机调整对比度的概率。默认为0.5。
  481. saturation_range (float): 饱和度因子的范围。默认为0.5。
  482. saturation_prob (float): 随机调整饱和度的概率。默认为0.5。
  483. hue_range (int): 色调因子的范围。默认为18。
  484. hue_prob (float): 随机调整色调的概率。默认为0.5。
  485. """
  486. def __init__(self,
  487. brightness_range=0.5,
  488. brightness_prob=0.5,
  489. contrast_range=0.5,
  490. contrast_prob=0.5,
  491. saturation_range=0.5,
  492. saturation_prob=0.5,
  493. hue_range=18,
  494. hue_prob=0.5):
  495. self.brightness_range = brightness_range
  496. self.brightness_prob = brightness_prob
  497. self.contrast_range = contrast_range
  498. self.contrast_prob = contrast_prob
  499. self.saturation_range = saturation_range
  500. self.saturation_prob = saturation_prob
  501. self.hue_range = hue_range
  502. self.hue_prob = hue_prob
  503. def __call__(self, im, im_info=None, label_info=None):
  504. """
  505. Args:
  506. im (np.ndarray): 图像np.ndarray数据。
  507. im_info (dict, 可选): 存储与图像相关的信息。
  508. label_info (dict, 可选): 存储与标注框相关的信息。
  509. Returns:
  510. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  511. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  512. 存储与标注框相关信息的字典。
  513. """
  514. brightness_lower = 1 - self.brightness_range
  515. brightness_upper = 1 + self.brightness_range
  516. contrast_lower = 1 - self.contrast_range
  517. contrast_upper = 1 + self.contrast_range
  518. saturation_lower = 1 - self.saturation_range
  519. saturation_upper = 1 + self.saturation_range
  520. hue_lower = -self.hue_range
  521. hue_upper = self.hue_range
  522. ops = [brightness, contrast, saturation, hue]
  523. random.shuffle(ops)
  524. params_dict = {
  525. 'brightness': {
  526. 'brightness_lower': brightness_lower,
  527. 'brightness_upper': brightness_upper
  528. },
  529. 'contrast': {
  530. 'contrast_lower': contrast_lower,
  531. 'contrast_upper': contrast_upper
  532. },
  533. 'saturation': {
  534. 'saturation_lower': saturation_lower,
  535. 'saturation_upper': saturation_upper
  536. },
  537. 'hue': {
  538. 'hue_lower': hue_lower,
  539. 'hue_upper': hue_upper
  540. }
  541. }
  542. prob_dict = {
  543. 'brightness': self.brightness_prob,
  544. 'contrast': self.contrast_prob,
  545. 'saturation': self.saturation_prob,
  546. 'hue': self.hue_prob
  547. }
  548. for id in range(4):
  549. params = params_dict[ops[id].__name__]
  550. prob = prob_dict[ops[id].__name__]
  551. params['im'] = im
  552. if np.random.uniform(0, 1) < prob:
  553. im = ops[id](**params)
  554. if label_info is None:
  555. return (im, im_info)
  556. else:
  557. return (im, im_info, label_info)
  558. class MixupImage(DetTransform):
  559. """对图像进行mixup操作,模型训练时的数据增强操作,目前仅YOLOv3模型支持该transform。
  560. 当label_info中不存在mixup字段时,直接返回,否则进行下述操作:
  561. 1. 从随机beta分布中抽取出随机因子factor。
  562. 2.
  563. - 当factor>=1.0时,去除label_info中的mixup字段,直接返回。
  564. - 当factor<=0.0时,直接返回label_info中的mixup字段,并在label_info中去除该字段。
  565. - 其余情况,执行下述操作:
  566. (1)原图像乘以factor,mixup图像乘以(1-factor),叠加2个结果。
  567. (2)拼接原图像标注框和mixup图像标注框。
  568. (3)拼接原图像标注框类别和mixup图像标注框类别。
  569. (4)原图像标注框混合得分乘以factor,mixup图像标注框混合得分乘以(1-factor),叠加2个结果。
  570. 3. 更新im_info中的image_shape信息。
  571. Args:
  572. alpha (float): 随机beta分布的下限。默认为1.5。
  573. beta (float): 随机beta分布的上限。默认为1.5。
  574. mixup_epoch (int): 在前mixup_epoch轮使用mixup增强操作;当该参数为-1时,该策略不会生效。
  575. 默认为-1。
  576. Raises:
  577. ValueError: 数据长度不匹配。
  578. """
  579. def __init__(self, alpha=1.5, beta=1.5, mixup_epoch=-1):
  580. self.alpha = alpha
  581. self.beta = beta
  582. if self.alpha <= 0.0:
  583. raise ValueError("alpha shold be positive in MixupImage")
  584. if self.beta <= 0.0:
  585. raise ValueError("beta shold be positive in MixupImage")
  586. self.mixup_epoch = mixup_epoch
  587. def _mixup_img(self, img1, img2, factor):
  588. h = max(img1.shape[0], img2.shape[0])
  589. w = max(img1.shape[1], img2.shape[1])
  590. img = np.zeros((h, w, img1.shape[2]), 'float32')
  591. img[:img1.shape[0], :img1.shape[1], :] = \
  592. img1.astype('float32') * factor
  593. img[:img2.shape[0], :img2.shape[1], :] += \
  594. img2.astype('float32') * (1.0 - factor)
  595. return img.astype('float32')
  596. def __call__(self, im, im_info=None, label_info=None):
  597. """
  598. Args:
  599. im (np.ndarray): 图像np.ndarray数据。
  600. im_info (dict, 可选): 存储与图像相关的信息。
  601. label_info (dict, 可选): 存储与标注框相关的信息。
  602. Returns:
  603. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  604. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  605. 存储与标注框相关信息的字典。
  606. 其中,im_info更新字段为:
  607. - image_shape (np.ndarray): mixup后的图像高、宽二者组成的np.ndarray,形状为(2,)。
  608. im_info删除的字段:
  609. - mixup (list): 与当前字段进行mixup的图像相关信息。
  610. label_info更新字段为:
  611. - gt_bbox (np.ndarray): mixup后真实标注框坐标,形状为(n, 4),
  612. 其中n代表真实标注框的个数。
  613. - gt_class (np.ndarray): mixup后每个真实标注框对应的类别序号,形状为(n, 1),
  614. 其中n代表真实标注框的个数。
  615. - gt_score (np.ndarray): mixup后每个真实标注框对应的混合得分,形状为(n, 1),
  616. 其中n代表真实标注框的个数。
  617. Raises:
  618. TypeError: 形参数据类型不满足需求。
  619. """
  620. if im_info is None:
  621. raise TypeError('Cannot do MixupImage! ' +
  622. 'Becasuse the im_info can not be None!')
  623. if 'mixup' not in im_info:
  624. if label_info is None:
  625. return (im, im_info)
  626. else:
  627. return (im, im_info, label_info)
  628. factor = np.random.beta(self.alpha, self.beta)
  629. factor = max(0.0, min(1.0, factor))
  630. if im_info['epoch'] > self.mixup_epoch \
  631. or factor >= 1.0:
  632. im_info.pop('mixup')
  633. if label_info is None:
  634. return (im, im_info)
  635. else:
  636. return (im, im_info, label_info)
  637. if factor <= 0.0:
  638. return im_info.pop('mixup')
  639. im = self._mixup_img(im, im_info['mixup'][0], factor)
  640. if label_info is None:
  641. raise TypeError('Cannot do MixupImage! ' +
  642. 'Becasuse the label_info can not be None!')
  643. if 'gt_bbox' not in label_info or \
  644. 'gt_class' not in label_info or \
  645. 'gt_score' not in label_info:
  646. raise TypeError('Cannot do MixupImage! ' + \
  647. 'Becasuse gt_bbox/gt_class/gt_score is not in label_info!')
  648. gt_bbox1 = label_info['gt_bbox']
  649. gt_bbox2 = im_info['mixup'][2]['gt_bbox']
  650. gt_class1 = label_info['gt_class']
  651. gt_class2 = im_info['mixup'][2]['gt_class']
  652. gt_score1 = label_info['gt_score']
  653. gt_score2 = im_info['mixup'][2]['gt_score']
  654. if 'gt_poly' in label_info:
  655. gt_poly1 = label_info['gt_poly']
  656. gt_poly2 = im_info['mixup'][2]['gt_poly']
  657. is_crowd1 = label_info['is_crowd']
  658. is_crowd2 = im_info['mixup'][2]['is_crowd']
  659. if 0 not in gt_class1 and 0 not in gt_class2:
  660. gt_bbox = np.concatenate((gt_bbox1, gt_bbox2), axis=0)
  661. gt_class = np.concatenate((gt_class1, gt_class2), axis=0)
  662. gt_score = np.concatenate(
  663. (gt_score1 * factor, gt_score2 * (1. - factor)), axis=0)
  664. if 'gt_poly' in label_info:
  665. label_info['gt_poly'] = gt_poly1 + gt_poly2
  666. is_crowd = np.concatenate((is_crowd1, is_crowd2), axis=0)
  667. elif 0 in gt_class1:
  668. gt_bbox = gt_bbox2
  669. gt_class = gt_class2
  670. gt_score = gt_score2 * (1. - factor)
  671. if 'gt_poly' in label_info:
  672. label_info['gt_poly'] = gt_poly2
  673. is_crowd = is_crowd2
  674. else:
  675. gt_bbox = gt_bbox1
  676. gt_class = gt_class1
  677. gt_score = gt_score1 * factor
  678. if 'gt_poly' in label_info:
  679. label_info['gt_poly'] = gt_poly1
  680. is_crowd = is_crowd1
  681. label_info['gt_bbox'] = gt_bbox
  682. label_info['gt_score'] = gt_score
  683. label_info['gt_class'] = gt_class
  684. label_info['is_crowd'] = is_crowd
  685. im_info['image_shape'] = np.array([im.shape[0],
  686. im.shape[1]]).astype('int32')
  687. im_info.pop('mixup')
  688. if label_info is None:
  689. return (im, im_info)
  690. else:
  691. return (im, im_info, label_info)
  692. class RandomExpand(DetTransform):
  693. """随机扩张图像,模型训练时的数据增强操作。
  694. 1. 随机选取扩张比例(扩张比例大于1时才进行扩张)。
  695. 2. 计算扩张后图像大小。
  696. 3. 初始化像素值为输入填充值的图像,并将原图像随机粘贴于该图像上。
  697. 4. 根据原图像粘贴位置换算出扩张后真实标注框的位置坐标。
  698. 5. 根据原图像粘贴位置换算出扩张后真实分割区域的位置坐标。
  699. Args:
  700. ratio (float): 图像扩张的最大比例。默认为4.0。
  701. prob (float): 随机扩张的概率。默认为0.5。
  702. fill_value (list): 扩张图像的初始填充值(0-255)。默认为[123.675, 116.28, 103.53]。
  703. """
  704. def __init__(self,
  705. ratio=4.,
  706. prob=0.5,
  707. fill_value=[123.675, 116.28, 103.53]):
  708. super(RandomExpand, self).__init__()
  709. assert ratio > 1.01, "expand ratio must be larger than 1.01"
  710. self.ratio = ratio
  711. self.prob = prob
  712. assert isinstance(fill_value, Sequence), \
  713. "fill value must be sequence"
  714. if not isinstance(fill_value, tuple):
  715. fill_value = tuple(fill_value)
  716. self.fill_value = fill_value
  717. def __call__(self, im, im_info=None, label_info=None):
  718. """
  719. Args:
  720. im (np.ndarray): 图像np.ndarray数据。
  721. im_info (dict, 可选): 存储与图像相关的信息。
  722. label_info (dict, 可选): 存储与标注框相关的信息。
  723. Returns:
  724. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  725. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  726. 存储与标注框相关信息的字典。
  727. 其中,im_info更新字段为:
  728. - image_shape (np.ndarray): 扩张后的图像高、宽二者组成的np.ndarray,形状为(2,)。
  729. label_info更新字段为:
  730. - gt_bbox (np.ndarray): 随机扩张后真实标注框坐标,形状为(n, 4),
  731. 其中n代表真实标注框的个数。
  732. - gt_class (np.ndarray): 随机扩张后每个真实标注框对应的类别序号,形状为(n, 1),
  733. 其中n代表真实标注框的个数。
  734. Raises:
  735. TypeError: 形参数据类型不满足需求。
  736. """
  737. if im_info is None or label_info is None:
  738. raise TypeError(
  739. 'Cannot do RandomExpand! ' +
  740. 'Becasuse the im_info and label_info can not be None!')
  741. if 'gt_bbox' not in label_info or \
  742. 'gt_class' not in label_info:
  743. raise TypeError('Cannot do RandomExpand! ' + \
  744. 'Becasuse gt_bbox/gt_class is not in label_info!')
  745. if np.random.uniform(0., 1.) < self.prob:
  746. return (im, im_info, label_info)
  747. if 'gt_class' in label_info and 0 in label_info['gt_class']:
  748. return (im, im_info, label_info)
  749. image_shape = im_info['image_shape']
  750. height = int(image_shape[0])
  751. width = int(image_shape[1])
  752. expand_ratio = np.random.uniform(1., self.ratio)
  753. h = int(height * expand_ratio)
  754. w = int(width * expand_ratio)
  755. if not h > height or not w > width:
  756. return (im, im_info, label_info)
  757. y = np.random.randint(0, h - height)
  758. x = np.random.randint(0, w - width)
  759. canvas = np.ones((h, w, 3), dtype=np.float32)
  760. canvas *= np.array(self.fill_value, dtype=np.float32)
  761. canvas[y:y + height, x:x + width, :] = im
  762. im_info['image_shape'] = np.array([h, w]).astype('int32')
  763. if 'gt_bbox' in label_info and len(label_info['gt_bbox']) > 0:
  764. label_info['gt_bbox'] += np.array([x, y] * 2, dtype=np.float32)
  765. if 'gt_poly' in label_info and len(label_info['gt_poly']) > 0:
  766. label_info['gt_poly'] = expand_segms(label_info['gt_poly'], x, y,
  767. height, width, expand_ratio)
  768. return (canvas, im_info, label_info)
  769. class RandomCrop(DetTransform):
  770. """随机裁剪图像。
  771. 1. 若allow_no_crop为True,则在thresholds加入’no_crop’。
  772. 2. 随机打乱thresholds。
  773. 3. 遍历thresholds中各元素:
  774. (1) 如果当前thresh为’no_crop’,则返回原始图像和标注信息。
  775. (2) 随机取出aspect_ratio和scaling中的值并由此计算出候选裁剪区域的高、宽、起始点。
  776. (3) 计算真实标注框与候选裁剪区域IoU,若全部真实标注框的IoU都小于thresh,则继续第3步。
  777. (4) 如果cover_all_box为True且存在真实标注框的IoU小于thresh,则继续第3步。
  778. (5) 筛选出位于候选裁剪区域内的真实标注框,若有效框的个数为0,则继续第3步,否则进行第4步。
  779. 4. 换算有效真值标注框相对候选裁剪区域的位置坐标。
  780. 5. 换算有效分割区域相对候选裁剪区域的位置坐标。
  781. Args:
  782. aspect_ratio (list): 裁剪后短边缩放比例的取值范围,以[min, max]形式表示。默认值为[.5, 2.]。
  783. thresholds (list): 判断裁剪候选区域是否有效所需的IoU阈值取值列表。默认值为[.0, .1, .3, .5, .7, .9]。
  784. scaling (list): 裁剪面积相对原面积的取值范围,以[min, max]形式表示。默认值为[.3, 1.]。
  785. num_attempts (int): 在放弃寻找有效裁剪区域前尝试的次数。默认值为50。
  786. allow_no_crop (bool): 是否允许未进行裁剪。默认值为True。
  787. cover_all_box (bool): 是否要求所有的真实标注框都必须在裁剪区域内。默认值为False。
  788. """
  789. def __init__(self,
  790. aspect_ratio=[.5, 2.],
  791. thresholds=[.0, .1, .3, .5, .7, .9],
  792. scaling=[.3, 1.],
  793. num_attempts=50,
  794. allow_no_crop=True,
  795. cover_all_box=False):
  796. self.aspect_ratio = aspect_ratio
  797. self.thresholds = thresholds
  798. self.scaling = scaling
  799. self.num_attempts = num_attempts
  800. self.allow_no_crop = allow_no_crop
  801. self.cover_all_box = cover_all_box
  802. def __call__(self, im, im_info=None, label_info=None):
  803. """
  804. Args:
  805. im (np.ndarray): 图像np.ndarray数据。
  806. im_info (dict, 可选): 存储与图像相关的信息。
  807. label_info (dict, 可选): 存储与标注框相关的信息。
  808. Returns:
  809. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  810. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  811. 存储与标注框相关信息的字典。
  812. 其中,im_info更新字段为:
  813. - image_shape (np.ndarray): 扩裁剪的图像高、宽二者组成的np.ndarray,形状为(2,)。
  814. label_info更新字段为:
  815. - gt_bbox (np.ndarray): 随机裁剪后真实标注框坐标,形状为(n, 4),
  816. 其中n代表真实标注框的个数。
  817. - gt_class (np.ndarray): 随机裁剪后每个真实标注框对应的类别序号,形状为(n, 1),
  818. 其中n代表真实标注框的个数。
  819. - gt_score (np.ndarray): 随机裁剪后每个真实标注框对应的混合得分,形状为(n, 1),
  820. 其中n代表真实标注框的个数。
  821. Raises:
  822. TypeError: 形参数据类型不满足需求。
  823. """
  824. if im_info is None or label_info is None:
  825. raise TypeError(
  826. 'Cannot do RandomCrop! ' +
  827. 'Becasuse the im_info and label_info can not be None!')
  828. if 'gt_bbox' not in label_info or \
  829. 'gt_class' not in label_info:
  830. raise TypeError('Cannot do RandomCrop! ' + \
  831. 'Becasuse gt_bbox/gt_class is not in label_info!')
  832. if len(label_info['gt_bbox']) == 0:
  833. return (im, im_info, label_info)
  834. if 'gt_class' in label_info and 0 in label_info['gt_class']:
  835. return (im, im_info, label_info)
  836. image_shape = im_info['image_shape']
  837. w = image_shape[1]
  838. h = image_shape[0]
  839. gt_bbox = label_info['gt_bbox']
  840. thresholds = list(self.thresholds)
  841. if self.allow_no_crop:
  842. thresholds.append('no_crop')
  843. np.random.shuffle(thresholds)
  844. for thresh in thresholds:
  845. if thresh == 'no_crop':
  846. return (im, im_info, label_info)
  847. found = False
  848. for i in range(self.num_attempts):
  849. scale = np.random.uniform(*self.scaling)
  850. min_ar, max_ar = self.aspect_ratio
  851. aspect_ratio = np.random.uniform(
  852. max(min_ar, scale**2), min(max_ar, scale**-2))
  853. crop_h = int(h * scale / np.sqrt(aspect_ratio))
  854. crop_w = int(w * scale * np.sqrt(aspect_ratio))
  855. crop_y = np.random.randint(0, h - crop_h)
  856. crop_x = np.random.randint(0, w - crop_w)
  857. crop_box = [crop_x, crop_y, crop_x + crop_w, crop_y + crop_h]
  858. iou = iou_matrix(
  859. gt_bbox, np.array(
  860. [crop_box], dtype=np.float32))
  861. if iou.max() < thresh:
  862. continue
  863. if self.cover_all_box and iou.min() < thresh:
  864. continue
  865. cropped_box, valid_ids = crop_box_with_center_constraint(
  866. gt_bbox, np.array(
  867. crop_box, dtype=np.float32))
  868. if valid_ids.size > 0:
  869. found = True
  870. break
  871. if found:
  872. if 'gt_poly' in label_info and len(label_info['gt_poly']) > 0:
  873. crop_polys = crop_segms(
  874. label_info['gt_poly'],
  875. valid_ids,
  876. np.array(
  877. crop_box, dtype=np.int64),
  878. h,
  879. w)
  880. if [] in crop_polys:
  881. delete_id = list()
  882. valid_polys = list()
  883. for id, crop_poly in enumerate(crop_polys):
  884. if crop_poly == []:
  885. delete_id.append(id)
  886. else:
  887. valid_polys.append(crop_poly)
  888. valid_ids = np.delete(valid_ids, delete_id)
  889. if len(valid_polys) == 0:
  890. return (im, im_info, label_info)
  891. label_info['gt_poly'] = valid_polys
  892. else:
  893. label_info['gt_poly'] = crop_polys
  894. im = crop_image(im, crop_box)
  895. label_info['gt_bbox'] = np.take(cropped_box, valid_ids, axis=0)
  896. label_info['gt_class'] = np.take(
  897. label_info['gt_class'], valid_ids, axis=0)
  898. im_info['image_shape'] = np.array(
  899. [crop_box[3] - crop_box[1],
  900. crop_box[2] - crop_box[0]]).astype('int32')
  901. if 'gt_score' in label_info:
  902. label_info['gt_score'] = np.take(
  903. label_info['gt_score'], valid_ids, axis=0)
  904. if 'is_crowd' in label_info:
  905. label_info['is_crowd'] = np.take(
  906. label_info['is_crowd'], valid_ids, axis=0)
  907. return (im, im_info, label_info)
  908. return (im, im_info, label_info)
  909. class ArrangeFasterRCNN(DetTransform):
  910. """获取FasterRCNN模型训练/验证/预测所需信息。
  911. Args:
  912. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  913. Raises:
  914. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  915. """
  916. def __init__(self, mode=None):
  917. if mode not in ['train', 'eval', 'test', 'quant']:
  918. raise ValueError(
  919. "mode must be in ['train', 'eval', 'test', 'quant']!")
  920. self.mode = mode
  921. def __call__(self, im, im_info=None, label_info=None):
  922. """
  923. Args:
  924. im (np.ndarray): 图像np.ndarray数据。
  925. im_info (dict, 可选): 存储与图像相关的信息。
  926. label_info (dict, 可选): 存储与标注框相关的信息。
  927. Returns:
  928. tuple: 当mode为'train'时,返回(im, im_resize_info, gt_bbox, gt_class, is_crowd),分别对应
  929. 图像np.ndarray数据、图像相当对于原图的resize信息、真实标注框、真实标注框对应的类别、真实标注框内是否是一组对象;
  930. 当mode为'eval'时,返回(im, im_resize_info, im_id, im_shape, gt_bbox, gt_class, is_difficult),
  931. 分别对应图像np.ndarray数据、图像相当对于原图的resize信息、图像id、图像大小信息、真实标注框、真实标注框对应的类别、
  932. 真实标注框是否为难识别对象;当mode为'test'或'quant'时,返回(im, im_resize_info, im_shape),分别对应图像np.ndarray数据、
  933. 图像相当对于原图的resize信息、图像大小信息。
  934. Raises:
  935. TypeError: 形参数据类型不满足需求。
  936. ValueError: 数据长度不匹配。
  937. """
  938. im = permute(im, False)
  939. if self.mode == 'train':
  940. if im_info is None or label_info is None:
  941. raise TypeError(
  942. 'Cannot do ArrangeFasterRCNN! ' +
  943. 'Becasuse the im_info and label_info can not be None!')
  944. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  945. raise ValueError("gt num mismatch: bbox and class.")
  946. im_resize_info = im_info['im_resize_info']
  947. gt_bbox = label_info['gt_bbox']
  948. gt_class = label_info['gt_class']
  949. is_crowd = label_info['is_crowd']
  950. outputs = (im, im_resize_info, gt_bbox, gt_class, is_crowd)
  951. elif self.mode == 'eval':
  952. if im_info is None or label_info is None:
  953. raise TypeError(
  954. 'Cannot do ArrangeFasterRCNN! ' +
  955. 'Becasuse the im_info and label_info can not be None!')
  956. im_resize_info = im_info['im_resize_info']
  957. im_id = im_info['im_id']
  958. im_shape = np.array(
  959. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  960. dtype=np.float32)
  961. gt_bbox = label_info['gt_bbox']
  962. gt_class = label_info['gt_class']
  963. is_difficult = label_info['difficult']
  964. outputs = (im, im_resize_info, im_id, im_shape, gt_bbox, gt_class,
  965. is_difficult)
  966. else:
  967. if im_info is None:
  968. raise TypeError('Cannot do ArrangeFasterRCNN! ' +
  969. 'Becasuse the im_info can not be None!')
  970. im_resize_info = im_info['im_resize_info']
  971. im_shape = np.array(
  972. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  973. dtype=np.float32)
  974. outputs = (im, im_resize_info, im_shape)
  975. return outputs
  976. class ArrangeMaskRCNN(DetTransform):
  977. """获取MaskRCNN模型训练/验证/预测所需信息。
  978. Args:
  979. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  980. Raises:
  981. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  982. """
  983. def __init__(self, mode=None):
  984. if mode not in ['train', 'eval', 'test', 'quant']:
  985. raise ValueError(
  986. "mode must be in ['train', 'eval', 'test', 'quant']!")
  987. self.mode = mode
  988. def __call__(self, im, im_info=None, label_info=None):
  989. """
  990. Args:
  991. im (np.ndarray): 图像np.ndarray数据。
  992. im_info (dict, 可选): 存储与图像相关的信息。
  993. label_info (dict, 可选): 存储与标注框相关的信息。
  994. Returns:
  995. tuple: 当mode为'train'时,返回(im, im_resize_info, gt_bbox, gt_class, is_crowd, gt_masks),分别对应
  996. 图像np.ndarray数据、图像相当对于原图的resize信息、真实标注框、真实标注框对应的类别、真实标注框内是否是一组对象、
  997. 真实分割区域;当mode为'eval'时,返回(im, im_resize_info, im_id, im_shape),分别对应图像np.ndarray数据、
  998. 图像相当对于原图的resize信息、图像id、图像大小信息;当mode为'test'或'quant'时,返回(im, im_resize_info, im_shape),
  999. 分别对应图像np.ndarray数据、图像相当对于原图的resize信息、图像大小信息。
  1000. Raises:
  1001. TypeError: 形参数据类型不满足需求。
  1002. ValueError: 数据长度不匹配。
  1003. """
  1004. im = permute(im, False)
  1005. if self.mode == 'train':
  1006. if im_info is None or label_info is None:
  1007. raise TypeError(
  1008. 'Cannot do ArrangeTrainMaskRCNN! ' +
  1009. 'Becasuse the im_info and label_info can not be None!')
  1010. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1011. raise ValueError("gt num mismatch: bbox and class.")
  1012. im_resize_info = im_info['im_resize_info']
  1013. gt_bbox = label_info['gt_bbox']
  1014. gt_class = label_info['gt_class']
  1015. is_crowd = label_info['is_crowd']
  1016. assert 'gt_poly' in label_info
  1017. segms = label_info['gt_poly']
  1018. if len(segms) != 0:
  1019. assert len(segms) == is_crowd.shape[0]
  1020. gt_masks = []
  1021. valid = True
  1022. for i in range(len(segms)):
  1023. segm = segms[i]
  1024. gt_segm = []
  1025. if is_crowd[i]:
  1026. gt_segm.append([[0, 0]])
  1027. else:
  1028. for poly in segm:
  1029. if len(poly) == 0:
  1030. valid = False
  1031. break
  1032. gt_segm.append(np.array(poly).reshape(-1, 2))
  1033. if (not valid) or len(gt_segm) == 0:
  1034. break
  1035. gt_masks.append(gt_segm)
  1036. outputs = (im, im_resize_info, gt_bbox, gt_class, is_crowd,
  1037. gt_masks)
  1038. else:
  1039. if im_info is None:
  1040. raise TypeError('Cannot do ArrangeMaskRCNN! ' +
  1041. 'Becasuse the im_info can not be None!')
  1042. im_resize_info = im_info['im_resize_info']
  1043. im_shape = np.array(
  1044. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  1045. dtype=np.float32)
  1046. if self.mode == 'eval':
  1047. im_id = im_info['im_id']
  1048. outputs = (im, im_resize_info, im_id, im_shape)
  1049. else:
  1050. outputs = (im, im_resize_info, im_shape)
  1051. return outputs
  1052. class ArrangeYOLOv3(DetTransform):
  1053. """获取YOLOv3模型训练/验证/预测所需信息。
  1054. Args:
  1055. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  1056. Raises:
  1057. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  1058. """
  1059. def __init__(self, mode=None):
  1060. if mode not in ['train', 'eval', 'test', 'quant']:
  1061. raise ValueError(
  1062. "mode must be in ['train', 'eval', 'test', 'quant']!")
  1063. self.mode = mode
  1064. def __call__(self, im, im_info=None, label_info=None):
  1065. """
  1066. Args:
  1067. im (np.ndarray): 图像np.ndarray数据。
  1068. im_info (dict, 可选): 存储与图像相关的信息。
  1069. label_info (dict, 可选): 存储与标注框相关的信息。
  1070. Returns:
  1071. tuple: 当mode为'train'时,返回(im, gt_bbox, gt_class, gt_score, im_shape),分别对应
  1072. 图像np.ndarray数据、真实标注框、真实标注框对应的类别、真实标注框混合得分、图像大小信息;
  1073. 当mode为'eval'时,返回(im, im_shape, im_id, gt_bbox, gt_class, difficult),
  1074. 分别对应图像np.ndarray数据、图像大小信息、图像id、真实标注框、真实标注框对应的类别、
  1075. 真实标注框是否为难识别对象;当mode为'test'或'quant'时,返回(im, im_shape),
  1076. 分别对应图像np.ndarray数据、图像大小信息。
  1077. Raises:
  1078. TypeError: 形参数据类型不满足需求。
  1079. ValueError: 数据长度不匹配。
  1080. """
  1081. im = permute(im, False)
  1082. if self.mode == 'train':
  1083. if im_info is None or label_info is None:
  1084. raise TypeError(
  1085. 'Cannot do ArrangeYolov3! ' +
  1086. 'Becasuse the im_info and label_info can not be None!')
  1087. im_shape = im_info['image_shape']
  1088. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1089. raise ValueError("gt num mismatch: bbox and class.")
  1090. if len(label_info['gt_bbox']) != len(label_info['gt_score']):
  1091. raise ValueError("gt num mismatch: bbox and score.")
  1092. gt_bbox = np.zeros((50, 4), dtype=im.dtype)
  1093. gt_class = np.zeros((50, ), dtype=np.int32)
  1094. gt_score = np.zeros((50, ), dtype=im.dtype)
  1095. gt_num = min(50, len(label_info['gt_bbox']))
  1096. if gt_num > 0:
  1097. label_info['gt_class'][:gt_num, 0] = label_info[
  1098. 'gt_class'][:gt_num, 0] - 1
  1099. if -1 not in label_info['gt_class']:
  1100. gt_bbox[:gt_num, :] = label_info['gt_bbox'][:gt_num, :]
  1101. gt_class[:gt_num] = label_info['gt_class'][:gt_num, 0]
  1102. gt_score[:gt_num] = label_info['gt_score'][:gt_num, 0]
  1103. # parse [x1, y1, x2, y2] to [x, y, w, h]
  1104. gt_bbox[:, 2:4] = gt_bbox[:, 2:4] - gt_bbox[:, :2]
  1105. gt_bbox[:, :2] = gt_bbox[:, :2] + gt_bbox[:, 2:4] / 2.
  1106. outputs = (im, gt_bbox, gt_class, gt_score, im_shape)
  1107. elif self.mode == 'eval':
  1108. if im_info is None or label_info is None:
  1109. raise TypeError(
  1110. 'Cannot do ArrangeYolov3! ' +
  1111. 'Becasuse the im_info and label_info can not be None!')
  1112. im_shape = im_info['image_shape']
  1113. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1114. raise ValueError("gt num mismatch: bbox and class.")
  1115. im_id = im_info['im_id']
  1116. gt_bbox = np.zeros((50, 4), dtype=im.dtype)
  1117. gt_class = np.zeros((50, ), dtype=np.int32)
  1118. difficult = np.zeros((50, ), dtype=np.int32)
  1119. gt_num = min(50, len(label_info['gt_bbox']))
  1120. if gt_num > 0:
  1121. label_info['gt_class'][:gt_num, 0] = label_info[
  1122. 'gt_class'][:gt_num, 0] - 1
  1123. gt_bbox[:gt_num, :] = label_info['gt_bbox'][:gt_num, :]
  1124. gt_class[:gt_num] = label_info['gt_class'][:gt_num, 0]
  1125. difficult[:gt_num] = label_info['difficult'][:gt_num, 0]
  1126. outputs = (im, im_shape, im_id, gt_bbox, gt_class, difficult)
  1127. else:
  1128. if im_info is None:
  1129. raise TypeError('Cannot do ArrangeYolov3! ' +
  1130. 'Becasuse the im_info can not be None!')
  1131. im_shape = im_info['image_shape']
  1132. outputs = (im, im_shape)
  1133. return outputs
  1134. class ComposedRCNNTransforms(Compose):
  1135. """ RCNN模型(faster-rcnn/mask-rcnn)图像处理流程,具体如下,
  1136. 训练阶段:
  1137. 1. 随机以0.5的概率将图像水平翻转
  1138. 2. 图像归一化
  1139. 3. 图像按比例Resize,scale计算方式如下
  1140. scale = min_max_size[0] / short_size_of_image
  1141. if max_size_of_image * scale > min_max_size[1]:
  1142. scale = min_max_size[1] / max_size_of_image
  1143. 4. 将3步骤的长宽进行padding,使得长宽为32的倍数
  1144. 验证阶段:
  1145. 1. 图像归一化
  1146. 2. 图像按比例Resize,scale计算方式同上训练阶段
  1147. 3. 将2步骤的长宽进行padding,使得长宽为32的倍数
  1148. Args:
  1149. mode(str): 图像处理流程所处阶段,训练/验证/预测,分别对应'train', 'eval', 'test'
  1150. min_max_size(list): 图像在缩放时,最小边和最大边的约束条件
  1151. mean(list): 图像均值
  1152. std(list): 图像方差
  1153. random_horizontal_flip(bool): 是否以0.5的概率使用随机水平翻转增强,该仅在mode为`train`时生效,默认为True
  1154. """
  1155. def __init__(self,
  1156. mode,
  1157. min_max_size=[800, 1333],
  1158. mean=[0.485, 0.456, 0.406],
  1159. std=[0.229, 0.224, 0.225],
  1160. random_horizontal_flip=True):
  1161. if mode == 'train':
  1162. # 训练时的transforms,包含数据增强
  1163. transforms = [
  1164. Normalize(
  1165. mean=mean, std=std), ResizeByShort(
  1166. short_size=min_max_size[0], max_size=min_max_size[1]),
  1167. Padding(coarsest_stride=32)
  1168. ]
  1169. if random_horizontal_flip:
  1170. transforms.insert(0, RandomHorizontalFlip())
  1171. else:
  1172. # 验证/预测时的transforms
  1173. transforms = [
  1174. Normalize(
  1175. mean=mean, std=std), ResizeByShort(
  1176. short_size=min_max_size[0], max_size=min_max_size[1]),
  1177. Padding(coarsest_stride=32)
  1178. ]
  1179. super(ComposedRCNNTransforms, self).__init__(transforms)
  1180. class ComposedYOLOv3Transforms(Compose):
  1181. """YOLOv3模型的图像预处理流程,具体如下,
  1182. 训练阶段:
  1183. 1. 在前mixup_epoch轮迭代中,使用MixupImage策略,见https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/det_transforms.html#mixupimage
  1184. 2. 对图像进行随机扰动,包括亮度,对比度,饱和度和色调
  1185. 3. 随机扩充图像,见https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/det_transforms.html#randomexpand
  1186. 4. 随机裁剪图像
  1187. 5. 将4步骤的输出图像Resize成shape参数的大小
  1188. 6. 随机0.5的概率水平翻转图像
  1189. 7. 图像归一化
  1190. 验证/预测阶段:
  1191. 1. 将图像Resize成shape参数大小
  1192. 2. 图像归一化
  1193. Args:
  1194. mode(str): 图像处理流程所处阶段,训练/验证/预测,分别对应'train', 'eval', 'test'
  1195. shape(list): 输入模型中图像的大小,输入模型的图像会被Resize成此大小
  1196. mixup_epoch(int): 模型训练过程中,前mixup_epoch会使用mixup策略, 若设为-1,则表示不使用该策略
  1197. mean(list): 图像均值
  1198. std(list): 图像方差
  1199. random_distort(bool): 数据增强方式,参数仅在mode为`train`时生效,表示是否在训练过程中随机扰动图像,默认为True
  1200. random_expand(bool): 数据增强方式,参数仅在mode为`train`时生效,表示是否在训练过程中随机扩张图像,默认为True
  1201. random_crop(bool): 数据增强方式,参数仅在mode为`train`时生效,表示是否在训练过程中随机裁剪图像,默认为True
  1202. random_horizontal_flip(bool): 数据增强方式,参数仅在mode为`train`时生效,表示是否在训练过程中随机水平翻转图像,默认为True
  1203. """
  1204. def __init__(self,
  1205. mode,
  1206. shape=[608, 608],
  1207. mixup_epoch=250,
  1208. mean=[0.485, 0.456, 0.406],
  1209. std=[0.229, 0.224, 0.225],
  1210. random_distort=True,
  1211. random_expand=True,
  1212. random_crop=True,
  1213. random_horizontal_flip=True):
  1214. width = shape
  1215. if isinstance(shape, list):
  1216. if shape[0] != shape[1]:
  1217. raise Exception(
  1218. "In YOLOv3 model, width and height should be equal")
  1219. width = shape[0]
  1220. if width % 32 != 0:
  1221. raise Exception(
  1222. "In YOLOv3 model, width and height should be multiple of 32, e.g 224、256、320...."
  1223. )
  1224. if mode == 'train':
  1225. # 训练时的transforms,包含数据增强
  1226. transforms = [
  1227. MixupImage(mixup_epoch=mixup_epoch), Resize(
  1228. target_size=width, interp='RANDOM'), Normalize(
  1229. mean=mean, std=std)
  1230. ]
  1231. if random_horizontal_flip:
  1232. transforms.insert(1, RandomHorizontalFlip())
  1233. if random_crop:
  1234. transforms.insert(1, RandomCrop())
  1235. if random_expand:
  1236. transforms.insert(1, RandomExpand())
  1237. if random_distort:
  1238. transforms.insert(1, RandomDistort())
  1239. else:
  1240. # 验证/预测时的transforms
  1241. transforms = [
  1242. Resize(
  1243. target_size=width, interp='CUBIC'), Normalize(
  1244. mean=mean, std=std)
  1245. ]
  1246. super(ComposedYOLOv3Transforms, self).__init__(transforms)