det_transforms.py 62 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361
  1. # copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. try:
  15. from collections.abc import Sequence
  16. except Exception:
  17. from collections import Sequence
  18. import random
  19. import os.path as osp
  20. import numpy as np
  21. import cv2
  22. from PIL import Image, ImageEnhance
  23. from .imgaug_support import execute_imgaug
  24. from .ops import *
  25. from .box_utils import *
  26. import paddlex.utils.logging as logging
  27. class DetTransform:
  28. """检测数据处理基类
  29. """
  30. def __init__(self):
  31. pass
  32. class Compose(DetTransform):
  33. """根据数据预处理/增强列表对输入数据进行操作。
  34. 所有操作的输入图像流形状均是[H, W, C],其中H为图像高,W为图像宽,C为图像通道数。
  35. Args:
  36. transforms (list): 数据预处理/增强列表。
  37. Raises:
  38. TypeError: 形参数据类型不满足需求。
  39. ValueError: 数据长度不匹配。
  40. """
  41. def __init__(self, transforms):
  42. if not isinstance(transforms, list):
  43. raise TypeError('The transforms must be a list!')
  44. if len(transforms) < 1:
  45. raise ValueError('The length of transforms ' + \
  46. 'must be equal or larger than 1!')
  47. self.transforms = transforms
  48. self.use_mixup = False
  49. for t in self.transforms:
  50. if type(t).__name__ == 'MixupImage':
  51. self.use_mixup = True
  52. # 检查transforms里面的操作,目前支持PaddleX定义的或者是imgaug操作
  53. for op in self.transforms:
  54. if not isinstance(op, DetTransform):
  55. import imgaug.augmenters as iaa
  56. if not isinstance(op, iaa.Augmenter):
  57. raise Exception(
  58. "Elements in transforms should be defined in 'paddlex.det.transforms' or class of imgaug.augmenters.Augmenter, see docs here: https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/"
  59. )
  60. def __call__(self, im, im_info=None, label_info=None, vdl_writer=None, step=0):
  61. """
  62. Args:
  63. im (str/np.ndarray): 图像路径/图像np.ndarray数据。
  64. im_info (dict): 存储与图像相关的信息,dict中的字段如下:
  65. - im_id (np.ndarray): 图像序列号,形状为(1,)。
  66. - image_shape (np.ndarray): 图像原始大小,形状为(2,),
  67. image_shape[0]为高,image_shape[1]为宽。
  68. - mixup (list): list为[im, im_info, label_info],分别对应
  69. 与当前图像进行mixup的图像np.ndarray数据、图像相关信息、标注框相关信息;
  70. 注意,当前epoch若无需进行mixup,则无该字段。
  71. label_info (dict): 存储与标注框相关的信息,dict中的字段如下:
  72. - gt_bbox (np.ndarray): 真实标注框坐标[x1, y1, x2, y2],形状为(n, 4),
  73. 其中n代表真实标注框的个数。
  74. - gt_class (np.ndarray): 每个真实标注框对应的类别序号,形状为(n, 1),
  75. 其中n代表真实标注框的个数。
  76. - gt_score (np.ndarray): 每个真实标注框对应的混合得分,形状为(n, 1),
  77. 其中n代表真实标注框的个数。
  78. - gt_poly (list): 每个真实标注框内的多边形分割区域,每个分割区域由点的x、y坐标组成,
  79. 长度为n,其中n代表真实标注框的个数。
  80. - is_crowd (np.ndarray): 每个真实标注框中是否是一组对象,形状为(n, 1),
  81. 其中n代表真实标注框的个数。
  82. - difficult (np.ndarray): 每个真实标注框中的对象是否为难识别对象,形状为(n, 1),
  83. 其中n代表真实标注框的个数。
  84. vdl_writer (visualdl.LogWriter): VisualDL存储器,日志信息将保存在其中。
  85. 当为None时,不对日志进行保存。默认为None。
  86. step (int): 数据预处理的轮数,当vdl_writer不为None时有效。默认为0。
  87. Returns:
  88. tuple: 根据网络所需字段所组成的tuple;
  89. 字段由transforms中的最后一个数据预处理操作决定。
  90. """
  91. def decode_image(im_file, im_info, label_info):
  92. if im_info is None:
  93. im_info = dict()
  94. if isinstance(im_file, np.ndarray):
  95. if len(im_file.shape) != 3:
  96. raise Exception(
  97. "im should be 3-dimensions, but now is {}-dimensions".
  98. format(len(im_file.shape)))
  99. im = im_file
  100. else:
  101. try:
  102. im = cv2.imread(im_file).astype('float32')
  103. except:
  104. raise TypeError('Can\'t read The image file {}!'.format(
  105. im_file))
  106. im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)
  107. # make default im_info with [h, w, 1]
  108. im_info['im_resize_info'] = np.array(
  109. [im.shape[0], im.shape[1], 1.], dtype=np.float32)
  110. im_info['image_shape'] = np.array([im.shape[0],
  111. im.shape[1]]).astype('int32')
  112. if not self.use_mixup:
  113. if 'mixup' in im_info:
  114. del im_info['mixup']
  115. # decode mixup image
  116. if 'mixup' in im_info:
  117. im_info['mixup'] = \
  118. decode_image(im_info['mixup'][0],
  119. im_info['mixup'][1],
  120. im_info['mixup'][2])
  121. if label_info is None:
  122. return (im, im_info)
  123. else:
  124. return (im, im_info, label_info)
  125. outputs = decode_image(im, im_info, label_info)
  126. im = outputs[0]
  127. im_info = outputs[1]
  128. if len(outputs) == 3:
  129. label_info = outputs[2]
  130. if vdl_writer is not None:
  131. vdl_writer.add_image(tag='0. origin image',
  132. img=im,
  133. step=step)
  134. op_id = 1
  135. for op in self.transforms:
  136. if im is None:
  137. return None
  138. if isinstance(op, DetTransform):
  139. outputs = op(im, im_info, label_info)
  140. im = outputs[0]
  141. else:
  142. im = execute_imgaug(op, im)
  143. if label_info is not None:
  144. outputs = (im, im_info, label_info)
  145. else:
  146. outputs = (im, im_info)
  147. if vdl_writer is not None:
  148. tag = str(op_id) + '. ' + op.__class__.__name__
  149. vdl_writer.add_image(tag=tag,
  150. img=im,
  151. step=step)
  152. op_id += 1
  153. return outputs
  154. def add_augmenters(self, augmenters):
  155. if not isinstance(augmenters, list):
  156. raise Exception(
  157. "augmenters should be list type in func add_augmenters()")
  158. transform_names = [type(x).__name__ for x in self.transforms]
  159. for aug in augmenters:
  160. if type(aug).__name__ in transform_names:
  161. logging.error("{} is already in ComposedTransforms, need to remove it from add_augmenters().".format(type(aug).__name__))
  162. self.transforms = augmenters + self.transforms
  163. class ResizeByShort(DetTransform):
  164. """根据图像的短边调整图像大小(resize)。
  165. 1. 获取图像的长边和短边长度。
  166. 2. 根据短边与short_size的比例,计算长边的目标长度,
  167. 此时高、宽的resize比例为short_size/原图短边长度。
  168. 3. 如果max_size>0,调整resize比例:
  169. 如果长边的目标长度>max_size,则高、宽的resize比例为max_size/原图长边长度。
  170. 4. 根据调整大小的比例对图像进行resize。
  171. Args:
  172. target_size (int): 短边目标长度。默认为800。
  173. max_size (int): 长边目标长度的最大限制。默认为1333。
  174. Raises:
  175. TypeError: 形参数据类型不满足需求。
  176. """
  177. def __init__(self, short_size=800, max_size=1333):
  178. self.max_size = int(max_size)
  179. if not isinstance(short_size, int):
  180. raise TypeError(
  181. "Type of short_size is invalid. Must be Integer, now is {}".
  182. format(type(short_size)))
  183. self.short_size = short_size
  184. if not (isinstance(self.max_size, int)):
  185. raise TypeError("max_size: input type is invalid.")
  186. def __call__(self, im, im_info=None, label_info=None):
  187. """
  188. Args:
  189. im (numnp.ndarraypy): 图像np.ndarray数据。
  190. im_info (dict, 可选): 存储与图像相关的信息。
  191. label_info (dict, 可选): 存储与标注框相关的信息。
  192. Returns:
  193. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  194. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  195. 存储与标注框相关信息的字典。
  196. 其中,im_info更新字段为:
  197. - im_resize_info (np.ndarray): resize后的图像高、resize后的图像宽、resize后的图像相对原始图的缩放比例
  198. 三者组成的np.ndarray,形状为(3,)。
  199. Raises:
  200. TypeError: 形参数据类型不满足需求。
  201. ValueError: 数据长度不匹配。
  202. """
  203. if im_info is None:
  204. im_info = dict()
  205. if not isinstance(im, np.ndarray):
  206. raise TypeError("ResizeByShort: image type is not numpy.")
  207. if len(im.shape) != 3:
  208. raise ValueError('ResizeByShort: image is not 3-dimensional.')
  209. im_short_size = min(im.shape[0], im.shape[1])
  210. im_long_size = max(im.shape[0], im.shape[1])
  211. scale = float(self.short_size) / im_short_size
  212. if self.max_size > 0 and np.round(scale *
  213. im_long_size) > self.max_size:
  214. scale = float(self.max_size) / float(im_long_size)
  215. resized_width = int(round(im.shape[1] * scale))
  216. resized_height = int(round(im.shape[0] * scale))
  217. im_resize_info = [resized_height, resized_width, scale]
  218. im = cv2.resize(
  219. im, (resized_width, resized_height),
  220. interpolation=cv2.INTER_LINEAR)
  221. im_info['im_resize_info'] = np.array(im_resize_info).astype(np.float32)
  222. if label_info is None:
  223. return (im, im_info)
  224. else:
  225. return (im, im_info, label_info)
  226. class Padding(DetTransform):
  227. """1.将图像的长和宽padding至coarsest_stride的倍数。如输入图像为[300, 640],
  228. `coarest_stride`为32,则由于300不为32的倍数,因此在图像最右和最下使用0值
  229. 进行padding,最终输出图像为[320, 640]。
  230. 2.或者,将图像的长和宽padding到target_size指定的shape,如输入的图像为[300,640],
  231. a. `target_size` = 960,在图像最右和最下使用0值进行padding,最终输出
  232. 图像为[960, 960]。
  233. b. `target_size` = [640, 960],在图像最右和最下使用0值进行padding,最终
  234. 输出图像为[640, 960]。
  235. 1. 如果coarsest_stride为1,target_size为None则直接返回。
  236. 2. 获取图像的高H、宽W。
  237. 3. 计算填充后图像的高H_new、宽W_new。
  238. 4. 构建大小为(H_new, W_new, 3)像素值为0的np.ndarray,
  239. 并将原图的np.ndarray粘贴于左上角。
  240. Args:
  241. coarsest_stride (int): 填充后的图像长、宽为该参数的倍数,默认为1。
  242. target_size (int|list|tuple): 填充后的图像长、宽,默认为None,coarset_stride优先级更高。
  243. Raises:
  244. TypeError: 形参`target_size`数据类型不满足需求。
  245. ValueError: 形参`target_size`为(list|tuple)时,长度不满足需求。
  246. """
  247. def __init__(self, coarsest_stride=1, target_size=None):
  248. self.coarsest_stride = coarsest_stride
  249. if target_size is not None:
  250. if not isinstance(target_size, int):
  251. if not isinstance(target_size, tuple) and not isinstance(
  252. target_size, list):
  253. raise TypeError(
  254. "Padding: Type of target_size must in (int|list|tuple)."
  255. )
  256. elif len(target_size) != 2:
  257. raise ValueError(
  258. "Padding: Length of target_size must equal 2.")
  259. self.target_size = target_size
  260. def __call__(self, im, im_info=None, label_info=None):
  261. """
  262. Args:
  263. im (numnp.ndarraypy): 图像np.ndarray数据。
  264. im_info (dict, 可选): 存储与图像相关的信息。
  265. label_info (dict, 可选): 存储与标注框相关的信息。
  266. Returns:
  267. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  268. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  269. 存储与标注框相关信息的字典。
  270. Raises:
  271. TypeError: 形参数据类型不满足需求。
  272. ValueError: 数据长度不匹配。
  273. ValueError: coarsest_stride,target_size需有且只有一个被指定。
  274. ValueError: target_size小于原图的大小。
  275. """
  276. if im_info is None:
  277. im_info = dict()
  278. if not isinstance(im, np.ndarray):
  279. raise TypeError("Padding: image type is not numpy.")
  280. if len(im.shape) != 3:
  281. raise ValueError('Padding: image is not 3-dimensional.')
  282. im_h, im_w, im_c = im.shape[:]
  283. if isinstance(self.target_size, int):
  284. padding_im_h = self.target_size
  285. padding_im_w = self.target_size
  286. elif isinstance(self.target_size, list) or isinstance(self.target_size,
  287. tuple):
  288. padding_im_w = self.target_size[0]
  289. padding_im_h = self.target_size[1]
  290. elif self.coarsest_stride > 0:
  291. padding_im_h = int(
  292. np.ceil(im_h / self.coarsest_stride) * self.coarsest_stride)
  293. padding_im_w = int(
  294. np.ceil(im_w / self.coarsest_stride) * self.coarsest_stride)
  295. else:
  296. raise ValueError(
  297. "coarsest_stridei(>1) or target_size(list|int) need setting in Padding transform"
  298. )
  299. pad_height = padding_im_h - im_h
  300. pad_width = padding_im_w - im_w
  301. if pad_height < 0 or pad_width < 0:
  302. raise ValueError(
  303. 'the size of image should be less than target_size, but the size of image ({}, {}), is larger than target_size ({}, {})'
  304. .format(im_w, im_h, padding_im_w, padding_im_h))
  305. padding_im = np.zeros(
  306. (padding_im_h, padding_im_w, im_c), dtype=np.float32)
  307. padding_im[:im_h, :im_w, :] = im
  308. if label_info is None:
  309. return (padding_im, im_info)
  310. else:
  311. return (padding_im, im_info, label_info)
  312. class Resize(DetTransform):
  313. """调整图像大小(resize)。
  314. - 当目标大小(target_size)类型为int时,根据插值方式,
  315. 将图像resize为[target_size, target_size]。
  316. - 当目标大小(target_size)类型为list或tuple时,根据插值方式,
  317. 将图像resize为target_size。
  318. 注意:当插值方式为“RANDOM”时,则随机选取一种插值方式进行resize。
  319. Args:
  320. target_size (int/list/tuple): 短边目标长度。默认为608。
  321. interp (str): resize的插值方式,与opencv的插值方式对应,取值范围为
  322. ['NEAREST', 'LINEAR', 'CUBIC', 'AREA', 'LANCZOS4', 'RANDOM']。默认为"LINEAR"。
  323. Raises:
  324. TypeError: 形参数据类型不满足需求。
  325. ValueError: 插值方式不在['NEAREST', 'LINEAR', 'CUBIC',
  326. 'AREA', 'LANCZOS4', 'RANDOM']中。
  327. """
  328. # The interpolation mode
  329. interp_dict = {
  330. 'NEAREST': cv2.INTER_NEAREST,
  331. 'LINEAR': cv2.INTER_LINEAR,
  332. 'CUBIC': cv2.INTER_CUBIC,
  333. 'AREA': cv2.INTER_AREA,
  334. 'LANCZOS4': cv2.INTER_LANCZOS4
  335. }
  336. def __init__(self, target_size=608, interp='LINEAR'):
  337. self.interp = interp
  338. if not (interp == "RANDOM" or interp in self.interp_dict):
  339. raise ValueError("interp should be one of {}".format(
  340. self.interp_dict.keys()))
  341. if isinstance(target_size, list) or isinstance(target_size, tuple):
  342. if len(target_size) != 2:
  343. raise TypeError(
  344. 'when target is list or tuple, it should include 2 elements, but it is {}'
  345. .format(target_size))
  346. elif not isinstance(target_size, int):
  347. raise TypeError(
  348. "Type of target_size is invalid. Must be Integer or List or tuple, now is {}"
  349. .format(type(target_size)))
  350. self.target_size = target_size
  351. def __call__(self, im, im_info=None, label_info=None):
  352. """
  353. Args:
  354. im (np.ndarray): 图像np.ndarray数据。
  355. im_info (dict, 可选): 存储与图像相关的信息。
  356. label_info (dict, 可选): 存储与标注框相关的信息。
  357. Returns:
  358. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  359. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  360. 存储与标注框相关信息的字典。
  361. Raises:
  362. TypeError: 形参数据类型不满足需求。
  363. ValueError: 数据长度不匹配。
  364. """
  365. if im_info is None:
  366. im_info = dict()
  367. if not isinstance(im, np.ndarray):
  368. raise TypeError("Resize: image type is not numpy.")
  369. if len(im.shape) != 3:
  370. raise ValueError('Resize: image is not 3-dimensional.')
  371. if self.interp == "RANDOM":
  372. interp = random.choice(list(self.interp_dict.keys()))
  373. else:
  374. interp = self.interp
  375. im = resize(im, self.target_size, self.interp_dict[interp])
  376. if label_info is None:
  377. return (im, im_info)
  378. else:
  379. return (im, im_info, label_info)
  380. class RandomHorizontalFlip(DetTransform):
  381. """随机翻转图像、标注框、分割信息,模型训练时的数据增强操作。
  382. 1. 随机采样一个0-1之间的小数,当小数小于水平翻转概率时,
  383. 执行2-4步操作,否则直接返回。
  384. 2. 水平翻转图像。
  385. 3. 计算翻转后的真实标注框的坐标,更新label_info中的gt_bbox信息。
  386. 4. 计算翻转后的真实分割区域的坐标,更新label_info中的gt_poly信息。
  387. Args:
  388. prob (float): 随机水平翻转的概率。默认为0.5。
  389. Raises:
  390. TypeError: 形参数据类型不满足需求。
  391. """
  392. def __init__(self, prob=0.5):
  393. self.prob = prob
  394. if not isinstance(self.prob, float):
  395. raise TypeError("RandomHorizontalFlip: input type is invalid.")
  396. def __call__(self, im, im_info=None, label_info=None):
  397. """
  398. Args:
  399. im (np.ndarray): 图像np.ndarray数据。
  400. im_info (dict, 可选): 存储与图像相关的信息。
  401. label_info (dict, 可选): 存储与标注框相关的信息。
  402. Returns:
  403. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  404. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  405. 存储与标注框相关信息的字典。
  406. 其中,im_info更新字段为:
  407. - gt_bbox (np.ndarray): 水平翻转后的标注框坐标[x1, y1, x2, y2],形状为(n, 4),
  408. 其中n代表真实标注框的个数。
  409. - gt_poly (list): 水平翻转后的多边形分割区域的x、y坐标,长度为n,
  410. 其中n代表真实标注框的个数。
  411. Raises:
  412. TypeError: 形参数据类型不满足需求。
  413. ValueError: 数据长度不匹配。
  414. """
  415. if not isinstance(im, np.ndarray):
  416. raise TypeError(
  417. "RandomHorizontalFlip: image is not a numpy array.")
  418. if len(im.shape) != 3:
  419. raise ValueError(
  420. "RandomHorizontalFlip: image is not 3-dimensional.")
  421. if im_info is None or label_info is None:
  422. raise TypeError(
  423. 'Cannot do RandomHorizontalFlip! ' +
  424. 'Becasuse the im_info and label_info can not be None!')
  425. if 'gt_bbox' not in label_info:
  426. raise TypeError('Cannot do RandomHorizontalFlip! ' + \
  427. 'Becasuse gt_bbox is not in label_info!')
  428. image_shape = im_info['image_shape']
  429. gt_bbox = label_info['gt_bbox']
  430. height = image_shape[0]
  431. width = image_shape[1]
  432. if np.random.uniform(0, 1) < self.prob:
  433. im = horizontal_flip(im)
  434. if gt_bbox.shape[0] == 0:
  435. if label_info is None:
  436. return (im, im_info)
  437. else:
  438. return (im, im_info, label_info)
  439. label_info['gt_bbox'] = box_horizontal_flip(gt_bbox, width)
  440. if 'gt_poly' in label_info and \
  441. len(label_info['gt_poly']) != 0:
  442. label_info['gt_poly'] = segms_horizontal_flip(
  443. label_info['gt_poly'], height, width)
  444. if label_info is None:
  445. return (im, im_info)
  446. else:
  447. return (im, im_info, label_info)
  448. class Normalize(DetTransform):
  449. """对图像进行标准化。
  450. 1. 归一化图像到到区间[0.0, 1.0]。
  451. 2. 对图像进行减均值除以标准差操作。
  452. Args:
  453. mean (list): 图像数据集的均值。默认为[0.485, 0.456, 0.406]。
  454. std (list): 图像数据集的标准差。默认为[0.229, 0.224, 0.225]。
  455. Raises:
  456. TypeError: 形参数据类型不满足需求。
  457. """
  458. def __init__(self, mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]):
  459. self.mean = mean
  460. self.std = std
  461. if not (isinstance(self.mean, list) and isinstance(self.std, list)):
  462. raise TypeError("NormalizeImage: input type is invalid.")
  463. from functools import reduce
  464. if reduce(lambda x, y: x * y, self.std) == 0:
  465. raise TypeError('NormalizeImage: std is invalid!')
  466. def __call__(self, im, im_info=None, label_info=None):
  467. """
  468. Args:
  469. im (numnp.ndarraypy): 图像np.ndarray数据。
  470. im_info (dict, 可选): 存储与图像相关的信息。
  471. label_info (dict, 可选): 存储与标注框相关的信息。
  472. Returns:
  473. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  474. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  475. 存储与标注框相关信息的字典。
  476. """
  477. mean = np.array(self.mean)[np.newaxis, np.newaxis, :]
  478. std = np.array(self.std)[np.newaxis, np.newaxis, :]
  479. im = normalize(im, mean, std)
  480. if label_info is None:
  481. return (im, im_info)
  482. else:
  483. return (im, im_info, label_info)
  484. class RandomDistort(DetTransform):
  485. """以一定的概率对图像进行随机像素内容变换,模型训练时的数据增强操作
  486. 1. 对变换的操作顺序进行随机化操作。
  487. 2. 按照1中的顺序以一定的概率在范围[-range, range]对图像进行随机像素内容变换。
  488. Args:
  489. brightness_range (float): 明亮度因子的范围。默认为0.5。
  490. brightness_prob (float): 随机调整明亮度的概率。默认为0.5。
  491. contrast_range (float): 对比度因子的范围。默认为0.5。
  492. contrast_prob (float): 随机调整对比度的概率。默认为0.5。
  493. saturation_range (float): 饱和度因子的范围。默认为0.5。
  494. saturation_prob (float): 随机调整饱和度的概率。默认为0.5。
  495. hue_range (int): 色调因子的范围。默认为18。
  496. hue_prob (float): 随机调整色调的概率。默认为0.5。
  497. """
  498. def __init__(self,
  499. brightness_range=0.5,
  500. brightness_prob=0.5,
  501. contrast_range=0.5,
  502. contrast_prob=0.5,
  503. saturation_range=0.5,
  504. saturation_prob=0.5,
  505. hue_range=18,
  506. hue_prob=0.5):
  507. self.brightness_range = brightness_range
  508. self.brightness_prob = brightness_prob
  509. self.contrast_range = contrast_range
  510. self.contrast_prob = contrast_prob
  511. self.saturation_range = saturation_range
  512. self.saturation_prob = saturation_prob
  513. self.hue_range = hue_range
  514. self.hue_prob = hue_prob
  515. def __call__(self, im, im_info=None, label_info=None):
  516. """
  517. Args:
  518. im (np.ndarray): 图像np.ndarray数据。
  519. im_info (dict, 可选): 存储与图像相关的信息。
  520. label_info (dict, 可选): 存储与标注框相关的信息。
  521. Returns:
  522. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  523. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  524. 存储与标注框相关信息的字典。
  525. """
  526. brightness_lower = 1 - self.brightness_range
  527. brightness_upper = 1 + self.brightness_range
  528. contrast_lower = 1 - self.contrast_range
  529. contrast_upper = 1 + self.contrast_range
  530. saturation_lower = 1 - self.saturation_range
  531. saturation_upper = 1 + self.saturation_range
  532. hue_lower = -self.hue_range
  533. hue_upper = self.hue_range
  534. ops = [brightness, contrast, saturation, hue]
  535. random.shuffle(ops)
  536. params_dict = {
  537. 'brightness': {
  538. 'brightness_lower': brightness_lower,
  539. 'brightness_upper': brightness_upper
  540. },
  541. 'contrast': {
  542. 'contrast_lower': contrast_lower,
  543. 'contrast_upper': contrast_upper
  544. },
  545. 'saturation': {
  546. 'saturation_lower': saturation_lower,
  547. 'saturation_upper': saturation_upper
  548. },
  549. 'hue': {
  550. 'hue_lower': hue_lower,
  551. 'hue_upper': hue_upper
  552. }
  553. }
  554. prob_dict = {
  555. 'brightness': self.brightness_prob,
  556. 'contrast': self.contrast_prob,
  557. 'saturation': self.saturation_prob,
  558. 'hue': self.hue_prob
  559. }
  560. for id in range(4):
  561. params = params_dict[ops[id].__name__]
  562. prob = prob_dict[ops[id].__name__]
  563. params['im'] = im
  564. if np.random.uniform(0, 1) < prob:
  565. im = ops[id](**params)
  566. im = im.astype('float32')
  567. if label_info is None:
  568. return (im, im_info)
  569. else:
  570. return (im, im_info, label_info)
  571. class MixupImage(DetTransform):
  572. """对图像进行mixup操作,模型训练时的数据增强操作,目前仅YOLOv3模型支持该transform。
  573. 当label_info中不存在mixup字段时,直接返回,否则进行下述操作:
  574. 1. 从随机beta分布中抽取出随机因子factor。
  575. 2.
  576. - 当factor>=1.0时,去除label_info中的mixup字段,直接返回。
  577. - 当factor<=0.0时,直接返回label_info中的mixup字段,并在label_info中去除该字段。
  578. - 其余情况,执行下述操作:
  579. (1)原图像乘以factor,mixup图像乘以(1-factor),叠加2个结果。
  580. (2)拼接原图像标注框和mixup图像标注框。
  581. (3)拼接原图像标注框类别和mixup图像标注框类别。
  582. (4)原图像标注框混合得分乘以factor,mixup图像标注框混合得分乘以(1-factor),叠加2个结果。
  583. 3. 更新im_info中的image_shape信息。
  584. Args:
  585. alpha (float): 随机beta分布的下限。默认为1.5。
  586. beta (float): 随机beta分布的上限。默认为1.5。
  587. mixup_epoch (int): 在前mixup_epoch轮使用mixup增强操作;当该参数为-1时,该策略不会生效。
  588. 默认为-1。
  589. Raises:
  590. ValueError: 数据长度不匹配。
  591. """
  592. def __init__(self, alpha=1.5, beta=1.5, mixup_epoch=-1):
  593. self.alpha = alpha
  594. self.beta = beta
  595. if self.alpha <= 0.0:
  596. raise ValueError("alpha shold be positive in MixupImage")
  597. if self.beta <= 0.0:
  598. raise ValueError("beta shold be positive in MixupImage")
  599. self.mixup_epoch = mixup_epoch
  600. def _mixup_img(self, img1, img2, factor):
  601. h = max(img1.shape[0], img2.shape[0])
  602. w = max(img1.shape[1], img2.shape[1])
  603. img = np.zeros((h, w, img1.shape[2]), 'float32')
  604. img[:img1.shape[0], :img1.shape[1], :] = \
  605. img1.astype('float32') * factor
  606. img[:img2.shape[0], :img2.shape[1], :] += \
  607. img2.astype('float32') * (1.0 - factor)
  608. return img.astype('float32')
  609. def __call__(self, im, im_info=None, label_info=None):
  610. """
  611. Args:
  612. im (np.ndarray): 图像np.ndarray数据。
  613. im_info (dict, 可选): 存储与图像相关的信息。
  614. label_info (dict, 可选): 存储与标注框相关的信息。
  615. Returns:
  616. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  617. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  618. 存储与标注框相关信息的字典。
  619. 其中,im_info更新字段为:
  620. - image_shape (np.ndarray): mixup后的图像高、宽二者组成的np.ndarray,形状为(2,)。
  621. im_info删除的字段:
  622. - mixup (list): 与当前字段进行mixup的图像相关信息。
  623. label_info更新字段为:
  624. - gt_bbox (np.ndarray): mixup后真实标注框坐标,形状为(n, 4),
  625. 其中n代表真实标注框的个数。
  626. - gt_class (np.ndarray): mixup后每个真实标注框对应的类别序号,形状为(n, 1),
  627. 其中n代表真实标注框的个数。
  628. - gt_score (np.ndarray): mixup后每个真实标注框对应的混合得分,形状为(n, 1),
  629. 其中n代表真实标注框的个数。
  630. Raises:
  631. TypeError: 形参数据类型不满足需求。
  632. """
  633. if im_info is None:
  634. raise TypeError('Cannot do MixupImage! ' +
  635. 'Becasuse the im_info can not be None!')
  636. if 'mixup' not in im_info:
  637. if label_info is None:
  638. return (im, im_info)
  639. else:
  640. return (im, im_info, label_info)
  641. factor = np.random.beta(self.alpha, self.beta)
  642. factor = max(0.0, min(1.0, factor))
  643. if im_info['epoch'] > self.mixup_epoch \
  644. or factor >= 1.0:
  645. im_info.pop('mixup')
  646. if label_info is None:
  647. return (im, im_info)
  648. else:
  649. return (im, im_info, label_info)
  650. if factor <= 0.0:
  651. return im_info.pop('mixup')
  652. im = self._mixup_img(im, im_info['mixup'][0], factor)
  653. if label_info is None:
  654. raise TypeError('Cannot do MixupImage! ' +
  655. 'Becasuse the label_info can not be None!')
  656. if 'gt_bbox' not in label_info or \
  657. 'gt_class' not in label_info or \
  658. 'gt_score' not in label_info:
  659. raise TypeError('Cannot do MixupImage! ' + \
  660. 'Becasuse gt_bbox/gt_class/gt_score is not in label_info!')
  661. gt_bbox1 = label_info['gt_bbox']
  662. gt_bbox2 = im_info['mixup'][2]['gt_bbox']
  663. gt_bbox = np.concatenate((gt_bbox1, gt_bbox2), axis=0)
  664. gt_class1 = label_info['gt_class']
  665. gt_class2 = im_info['mixup'][2]['gt_class']
  666. gt_class = np.concatenate((gt_class1, gt_class2), axis=0)
  667. gt_score1 = label_info['gt_score']
  668. gt_score2 = im_info['mixup'][2]['gt_score']
  669. gt_score = np.concatenate(
  670. (gt_score1 * factor, gt_score2 * (1. - factor)), axis=0)
  671. if 'gt_poly' in label_info:
  672. gt_poly1 = label_info['gt_poly']
  673. gt_poly2 = im_info['mixup'][2]['gt_poly']
  674. label_info['gt_poly'] = gt_poly1 + gt_poly2
  675. is_crowd1 = label_info['is_crowd']
  676. is_crowd2 = im_info['mixup'][2]['is_crowd']
  677. is_crowd = np.concatenate((is_crowd1, is_crowd2), axis=0)
  678. label_info['gt_bbox'] = gt_bbox
  679. label_info['gt_score'] = gt_score
  680. label_info['gt_class'] = gt_class
  681. label_info['is_crowd'] = is_crowd
  682. im_info['image_shape'] = np.array([im.shape[0],
  683. im.shape[1]]).astype('int32')
  684. im_info.pop('mixup')
  685. if label_info is None:
  686. return (im, im_info)
  687. else:
  688. return (im, im_info, label_info)
  689. class RandomExpand(DetTransform):
  690. """随机扩张图像,模型训练时的数据增强操作。
  691. 1. 随机选取扩张比例(扩张比例大于1时才进行扩张)。
  692. 2. 计算扩张后图像大小。
  693. 3. 初始化像素值为输入填充值的图像,并将原图像随机粘贴于该图像上。
  694. 4. 根据原图像粘贴位置换算出扩张后真实标注框的位置坐标。
  695. 5. 根据原图像粘贴位置换算出扩张后真实分割区域的位置坐标。
  696. Args:
  697. ratio (float): 图像扩张的最大比例。默认为4.0。
  698. prob (float): 随机扩张的概率。默认为0.5。
  699. fill_value (list): 扩张图像的初始填充值(0-255)。默认为[123.675, 116.28, 103.53]。
  700. """
  701. def __init__(self,
  702. ratio=4.,
  703. prob=0.5,
  704. fill_value=[123.675, 116.28, 103.53]):
  705. super(RandomExpand, self).__init__()
  706. assert ratio > 1.01, "expand ratio must be larger than 1.01"
  707. self.ratio = ratio
  708. self.prob = prob
  709. assert isinstance(fill_value, Sequence), \
  710. "fill value must be sequence"
  711. if not isinstance(fill_value, tuple):
  712. fill_value = tuple(fill_value)
  713. self.fill_value = fill_value
  714. def __call__(self, im, im_info=None, label_info=None):
  715. """
  716. Args:
  717. im (np.ndarray): 图像np.ndarray数据。
  718. im_info (dict, 可选): 存储与图像相关的信息。
  719. label_info (dict, 可选): 存储与标注框相关的信息。
  720. Returns:
  721. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  722. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  723. 存储与标注框相关信息的字典。
  724. 其中,im_info更新字段为:
  725. - image_shape (np.ndarray): 扩张后的图像高、宽二者组成的np.ndarray,形状为(2,)。
  726. label_info更新字段为:
  727. - gt_bbox (np.ndarray): 随机扩张后真实标注框坐标,形状为(n, 4),
  728. 其中n代表真实标注框的个数。
  729. - gt_class (np.ndarray): 随机扩张后每个真实标注框对应的类别序号,形状为(n, 1),
  730. 其中n代表真实标注框的个数。
  731. Raises:
  732. TypeError: 形参数据类型不满足需求。
  733. """
  734. if im_info is None or label_info is None:
  735. raise TypeError(
  736. 'Cannot do RandomExpand! ' +
  737. 'Becasuse the im_info and label_info can not be None!')
  738. if 'gt_bbox' not in label_info or \
  739. 'gt_class' not in label_info:
  740. raise TypeError('Cannot do RandomExpand! ' + \
  741. 'Becasuse gt_bbox/gt_class is not in label_info!')
  742. if np.random.uniform(0., 1.) < self.prob:
  743. return (im, im_info, label_info)
  744. image_shape = im_info['image_shape']
  745. height = int(image_shape[0])
  746. width = int(image_shape[1])
  747. expand_ratio = np.random.uniform(1., self.ratio)
  748. h = int(height * expand_ratio)
  749. w = int(width * expand_ratio)
  750. if not h > height or not w > width:
  751. return (im, im_info, label_info)
  752. y = np.random.randint(0, h - height)
  753. x = np.random.randint(0, w - width)
  754. canvas = np.ones((h, w, 3), dtype=np.float32)
  755. canvas *= np.array(self.fill_value, dtype=np.float32)
  756. canvas[y:y + height, x:x + width, :] = im
  757. im_info['image_shape'] = np.array([h, w]).astype('int32')
  758. if 'gt_bbox' in label_info and len(label_info['gt_bbox']) > 0:
  759. label_info['gt_bbox'] += np.array([x, y] * 2, dtype=np.float32)
  760. if 'gt_poly' in label_info and len(label_info['gt_poly']) > 0:
  761. label_info['gt_poly'] = expand_segms(label_info['gt_poly'], x, y,
  762. height, width, expand_ratio)
  763. return (canvas, im_info, label_info)
  764. class RandomCrop(DetTransform):
  765. """随机裁剪图像。
  766. 1. 若allow_no_crop为True,则在thresholds加入’no_crop’。
  767. 2. 随机打乱thresholds。
  768. 3. 遍历thresholds中各元素:
  769. (1) 如果当前thresh为’no_crop’,则返回原始图像和标注信息。
  770. (2) 随机取出aspect_ratio和scaling中的值并由此计算出候选裁剪区域的高、宽、起始点。
  771. (3) 计算真实标注框与候选裁剪区域IoU,若全部真实标注框的IoU都小于thresh,则继续第3步。
  772. (4) 如果cover_all_box为True且存在真实标注框的IoU小于thresh,则继续第3步。
  773. (5) 筛选出位于候选裁剪区域内的真实标注框,若有效框的个数为0,则继续第3步,否则进行第4步。
  774. 4. 换算有效真值标注框相对候选裁剪区域的位置坐标。
  775. 5. 换算有效分割区域相对候选裁剪区域的位置坐标。
  776. Args:
  777. aspect_ratio (list): 裁剪后短边缩放比例的取值范围,以[min, max]形式表示。默认值为[.5, 2.]。
  778. thresholds (list): 判断裁剪候选区域是否有效所需的IoU阈值取值列表。默认值为[.0, .1, .3, .5, .7, .9]。
  779. scaling (list): 裁剪面积相对原面积的取值范围,以[min, max]形式表示。默认值为[.3, 1.]。
  780. num_attempts (int): 在放弃寻找有效裁剪区域前尝试的次数。默认值为50。
  781. allow_no_crop (bool): 是否允许未进行裁剪。默认值为True。
  782. cover_all_box (bool): 是否要求所有的真实标注框都必须在裁剪区域内。默认值为False。
  783. """
  784. def __init__(self,
  785. aspect_ratio=[.5, 2.],
  786. thresholds=[.0, .1, .3, .5, .7, .9],
  787. scaling=[.3, 1.],
  788. num_attempts=50,
  789. allow_no_crop=True,
  790. cover_all_box=False):
  791. self.aspect_ratio = aspect_ratio
  792. self.thresholds = thresholds
  793. self.scaling = scaling
  794. self.num_attempts = num_attempts
  795. self.allow_no_crop = allow_no_crop
  796. self.cover_all_box = cover_all_box
  797. def __call__(self, im, im_info=None, label_info=None):
  798. """
  799. Args:
  800. im (np.ndarray): 图像np.ndarray数据。
  801. im_info (dict, 可选): 存储与图像相关的信息。
  802. label_info (dict, 可选): 存储与标注框相关的信息。
  803. Returns:
  804. tuple: 当label_info为空时,返回的tuple为(im, im_info),分别对应图像np.ndarray数据、存储与图像相关信息的字典;
  805. 当label_info不为空时,返回的tuple为(im, im_info, label_info),分别对应图像np.ndarray数据、
  806. 存储与标注框相关信息的字典。
  807. 其中,im_info更新字段为:
  808. - image_shape (np.ndarray): 扩裁剪的图像高、宽二者组成的np.ndarray,形状为(2,)。
  809. label_info更新字段为:
  810. - gt_bbox (np.ndarray): 随机裁剪后真实标注框坐标,形状为(n, 4),
  811. 其中n代表真实标注框的个数。
  812. - gt_class (np.ndarray): 随机裁剪后每个真实标注框对应的类别序号,形状为(n, 1),
  813. 其中n代表真实标注框的个数。
  814. - gt_score (np.ndarray): 随机裁剪后每个真实标注框对应的混合得分,形状为(n, 1),
  815. 其中n代表真实标注框的个数。
  816. Raises:
  817. TypeError: 形参数据类型不满足需求。
  818. """
  819. if im_info is None or label_info is None:
  820. raise TypeError(
  821. 'Cannot do RandomCrop! ' +
  822. 'Becasuse the im_info and label_info can not be None!')
  823. if 'gt_bbox' not in label_info or \
  824. 'gt_class' not in label_info:
  825. raise TypeError('Cannot do RandomCrop! ' + \
  826. 'Becasuse gt_bbox/gt_class is not in label_info!')
  827. if len(label_info['gt_bbox']) == 0:
  828. return (im, im_info, label_info)
  829. image_shape = im_info['image_shape']
  830. w = image_shape[1]
  831. h = image_shape[0]
  832. gt_bbox = label_info['gt_bbox']
  833. thresholds = list(self.thresholds)
  834. if self.allow_no_crop:
  835. thresholds.append('no_crop')
  836. np.random.shuffle(thresholds)
  837. for thresh in thresholds:
  838. if thresh == 'no_crop':
  839. return (im, im_info, label_info)
  840. found = False
  841. for i in range(self.num_attempts):
  842. scale = np.random.uniform(*self.scaling)
  843. min_ar, max_ar = self.aspect_ratio
  844. aspect_ratio = np.random.uniform(
  845. max(min_ar, scale**2), min(max_ar, scale**-2))
  846. crop_h = int(h * scale / np.sqrt(aspect_ratio))
  847. crop_w = int(w * scale * np.sqrt(aspect_ratio))
  848. crop_y = np.random.randint(0, h - crop_h)
  849. crop_x = np.random.randint(0, w - crop_w)
  850. crop_box = [crop_x, crop_y, crop_x + crop_w, crop_y + crop_h]
  851. iou = iou_matrix(
  852. gt_bbox, np.array(
  853. [crop_box], dtype=np.float32))
  854. if iou.max() < thresh:
  855. continue
  856. if self.cover_all_box and iou.min() < thresh:
  857. continue
  858. cropped_box, valid_ids = crop_box_with_center_constraint(
  859. gt_bbox, np.array(
  860. crop_box, dtype=np.float32))
  861. if valid_ids.size > 0:
  862. found = True
  863. break
  864. if found:
  865. if 'gt_poly' in label_info and len(label_info['gt_poly']) > 0:
  866. crop_polys = crop_segms(
  867. label_info['gt_poly'],
  868. valid_ids,
  869. np.array(
  870. crop_box, dtype=np.int64),
  871. h,
  872. w)
  873. if [] in crop_polys:
  874. delete_id = list()
  875. valid_polys = list()
  876. for id, crop_poly in enumerate(crop_polys):
  877. if crop_poly == []:
  878. delete_id.append(id)
  879. else:
  880. valid_polys.append(crop_poly)
  881. valid_ids = np.delete(valid_ids, delete_id)
  882. if len(valid_polys) == 0:
  883. return (im, im_info, label_info)
  884. label_info['gt_poly'] = valid_polys
  885. else:
  886. label_info['gt_poly'] = crop_polys
  887. im = crop_image(im, crop_box)
  888. label_info['gt_bbox'] = np.take(cropped_box, valid_ids, axis=0)
  889. label_info['gt_class'] = np.take(
  890. label_info['gt_class'], valid_ids, axis=0)
  891. im_info['image_shape'] = np.array(
  892. [crop_box[3] - crop_box[1],
  893. crop_box[2] - crop_box[0]]).astype('int32')
  894. if 'gt_score' in label_info:
  895. label_info['gt_score'] = np.take(
  896. label_info['gt_score'], valid_ids, axis=0)
  897. if 'is_crowd' in label_info:
  898. label_info['is_crowd'] = np.take(
  899. label_info['is_crowd'], valid_ids, axis=0)
  900. return (im, im_info, label_info)
  901. return (im, im_info, label_info)
  902. class ArrangeFasterRCNN(DetTransform):
  903. """获取FasterRCNN模型训练/验证/预测所需信息。
  904. Args:
  905. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  906. Raises:
  907. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  908. """
  909. def __init__(self, mode=None):
  910. if mode not in ['train', 'eval', 'test', 'quant']:
  911. raise ValueError(
  912. "mode must be in ['train', 'eval', 'test', 'quant']!")
  913. self.mode = mode
  914. def __call__(self, im, im_info=None, label_info=None):
  915. """
  916. Args:
  917. im (np.ndarray): 图像np.ndarray数据。
  918. im_info (dict, 可选): 存储与图像相关的信息。
  919. label_info (dict, 可选): 存储与标注框相关的信息。
  920. Returns:
  921. tuple: 当mode为'train'时,返回(im, im_resize_info, gt_bbox, gt_class, is_crowd),分别对应
  922. 图像np.ndarray数据、图像相当对于原图的resize信息、真实标注框、真实标注框对应的类别、真实标注框内是否是一组对象;
  923. 当mode为'eval'时,返回(im, im_resize_info, im_id, im_shape, gt_bbox, gt_class, is_difficult),
  924. 分别对应图像np.ndarray数据、图像相当对于原图的resize信息、图像id、图像大小信息、真实标注框、真实标注框对应的类别、
  925. 真实标注框是否为难识别对象;当mode为'test'或'quant'时,返回(im, im_resize_info, im_shape),分别对应图像np.ndarray数据、
  926. 图像相当对于原图的resize信息、图像大小信息。
  927. Raises:
  928. TypeError: 形参数据类型不满足需求。
  929. ValueError: 数据长度不匹配。
  930. """
  931. im = permute(im, False)
  932. if self.mode == 'train':
  933. if im_info is None or label_info is None:
  934. raise TypeError(
  935. 'Cannot do ArrangeFasterRCNN! ' +
  936. 'Becasuse the im_info and label_info can not be None!')
  937. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  938. raise ValueError("gt num mismatch: bbox and class.")
  939. im_resize_info = im_info['im_resize_info']
  940. gt_bbox = label_info['gt_bbox']
  941. gt_class = label_info['gt_class']
  942. is_crowd = label_info['is_crowd']
  943. outputs = (im, im_resize_info, gt_bbox, gt_class, is_crowd)
  944. elif self.mode == 'eval':
  945. if im_info is None or label_info is None:
  946. raise TypeError(
  947. 'Cannot do ArrangeFasterRCNN! ' +
  948. 'Becasuse the im_info and label_info can not be None!')
  949. im_resize_info = im_info['im_resize_info']
  950. im_id = im_info['im_id']
  951. im_shape = np.array(
  952. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  953. dtype=np.float32)
  954. gt_bbox = label_info['gt_bbox']
  955. gt_class = label_info['gt_class']
  956. is_difficult = label_info['difficult']
  957. outputs = (im, im_resize_info, im_id, im_shape, gt_bbox, gt_class,
  958. is_difficult)
  959. else:
  960. if im_info is None:
  961. raise TypeError('Cannot do ArrangeFasterRCNN! ' +
  962. 'Becasuse the im_info can not be None!')
  963. im_resize_info = im_info['im_resize_info']
  964. im_shape = np.array(
  965. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  966. dtype=np.float32)
  967. outputs = (im, im_resize_info, im_shape)
  968. return outputs
  969. class ArrangeMaskRCNN(DetTransform):
  970. """获取MaskRCNN模型训练/验证/预测所需信息。
  971. Args:
  972. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  973. Raises:
  974. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  975. """
  976. def __init__(self, mode=None):
  977. if mode not in ['train', 'eval', 'test', 'quant']:
  978. raise ValueError(
  979. "mode must be in ['train', 'eval', 'test', 'quant']!")
  980. self.mode = mode
  981. def __call__(self, im, im_info=None, label_info=None):
  982. """
  983. Args:
  984. im (np.ndarray): 图像np.ndarray数据。
  985. im_info (dict, 可选): 存储与图像相关的信息。
  986. label_info (dict, 可选): 存储与标注框相关的信息。
  987. Returns:
  988. tuple: 当mode为'train'时,返回(im, im_resize_info, gt_bbox, gt_class, is_crowd, gt_masks),分别对应
  989. 图像np.ndarray数据、图像相当对于原图的resize信息、真实标注框、真实标注框对应的类别、真实标注框内是否是一组对象、
  990. 真实分割区域;当mode为'eval'时,返回(im, im_resize_info, im_id, im_shape),分别对应图像np.ndarray数据、
  991. 图像相当对于原图的resize信息、图像id、图像大小信息;当mode为'test'或'quant'时,返回(im, im_resize_info, im_shape),
  992. 分别对应图像np.ndarray数据、图像相当对于原图的resize信息、图像大小信息。
  993. Raises:
  994. TypeError: 形参数据类型不满足需求。
  995. ValueError: 数据长度不匹配。
  996. """
  997. im = permute(im, False)
  998. if self.mode == 'train':
  999. if im_info is None or label_info is None:
  1000. raise TypeError(
  1001. 'Cannot do ArrangeTrainMaskRCNN! ' +
  1002. 'Becasuse the im_info and label_info can not be None!')
  1003. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1004. raise ValueError("gt num mismatch: bbox and class.")
  1005. im_resize_info = im_info['im_resize_info']
  1006. gt_bbox = label_info['gt_bbox']
  1007. gt_class = label_info['gt_class']
  1008. is_crowd = label_info['is_crowd']
  1009. assert 'gt_poly' in label_info
  1010. segms = label_info['gt_poly']
  1011. if len(segms) != 0:
  1012. assert len(segms) == is_crowd.shape[0]
  1013. gt_masks = []
  1014. valid = True
  1015. for i in range(len(segms)):
  1016. segm = segms[i]
  1017. gt_segm = []
  1018. if is_crowd[i]:
  1019. gt_segm.append([[0, 0]])
  1020. else:
  1021. for poly in segm:
  1022. if len(poly) == 0:
  1023. valid = False
  1024. break
  1025. gt_segm.append(np.array(poly).reshape(-1, 2))
  1026. if (not valid) or len(gt_segm) == 0:
  1027. break
  1028. gt_masks.append(gt_segm)
  1029. outputs = (im, im_resize_info, gt_bbox, gt_class, is_crowd,
  1030. gt_masks)
  1031. else:
  1032. if im_info is None:
  1033. raise TypeError('Cannot do ArrangeMaskRCNN! ' +
  1034. 'Becasuse the im_info can not be None!')
  1035. im_resize_info = im_info['im_resize_info']
  1036. im_shape = np.array(
  1037. (im_info['image_shape'][0], im_info['image_shape'][1], 1),
  1038. dtype=np.float32)
  1039. if self.mode == 'eval':
  1040. im_id = im_info['im_id']
  1041. outputs = (im, im_resize_info, im_id, im_shape)
  1042. else:
  1043. outputs = (im, im_resize_info, im_shape)
  1044. return outputs
  1045. class ArrangeYOLOv3(DetTransform):
  1046. """获取YOLOv3模型训练/验证/预测所需信息。
  1047. Args:
  1048. mode (str): 指定数据用于何种用途,取值范围为['train', 'eval', 'test', 'quant']。
  1049. Raises:
  1050. ValueError: mode的取值不在['train', 'eval', 'test', 'quant']之内。
  1051. """
  1052. def __init__(self, mode=None):
  1053. if mode not in ['train', 'eval', 'test', 'quant']:
  1054. raise ValueError(
  1055. "mode must be in ['train', 'eval', 'test', 'quant']!")
  1056. self.mode = mode
  1057. def __call__(self, im, im_info=None, label_info=None):
  1058. """
  1059. Args:
  1060. im (np.ndarray): 图像np.ndarray数据。
  1061. im_info (dict, 可选): 存储与图像相关的信息。
  1062. label_info (dict, 可选): 存储与标注框相关的信息。
  1063. Returns:
  1064. tuple: 当mode为'train'时,返回(im, gt_bbox, gt_class, gt_score, im_shape),分别对应
  1065. 图像np.ndarray数据、真实标注框、真实标注框对应的类别、真实标注框混合得分、图像大小信息;
  1066. 当mode为'eval'时,返回(im, im_shape, im_id, gt_bbox, gt_class, difficult),
  1067. 分别对应图像np.ndarray数据、图像大小信息、图像id、真实标注框、真实标注框对应的类别、
  1068. 真实标注框是否为难识别对象;当mode为'test'或'quant'时,返回(im, im_shape),
  1069. 分别对应图像np.ndarray数据、图像大小信息。
  1070. Raises:
  1071. TypeError: 形参数据类型不满足需求。
  1072. ValueError: 数据长度不匹配。
  1073. """
  1074. im = permute(im, False)
  1075. if self.mode == 'train':
  1076. if im_info is None or label_info is None:
  1077. raise TypeError(
  1078. 'Cannot do ArrangeYolov3! ' +
  1079. 'Becasuse the im_info and label_info can not be None!')
  1080. im_shape = im_info['image_shape']
  1081. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1082. raise ValueError("gt num mismatch: bbox and class.")
  1083. if len(label_info['gt_bbox']) != len(label_info['gt_score']):
  1084. raise ValueError("gt num mismatch: bbox and score.")
  1085. gt_bbox = np.zeros((50, 4), dtype=im.dtype)
  1086. gt_class = np.zeros((50, ), dtype=np.int32)
  1087. gt_score = np.zeros((50, ), dtype=im.dtype)
  1088. gt_num = min(50, len(label_info['gt_bbox']))
  1089. if gt_num > 0:
  1090. label_info['gt_class'][:gt_num, 0] = label_info[
  1091. 'gt_class'][:gt_num, 0] - 1
  1092. gt_bbox[:gt_num, :] = label_info['gt_bbox'][:gt_num, :]
  1093. gt_class[:gt_num] = label_info['gt_class'][:gt_num, 0]
  1094. gt_score[:gt_num] = label_info['gt_score'][:gt_num, 0]
  1095. # parse [x1, y1, x2, y2] to [x, y, w, h]
  1096. gt_bbox[:, 2:4] = gt_bbox[:, 2:4] - gt_bbox[:, :2]
  1097. gt_bbox[:, :2] = gt_bbox[:, :2] + gt_bbox[:, 2:4] / 2.
  1098. outputs = (im, gt_bbox, gt_class, gt_score, im_shape)
  1099. elif self.mode == 'eval':
  1100. if im_info is None or label_info is None:
  1101. raise TypeError(
  1102. 'Cannot do ArrangeYolov3! ' +
  1103. 'Becasuse the im_info and label_info can not be None!')
  1104. im_shape = im_info['image_shape']
  1105. if len(label_info['gt_bbox']) != len(label_info['gt_class']):
  1106. raise ValueError("gt num mismatch: bbox and class.")
  1107. im_id = im_info['im_id']
  1108. gt_bbox = np.zeros((50, 4), dtype=im.dtype)
  1109. gt_class = np.zeros((50, ), dtype=np.int32)
  1110. difficult = np.zeros((50, ), dtype=np.int32)
  1111. gt_num = min(50, len(label_info['gt_bbox']))
  1112. if gt_num > 0:
  1113. label_info['gt_class'][:gt_num, 0] = label_info[
  1114. 'gt_class'][:gt_num, 0] - 1
  1115. gt_bbox[:gt_num, :] = label_info['gt_bbox'][:gt_num, :]
  1116. gt_class[:gt_num] = label_info['gt_class'][:gt_num, 0]
  1117. difficult[:gt_num] = label_info['difficult'][:gt_num, 0]
  1118. outputs = (im, im_shape, im_id, gt_bbox, gt_class, difficult)
  1119. else:
  1120. if im_info is None:
  1121. raise TypeError('Cannot do ArrangeYolov3! ' +
  1122. 'Becasuse the im_info can not be None!')
  1123. im_shape = im_info['image_shape']
  1124. outputs = (im, im_shape)
  1125. return outputs
  1126. class ComposedRCNNTransforms(Compose):
  1127. """ RCNN模型(faster-rcnn/mask-rcnn)图像处理流程,具体如下,
  1128. 训练阶段:
  1129. 1. 随机以0.5的概率将图像水平翻转
  1130. 2. 图像归一化
  1131. 3. 图像按比例Resize,scale计算方式如下
  1132. scale = min_max_size[0] / short_size_of_image
  1133. if max_size_of_image * scale > min_max_size[1]:
  1134. scale = min_max_size[1] / max_size_of_image
  1135. 4. 将3步骤的长宽进行padding,使得长宽为32的倍数
  1136. 验证阶段:
  1137. 1. 图像归一化
  1138. 2. 图像按比例Resize,scale计算方式同上训练阶段
  1139. 3. 将2步骤的长宽进行padding,使得长宽为32的倍数
  1140. Args:
  1141. mode(str): 图像处理流程所处阶段,训练/验证/预测,分别对应'train', 'eval', 'test'
  1142. min_max_size(list): 图像在缩放时,最小边和最大边的约束条件
  1143. mean(list): 图像均值
  1144. std(list): 图像方差
  1145. """
  1146. def __init__(self,
  1147. mode,
  1148. min_max_size=[800, 1333],
  1149. mean=[0.485, 0.456, 0.406],
  1150. std=[0.229, 0.224, 0.225]):
  1151. if mode == 'train':
  1152. # 训练时的transforms,包含数据增强
  1153. transforms = [
  1154. RandomHorizontalFlip(prob=0.5), Normalize(
  1155. mean=mean, std=std), ResizeByShort(
  1156. short_size=min_max_size[0], max_size=min_max_size[1]),
  1157. Padding(coarsest_stride=32)
  1158. ]
  1159. else:
  1160. # 验证/预测时的transforms
  1161. transforms = [
  1162. Normalize(
  1163. mean=mean, std=std), ResizeByShort(
  1164. short_size=min_max_size[0], max_size=min_max_size[1]),
  1165. Padding(coarsest_stride=32)
  1166. ]
  1167. super(ComposedRCNNTransforms, self).__init__(transforms)
  1168. class ComposedYOLOv3Transforms(Compose):
  1169. """YOLOv3模型的图像预处理流程,具体如下,
  1170. 训练阶段:
  1171. 1. 在前mixup_epoch轮迭代中,使用MixupImage策略,见https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/det_transforms.html#mixupimage
  1172. 2. 对图像进行随机扰动,包括亮度,对比度,饱和度和色调
  1173. 3. 随机扩充图像,见https://paddlex.readthedocs.io/zh_CN/latest/apis/transforms/det_transforms.html#randomexpand
  1174. 4. 随机裁剪图像
  1175. 5. 将4步骤的输出图像Resize成shape参数的大小
  1176. 6. 随机0.5的概率水平翻转图像
  1177. 7. 图像归一化
  1178. 验证/预测阶段:
  1179. 1. 将图像Resize成shape参数大小
  1180. 2. 图像归一化
  1181. Args:
  1182. mode(str): 图像处理流程所处阶段,训练/验证/预测,分别对应'train', 'eval', 'test'
  1183. shape(list): 输入模型中图像的大小,输入模型的图像会被Resize成此大小
  1184. mixup_epoch(int): 模型训练过程中,前mixup_epoch会使用mixup策略
  1185. mean(list): 图像均值
  1186. std(list): 图像方差
  1187. """
  1188. def __init__(self,
  1189. mode,
  1190. shape=[608, 608],
  1191. mixup_epoch=250,
  1192. mean=[0.485, 0.456, 0.406],
  1193. std=[0.229, 0.224, 0.225]):
  1194. width = shape
  1195. if isinstance(shape, list):
  1196. if shape[0] != shape[1]:
  1197. raise Exception(
  1198. "In YOLOv3 model, width and height should be equal")
  1199. width = shape[0]
  1200. if width % 32 != 0:
  1201. raise Exception(
  1202. "In YOLOv3 model, width and height should be multiple of 32, e.g 224、256、320...."
  1203. )
  1204. if mode == 'train':
  1205. # 训练时的transforms,包含数据增强
  1206. transforms = [
  1207. MixupImage(mixup_epoch=mixup_epoch), RandomDistort(),
  1208. RandomExpand(), RandomCrop(), Resize(
  1209. target_size=width,
  1210. interp='RANDOM'), RandomHorizontalFlip(), Normalize(
  1211. mean=mean, std=std)
  1212. ]
  1213. else:
  1214. # 验证/预测时的transforms
  1215. transforms = [
  1216. Resize(
  1217. target_size=width, interp='CUBIC'), Normalize(
  1218. mean=mean, std=std)
  1219. ]
  1220. super(ComposedYOLOv3Transforms, self).__init__(transforms)