processors.py 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411
  1. # Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. from typing import List, Optional, Sequence, Tuple, Union
  15. import cv2
  16. import lazy_paddle
  17. import numpy as np
  18. from ...utils.benchmark import benchmark
  19. @benchmark.timeit
  20. class Scale:
  21. """Scale images."""
  22. def __init__(
  23. self,
  24. short_size: int,
  25. fixed_ratio: bool = True,
  26. keep_ratio: Union[bool, None] = None,
  27. do_round: bool = False,
  28. ) -> None:
  29. """
  30. Initializes the Scale class.
  31. Args:
  32. short_size (int): The target size for the shorter side of the image.
  33. fixed_ratio (bool): Whether to maintain a fixed aspect ratio of 4:3.
  34. keep_ratio (Union[bool, None]): Whether to keep the aspect ratio. Cannot be True if fixed_ratio is True.
  35. do_round (bool): Whether to round the scaling factor.
  36. """
  37. super().__init__()
  38. self.short_size = short_size
  39. assert (fixed_ratio and not keep_ratio) or (
  40. not fixed_ratio
  41. ), f"fixed_ratio and keep_ratio cannot be true at the same time"
  42. self.fixed_ratio = fixed_ratio
  43. self.keep_ratio = keep_ratio
  44. self.do_round = do_round
  45. def scale(self, video: List[np.ndarray]) -> List[np.ndarray]:
  46. """
  47. Performs resize operations on a sequence of images.
  48. Args:
  49. video (List[np.ndarray]): List where each item is an image, as a numpy array.
  50. For example, [np.ndarray0, np.ndarray1, np.ndarray2, ...]
  51. Returns:
  52. List[np.ndarray]: List where each item is a np.ndarray after scaling.
  53. """
  54. imgs = video
  55. resized_imgs = []
  56. for i in range(len(imgs)):
  57. img = imgs[i]
  58. if isinstance(img, np.ndarray):
  59. h, w, _ = img.shape
  60. else:
  61. raise NotImplementedError
  62. if (w <= h and w == self.short_size) or (h <= w and h == self.short_size):
  63. resized_imgs.append(img)
  64. continue
  65. if w <= h:
  66. ow = self.short_size
  67. if self.fixed_ratio:
  68. oh = int(self.short_size * 4.0 / 3.0)
  69. elif self.keep_ratio is False:
  70. oh = self.short_size
  71. else:
  72. scale_factor = self.short_size / w
  73. oh = (
  74. int(h * float(scale_factor) + 0.5)
  75. if self.do_round
  76. else int(h * self.short_size / w)
  77. )
  78. ow = (
  79. int(w * float(scale_factor) + 0.5)
  80. if self.do_round
  81. else self.short_size
  82. )
  83. else:
  84. oh = self.short_size
  85. if self.fixed_ratio:
  86. ow = int(self.short_size * 4.0 / 3.0)
  87. elif self.keep_ratio is False:
  88. ow = self.short_size
  89. else:
  90. scale_factor = self.short_size / h
  91. oh = (
  92. int(h * float(scale_factor) + 0.5)
  93. if self.do_round
  94. else self.short_size
  95. )
  96. ow = (
  97. int(w * float(scale_factor) + 0.5)
  98. if self.do_round
  99. else int(w * self.short_size / h)
  100. )
  101. resized_imgs.append(
  102. cv2.resize(img, (ow, oh), interpolation=cv2.INTER_LINEAR)
  103. )
  104. imgs = resized_imgs
  105. return imgs
  106. def __call__(self, videos: List[np.ndarray]) -> List[np.ndarray]:
  107. """
  108. Apply the scaling operation to a list of videos.
  109. Args:
  110. videos (List[np.ndarray]): A list of videos, where each video is a sequence
  111. of images.
  112. Returns:
  113. List[np.ndarray]: A list of videos after scaling, where each video is a list of images.
  114. """
  115. return [self.scale(video) for video in videos]
  116. @benchmark.timeit
  117. class CenterCrop:
  118. """Center crop images."""
  119. def __init__(self, target_size: int, do_round: bool = True) -> None:
  120. """
  121. Initializes the CenterCrop class.
  122. Args:
  123. target_size (int): The size of the cropped area.
  124. do_round (bool): Whether to round the crop coordinates.
  125. """
  126. super().__init__()
  127. self.target_size = target_size
  128. self.do_round = do_round
  129. def center_crop(self, imgs: List[np.ndarray]) -> List[np.ndarray]:
  130. """
  131. Performs center crop operations on images.
  132. Args:
  133. imgs (List[np.ndarray]): A sequence of images (a numpy array).
  134. Returns:
  135. List[np.ndarray]: A list of images after center cropping or a cropped numpy array.
  136. """
  137. crop_imgs = []
  138. th, tw = self.target_size, self.target_size
  139. if isinstance(imgs, lazy_paddle.Tensor):
  140. h, w = imgs.shape[-2:]
  141. x1 = int(round((w - tw) / 2.0)) if self.do_round else (w - tw) // 2
  142. y1 = int(round((h - th) / 2.0)) if self.do_round else (h - th) // 2
  143. crop_imgs = imgs[:, :, y1 : y1 + th, x1 : x1 + tw]
  144. else:
  145. for img in imgs:
  146. h, w, _ = img.shape
  147. assert (w >= self.target_size) and (
  148. h >= self.target_size
  149. ), "image width({}) and height({}) should be larger than crop size".format(
  150. w, h, self.target_size
  151. )
  152. x1 = int(round((w - tw) / 2.0)) if self.do_round else (w - tw) // 2
  153. y1 = int(round((h - th) / 2.0)) if self.do_round else (h - th) // 2
  154. crop_imgs.append(img[y1 : y1 + th, x1 : x1 + tw])
  155. return crop_imgs
  156. def __call__(self, videos: List[np.ndarray]) -> List[np.ndarray]:
  157. """
  158. Apply the center crop operation to a list of videos.
  159. Args:
  160. videos (List[np.ndarray]): A list of videos, where each video is a sequence of images.
  161. Returns:
  162. List[np.ndarray]: A list of videos after center cropping.
  163. """
  164. return [self.center_crop(video) for video in videos]
  165. @benchmark.timeit
  166. class Image2Array:
  167. """Convert a sequence of images to a numpy array with optional transposition."""
  168. def __init__(self, transpose: bool = True, data_format: str = "tchw") -> None:
  169. """
  170. Initializes the Image2Array class.
  171. Args:
  172. transpose (bool): Whether to transpose the resulting numpy array.
  173. data_format (str): The format to transpose to, either 'tchw' or 'cthw'.
  174. Raises:
  175. AssertionError: If data_format is not one of the allowed values.
  176. """
  177. super().__init__()
  178. assert data_format in [
  179. "tchw",
  180. "cthw",
  181. ], f"Target format must in ['tchw', 'cthw'], but got {data_format}"
  182. self.transpose = transpose
  183. self.data_format = data_format
  184. def img2array(self, imgs: List[np.ndarray]) -> np.ndarray:
  185. """
  186. Converts a sequence of images to a numpy array and optionally transposes it.
  187. Args:
  188. imgs (List[np.ndarray]): A list of images to be converted to a numpy array.
  189. Returns:
  190. np.ndarray: A numpy array representation of the images.
  191. """
  192. t_imgs = np.stack(imgs).astype("float32")
  193. if self.transpose:
  194. if self.data_format == "tchw":
  195. t_imgs = t_imgs.transpose([0, 3, 1, 2]) # tchw
  196. else:
  197. t_imgs = t_imgs.transpose([3, 0, 1, 2]) # cthw
  198. return t_imgs
  199. def __call__(self, videos: List[np.ndarray]) -> List[np.ndarray]:
  200. """
  201. Apply the image to array conversion to a list of videos.
  202. Args:
  203. videos (List[Sequence[np.ndarray]]): A list of videos, where each video is a sequence of images.
  204. Returns:
  205. List[np.ndarray]: A list of numpy arrays, one for each video.
  206. """
  207. return [self.img2array(video) for video in videos]
  208. @benchmark.timeit
  209. class NormalizeVideo:
  210. """
  211. Normalize video frames by subtracting the mean and dividing by the standard deviation.
  212. """
  213. def __init__(
  214. self,
  215. mean: Sequence[float],
  216. std: Sequence[float],
  217. tensor_shape: Sequence[int] = [3, 1, 1],
  218. inplace: bool = False,
  219. ) -> None:
  220. """
  221. Initializes the NormalizeVideo class.
  222. Args:
  223. mean (Sequence[float]): The mean values for each channel.
  224. std (Sequence[float]): The standard deviation values for each channel.
  225. tensor_shape (Sequence[int]): The shape of the mean and std tensors.
  226. inplace (bool): Whether to perform normalization in place.
  227. """
  228. super().__init__()
  229. self.inplace = inplace
  230. if not inplace:
  231. self.mean = np.array(mean).reshape(tensor_shape).astype(np.float32)
  232. self.std = np.array(std).reshape(tensor_shape).astype(np.float32)
  233. else:
  234. self.mean = np.array(mean, dtype=np.float32)
  235. self.std = np.array(std, dtype=np.float32)
  236. def normalize_video(self, imgs: np.ndarray) -> np.ndarray:
  237. """
  238. Normalizes a sequence of images.
  239. Args:
  240. imgs (np.ndarray): A numpy array of images to be normalized.
  241. Returns:
  242. np.ndarray: The normalized images as a numpy array.
  243. """
  244. if self.inplace:
  245. n = len(imgs)
  246. h, w, c = imgs[0].shape
  247. norm_imgs = np.empty((n, h, w, c), dtype=np.float32)
  248. for i, img in enumerate(imgs):
  249. norm_imgs[i] = img
  250. for img in norm_imgs: # [n,h,w,c]
  251. mean = np.float64(self.mean.reshape(1, -1)) # [1, 3]
  252. stdinv = 1 / np.float64(self.std.reshape(1, -1)) # [1, 3]
  253. cv2.subtract(img, mean, img)
  254. cv2.multiply(img, stdinv, img)
  255. else:
  256. imgs = imgs
  257. norm_imgs = imgs / 255.0
  258. norm_imgs -= self.mean
  259. norm_imgs /= self.std
  260. imgs = norm_imgs
  261. imgs = np.expand_dims(imgs, axis=0).copy()
  262. return imgs
  263. def __call__(self, videos: List[np.ndarray]) -> List[np.ndarray]:
  264. """
  265. Apply normalization to a list of videos.
  266. Args:
  267. videos (List[np.ndarray]): A list of videos, where each video is a numpy array of images.
  268. Returns:
  269. List[np.ndarray]: A list of normalized videos as numpy arrays.
  270. """
  271. return [self.normalize_video(video) for video in videos]
  272. @benchmark.timeit
  273. class VideoClasTopk:
  274. """Applies a top-k transformation on video classification predictions."""
  275. def __init__(self, class_ids: Optional[Sequence[Union[str, int]]] = None) -> None:
  276. """
  277. Initializes the VideoClasTopk class.
  278. Args:
  279. class_ids (Optional[Sequence[Union[str, int]]]): A list of class labels corresponding to class indices.
  280. """
  281. super().__init__()
  282. self.class_id_map = self._parse_class_id_map(class_ids)
  283. def softmax(self, data: np.ndarray) -> np.ndarray:
  284. """
  285. Applies the softmax function to an array of data.
  286. Args:
  287. data (np.ndarray): An array of data for which to compute softmax.
  288. Returns:
  289. np.ndarray: The softmax-transformed data.
  290. """
  291. x_max = np.max(data, axis=-1, keepdims=True)
  292. e_x = np.exp(data - x_max)
  293. return e_x / np.sum(e_x, axis=-1, keepdims=True)
  294. def _parse_class_id_map(
  295. self, class_ids: Optional[Sequence[Union[str, int]]]
  296. ) -> Optional[dict]:
  297. """
  298. Parses a list of class IDs into a mapping from class index to class label.
  299. Args:
  300. class_ids (Optional[Sequence[Union[str, int]]]): A list of class labels.
  301. Returns:
  302. Optional[dict]: A dictionary mapping class indices to labels, or None if no class_ids are provided.
  303. """
  304. if class_ids is None:
  305. return None
  306. class_id_map = {id: str(lb) for id, lb in enumerate(class_ids)}
  307. return class_id_map
  308. def __call__(
  309. self, preds: np.ndarray, topk: int = 5
  310. ) -> Tuple[np.ndarray, List[np.ndarray], List[List[str]]]:
  311. """
  312. Selects the top-k predictions from the classification output.
  313. Args:
  314. preds (np.ndarray): A 2D array of prediction scores.
  315. topk (int): The number of top predictions to return.
  316. Returns:
  317. Tuple[np.ndarray, List[np.ndarray], List[List[str]]]: A tuple containing:
  318. - An array of indices of the top-k predictions.
  319. - A list of arrays of scores for the top-k predictions.
  320. - A list of lists of label names for the top-k predictions.
  321. """
  322. preds[0] = self.softmax(preds[0])
  323. indexes = preds[0].argsort(axis=1)[:, -topk:][:, ::-1].astype("int32")
  324. scores = [
  325. list(np.around(pred[index], decimals=5))
  326. for pred, index in zip(preds[0], indexes)
  327. ]
  328. label_names = [[self.class_id_map[i] for i in index] for index in indexes]
  329. return indexes, scores, label_names
  330. @benchmark.timeit
  331. class ToBatch:
  332. """A class for batching videos."""
  333. def __call__(self, videos: List[np.ndarray]) -> List[np.ndarray]:
  334. """Call method to stack videos into a batch.
  335. Args:
  336. videos (list of np.ndarrays): List of videos to process.
  337. Returns:
  338. list of np.ndarrays: List containing a stacked tensor of the videos.
  339. """
  340. return [np.concatenate(videos, axis=0).astype(dtype=np.float32, copy=False)]