_ppocrvl.py 7.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205
  1. # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # This file is based on https://github.com/Kwai-Keye/Keye/blob/main/keye-vl-8b-preview/processing_keye.py
  15. # Original header:
  16. # Copyright 2025 The Keye Team and The HuggingFace Inc. team. All rights reserved.
  17. #
  18. # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
  19. # and OPT implementations in this library. It has been modified from its
  20. # original forms to accommodate minor architectural differences compared
  21. # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
  22. #
  23. # Licensed under the Apache License, Version 2.0 (the "License");
  24. # you may not use this file except in compliance with the License.
  25. # You may obtain a copy of the License at
  26. #
  27. # http://www.apache.org/licenses/LICENSE-2.0
  28. #
  29. # Unless required by applicable law or agreed to in writing, software
  30. # distributed under the License is distributed on an "AS IS" BASIS,
  31. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  32. # See the License for the specific language governing permissions and
  33. # limitations under the License.
  34. import copy
  35. from typing import List
  36. import paddle
  37. from .....utils.benchmark import benchmark
  38. from ..common import BatchFeature, fetch_image
  39. class PPOCRVLProcessor(object):
  40. _DEFAULT_TEXT_KWARGS = {
  41. "padding": False,
  42. "return_tensors": "pd",
  43. }
  44. _DEFAULT_VIDEO_KWARGS = {
  45. "fps": 2.0,
  46. "return_tensors": "pd",
  47. }
  48. def __init__(
  49. self,
  50. image_processor=None,
  51. tokenizer=None,
  52. ):
  53. self.image_token = (
  54. "<|IMAGE_PLACEHOLDER|>"
  55. if not hasattr(tokenizer, "image_token")
  56. else tokenizer.image_token
  57. )
  58. self.video_token = (
  59. "<|video_pad|>"
  60. if not hasattr(tokenizer, "video_token")
  61. else tokenizer.video_token
  62. )
  63. self.image_processor = image_processor
  64. self.tokenizer = tokenizer
  65. @benchmark.timeit
  66. def preprocess(
  67. self,
  68. input_dicts,
  69. ):
  70. images = [fetch_image(input_dict["image"]) for input_dict in input_dicts]
  71. text = []
  72. for input_dict in input_dicts:
  73. messages = [
  74. {
  75. "role": "user",
  76. "content": input_dict["query"],
  77. }
  78. ]
  79. prompt = self.tokenizer.apply_chat_template(messages, tokenize=False)
  80. text.append(prompt)
  81. videos = None
  82. output_kwargs = {
  83. "tokenizer_init_kwargs": self.tokenizer.init_kwargs,
  84. "text_kwargs": copy.deepcopy(self._DEFAULT_TEXT_KWARGS),
  85. "video_kwargs": copy.deepcopy(self._DEFAULT_VIDEO_KWARGS),
  86. }
  87. if images is not None:
  88. image_inputs = self.image_processor(images=images, return_tensors="pd")
  89. image_inputs["pixel_values"] = image_inputs["pixel_values"]
  90. image_grid_thw = image_inputs["image_grid_thw"]
  91. else:
  92. image_inputs = {}
  93. image_grid_thw = None
  94. if videos is not None:
  95. # TODO: add video processing
  96. videos_inputs = self.image_processor(
  97. images=None, videos=videos, **output_kwargs["images_kwargs"]
  98. )
  99. video_grid_thw = videos_inputs["video_grid_thw"]
  100. fps = output_kwargs["videos_kwargs"].pop("fps", 2.0)
  101. if isinstance(fps, (int, float)):
  102. second_per_grid_ts = [
  103. self.image_processor.temporal_patch_size / fps
  104. ] * len(video_grid_thw)
  105. elif hasattr(fps, "__len__") and len(fps) == len(video_grid_thw):
  106. second_per_grid_ts = [
  107. self.image_processor.temporal_patch_size / tmp for tmp in fps
  108. ]
  109. else:
  110. raise ValueError(
  111. f"The length of fps ({len(fps) if hasattr(fps, '__len__') else fps}) must be equal to the length of video_grid_thw ({len(video_grid_thw)}) or fps should be a single number."
  112. )
  113. videos_inputs.update(
  114. {"second_per_grid_ts": paddle.to_tensor(second_per_grid_ts)}
  115. )
  116. else:
  117. videos_inputs = {}
  118. video_grid_thw = None
  119. if not isinstance(text, list):
  120. text = [text]
  121. if image_grid_thw is not None:
  122. index = 0
  123. for i in range(len(text)):
  124. while self.image_token in text[i]:
  125. text[i] = text[i].replace(
  126. self.image_token,
  127. "<|placeholder|>"
  128. * int(
  129. image_grid_thw[index].prod()
  130. // self.image_processor.merge_size
  131. // self.image_processor.merge_size
  132. ),
  133. 1,
  134. )
  135. index += 1
  136. text[i] = text[i].replace("<|placeholder|>", self.image_token)
  137. if video_grid_thw is not None:
  138. index = 0
  139. for i in range(len(text)):
  140. while self.video_token in text[i]:
  141. text[i] = text[i].replace(
  142. self.video_token,
  143. "<|placeholder|>"
  144. * (
  145. video_grid_thw[index].prod()
  146. // self.image_processor.merge_size
  147. // self.image_processor.merge_size
  148. ),
  149. 1,
  150. )
  151. index += 1
  152. text[i] = text[i].replace("<|placeholder|>", self.video_token)
  153. text_inputs = self.tokenizer(text, **output_kwargs["text_kwargs"])
  154. return BatchFeature(data={**text_inputs, **image_inputs, **videos_inputs})
  155. @benchmark.timeit
  156. def postprocess(self, model_pred, **kwargs) -> List[str]:
  157. return self.tokenizer.batch_decode(
  158. model_pred[0],
  159. skip_special_tokens=kwargs.get("skip_special_tokens", True),
  160. spaces_between_special_tokens=False,
  161. )
  162. def batch_decode(self, *args, **kwargs):
  163. """
  164. This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.batch_decode`]. Please
  165. refer to the docstring of this method for more information.
  166. """
  167. return self.tokenizer.batch_decode(*args, **kwargs)
  168. def decode(self, *args, **kwargs):
  169. """
  170. This method forwards all its arguments to Qwen2TokenizerFast's [`~PreTrainedTokenizer.decode`]. Please refer to
  171. the docstring of this method for more information.
  172. """
  173. return self.tokenizer.decode(*args, **kwargs)
  174. @property
  175. def model_input_names(self):
  176. tokenizer_input_names = self.tokenizer.model_input_names
  177. image_processor_input_names = self.image_processor.model_input_names
  178. names_from_processor = list(
  179. dict.fromkeys(tokenizer_input_names + image_processor_input_names)
  180. )
  181. return names_from_processor + ["second_per_grid_ts"]