_projector.py 4.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116
  1. # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
  2. #
  3. # Licensed under the Apache License, Version 2.0 (the "License");
  4. # you may not use this file except in compliance with the License.
  5. # You may obtain a copy of the License at
  6. #
  7. # http://www.apache.org/licenses/LICENSE-2.0
  8. #
  9. # Unless required by applicable law or agreed to in writing, software
  10. # distributed under the License is distributed on an "AS IS" BASIS,
  11. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. # See the License for the specific language governing permissions and
  13. # limitations under the License.
  14. # This file is based on https://github.com/Kwai-Keye/Keye/blob/main/keye-vl-8b-preview/modeling_keye.py
  15. # Original header:
  16. # Copyright 2025 The Keye Team and The HuggingFace Inc. team. All rights reserved.
  17. #
  18. # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
  19. # and OPT implementations in this library. It has been modified from its
  20. # original forms to accommodate minor architectural differences compared
  21. # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
  22. #
  23. # Licensed under the Apache License, Version 2.0 (the "License");
  24. # you may not use this file except in compliance with the License.
  25. # You may obtain a copy of the License at
  26. #
  27. # http://www.apache.org/licenses/LICENSE-2.0
  28. #
  29. # Unless required by applicable law or agreed to in writing, software
  30. # distributed under the License is distributed on an "AS IS" BASIS,
  31. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  32. # See the License for the specific language governing permissions and
  33. # limitations under the License.
  34. import math
  35. import paddle
  36. import paddle.nn as nn
  37. class GELUActivation(nn.Layer):
  38. """
  39. Original Implementation of the GELU activation function in Google BERT repo when initially created. For
  40. information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
  41. torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
  42. Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
  43. """
  44. def __init__(self, use_gelu_python: bool = False):
  45. super().__init__()
  46. if use_gelu_python:
  47. self.act = self._gelu_python
  48. else:
  49. self.act = nn.functional.gelu
  50. def _gelu_python(self, input):
  51. return input * 0.5 * (1.0 + paddle.erf(input / math.sqrt(2.0)))
  52. def forward(self, input):
  53. return self.act(input)
  54. class Projector(nn.Layer):
  55. def __init__(self, text_config, vision_config):
  56. super().__init__()
  57. self.text_config = text_config
  58. self.vision_config = vision_config
  59. self.merge_kernel_size = (2, 2)
  60. self.hidden_size = (
  61. self.vision_config.hidden_size
  62. * self.merge_kernel_size[0]
  63. * self.merge_kernel_size[1]
  64. )
  65. self.pre_norm = nn.LayerNorm(self.vision_config.hidden_size, epsilon=1e-05)
  66. self.linear_1 = nn.Linear(self.hidden_size, self.hidden_size)
  67. self.act = GELUActivation()
  68. self.linear_2 = nn.Linear(self.hidden_size, self.text_config.hidden_size)
  69. def forward(self, image_features, image_grid_thw):
  70. m1, m2 = self.merge_kernel_size
  71. if isinstance(image_features, (list, tuple)):
  72. processed_features = list()
  73. for image_feature, image_grid in zip(image_features, image_grid_thw):
  74. image_feature = self.pre_norm(image_feature) # shape: (T*H*W, D)
  75. t, h, w = image_grid
  76. from einops import rearrange
  77. image_feature = rearrange(
  78. image_feature,
  79. "(t h p1 w p2) d -> (t h w) (p1 p2 d)",
  80. t=int(t),
  81. h=int(h // m1),
  82. p1=int(m1),
  83. w=int(w // m2),
  84. p2=int(m2),
  85. )
  86. hidden_states = self.linear_1(image_feature)
  87. hidden_states = self.act(hidden_states)
  88. hidden_states = self.linear_2(hidden_states)
  89. processed_features.append(hidden_states)
  90. return processed_features
  91. dims = image_features.shape[:-1]
  92. dim = image_features.shape[-1]
  93. image_features = paddle.reshape(image_features, [-1, dim])
  94. hidden_states = self.pre_norm(image_features)
  95. hidden_states = paddle.reshape(hidden_states, [-1, self.hidden_size])
  96. hidden_states = self.linear_1(hidden_states)
  97. hidden_states = self.act(hidden_states)
  98. hidden_states = self.linear_2(hidden_states)
  99. return paddle.reshape(hidden_states, [*dims, -1])