| 123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116 |
- # Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- # This file is based on https://github.com/Kwai-Keye/Keye/blob/main/keye-vl-8b-preview/modeling_keye.py
- # Original header:
- # Copyright 2025 The Keye Team and The HuggingFace Inc. team. All rights reserved.
- #
- # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
- # and OPT implementations in this library. It has been modified from its
- # original forms to accommodate minor architectural differences compared
- # to GPT-NeoX and OPT used by the Meta AI team that trained the model.
- #
- # Licensed under the Apache License, Version 2.0 (the "License");
- # you may not use this file except in compliance with the License.
- # You may obtain a copy of the License at
- #
- # http://www.apache.org/licenses/LICENSE-2.0
- #
- # Unless required by applicable law or agreed to in writing, software
- # distributed under the License is distributed on an "AS IS" BASIS,
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- # See the License for the specific language governing permissions and
- # limitations under the License.
- import math
- import paddle
- import paddle.nn as nn
- class GELUActivation(nn.Layer):
- """
- Original Implementation of the GELU activation function in Google BERT repo when initially created. For
- information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +
- torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional
- Also see the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415
- """
- def __init__(self, use_gelu_python: bool = False):
- super().__init__()
- if use_gelu_python:
- self.act = self._gelu_python
- else:
- self.act = nn.functional.gelu
- def _gelu_python(self, input):
- return input * 0.5 * (1.0 + paddle.erf(input / math.sqrt(2.0)))
- def forward(self, input):
- return self.act(input)
- class Projector(nn.Layer):
- def __init__(self, text_config, vision_config):
- super().__init__()
- self.text_config = text_config
- self.vision_config = vision_config
- self.merge_kernel_size = (2, 2)
- self.hidden_size = (
- self.vision_config.hidden_size
- * self.merge_kernel_size[0]
- * self.merge_kernel_size[1]
- )
- self.pre_norm = nn.LayerNorm(self.vision_config.hidden_size, epsilon=1e-05)
- self.linear_1 = nn.Linear(self.hidden_size, self.hidden_size)
- self.act = GELUActivation()
- self.linear_2 = nn.Linear(self.hidden_size, self.text_config.hidden_size)
- def forward(self, image_features, image_grid_thw):
- m1, m2 = self.merge_kernel_size
- if isinstance(image_features, (list, tuple)):
- processed_features = list()
- for image_feature, image_grid in zip(image_features, image_grid_thw):
- image_feature = self.pre_norm(image_feature) # shape: (T*H*W, D)
- t, h, w = image_grid
- from einops import rearrange
- image_feature = rearrange(
- image_feature,
- "(t h p1 w p2) d -> (t h w) (p1 p2 d)",
- t=int(t),
- h=int(h // m1),
- p1=int(m1),
- w=int(w // m2),
- p2=int(m2),
- )
- hidden_states = self.linear_1(image_feature)
- hidden_states = self.act(hidden_states)
- hidden_states = self.linear_2(hidden_states)
- processed_features.append(hidden_states)
- return processed_features
- dims = image_features.shape[:-1]
- dim = image_features.shape[-1]
- image_features = paddle.reshape(image_features, [-1, dim])
- hidden_states = self.pre_norm(image_features)
- hidden_states = paddle.reshape(hidden_states, [-1, self.hidden_size])
- hidden_states = self.linear_1(hidden_states)
- hidden_states = self.act(hidden_states)
- hidden_states = self.linear_2(hidden_states)
- return paddle.reshape(hidden_states, [*dims, -1])
|