|
@@ -1,3 +1,17 @@
|
|
|
|
|
+# copyright (c) 2024 PaddlePaddle Authors. All Rights Reserve.
|
|
|
|
|
+#
|
|
|
|
|
+# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
+# you may not use this file except in compliance with the License.
|
|
|
|
|
+# You may obtain a copy of the License at
|
|
|
|
|
+#
|
|
|
|
|
+# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
+#
|
|
|
|
|
+# Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
+# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
+# See the License for the specific language governing permissions and
|
|
|
|
|
+# limitations under the License.
|
|
|
|
|
+
|
|
|
import os, sys
|
|
import os, sys
|
|
|
import numpy as np
|
|
import numpy as np
|
|
|
from numpy import cos, sin, arctan, sqrt
|
|
from numpy import cos, sin, arctan, sqrt
|
|
@@ -6,24 +20,15 @@ import copy
|
|
|
import time
|
|
import time
|
|
|
|
|
|
|
|
|
|
|
|
|
-def Homography(image, img_points, world_width, world_height,
|
|
|
|
|
- interpolation=cv2.INTER_CUBIC, ratio_width=1.0, ratio_height=1.0):
|
|
|
|
|
- """
|
|
|
|
|
- 将图像透视变换到新的视角,返回变换后的图像。
|
|
|
|
|
-
|
|
|
|
|
- Args:
|
|
|
|
|
- image (np.ndarray): 输入的图像,应为numpy数组类型。
|
|
|
|
|
- img_points (List[Tuple[int, int]]): 图像上的四个点的坐标,顺序为左上角、右上角、右下角、左下角。
|
|
|
|
|
- world_width (int): 变换后图像在世界坐标系中的宽度。
|
|
|
|
|
- world_height (int): 变换后图像在世界坐标系中的高度。
|
|
|
|
|
- interpolation (int, optional): 插值方式,默认为cv2.INTER_CUBIC。
|
|
|
|
|
- ratio_width (float, optional): 变换后图像在x轴上的缩放比例,默认为1.0。
|
|
|
|
|
- ratio_height (float, optional): 变换后图像在y轴上的缩放比例,默认为1.0。
|
|
|
|
|
-
|
|
|
|
|
- Returns:
|
|
|
|
|
- np.ndarray: 变换后的图像,为numpy数组类型。
|
|
|
|
|
-
|
|
|
|
|
- """
|
|
|
|
|
|
|
+def Homography(
|
|
|
|
|
+ image,
|
|
|
|
|
+ img_points,
|
|
|
|
|
+ world_width,
|
|
|
|
|
+ world_height,
|
|
|
|
|
+ interpolation=cv2.INTER_CUBIC,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+):
|
|
|
_points = np.array(img_points).reshape(-1, 2).astype(np.float32)
|
|
_points = np.array(img_points).reshape(-1, 2).astype(np.float32)
|
|
|
|
|
|
|
|
expand_x = int(0.5 * world_width * (ratio_width - 1))
|
|
expand_x = int(0.5 * world_width * (ratio_width - 1))
|
|
@@ -34,8 +39,7 @@ def Homography(image, img_points, world_width, world_height,
|
|
|
pt_leftbottom = [expand_x + world_width, expand_y + world_height]
|
|
pt_leftbottom = [expand_x + world_width, expand_y + world_height]
|
|
|
pt_rightbottom = [expand_x, expand_y + world_height]
|
|
pt_rightbottom = [expand_x, expand_y + world_height]
|
|
|
|
|
|
|
|
- pts_std = np.float32([pt_lefttop, pt_righttop,
|
|
|
|
|
- pt_leftbottom, pt_rightbottom])
|
|
|
|
|
|
|
+ pts_std = np.float32([pt_lefttop, pt_righttop, pt_leftbottom, pt_rightbottom])
|
|
|
|
|
|
|
|
img_crop_width = int(world_width * ratio_width)
|
|
img_crop_width = int(world_width * ratio_width)
|
|
|
img_crop_height = int(world_height * ratio_height)
|
|
img_crop_height = int(world_height * ratio_height)
|
|
@@ -44,21 +48,89 @@ def Homography(image, img_points, world_width, world_height,
|
|
|
|
|
|
|
|
dst_img = cv2.warpPerspective(
|
|
dst_img = cv2.warpPerspective(
|
|
|
image,
|
|
image,
|
|
|
- M, (img_crop_width, img_crop_height),
|
|
|
|
|
|
|
+ M,
|
|
|
|
|
+ (img_crop_width, img_crop_height),
|
|
|
borderMode=cv2.BORDER_CONSTANT, # BORDER_CONSTANT BORDER_REPLICATE
|
|
borderMode=cv2.BORDER_CONSTANT, # BORDER_CONSTANT BORDER_REPLICATE
|
|
|
- flags=interpolation)
|
|
|
|
|
|
|
+ flags=interpolation,
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return dst_img
|
|
return dst_img
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
+class PlanB:
|
|
|
|
|
+ def __call__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ image,
|
|
|
|
|
+ points,
|
|
|
|
|
+ curveTextRectifier,
|
|
|
|
|
+ interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ loss_thresh=5.0,
|
|
|
|
|
+ square=False,
|
|
|
|
|
+ ):
|
|
|
|
|
+ """
|
|
|
|
|
+ Plan B using sub-image when it failed in original image
|
|
|
|
|
+ :param image:
|
|
|
|
|
+ :param points:
|
|
|
|
|
+ :param curveTextRectifier: CurveTextRectifier
|
|
|
|
|
+ :param interpolation: cv2.INTER_NEAREST, cv2.INTER_LINEAR, cv2.INTER_AREA, cv2.INTER_CUBIC, cv2.INTER_LANCZOS4
|
|
|
|
|
+ :param ratio_width: roi_image width expansion. It should not be smaller than 1.0
|
|
|
|
|
+ :param ratio_height: roi_image height expansion. It should not be smaller than 1.0
|
|
|
|
|
+ :param loss_thresh: if loss greater than loss_thresh --> get_rotate_crop_image
|
|
|
|
|
+ :param square: crop square image or not. True or False. The default is False
|
|
|
|
|
+ :return:
|
|
|
|
|
+ """
|
|
|
|
|
+ h, w = image.shape[:2]
|
|
|
|
|
+ _points = np.array(points).reshape(-1, 2).astype(np.float32)
|
|
|
|
|
+ x_min = int(np.min(_points[:, 0]))
|
|
|
|
|
+ y_min = int(np.min(_points[:, 1]))
|
|
|
|
|
+ x_max = int(np.max(_points[:, 0]))
|
|
|
|
|
+ y_max = int(np.max(_points[:, 1]))
|
|
|
|
|
+ dx = x_max - x_min
|
|
|
|
|
+ dy = y_max - y_min
|
|
|
|
|
+ max_d = max(dx, dy)
|
|
|
|
|
+ mean_pt = np.mean(_points, 0)
|
|
|
|
|
+
|
|
|
|
|
+ expand_x = (ratio_width - 1.0) * 0.5 * max_d
|
|
|
|
|
+ expand_y = (ratio_height - 1.0) * 0.5 * max_d
|
|
|
|
|
+
|
|
|
|
|
+ if square:
|
|
|
|
|
+ x_min = np.clip(int(mean_pt[0] - max_d - expand_x), 0, w - 1)
|
|
|
|
|
+ y_min = np.clip(int(mean_pt[1] - max_d - expand_y), 0, h - 1)
|
|
|
|
|
+ x_max = np.clip(int(mean_pt[0] + max_d + expand_x), 0, w - 1)
|
|
|
|
|
+ y_max = np.clip(int(mean_pt[1] + max_d + expand_y), 0, h - 1)
|
|
|
|
|
+ else:
|
|
|
|
|
+ x_min = np.clip(int(x_min - expand_x), 0, w - 1)
|
|
|
|
|
+ y_min = np.clip(int(y_min - expand_y), 0, h - 1)
|
|
|
|
|
+ x_max = np.clip(int(x_max + expand_x), 0, w - 1)
|
|
|
|
|
+ y_max = np.clip(int(y_max + expand_y), 0, h - 1)
|
|
|
|
|
+
|
|
|
|
|
+ new_image = image[y_min:y_max, x_min:x_max, :].copy()
|
|
|
|
|
+ new_points = _points.copy()
|
|
|
|
|
+ new_points[:, 0] -= x_min
|
|
|
|
|
+ new_points[:, 1] -= y_min
|
|
|
|
|
+
|
|
|
|
|
+ dst_img, loss = curveTextRectifier(
|
|
|
|
|
+ new_image,
|
|
|
|
|
+ new_points,
|
|
|
|
|
+ interpolation,
|
|
|
|
|
+ ratio_width,
|
|
|
|
|
+ ratio_height,
|
|
|
|
|
+ mode="calibration",
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ return dst_img, loss
|
|
|
|
|
+
|
|
|
|
|
+
|
|
|
class CurveTextRectifier:
|
|
class CurveTextRectifier:
|
|
|
"""
|
|
"""
|
|
|
spatial transformer via monocular vision
|
|
spatial transformer via monocular vision
|
|
|
"""
|
|
"""
|
|
|
|
|
+
|
|
|
def __init__(self):
|
|
def __init__(self):
|
|
|
self.get_virtual_camera_parameter()
|
|
self.get_virtual_camera_parameter()
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def get_virtual_camera_parameter(self):
|
|
def get_virtual_camera_parameter(self):
|
|
|
vcam_thz = 0
|
|
vcam_thz = 0
|
|
|
vcam_thx1 = 180
|
|
vcam_thx1 = 180
|
|
@@ -84,21 +156,33 @@ class CurveTextRectifier:
|
|
|
fv = 100
|
|
fv = 100
|
|
|
|
|
|
|
|
matT = np.zeros((4, 4))
|
|
matT = np.zeros((4, 4))
|
|
|
- matT[0, 0] = cos(angle_z) * cos(angle_y) - sin(angle_z) * sin(angle_x1) * sin(angle_y)
|
|
|
|
|
|
|
+ matT[0, 0] = cos(angle_z) * cos(angle_y) - sin(angle_z) * sin(angle_x1) * sin(
|
|
|
|
|
+ angle_y
|
|
|
|
|
+ )
|
|
|
matT[0, 1] = cos(angle_z) * sin(angle_y) * sin(angle_x2) - sin(angle_z) * (
|
|
matT[0, 1] = cos(angle_z) * sin(angle_y) * sin(angle_x2) - sin(angle_z) * (
|
|
|
- cos(angle_x1) * cos(angle_x2) - sin(angle_x1) * cos(angle_y) * sin(angle_x2))
|
|
|
|
|
|
|
+ cos(angle_x1) * cos(angle_x2) - sin(angle_x1) * cos(angle_y) * sin(angle_x2)
|
|
|
|
|
+ )
|
|
|
matT[0, 2] = cos(angle_z) * sin(angle_y) * cos(angle_x2) + sin(angle_z) * (
|
|
matT[0, 2] = cos(angle_z) * sin(angle_y) * cos(angle_x2) + sin(angle_z) * (
|
|
|
- cos(angle_x1) * sin(angle_x2) + sin(angle_x1) * cos(angle_y) * cos(angle_x2))
|
|
|
|
|
|
|
+ cos(angle_x1) * sin(angle_x2) + sin(angle_x1) * cos(angle_y) * cos(angle_x2)
|
|
|
|
|
+ )
|
|
|
matT[0, 3] = optic_x
|
|
matT[0, 3] = optic_x
|
|
|
- matT[1, 0] = sin(angle_z) * cos(angle_y) + cos(angle_z) * sin(angle_x1) * sin(angle_y)
|
|
|
|
|
|
|
+ matT[1, 0] = sin(angle_z) * cos(angle_y) + cos(angle_z) * sin(angle_x1) * sin(
|
|
|
|
|
+ angle_y
|
|
|
|
|
+ )
|
|
|
matT[1, 1] = sin(angle_z) * sin(angle_y) * sin(angle_x2) + cos(angle_z) * (
|
|
matT[1, 1] = sin(angle_z) * sin(angle_y) * sin(angle_x2) + cos(angle_z) * (
|
|
|
- cos(angle_x1) * cos(angle_x2) - sin(angle_x1) * cos(angle_y) * sin(angle_x2))
|
|
|
|
|
|
|
+ cos(angle_x1) * cos(angle_x2) - sin(angle_x1) * cos(angle_y) * sin(angle_x2)
|
|
|
|
|
+ )
|
|
|
matT[1, 2] = sin(angle_z) * sin(angle_y) * cos(angle_x2) - cos(angle_z) * (
|
|
matT[1, 2] = sin(angle_z) * sin(angle_y) * cos(angle_x2) - cos(angle_z) * (
|
|
|
- cos(angle_x1) * sin(angle_x2) + sin(angle_x1) * cos(angle_y) * cos(angle_x2))
|
|
|
|
|
|
|
+ cos(angle_x1) * sin(angle_x2) + sin(angle_x1) * cos(angle_y) * cos(angle_x2)
|
|
|
|
|
+ )
|
|
|
matT[1, 3] = optic_y
|
|
matT[1, 3] = optic_y
|
|
|
matT[2, 0] = -cos(angle_x1) * sin(angle_y)
|
|
matT[2, 0] = -cos(angle_x1) * sin(angle_y)
|
|
|
- matT[2, 1] = cos(angle_x1) * cos(angle_y) * sin(angle_x2) + sin(angle_x1) * cos(angle_x2)
|
|
|
|
|
- matT[2, 2] = cos(angle_x1) * cos(angle_y) * cos(angle_x2) - sin(angle_x1) * sin(angle_x2)
|
|
|
|
|
|
|
+ matT[2, 1] = cos(angle_x1) * cos(angle_y) * sin(angle_x2) + sin(angle_x1) * cos(
|
|
|
|
|
+ angle_x2
|
|
|
|
|
+ )
|
|
|
|
|
+ matT[2, 2] = cos(angle_x1) * cos(angle_y) * cos(angle_x2) - sin(angle_x1) * sin(
|
|
|
|
|
+ angle_x2
|
|
|
|
|
+ )
|
|
|
matT[2, 3] = optic_z
|
|
matT[2, 3] = optic_z
|
|
|
matT[3, 0] = 0
|
|
matT[3, 0] = 0
|
|
|
matT[3, 1] = 0
|
|
matT[3, 1] = 0
|
|
@@ -117,7 +201,6 @@ class CurveTextRectifier:
|
|
|
self.K = np.dot(matT.T, matS)
|
|
self.K = np.dot(matT.T, matS)
|
|
|
self.K = np.dot(self.K, matT)
|
|
self.K = np.dot(self.K, matT)
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def vertical_text_process(self, points, org_size):
|
|
def vertical_text_process(self, points, org_size):
|
|
|
"""
|
|
"""
|
|
|
change sequence amd process
|
|
change sequence amd process
|
|
@@ -134,7 +217,9 @@ class CurveTextRectifier:
|
|
|
adjusted_points[:, 0] = _points[:, 1]
|
|
adjusted_points[:, 0] = _points[:, 1]
|
|
|
adjusted_points[:, 1] = org_h - _points[:, 0] - 1
|
|
adjusted_points[:, 1] = org_h - _points[:, 0] - 1
|
|
|
|
|
|
|
|
- _image_coord, _world_coord, _new_image_size = self.horizontal_text_process(adjusted_points)
|
|
|
|
|
|
|
+ _image_coord, _world_coord, _new_image_size = self.horizontal_text_process(
|
|
|
|
|
+ adjusted_points
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
# # convert to vertical points back
|
|
# # convert to vertical points back
|
|
|
image_coord = _points.reshape(1, -1, 2)
|
|
image_coord = _points.reshape(1, -1, 2)
|
|
@@ -146,7 +231,6 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
return image_coord, world_coord, new_image_size
|
|
return image_coord, world_coord, new_image_size
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def horizontal_text_process(self, points):
|
|
def horizontal_text_process(self, points):
|
|
|
"""
|
|
"""
|
|
|
get image coordinate and world coordinate
|
|
get image coordinate and world coordinate
|
|
@@ -160,16 +244,19 @@ class CurveTextRectifier:
|
|
|
for i in range(1, len(poly) // 2):
|
|
for i in range(1, len(poly) // 2):
|
|
|
xdx = poly[i * 2] - poly[(i - 1) * 2]
|
|
xdx = poly[i * 2] - poly[(i - 1) * 2]
|
|
|
xdy = poly[i * 2 + 1] - poly[(i - 1) * 2 + 1]
|
|
xdy = poly[i * 2 + 1] - poly[(i - 1) * 2 + 1]
|
|
|
- d = sqrt(xdx ** 2 + xdy ** 2)
|
|
|
|
|
|
|
+ d = sqrt(xdx**2 + xdy**2)
|
|
|
dx_list.append(d)
|
|
dx_list.append(d)
|
|
|
|
|
|
|
|
for i in range(0, len(poly) // 4):
|
|
for i in range(0, len(poly) // 4):
|
|
|
ydx = poly[i * 2] - poly[len(poly) - 1 - (i * 2 + 1)]
|
|
ydx = poly[i * 2] - poly[len(poly) - 1 - (i * 2 + 1)]
|
|
|
ydy = poly[i * 2 + 1] - poly[len(poly) - 1 - (i * 2)]
|
|
ydy = poly[i * 2 + 1] - poly[len(poly) - 1 - (i * 2)]
|
|
|
- d = sqrt(ydx ** 2 + ydy ** 2)
|
|
|
|
|
|
|
+ d = sqrt(ydx**2 + ydy**2)
|
|
|
dy_list.append(d)
|
|
dy_list.append(d)
|
|
|
|
|
|
|
|
- dx_list = [(dx_list[i] + dx_list[len(dx_list) - 1 - i]) / 2 for i in range(len(dx_list) // 2)]
|
|
|
|
|
|
|
+ dx_list = [
|
|
|
|
|
+ (dx_list[i] + dx_list[len(dx_list) - 1 - i]) / 2
|
|
|
|
|
+ for i in range(len(dx_list) // 2)
|
|
|
|
|
+ ]
|
|
|
|
|
|
|
|
height = np.around(np.mean(dy_list))
|
|
height = np.around(np.mean(dy_list))
|
|
|
|
|
|
|
@@ -210,7 +297,6 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
return image_coord, world_coord, new_image_size
|
|
return image_coord, world_coord, new_image_size
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def horizontal_text_estimate(self, points):
|
|
def horizontal_text_estimate(self, points):
|
|
|
"""
|
|
"""
|
|
|
horizontal or vertical text
|
|
horizontal or vertical text
|
|
@@ -225,11 +311,10 @@ class CurveTextRectifier:
|
|
|
x = x_max - x_min
|
|
x = x_max - x_min
|
|
|
y = y_max - y_min
|
|
y = y_max - y_min
|
|
|
is_horizontal_text = True
|
|
is_horizontal_text = True
|
|
|
- if y / x > 1.5: # vertical text condition
|
|
|
|
|
|
|
+ if y / x > 1.5: # vertical text condition
|
|
|
is_horizontal_text = False
|
|
is_horizontal_text = False
|
|
|
return is_horizontal_text
|
|
return is_horizontal_text
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def virtual_camera_to_world(self, size):
|
|
def virtual_camera_to_world(self, size):
|
|
|
ifu, ifv = self.ifu, self.ifv
|
|
ifu, ifv = self.ifu, self.ifv
|
|
|
K, matT = self.K, self.matT
|
|
K, matT = self.K, self.matT
|
|
@@ -255,24 +340,41 @@ class CurveTextRectifier:
|
|
|
D0[xp <= 0] = -D0[xp <= 0]
|
|
D0[xp <= 0] = -D0[xp <= 0]
|
|
|
D1[xp <= 0] = -D1[xp <= 0]
|
|
D1[xp <= 0] = -D1[xp <= 0]
|
|
|
|
|
|
|
|
- ratio_a = K[0, 0] * D0 * D0 + K[1, 1] * D1 * D1 + K[2, 2] * D2 * D2 + \
|
|
|
|
|
- (K[0, 1] + K[1, 0]) * D0 * D1 + (K[0, 2] + K[2, 0]) * D0 * D2 + (K[1, 2] + K[2, 1]) * D1 * D2
|
|
|
|
|
- ratio_b = (K[0, 3] + K[3, 0]) * D0 + (K[1, 3] + K[3, 1]) * D1 + (K[2, 3] + K[3, 2]) * D2
|
|
|
|
|
|
|
+ ratio_a = (
|
|
|
|
|
+ K[0, 0] * D0 * D0
|
|
|
|
|
+ + K[1, 1] * D1 * D1
|
|
|
|
|
+ + K[2, 2] * D2 * D2
|
|
|
|
|
+ + (K[0, 1] + K[1, 0]) * D0 * D1
|
|
|
|
|
+ + (K[0, 2] + K[2, 0]) * D0 * D2
|
|
|
|
|
+ + (K[1, 2] + K[2, 1]) * D1 * D2
|
|
|
|
|
+ )
|
|
|
|
|
+ ratio_b = (
|
|
|
|
|
+ (K[0, 3] + K[3, 0]) * D0
|
|
|
|
|
+ + (K[1, 3] + K[3, 1]) * D1
|
|
|
|
|
+ + (K[2, 3] + K[3, 2]) * D2
|
|
|
|
|
+ )
|
|
|
ratio_c = K[3, 3] * np.ones(ratio_b.shape)
|
|
ratio_c = K[3, 3] * np.ones(ratio_b.shape)
|
|
|
|
|
|
|
|
delta = ratio_b * ratio_b - 4 * ratio_a * ratio_c
|
|
delta = ratio_b * ratio_b - 4 * ratio_a * ratio_c
|
|
|
t = np.zeros(delta.shape)
|
|
t = np.zeros(delta.shape)
|
|
|
t[ratio_a == 0] = -ratio_c[ratio_a == 0] / ratio_b[ratio_a == 0]
|
|
t[ratio_a == 0] = -ratio_c[ratio_a == 0] / ratio_b[ratio_a == 0]
|
|
|
- t[ratio_a != 0] = (-ratio_b[ratio_a != 0] + sqrt(delta[ratio_a != 0])) / (2 * ratio_a[ratio_a != 0])
|
|
|
|
|
|
|
+ t[ratio_a != 0] = (-ratio_b[ratio_a != 0] + sqrt(delta[ratio_a != 0])) / (
|
|
|
|
|
+ 2 * ratio_a[ratio_a != 0]
|
|
|
|
|
+ )
|
|
|
t[delta < 0] = 0
|
|
t[delta < 0] = 0
|
|
|
|
|
|
|
|
- P[:, :, 0] = matT[0, 3] + t * (matT[0, 0] * D0 + matT[0, 1] * D1 + matT[0, 2] * D2)
|
|
|
|
|
- P[:, :, 1] = matT[1, 3] + t * (matT[1, 0] * D0 + matT[1, 1] * D1 + matT[1, 2] * D2)
|
|
|
|
|
- P[:, :, 2] = matT[2, 3] + t * (matT[2, 0] * D0 + matT[2, 1] * D1 + matT[2, 2] * D2)
|
|
|
|
|
|
|
+ P[:, :, 0] = matT[0, 3] + t * (
|
|
|
|
|
+ matT[0, 0] * D0 + matT[0, 1] * D1 + matT[0, 2] * D2
|
|
|
|
|
+ )
|
|
|
|
|
+ P[:, :, 1] = matT[1, 3] + t * (
|
|
|
|
|
+ matT[1, 0] * D0 + matT[1, 1] * D1 + matT[1, 2] * D2
|
|
|
|
|
+ )
|
|
|
|
|
+ P[:, :, 2] = matT[2, 3] + t * (
|
|
|
|
|
+ matT[2, 0] * D0 + matT[2, 1] * D1 + matT[2, 2] * D2
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return P
|
|
return P
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def world_to_image(self, image_size, world, intrinsic, distCoeffs, rotation, tvec):
|
|
def world_to_image(self, image_size, world, intrinsic, distCoeffs, rotation, tvec):
|
|
|
r11 = rotation[0, 0]
|
|
r11 = rotation[0, 0]
|
|
|
r12 = rotation[0, 1]
|
|
r12 = rotation[0, 1]
|
|
@@ -343,9 +445,15 @@ class CurveTextRectifier:
|
|
|
r4 = r2 * r2
|
|
r4 = r2 * r2
|
|
|
r6 = r2 * r4
|
|
r6 = r2 * r4
|
|
|
|
|
|
|
|
- radial_distortion = (1 + k1 * r2 + k2 * r4 + k3 * r6) / (1 + k4 * r2 + k5 * r4 + k6 * r6)
|
|
|
|
|
- x2 = x1 * radial_distortion + p1 * x1y1 + p2 * (r2 + 2 * x12) + s1 * r2 + s2 * r4
|
|
|
|
|
- y2 = y1 * radial_distortion + p2 * x1y1 + p1 * (r2 + 2 * y12) + s3 * r2 + s4 * r4
|
|
|
|
|
|
|
+ radial_distortion = (1 + k1 * r2 + k2 * r4 + k3 * r6) / (
|
|
|
|
|
+ 1 + k4 * r2 + k5 * r4 + k6 * r6
|
|
|
|
|
+ )
|
|
|
|
|
+ x2 = (
|
|
|
|
|
+ x1 * radial_distortion + p1 * x1y1 + p2 * (r2 + 2 * x12) + s1 * r2 + s2 * r4
|
|
|
|
|
+ )
|
|
|
|
|
+ y2 = (
|
|
|
|
|
+ y1 * radial_distortion + p2 * x1y1 + p1 * (r2 + 2 * y12) + s3 * r2 + s4 * r4
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
x3 = tao11 * x2 + tao12 * y2 + tao13
|
|
x3 = tao11 * x2 + tao12 * y2 + tao13
|
|
|
y3 = tao21 * x2 + tao22 * y2 + tao23
|
|
y3 = tao21 * x2 + tao22 * y2 + tao23
|
|
@@ -356,16 +464,20 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
return P
|
|
return P
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- def spatial_transform(self, image_data, new_image_size, mtx, dist, rvecs, tvecs, interpolation):
|
|
|
|
|
|
|
+ def spatial_transform(
|
|
|
|
|
+ self, image_data, new_image_size, mtx, dist, rvecs, tvecs, interpolation
|
|
|
|
|
+ ):
|
|
|
rotation, _ = cv2.Rodrigues(rvecs)
|
|
rotation, _ = cv2.Rodrigues(rvecs)
|
|
|
world_map = self.virtual_camera_to_world(new_image_size)
|
|
world_map = self.virtual_camera_to_world(new_image_size)
|
|
|
- image_map = self.world_to_image(new_image_size, world_map, mtx, dist, rotation, tvecs)
|
|
|
|
|
|
|
+ image_map = self.world_to_image(
|
|
|
|
|
+ new_image_size, world_map, mtx, dist, rotation, tvecs
|
|
|
|
|
+ )
|
|
|
image_map = image_map.astype(np.float32)
|
|
image_map = image_map.astype(np.float32)
|
|
|
- dst = cv2.remap(image_data, image_map[:, :, 0], image_map[:, :, 1], interpolation)
|
|
|
|
|
|
|
+ dst = cv2.remap(
|
|
|
|
|
+ image_data, image_map[:, :, 0], image_map[:, :, 1], interpolation
|
|
|
|
|
+ )
|
|
|
return dst
|
|
return dst
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def calibrate(self, org_size, image_coord, world_coord):
|
|
def calibrate(self, org_size, image_coord, world_coord):
|
|
|
"""
|
|
"""
|
|
|
calibration
|
|
calibration
|
|
@@ -378,36 +490,55 @@ class CurveTextRectifier:
|
|
|
flag = cv2.CALIB_RATIONAL_MODEL
|
|
flag = cv2.CALIB_RATIONAL_MODEL
|
|
|
flag2 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_TILTED_MODEL
|
|
flag2 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_TILTED_MODEL
|
|
|
flag3 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_THIN_PRISM_MODEL
|
|
flag3 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_THIN_PRISM_MODEL
|
|
|
- flag4 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_ZERO_TANGENT_DIST | cv2.CALIB_FIX_ASPECT_RATIO
|
|
|
|
|
- flag5 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_TILTED_MODEL | cv2.CALIB_ZERO_TANGENT_DIST
|
|
|
|
|
|
|
+ flag4 = (
|
|
|
|
|
+ cv2.CALIB_RATIONAL_MODEL
|
|
|
|
|
+ | cv2.CALIB_ZERO_TANGENT_DIST
|
|
|
|
|
+ | cv2.CALIB_FIX_ASPECT_RATIO
|
|
|
|
|
+ )
|
|
|
|
|
+ flag5 = (
|
|
|
|
|
+ cv2.CALIB_RATIONAL_MODEL
|
|
|
|
|
+ | cv2.CALIB_TILTED_MODEL
|
|
|
|
|
+ | cv2.CALIB_ZERO_TANGENT_DIST
|
|
|
|
|
+ )
|
|
|
flag6 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_FIX_ASPECT_RATIO
|
|
flag6 = cv2.CALIB_RATIONAL_MODEL | cv2.CALIB_FIX_ASPECT_RATIO
|
|
|
flag_list = [flag2, flag3, flag4, flag5, flag6]
|
|
flag_list = [flag2, flag3, flag4, flag5, flag6]
|
|
|
|
|
|
|
|
- ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(world_coord.astype(np.float32),
|
|
|
|
|
- image_coord.astype(np.float32),
|
|
|
|
|
- org_size,
|
|
|
|
|
- None,
|
|
|
|
|
- None,
|
|
|
|
|
- flags=flag)
|
|
|
|
|
|
|
+ ret, mtx, dist, rvecs, tvecs = cv2.calibrateCamera(
|
|
|
|
|
+ world_coord.astype(np.float32),
|
|
|
|
|
+ image_coord.astype(np.float32),
|
|
|
|
|
+ org_size,
|
|
|
|
|
+ None,
|
|
|
|
|
+ None,
|
|
|
|
|
+ flags=flag,
|
|
|
|
|
+ )
|
|
|
if ret > 2:
|
|
if ret > 2:
|
|
|
# strategies
|
|
# strategies
|
|
|
min_ret = ret
|
|
min_ret = ret
|
|
|
for i, flag in enumerate(flag_list):
|
|
for i, flag in enumerate(flag_list):
|
|
|
- _ret, _mtx, _dist, _rvecs, _tvecs = cv2.calibrateCamera(world_coord.astype(np.float32),
|
|
|
|
|
- image_coord.astype(np.float32),
|
|
|
|
|
- org_size,
|
|
|
|
|
- None,
|
|
|
|
|
- None,
|
|
|
|
|
- flags=flag)
|
|
|
|
|
|
|
+ _ret, _mtx, _dist, _rvecs, _tvecs = cv2.calibrateCamera(
|
|
|
|
|
+ world_coord.astype(np.float32),
|
|
|
|
|
+ image_coord.astype(np.float32),
|
|
|
|
|
+ org_size,
|
|
|
|
|
+ None,
|
|
|
|
|
+ None,
|
|
|
|
|
+ flags=flag,
|
|
|
|
|
+ )
|
|
|
if _ret < min_ret:
|
|
if _ret < min_ret:
|
|
|
min_ret = _ret
|
|
min_ret = _ret
|
|
|
ret, mtx, dist, rvecs, tvecs = _ret, _mtx, _dist, _rvecs, _tvecs
|
|
ret, mtx, dist, rvecs, tvecs = _ret, _mtx, _dist, _rvecs, _tvecs
|
|
|
|
|
|
|
|
return ret, mtx, dist, rvecs, tvecs
|
|
return ret, mtx, dist, rvecs, tvecs
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- def dc_homo(self, img, img_points, obj_points, is_horizontal_text, interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
- ratio_width=1.0, ratio_height=1.0):
|
|
|
|
|
|
|
+ def dc_homo(
|
|
|
|
|
+ self,
|
|
|
|
|
+ img,
|
|
|
|
|
+ img_points,
|
|
|
|
|
+ obj_points,
|
|
|
|
|
+ is_horizontal_text,
|
|
|
|
|
+ interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ ):
|
|
|
"""
|
|
"""
|
|
|
divide and conquer: homography
|
|
divide and conquer: homography
|
|
|
# ratio_width and ratio_height must be 1.0 here
|
|
# ratio_width and ratio_height must be 1.0 here
|
|
@@ -423,11 +554,11 @@ class CurveTextRectifier:
|
|
|
new_img_points = np.zeros((4, 2)).astype(np.float32)
|
|
new_img_points = np.zeros((4, 2)).astype(np.float32)
|
|
|
new_obj_points = np.zeros((4, 2)).astype(np.float32)
|
|
new_obj_points = np.zeros((4, 2)).astype(np.float32)
|
|
|
|
|
|
|
|
- new_img_points[0:2, :] = _img_points[i:(i + 2), :2]
|
|
|
|
|
- new_img_points[2:4, :] = _img_points[::-1, :][i:(i + 2), :2][::-1, :]
|
|
|
|
|
|
|
+ new_img_points[0:2, :] = _img_points[i : (i + 2), :2]
|
|
|
|
|
+ new_img_points[2:4, :] = _img_points[::-1, :][i : (i + 2), :2][::-1, :]
|
|
|
|
|
|
|
|
- new_obj_points[0:2, :] = _obj_points[i:(i + 2), :2]
|
|
|
|
|
- new_obj_points[2:4, :] = _obj_points[::-1, :][i:(i + 2), :2][::-1, :]
|
|
|
|
|
|
|
+ new_obj_points[0:2, :] = _obj_points[i : (i + 2), :2]
|
|
|
|
|
+ new_obj_points[2:4, :] = _obj_points[::-1, :][i : (i + 2), :2][::-1, :]
|
|
|
|
|
|
|
|
if is_horizontal_text:
|
|
if is_horizontal_text:
|
|
|
world_width = np.abs(new_obj_points[1, 0] - new_obj_points[0, 0])
|
|
world_width = np.abs(new_obj_points[1, 0] - new_obj_points[0, 0])
|
|
@@ -436,9 +567,15 @@ class CurveTextRectifier:
|
|
|
world_width = np.abs(new_obj_points[1, 1] - new_obj_points[0, 1])
|
|
world_width = np.abs(new_obj_points[1, 1] - new_obj_points[0, 1])
|
|
|
world_height = np.abs(new_obj_points[3, 0] - new_obj_points[0, 0])
|
|
world_height = np.abs(new_obj_points[3, 0] - new_obj_points[0, 0])
|
|
|
|
|
|
|
|
- homo_img = Homography(img, new_img_points, world_width, world_height,
|
|
|
|
|
- interpolation=interpolation,
|
|
|
|
|
- ratio_width=ratio_width, ratio_height=ratio_height)
|
|
|
|
|
|
|
+ homo_img = Homography(
|
|
|
|
|
+ img,
|
|
|
|
|
+ new_img_points,
|
|
|
|
|
+ world_width,
|
|
|
|
|
+ world_height,
|
|
|
|
|
+ interpolation=interpolation,
|
|
|
|
|
+ ratio_width=ratio_width,
|
|
|
|
|
+ ratio_height=ratio_height,
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
homo_img_list.append(homo_img)
|
|
homo_img_list.append(homo_img)
|
|
|
_h, _w = homo_img.shape[:2]
|
|
_h, _w = homo_img.shape[:2]
|
|
@@ -446,11 +583,13 @@ class CurveTextRectifier:
|
|
|
height_list.append(_h)
|
|
height_list.append(_h)
|
|
|
|
|
|
|
|
# stitching
|
|
# stitching
|
|
|
- rectified_image = np.zeros((np.max(height_list), sum(width_list), 3)).astype(np.uint8)
|
|
|
|
|
|
|
+ rectified_image = np.zeros((np.max(height_list), sum(width_list), 3)).astype(
|
|
|
|
|
+ np.uint8
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
st = 0
|
|
st = 0
|
|
|
- for (homo_img, w, h) in zip(homo_img_list, width_list, height_list):
|
|
|
|
|
- rectified_image[:h, st:st + w, :] = homo_img
|
|
|
|
|
|
|
+ for homo_img, w, h in zip(homo_img_list, width_list, height_list):
|
|
|
|
|
+ rectified_image[:h, st : st + w, :] = homo_img
|
|
|
st += w
|
|
st += w
|
|
|
|
|
|
|
|
if not is_horizontal_text:
|
|
if not is_horizontal_text:
|
|
@@ -459,24 +598,16 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
return rectified_image
|
|
return rectified_image
|
|
|
|
|
|
|
|
- def Homography(self, image, img_points, world_width, world_height,
|
|
|
|
|
- interpolation=cv2.INTER_CUBIC, ratio_width=1.0, ratio_height=1.0):
|
|
|
|
|
- """
|
|
|
|
|
- 将图像透视变换到新的视角,返回变换后的图像。
|
|
|
|
|
-
|
|
|
|
|
- Args:
|
|
|
|
|
- image (np.ndarray): 输入的图像,应为numpy数组类型。
|
|
|
|
|
- img_points (List[Tuple[int, int]]): 图像上的四个点的坐标,顺序为左上角、右上角、右下角、左下角。
|
|
|
|
|
- world_width (int): 变换后图像在世界坐标系中的宽度。
|
|
|
|
|
- world_height (int): 变换后图像在世界坐标系中的高度。
|
|
|
|
|
- interpolation (int, optional): 插值方式,默认为cv2.INTER_CUBIC。
|
|
|
|
|
- ratio_width (float, optional): 变换后图像在x轴上的缩放比例,默认为1.0。
|
|
|
|
|
- ratio_height (float, optional): 变换后图像在y轴上的缩放比例,默认为1.0。
|
|
|
|
|
-
|
|
|
|
|
- Returns:
|
|
|
|
|
- np.ndarray: 变换后的图像,为numpy数组类型。
|
|
|
|
|
-
|
|
|
|
|
- """
|
|
|
|
|
|
|
+ def Homography(
|
|
|
|
|
+ self,
|
|
|
|
|
+ image,
|
|
|
|
|
+ img_points,
|
|
|
|
|
+ world_width,
|
|
|
|
|
+ world_height,
|
|
|
|
|
+ interpolation=cv2.INTER_CUBIC,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ ):
|
|
|
_points = np.array(img_points).reshape(-1, 2).astype(np.float32)
|
|
_points = np.array(img_points).reshape(-1, 2).astype(np.float32)
|
|
|
|
|
|
|
|
expand_x = int(0.5 * world_width * (ratio_width - 1))
|
|
expand_x = int(0.5 * world_width * (ratio_width - 1))
|
|
@@ -487,8 +618,7 @@ class CurveTextRectifier:
|
|
|
pt_leftbottom = [expand_x + world_width, expand_y + world_height]
|
|
pt_leftbottom = [expand_x + world_width, expand_y + world_height]
|
|
|
pt_rightbottom = [expand_x, expand_y + world_height]
|
|
pt_rightbottom = [expand_x, expand_y + world_height]
|
|
|
|
|
|
|
|
- pts_std = np.float32([pt_lefttop, pt_righttop,
|
|
|
|
|
- pt_leftbottom, pt_rightbottom])
|
|
|
|
|
|
|
+ pts_std = np.float32([pt_lefttop, pt_righttop, pt_leftbottom, pt_rightbottom])
|
|
|
|
|
|
|
|
img_crop_width = int(world_width * ratio_width)
|
|
img_crop_width = int(world_width * ratio_width)
|
|
|
img_crop_height = int(world_height * ratio_height)
|
|
img_crop_height = int(world_height * ratio_height)
|
|
@@ -497,14 +627,23 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
dst_img = cv2.warpPerspective(
|
|
dst_img = cv2.warpPerspective(
|
|
|
image,
|
|
image,
|
|
|
- M, (img_crop_width, img_crop_height),
|
|
|
|
|
|
|
+ M,
|
|
|
|
|
+ (img_crop_width, img_crop_height),
|
|
|
borderMode=cv2.BORDER_CONSTANT, # BORDER_CONSTANT BORDER_REPLICATE
|
|
borderMode=cv2.BORDER_CONSTANT, # BORDER_CONSTANT BORDER_REPLICATE
|
|
|
- flags=interpolation)
|
|
|
|
|
|
|
+ flags=interpolation,
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return dst_img
|
|
return dst_img
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- def __call__(self, image_data, points, interpolation=cv2.INTER_LINEAR, ratio_width=1.0, ratio_height=1.0, mode='calibration'):
|
|
|
|
|
|
|
+ def __call__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points,
|
|
|
|
|
+ interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ mode="calibration",
|
|
|
|
|
+ ):
|
|
|
"""
|
|
"""
|
|
|
spatial transform for a poly text
|
|
spatial transform for a poly text
|
|
|
:param image_data:
|
|
:param image_data:
|
|
@@ -521,22 +660,42 @@ class CurveTextRectifier:
|
|
|
|
|
|
|
|
is_horizontal_text = self.horizontal_text_estimate(points)
|
|
is_horizontal_text = self.horizontal_text_estimate(points)
|
|
|
if is_horizontal_text:
|
|
if is_horizontal_text:
|
|
|
- image_coord, world_coord, new_image_size = self.horizontal_text_process(points)
|
|
|
|
|
|
|
+ image_coord, world_coord, new_image_size = self.horizontal_text_process(
|
|
|
|
|
+ points
|
|
|
|
|
+ )
|
|
|
else:
|
|
else:
|
|
|
- image_coord, world_coord, new_image_size = self.vertical_text_process(points, org_size)
|
|
|
|
|
-
|
|
|
|
|
- if mode.lower() == 'calibration':
|
|
|
|
|
- ret, mtx, dist, rvecs, tvecs = self.calibrate(org_size, image_coord, world_coord)
|
|
|
|
|
-
|
|
|
|
|
- st_size = (int(new_image_size[0]*ratio_width), int(new_image_size[1]*ratio_height))
|
|
|
|
|
- dst = self.spatial_transform(image_data, st_size, mtx, dist[0], rvecs[0], tvecs[0], interpolation)
|
|
|
|
|
- elif mode.lower() == 'homography':
|
|
|
|
|
|
|
+ image_coord, world_coord, new_image_size = self.vertical_text_process(
|
|
|
|
|
+ points, org_size
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if mode.lower() == "calibration":
|
|
|
|
|
+ ret, mtx, dist, rvecs, tvecs = self.calibrate(
|
|
|
|
|
+ org_size, image_coord, world_coord
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ st_size = (
|
|
|
|
|
+ int(new_image_size[0] * ratio_width),
|
|
|
|
|
+ int(new_image_size[1] * ratio_height),
|
|
|
|
|
+ )
|
|
|
|
|
+ dst = self.spatial_transform(
|
|
|
|
|
+ image_data, st_size, mtx, dist[0], rvecs[0], tvecs[0], interpolation
|
|
|
|
|
+ )
|
|
|
|
|
+ elif mode.lower() == "homography":
|
|
|
# ratio_width and ratio_height must be 1.0 here and ret set to 0.01 without loss manually
|
|
# ratio_width and ratio_height must be 1.0 here and ret set to 0.01 without loss manually
|
|
|
ret = 0.01
|
|
ret = 0.01
|
|
|
- dst = self.dc_homo(image_data, image_coord, world_coord, is_horizontal_text,
|
|
|
|
|
- interpolation=interpolation, ratio_width=1.0, ratio_height=1.0)
|
|
|
|
|
|
|
+ dst = self.dc_homo(
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ image_coord,
|
|
|
|
|
+ world_coord,
|
|
|
|
|
+ is_horizontal_text,
|
|
|
|
|
+ interpolation=interpolation,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ )
|
|
|
else:
|
|
else:
|
|
|
- raise ValueError('mode must be ["calibration", "homography"], but got {}'.format(mode))
|
|
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ 'mode must be ["calibration", "homography"], but got {}'.format(mode)
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return dst, ret
|
|
return dst, ret
|
|
|
|
|
|
|
@@ -547,7 +706,9 @@ class AutoRectifier:
|
|
|
self.curveTextRectifier = CurveTextRectifier()
|
|
self.curveTextRectifier = CurveTextRectifier()
|
|
|
|
|
|
|
|
@staticmethod
|
|
@staticmethod
|
|
|
- def get_rotate_crop_image(img, points, interpolation=cv2.INTER_CUBIC, ratio_width=1.0, ratio_height=1.0):
|
|
|
|
|
|
|
+ def get_rotate_crop_image(
|
|
|
|
|
+ img, points, interpolation=cv2.INTER_CUBIC, ratio_width=1.0, ratio_height=1.0
|
|
|
|
|
+ ):
|
|
|
"""
|
|
"""
|
|
|
crop or homography
|
|
crop or homography
|
|
|
:param img:
|
|
:param img:
|
|
@@ -579,33 +740,65 @@ class AutoRectifier:
|
|
|
img_crop_width = int(
|
|
img_crop_width = int(
|
|
|
max(
|
|
max(
|
|
|
np.linalg.norm(_points[0] - _points[1]),
|
|
np.linalg.norm(_points[0] - _points[1]),
|
|
|
- np.linalg.norm(_points[2] - _points[3])))
|
|
|
|
|
|
|
+ np.linalg.norm(_points[2] - _points[3]),
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
img_crop_height = int(
|
|
img_crop_height = int(
|
|
|
max(
|
|
max(
|
|
|
np.linalg.norm(_points[0] - _points[3]),
|
|
np.linalg.norm(_points[0] - _points[3]),
|
|
|
- np.linalg.norm(_points[1] - _points[2])))
|
|
|
|
|
-
|
|
|
|
|
- dst_img = Homography(img, _points, img_crop_width, img_crop_height, interpolation, ratio_width, ratio_height)
|
|
|
|
|
|
|
+ np.linalg.norm(_points[1] - _points[2]),
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ dst_img = Homography(
|
|
|
|
|
+ img,
|
|
|
|
|
+ _points,
|
|
|
|
|
+ img_crop_width,
|
|
|
|
|
+ img_crop_height,
|
|
|
|
|
+ interpolation,
|
|
|
|
|
+ ratio_width,
|
|
|
|
|
+ ratio_height,
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return dst_img
|
|
return dst_img
|
|
|
|
|
|
|
|
-
|
|
|
|
|
def visualize(self, image_data, points_list):
|
|
def visualize(self, image_data, points_list):
|
|
|
visualization = image_data.copy()
|
|
visualization = image_data.copy()
|
|
|
|
|
|
|
|
for box in points_list:
|
|
for box in points_list:
|
|
|
box = np.array(box).reshape(-1, 2).astype(np.int32)
|
|
box = np.array(box).reshape(-1, 2).astype(np.int32)
|
|
|
- cv2.drawContours(visualization, [np.array(box).reshape((-1, 1, 2))], -1, (0, 0, 255), 2)
|
|
|
|
|
|
|
+ cv2.drawContours(
|
|
|
|
|
+ visualization, [np.array(box).reshape((-1, 1, 2))], -1, (0, 0, 255), 2
|
|
|
|
|
+ )
|
|
|
for i, p in enumerate(box):
|
|
for i, p in enumerate(box):
|
|
|
if i != 0:
|
|
if i != 0:
|
|
|
- cv2.circle(visualization, tuple(p), radius=1, color=(255, 0, 0), thickness=2)
|
|
|
|
|
|
|
+ cv2.circle(
|
|
|
|
|
+ visualization,
|
|
|
|
|
+ tuple(p),
|
|
|
|
|
+ radius=1,
|
|
|
|
|
+ color=(255, 0, 0),
|
|
|
|
|
+ thickness=2,
|
|
|
|
|
+ )
|
|
|
else:
|
|
else:
|
|
|
- cv2.circle(visualization, tuple(p), radius=1, color=(255, 255, 0), thickness=2)
|
|
|
|
|
|
|
+ cv2.circle(
|
|
|
|
|
+ visualization,
|
|
|
|
|
+ tuple(p),
|
|
|
|
|
+ radius=1,
|
|
|
|
|
+ color=(255, 255, 0),
|
|
|
|
|
+ thickness=2,
|
|
|
|
|
+ )
|
|
|
return visualization
|
|
return visualization
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- def __call__(self, image_data, points, interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
- ratio_width=1.0, ratio_height=1.0, loss_thresh=5.0, mode='calibration'):
|
|
|
|
|
|
|
+ def __call__(
|
|
|
|
|
+ self,
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points,
|
|
|
|
|
+ interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ loss_thresh=5.0,
|
|
|
|
|
+ mode="calibration",
|
|
|
|
|
+ ):
|
|
|
"""
|
|
"""
|
|
|
rectification in strategies for a poly text
|
|
rectification in strategies for a poly text
|
|
|
:param image_data:
|
|
:param image_data:
|
|
@@ -617,26 +810,41 @@ class AutoRectifier:
|
|
|
:param mode: 'calibration' or 'homography'. when homography, ratio_width and ratio_height must be 1.0
|
|
:param mode: 'calibration' or 'homography'. when homography, ratio_width and ratio_height must be 1.0
|
|
|
:return:
|
|
:return:
|
|
|
"""
|
|
"""
|
|
|
- _points = np.array(points).reshape(-1,2)
|
|
|
|
|
|
|
+ _points = np.array(points).reshape(-1, 2)
|
|
|
if len(_points) >= self.npoints and len(_points) % 2 == 0:
|
|
if len(_points) >= self.npoints and len(_points) % 2 == 0:
|
|
|
try:
|
|
try:
|
|
|
curveTextRectifier = CurveTextRectifier()
|
|
curveTextRectifier = CurveTextRectifier()
|
|
|
|
|
|
|
|
- dst_img, loss = curveTextRectifier(image_data, points, interpolation, ratio_width, ratio_height, mode)
|
|
|
|
|
|
|
+ dst_img, loss = curveTextRectifier(
|
|
|
|
|
+ image_data, points, interpolation, ratio_width, ratio_height, mode
|
|
|
|
|
+ )
|
|
|
if loss >= 2:
|
|
if loss >= 2:
|
|
|
# for robust
|
|
# for robust
|
|
|
# large loss means it cannot be reconstruct correctly, we must find other way to reconstruct
|
|
# large loss means it cannot be reconstruct correctly, we must find other way to reconstruct
|
|
|
img_list, loss_list = [dst_img], [loss]
|
|
img_list, loss_list = [dst_img], [loss]
|
|
|
- _dst_img, _loss = PlanB()(image_data, points, curveTextRectifier,
|
|
|
|
|
- interpolation, ratio_width, ratio_height,
|
|
|
|
|
- loss_thresh=loss_thresh,
|
|
|
|
|
- square=True)
|
|
|
|
|
|
|
+ _dst_img, _loss = PlanB()(
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points,
|
|
|
|
|
+ curveTextRectifier,
|
|
|
|
|
+ interpolation,
|
|
|
|
|
+ ratio_width,
|
|
|
|
|
+ ratio_height,
|
|
|
|
|
+ loss_thresh=loss_thresh,
|
|
|
|
|
+ square=True,
|
|
|
|
|
+ )
|
|
|
img_list += [_dst_img]
|
|
img_list += [_dst_img]
|
|
|
loss_list += [_loss]
|
|
loss_list += [_loss]
|
|
|
|
|
|
|
|
- _dst_img, _loss = PlanB()(image_data, points, curveTextRectifier,
|
|
|
|
|
- interpolation, ratio_width, ratio_height,
|
|
|
|
|
- loss_thresh=loss_thresh, square=False)
|
|
|
|
|
|
|
+ _dst_img, _loss = PlanB()(
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points,
|
|
|
|
|
+ curveTextRectifier,
|
|
|
|
|
+ interpolation,
|
|
|
|
|
+ ratio_width,
|
|
|
|
|
+ ratio_height,
|
|
|
|
|
+ loss_thresh=loss_thresh,
|
|
|
|
|
+ square=False,
|
|
|
|
|
+ )
|
|
|
img_list += [_dst_img]
|
|
img_list += [_dst_img]
|
|
|
loss_list += [_loss]
|
|
loss_list += [_loss]
|
|
|
|
|
|
|
@@ -644,20 +852,37 @@ class AutoRectifier:
|
|
|
dst_img = img_list[loss_list.index(min_loss)]
|
|
dst_img = img_list[loss_list.index(min_loss)]
|
|
|
|
|
|
|
|
if min_loss >= loss_thresh:
|
|
if min_loss >= loss_thresh:
|
|
|
- print('calibration loss: {} is too large for spatial transformer. It is failed. Using get_rotate_crop_image'.format(loss))
|
|
|
|
|
- dst_img = self.get_rotate_crop_image(image_data, points, interpolation, ratio_width, ratio_height)
|
|
|
|
|
- print('here')
|
|
|
|
|
|
|
+ print(
|
|
|
|
|
+ "calibration loss: {} is too large for spatial transformer. It is failed. Using get_rotate_crop_image".format(
|
|
|
|
|
+ loss
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
+ dst_img = self.get_rotate_crop_image(
|
|
|
|
|
+ image_data, points, interpolation, ratio_width, ratio_height
|
|
|
|
|
+ )
|
|
|
|
|
+ print("here")
|
|
|
except Exception as e:
|
|
except Exception as e:
|
|
|
print(e)
|
|
print(e)
|
|
|
- dst_img = self.get_rotate_crop_image(image_data, points, interpolation, ratio_width, ratio_height)
|
|
|
|
|
|
|
+ dst_img = self.get_rotate_crop_image(
|
|
|
|
|
+ image_data, points, interpolation, ratio_width, ratio_height
|
|
|
|
|
+ )
|
|
|
else:
|
|
else:
|
|
|
- dst_img = self.get_rotate_crop_image(image_data, _points, interpolation, ratio_width, ratio_height)
|
|
|
|
|
|
|
+ dst_img = self.get_rotate_crop_image(
|
|
|
|
|
+ image_data, _points, interpolation, ratio_width, ratio_height
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
return dst_img
|
|
return dst_img
|
|
|
|
|
|
|
|
-
|
|
|
|
|
- def run(self, image_data, points_list, interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
- ratio_width=1.0, ratio_height=1.0, loss_thresh=5.0, mode='calibration'):
|
|
|
|
|
|
|
+ def run(
|
|
|
|
|
+ self,
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points_list,
|
|
|
|
|
+ interpolation=cv2.INTER_LINEAR,
|
|
|
|
|
+ ratio_width=1.0,
|
|
|
|
|
+ ratio_height=1.0,
|
|
|
|
|
+ loss_thresh=5.0,
|
|
|
|
|
+ mode="calibration",
|
|
|
|
|
+ ):
|
|
|
"""
|
|
"""
|
|
|
run for texts in an image
|
|
run for texts in an image
|
|
|
:param image_data: numpy.ndarray. The shape is [h, w, 3]
|
|
:param image_data: numpy.ndarray. The shape is [h, w, 3]
|
|
@@ -678,22 +903,37 @@ class AutoRectifier:
|
|
|
raise ValueError
|
|
raise ValueError
|
|
|
|
|
|
|
|
if ratio_width < 1.0 or ratio_height < 1.0:
|
|
if ratio_width < 1.0 or ratio_height < 1.0:
|
|
|
- raise ValueError('ratio_width and ratio_height cannot be smaller than 1, but got {}', (ratio_width, ratio_height))
|
|
|
|
|
-
|
|
|
|
|
- if mode.lower() != 'calibration' and mode.lower() != 'homography':
|
|
|
|
|
- raise ValueError('mode must be ["calibration", "homography"], but got {}'.format(mode))
|
|
|
|
|
-
|
|
|
|
|
- if mode.lower() == 'homography' and ratio_width != 1.0 and ratio_height != 1.0:
|
|
|
|
|
- raise ValueError('ratio_width and ratio_height must be 1.0 when mode is homography, but got mode:{}, ratio:({},{})'.format(mode, ratio_width, ratio_height))
|
|
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "ratio_width and ratio_height cannot be smaller than 1, but got {}",
|
|
|
|
|
+ (ratio_width, ratio_height),
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if mode.lower() != "calibration" and mode.lower() != "homography":
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ 'mode must be ["calibration", "homography"], but got {}'.format(mode)
|
|
|
|
|
+ )
|
|
|
|
|
+
|
|
|
|
|
+ if mode.lower() == "homography" and ratio_width != 1.0 and ratio_height != 1.0:
|
|
|
|
|
+ raise ValueError(
|
|
|
|
|
+ "ratio_width and ratio_height must be 1.0 when mode is homography, but got mode:{}, ratio:({},{})".format(
|
|
|
|
|
+ mode, ratio_width, ratio_height
|
|
|
|
|
+ )
|
|
|
|
|
+ )
|
|
|
|
|
|
|
|
res = []
|
|
res = []
|
|
|
for points in points_list:
|
|
for points in points_list:
|
|
|
- rectified_img = self(image_data, points, interpolation, ratio_width, ratio_height,
|
|
|
|
|
- loss_thresh=loss_thresh, mode=mode)
|
|
|
|
|
|
|
+ rectified_img = self(
|
|
|
|
|
+ image_data,
|
|
|
|
|
+ points,
|
|
|
|
|
+ interpolation,
|
|
|
|
|
+ ratio_width,
|
|
|
|
|
+ ratio_height,
|
|
|
|
|
+ loss_thresh=loss_thresh,
|
|
|
|
|
+ mode=mode,
|
|
|
|
|
+ )
|
|
|
res.append(rectified_img)
|
|
res.append(rectified_img)
|
|
|
|
|
|
|
|
# visualize
|
|
# visualize
|
|
|
visualized_image = self.visualize(image_data, points_list)
|
|
visualized_image = self.visualize(image_data, points_list)
|
|
|
|
|
|
|
|
return res, visualized_image
|
|
return res, visualized_image
|
|
|
-
|
|
|