1.py

"""
Copyright (c) 2019-present NAVER Corp.
MIT License
"""

# -*- coding: utf-8 -*-
import base64

import cv2
import numpy as np
import math

def distance(start, end):
    x1, y1 = start
    x2, y2 = end
    return math.sqrt((x2 - x1) ** 2 + (y2 - y1) ** 2)

def deskew(boxes):
    if len(boxes) == 0:
        return 0
    rectpoints = max(boxes, key=cv2.contourArea)
    h = distance(rectpoints[ 0 ], rectpoints[ 3 ])
    w = distance(rectpoints[ 2 ], rectpoints[ 3 ])
    angle = 0
    if h > w:
        angle = math.atan2(rectpoints[ 3 ][ 1 ] - rectpoints[ 0 ][ 1 ], rectpoints[ 0 ][ 0 ] - rectpoints[ 3 ][ 0 ])
    else:
        angle = math.atan2(rectpoints[ 3 ][ 1 ] - rectpoints[ 2 ][ 1 ], rectpoints[ 2 ][ 0 ] - rectpoints[ 3 ][ 0 ])
    angle = angle * 180 / 3.1415926535
    angle = -1 * angle
    return angle


def rotate_image(Inimg, angle):
    image_center = tuple(np.array(Inimg.shape[ 1::-1 ]) / 2)
    rot_mat = cv2.getRotationMatrix2D(image_center, angle, 1.0)
    result = cv2.warpAffine(Inimg, rot_mat, Inimg.shape[ 1::-1 ], flags=cv2.INTER_LINEAR)
    return result

def getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold, low_text):
    # prepare data
    linkmap = linkmap.copy()
    textmap = textmap.copy()
    img_h, img_w = textmap.shape

    """ labeling method """
    ret, text_score = cv2.threshold(textmap, low_text, 1, 0)
    ret, link_score = cv2.threshold(linkmap, link_threshold, 1, 0)

    text_score_comb = np.clip(text_score + link_score, 0, 1)
    nLabels, labels, stats, centroids = cv2.connectedComponentsWithStats(
        text_score_comb.astype(np.uint8), connectivity=4)

    det = []
    mapper = []
    for k in range(1, nLabels):
        # size filtering
        size = stats[k, cv2.CC_STAT_AREA]
        if size < 10: continue

        # thresholding
        if np.max(textmap[labels == k]) < text_threshold: continue

        # make segmentation map
        segmap = np.zeros(textmap.shape, dtype=np.uint8)
        segmap[labels == k] = 255
        segmap[np.logical_and(link_score == 1, text_score == 0)] = 0  # remove link area
        x, y = stats[k, cv2.CC_STAT_LEFT], stats[k, cv2.CC_STAT_TOP]
        w, h = stats[k, cv2.CC_STAT_WIDTH], stats[k, cv2.CC_STAT_HEIGHT]
        niter = int(math.sqrt(size * min(w, h) / (w * h)) * 2)
        sx, ex, sy, ey = x - niter, x + w + niter + 1, y - niter, y + h + niter + 1
        # boundary check
        if sx < 0: sx = 0
        if sy < 0: sy = 0
        if ex >= img_w: ex = img_w
        if ey >= img_h: ey = img_h
        kernel = cv2.getStructuringElement(cv2.MORPH_RECT, (1 + niter, 1 + niter))
        segmap[sy:ey, sx:ex] = cv2.dilate(segmap[sy:ey, sx:ex], kernel)

        # make box
        np_contours = np.roll(np.array(np.where(segmap != 0)), 1, axis=0).transpose().reshape(-1, 2)
        rectangle = cv2.minAreaRect(np_contours)
        box = cv2.boxPoints(rectangle)

        # align diamond-shape
        w, h = np.linalg.norm(box[0] - box[1]), np.linalg.norm(box[1] - box[2])
        box_ratio = max(w, h) / (min(w, h) + 1e-5)
        if abs(1 - box_ratio) <= 0.1:
            l, r = min(np_contours[:, 0]), max(np_contours[:, 0])
            t, b = min(np_contours[:, 1]), max(np_contours[:, 1])
            box = np.array([[l, t], [r, t], [r, b], [l, b]], dtype=np.float32)

        # make clock-wise order
        startidx = box.sum(axis=1).argmin()
        box = np.roll(box, 4 - startidx, 0)
        box = np.array(box)

        det.append(box)
        mapper.append(k)

    return det, labels, mapper


def getPoly_core(boxes, labels, mapper, linkmap):
    # configs
    num_cp = 5
    max_len_ratio = 0.7
    expand_ratio = 1.45
    max_r = 2.0
    step_r = 0.2

    polys = []
    for k, box in enumerate(boxes):
        # size filter for small instance
        w, h = int(np.linalg.norm(box[0] - box[1]) + 1), int(np.linalg.norm(box[1] - box[2]) + 1)
        if w < 10 or h < 10:
            polys.append(None);
            continue

        # warp image
        tar = np.float32([[0, 0], [w, 0], [w, h], [0, h]])
        M = cv2.getPerspectiveTransform(box, tar)
        word_label = cv2.warpPerspective(labels, M, (w, h), flags=cv2.INTER_NEAREST)
        try:
            Minv = np.linalg.inv(M)
        except:
            polys.append(None);
            continue

        # binarization for selected label
        cur_label = mapper[k]
        word_label[word_label != cur_label] = 0
        word_label[word_label > 0] = 1

        """ Polygon generation """
        # find top/bottom contours
        cp = []
        max_len = -1
        for i in range(w):
            region = np.where(word_label[:, i] != 0)[0]
            if len(region) < 2: continue
            cp.append((i, region[0], region[-1]))
            length = region[-1] - region[0] + 1
            if length > max_len: max_len = length

        # pass if max_len is similar to h
        if h * max_len_ratio < max_len:
            polys.append(None);
            continue

        # get pivot points with fixed length
        tot_seg = num_cp * 2 + 1
        seg_w = w / tot_seg  # segment width
        pp = [None] * num_cp  # init pivot points
        cp_section = [[0, 0]] * tot_seg
        seg_height = [0] * num_cp
        seg_num = 0
        num_sec = 0
        prev_h = -1
        for i in range(0, len(cp)):
            (x, sy, ey) = cp[i]
            if (seg_num + 1) * seg_w <= x and seg_num <= tot_seg:
                # average previous segment
                if num_sec == 0: break
                cp_section[seg_num] = [cp_section[seg_num][0] / num_sec,
                                       cp_section[seg_num][1] / num_sec]
                num_sec = 0

                # reset variables
                seg_num += 1
                prev_h = -1

            # accumulate center points
            cy = (sy + ey) * 0.5
            cur_h = ey - sy + 1
            cp_section[seg_num] = [cp_section[seg_num][0] + x, cp_section[seg_num][1] + cy]
            num_sec += 1

            if seg_num % 2 == 0: continue  # No polygon area

            if prev_h < cur_h:
                pp[int((seg_num - 1) / 2)] = (x, cy)
                seg_height[int((seg_num - 1) / 2)] = cur_h
                prev_h = cur_h

        # processing last segment
        if num_sec != 0:
            cp_section[-1] = [cp_section[-1][0] / num_sec, cp_section[-1][1] / num_sec]

        # pass if num of pivots is not sufficient or segment widh is smaller than character height
        if None in pp or seg_w < np.max(seg_height) * 0.25:
            polys.append(None);
            continue

        # calc median maximum of pivot points
        half_char_h = np.median(seg_height) * expand_ratio / 2

        # calc gradiant and apply to make horizontal pivots
        new_pp = []
        for i, (x, cy) in enumerate(pp):
            dx = cp_section[i * 2 + 2][0] - cp_section[i * 2][0]
            dy = cp_section[i * 2 + 2][1] - cp_section[i * 2][1]
            if dx == 0:  # gradient if zero
                new_pp.append([x, cy - half_char_h, x, cy + half_char_h])
                continue
            rad = - math.atan2(dy, dx)
            c, s = half_char_h * math.cos(rad), half_char_h * math.sin(rad)
            new_pp.append([x - s, cy - c, x + s, cy + c])

        # get edge points to cover character heatmaps
        isSppFound, isEppFound = False, False
        grad_s = (pp[1][1] - pp[0][1]) / (pp[1][0] - pp[0][0]) + (pp[2][1] - pp[1][1]) / (
                    pp[2][0] - pp[1][0])
        grad_e = (pp[-2][1] - pp[-1][1]) / (pp[-2][0] - pp[-1][0]) + (pp[-3][1] - pp[-2][1]) / (
                    pp[-3][0] - pp[-2][0])
        for r in np.arange(0.5, max_r, step_r):
            dx = 2 * half_char_h * r
            if not isSppFound:
                line_img = np.zeros(word_label.shape, dtype=np.uint8)
                dy = grad_s * dx
                p = np.array(new_pp[0]) - np.array([dx, dy, dx, dy])
                cv2.line(line_img, (int(p[0]), int(p[1])), (int(p[2]), int(p[3])), 1, thickness=1)
                if np.sum(np.logical_and(word_label, line_img)) == 0 or r + 2 * step_r >= max_r:
                    spp = p
                    isSppFound = True
            if not isEppFound:
                line_img = np.zeros(word_label.shape, dtype=np.uint8)
                dy = grad_e * dx
                p = np.array(new_pp[-1]) + np.array([dx, dy, dx, dy])
                cv2.line(line_img, (int(p[0]), int(p[1])), (int(p[2]), int(p[3])), 1, thickness=1)
                if np.sum(np.logical_and(word_label, line_img)) == 0 or r + 2 * step_r >= max_r:
                    epp = p
                    isEppFound = True
            if isSppFound and isEppFound:
                break

        # pass if boundary of polygon is not found
        if not (isSppFound and isEppFound):
            polys.append(None);
            continue

        # make final polygon
        poly = []
        poly.append(warpCoord(Minv, (spp[0], spp[1])))
        for p in new_pp:
            poly.append(warpCoord(Minv, (p[0], p[1])))
        poly.append(warpCoord(Minv, (epp[0], epp[1])))
        poly.append(warpCoord(Minv, (epp[2], epp[3])))
        for p in reversed(new_pp):
            poly.append(warpCoord(Minv, (p[2], p[3])))
        poly.append(warpCoord(Minv, (spp[2], spp[3])))

        # add to final result
        polys.append(np.array(poly))

    return polys


def getDetBoxes(textmap, linkmap, text_threshold, link_threshold, low_text, poly=False):
    boxes, labels, mapper = getDetBoxes_core(textmap, linkmap, text_threshold, link_threshold,
                                             low_text)

    if poly:
        polys = getPoly_core(boxes, labels, mapper, linkmap)
    else:
        polys = [None] * len(boxes)

    return boxes, polys


def adjustResultCoordinates(polys, ratio_w, ratio_h, ratio_net=2):
    if len(polys) > 0:
        polys = np.array(polys)
        for k in range(len(polys)):
            if polys[k] is not None:
                polys[k] *= (ratio_w * ratio_net, ratio_h * ratio_net)
    return polys


def normalizeMeanVariance(in_img, mean=(0.485, 0.456, 0.406), variance=(0.229, 0.224, 0.225)):
    # should be RGB order
    img = in_img.copy().astype(np.float32)

    img -= np.array([mean[0] * 255.0, mean[1] * 255.0, mean[2] * 255.0], dtype=np.float32)
    img /= np.array([variance[0] * 255.0, variance[1] * 255.0, variance[2] * 255.0], dtype=np.float32)
    h, w, _ = img.shape
    img = cv2.CreateMat(h, w, cv2.CV_32FC3)
    return img


def denormalizeMeanVariance(in_img, mean=(0.485, 0.456, 0.406), variance=(0.229, 0.224, 0.225)):
    # should be RGB order
    img = in_img.copy()
    img *= variance
    img += mean
    img *= 255.0
    img = np.clip(img, 0, 255).astype(np.uint8)
    return img


def resize_aspect_ratio(img, square_size, interpolation, mag_ratio=1):
    height, width, channel = img.shape

    # magnify image size
    target_size = mag_ratio * max(height, width)

    # set original image size
    if target_size > square_size:
        target_size = square_size

    ratio = target_size / max(height, width)

    target_h, target_w = int(height * ratio), int(width * ratio)
    proc = cv2.resize(img, (target_w, target_h), interpolation=interpolation)

    # make canvas and paste image
    target_h32, target_w32 = target_h, target_w
    if target_h % 32 != 0:
        target_h32 = target_h + (32 - target_h % 32)
    if target_w % 32 != 0:
        target_w32 = target_w + (32 - target_w % 32)
    resized = np.zeros((target_h32, target_w32, channel), dtype=np.float32)
    resized[0:target_h, 0:target_w, :] = proc
    target_h, target_w = target_h32, target_w32

    size_heatmap = (int(target_w / 2), int(target_h / 2))

    return resized, ratio, size_heatmap


def showReturn(data):
    decoded_data = base64.b64decode(data)
    np_data = np.fromstring(decoded_data, np.uint8)
    img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
    return (img.shape, type(img))


def pre_processing(data):
    # img = imgproc.loadImage(image_path)
    decoded_data = base64.b64decode(data)
    np_data = np.fromstring(decoded_data, np.uint8)
    img = cv2.imdecode(np_data, cv2.IMREAD_UNCHANGED)
    # IMP RESIZE
    img = cv2.resize(img, (182, 190), interpolation=cv2.INTER_LINEAR)
    img_resized, target_ratio, size_heatmap = resize_aspect_ratio(img, 1280,
                                                                  interpolation=cv2.INTER_LINEAR,
                                                                  mag_ratio=1.5)
    ratio_h = ratio_w = 1 / target_ratio
    # x = normalizeMeanVariance(img_resized)

    # RESHAPE INPUT IMAGE
    # x = torch.from_numpy(x).permute(2, 0, 1)    # [height, width, channels] to [channels, height, width]
    # ADD one more dimension
    # x = x.unsqueeze(0)                          # [channels, height, width] to [batch=1, channels, height, width]
    return (x, type(x))


def post_processing(data):
    # extracting from the [1, 144, 144, 2] array
    # : -> all content of that dimension
    y = np.array(data)
    y = y.reshape(1, 144, 144, 2)
    score_text = y[0, :, :, 0]
    score_link = y[0, :, :, 1]

    # Post-processing
    # FUNCTION call from: https://github.com/clovaai/CRAFT-pytorch/blob/e332dd8b718e291f51b66ff8f9ef2c98ee4474c8/craft_utils.py#L227
    # return getcharboxes(score_text, 0.4)

    #### Donot require this code anymore
    # data.numpy() -> convert the tensorf to a array (float)
    boxes, polys = getDetBoxes(score_text, score_link, 0.7, 0.4, 0.4, False)
    # coordinate adjustment: https://github.com/clovaai/CRAFT-pytorch/blob/e332dd8b718e291f51b66ff8f9ef2c98ee4474c8/craft_utils.py#L237
    boxes = adjustResultCoordinates(boxes, 1, 1)
    polys = adjustResultCoordinates(polys, 1, 1)
    for k in range(len(polys)):
        if polys[k] is None: polys[k] = boxes[k]
    # =============================================================================================
    # t1 = time.time() - t1

    angle = deskew(boxes)
    # rotate_image(data, boxes)
    deskewed_score_text = rotate_image(score_text, angle)
    return [getcharboxes(deskewed_score_text, 0.4), angle]
    # file_utils.saveResult(image_path, img[:,:,::-1], polys, dirname=OUTPUT_FOLDER)
    # print(f"{idx+1}) {image_path}-{img.shape}, {img_resized.shape}, {target_ratio}, {ratio_h}, {ratio_w}")
    # break


def sort_contours(cnts):
    reverse = False
    boundingBoxes = [cv2.boundingRect(c) for c in cnts]
    cnts, boundingBoxes = zip(
        *sorted(zip(cnts, boundingBoxes), key=lambda b: b[1][1] + (b[1][3] / 2), reverse=reverse))
    return cnts


def findminy(roi):
    roi = 255 - roi
    min_wp = 30000
    min_index = 0
    for col in range(10, roi.shape[1] - 10):
        curr_wp = 0
        for row in range(roi.shape[0]):
            if roi[row][col] > 0:
                curr_wp += 1
        if curr_wp < min_wp:
            min_wp = curr_wp
            min_index = col
    return min_index


def medianfind(lst):
    sortedLst = sorted(lst)
    n = len(lst)
    if n % 2:
        return sortedLst[int((n - 1) / 2)]
    else:
        return sortedLst[int((n / 2) - 1)]


def getcharboxes(textmap, low_text):
    img_h, img_w = textmap.shape
    ret, textscore = cv2.threshold(textmap, low_text, 1, 0)
    textscore = cv2.convertScaleAbs(textscore, alpha=255.0)
    textscore_org = textscore
    if img_w < 120:
        kernel = np.ones((3, 3), np.uint8)
        iterations = 1
    elif 119 < img_w < 170:
        kernel = np.ones((4, 4), np.uint8)
        iterations = 1
    elif 169 < img_w < 399:
        kernel = np.ones((5, 5), np.uint8)
        iterations = 1
    else:
        kernel = np.ones((7, 7), np.uint8)
        iterations = 2
    textscore = cv2.erode(textscore, kernel=kernel, iterations=iterations)
    textscore = cv2.dilate(textscore, kernel=kernel, iterations=iterations)
    textscore = 255 - textscore
    contours, hierarchy = cv2.findContours(textscore, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
    contours = sort_contours(contours)
    row_boxes, row1, final = [], [], []
    curr, prevmidy, prevh, prevw = 0, 0, 0, 0
    for cnt in contours:
        x, y, w, h = cv2.boundingRect(cnt)
        if w > 0.4 * textmap.shape[0] or h > 0.4 * textmap.shape[1] or w < 0.01 * textmap.shape[
            0] or h < 0.01 * textmap.shape[1]:
            continue
        midx = int(x + w / 2)
        midy = int(y + h / 2)
        if len(row_boxes) == 0 and len(row1) == 0:
            row1.append([x, y, w, h])
            prevmidy = midy
            prevmidx = midx
            prevh = h
            prevw = w
            continue
        diffy = abs(midy - prevmidy)
        if diffy > 0.62 * prevh:
            row_boxes.append(row1)
            row1 = []
            row1.append([x, y, w, h])
        else:
            row1.append([x, y, w, h])
        prevmidy = midy
        prevh = h
        prevw = w
    row_boxes.append(row1)
    row_len = [len(row) for row in row_boxes]

    to_add = {"1": [], "2": [], "3": [], '4': [], '0': [], '5': [], '6': []}
    to_del = {i: 0 for i in range(len(row_boxes))}
    for k in range(len(row_boxes)):
        row_boxes[k] = sorted(row_boxes[k], key=lambda a: a[0])
        row2 = row_boxes[k]
        width = 0
        errors = 0
        to_del_box = []
        if len(row2) < 3 or len(row2) > 11:
            to_del[k] = 10
            continue
        mean_w = int(medianfind([box[2] for box in row2]))
        mean_h = int(medianfind([box[3] for box in row2]))
        mean_y = int(medianfind([box[1] for box in row2]))
        for j in range(len(row2)):
            if j != 0:
                if width == 0:
                    gap = row2[j][0] - row2[j - 1][0] - row2[j - 1][2]
                    intersection = row2[j][0] + row2[j][2] - row2[j - 1][0] - row2[j - 1][2]
                else:
                    gap = row2[j][0] - row2[j - 1][0] - width
                    intersection = row2[j][0] + row2[j][2] - row2[j - 1][0] - width
                    width = 0
                if intersection < 0.2 * mean_w:
                    errors += 1
                if gap > mean_w:
                    errors += 1
                    if gap > 3 * mean_w:
                        errors += 1
                    else:
                        if len(row2) + len(to_add[str(k)]) in [5, 3]:
                            to_add[str(k)].append(
                                [row2[j - 1][0] + row2[j - 1][2], row2[j][1], mean_w, row2[j][3]])

            if row2[j][3] > 1.5 * mean_h:
                errors += 1
                row2[j][1] = mean_y
                row2[j][3] = mean_h
            if row2[j][2] > 1.5 * mean_w:
                errors += 1
                width = row2[j][2]
                roi = textscore[row2[j][1]:row2[j][1] + row2[j][3],
                      row2[j][0]:row2[j][0] + row2[j][2]]
                new_w = findminy(roi)
                if new_w > 1.5 * mean_w:
                    row2[j][2] = int(new_w / 2)
                    to_add[str(k)].append(
                        [row2[j][0] + int(new_w / 2), row2[j][1], int(new_w / 2), row2[j][3]])
                    to_add[str(k)].append(
                        [row2[j][0] + new_w, row2[j][1], width - new_w, row2[j][3]])
                elif width - new_w > 1.7 * mean_w:
                    row2[j][2] = new_w
                    to_add[str(k)].append(
                        [row2[j][0] + row2[j][2], row2[j][1], int((width - new_w) / 2), row2[j][3]])
                    to_add[str(k)].append(
                        [row2[j][0] + row2[j][2] + int(new_w / 2), row2[j][1],
                         int((width - new_w) / 2),
                         row2[j][3]])
                else:
                    row2[j][2] = new_w
                    to_add[str(k)].append(
                        [row2[j][0] + new_w, row2[j][1], width - new_w, row2[j][3]])

            if row2[j][2] < 0.5 * mean_w or row2[j][3] < 0.5 * mean_h:
                errors += 1
                if len(row2) + len(to_add[str(k)]) in [7, 9]:
                    to_del_box.append(j)
                elif len(row2) + len(to_add[str(k)]) in [5, 3]:
                    row2[j][0] = int(row2[j][0] - 0.15 * row2[j][2])
                    row2[j][1] = int(row2[j][1] - 0.15 * row2[j][3])
                    row2[j][2] = int(1.15 * row2[j][2])
                    row2[j][3] = int(1.15 * row2[j][3])
    if errors < 4:
        if len(row2) + len(to_add[str(k)]) in [5, 7]:
            if row2[0][0] - mean_w < 0:
                to_add[str(k)].append([row2[-1][0] + mean_w, mean_y, mean_w, mean_h])
            elif row2[-1][0] + 2 * mean_w > img_w:
                to_add[str(k)].append([row2[0][0] - mean_w, mean_y, mean_w, mean_h])
            else:
                roi_left = textscore_org[mean_y:mean_y + mean_h,
                           row2[0][0] - mean_w:row2[0][0] - int(0.2 * mean_w)]
                roi_right = textscore_org[mean_y:mean_y + mean_h,
                            row2[-1][0] + mean_w:row2[-1][0] + 2 * mean_w]
                roi_left_wp = np.sum(roi_left == 255)
                roi_right_wp = np.sum(roi_right == 255)
                if roi_left_wp - roi_right_wp > 26:
                    to_add[str(k)].append([row2[0][0] - mean_w, mean_y, mean_w, mean_h])
                elif roi_right_wp - roi_left_wp > 26:
                    to_add[str(k)].append([row2[-1][0] + mean_w, mean_y, mean_w, mean_h])
                else:
                    errors += 2
                    to_add[str(k)].append([row2[-1][0] + mean_w, mean_y, mean_w, mean_h])

        to_del[k] = errors
        row_boxes[k] = [row_boxes[k][j] for j in range(len(row_boxes[k])) if j not in to_del_box]

    for er, eb in to_add.items():
        prev_box = []
        for b in eb:
            if len(prev_box) == 0:
                row_boxes[int(er)].append(b)
                prev_box = b
                continue
            if b[0] - prev_box[0] < 0.5 * prev_box[2]:
                continue
            row_boxes[int(er)].append(b)
            prev_box = b
    templ = list()
    for er, err in to_del.items():
        if err > 4:
            row_boxes[er] = []

    row_len = [len(row) for row in row_boxes]
    if len(row_len) == 0:
        return None
    pairs = []
    for i in range(len(row_len)):
        for j in range(len(row_len)):
            if i == j:
                continue
            if row_len[i] + row_len[j] == 12 and row_len[i] != 0 and row_len[j] != 0:
                if [j, i] not in pairs:
                    pairs.append([i, j])

    final_pair = list()
    if len(pairs) == 0:
        if len(row_boxes) == 1:
            return row_boxes
        min_err = 200
        max_char = 0
        lst_err_pair = [0, 1]
        for i in range(len(row_boxes)):
            for j in range(i + 1, len(row_boxes)):
                curr_err = to_del[i] + to_del[j]
                curr_char = len(row_boxes[i]) + len(row_boxes[j])
                if curr_err < min_err:
                    min_err = curr_err
                    lst_err_pair = [i, j]
                if curr_char > max_char:
                    max_char = curr_char
                    max_chr_pair = [i, j]
        if len(row_boxes[lst_err_pair[0]]) + len(row_boxes[lst_err_pair[0]]) < 10:
            final_pair = max_chr_pair
        else:
            final_pair = lst_err_pair

    else:
        pairs_error = list()
        for i in range(len(pairs)):
            temp = to_del[pairs[i][0]] + to_del[pairs[i][1]]
            pairs_error.append([pairs[i], temp])
        pairs_error = [sorted(pairs_error, key=lambda item: item[1])]
        # pairs_error = [pe for pe in (y for y in pairs_error if y[0][1] == pairs_error[0][0][1])]
        if len(pairs_error[0]) > 1:
            if pairs_error[0][0][1] == pairs_error[0][1][1]:
                final_pair = pairs_error[0][1][0]
        else:
            final_pair = pairs_error[0][0][0]

    if len(row_boxes) == 1:
        final = row_boxes
    else:
        final.append(row_boxes[final_pair[0]])
        final.append(row_boxes[final_pair[1]])

    for row in final:
        for box in row:
            box[ 0 ] = int(box[ 0 ] - 0.15 * box[ 2 ])
            box[ 1 ] = int(box[ 1 ] - 0.15 * box[ 3 ])
            box[ 2 ] = int(1.12 * box[ 2 ])
            box[ 3 ] = int(1.30 * box[ 3 ])

    return final


# function call
# bboxes_char = getcharboxes(score_text, low_text)