Speed up post-processing with C++ bindings and add char detection boxes

.gitignore

@@ -3,4 +3,6 @@
 *.pkl
 *.pth
 result*
-weights*
+weights*
+
+cpp_bindings/build/**

build_cpp_bindings_unix.sh

@@ -0,0 +1,4 @@
+mkdir cpp_bindings/build/
+cd cpp_bindings/build/
+cmake .. -DCMAKE_BUILD_TYPE=RELEASE
+make

cpp_bindings/CMakeLists.txt

@@ -0,0 +1,17 @@
+cmake_minimum_required(VERSION 3.2)
+
+project(cpp_bindings VERSION 0.0.1)
+
+set_property(GLOBAL PROPERTY USE_FOLDERS ON)
+
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Wfatal-errors")
+
+include_directories(
+    "src/"
+)
+
+file(GLOB all_cpp_bindings_src
+    "src/*.cpp"
+)
+
+add_library(fast_boxes SHARED ${all_cpp_bindings_src})

cpp_bindings/cpp_bindings.py

@@ -0,0 +1,82 @@
+import ctypes
+import numpy as np
+from numpy.ctypeslib import ndpointer
+import cv2
+
+lib = ctypes.CDLL('cpp_bindings/build/libfast_boxes.so')
+
+c_find_char_boxes = lib.findMinMaxBoxes
+c_find_char_boxes.argtypes = [
+    ndpointer(ctypes.c_int, flags='C_CONTIGUOUS'),
+    ctypes.c_int,
+    ctypes.c_int,
+    ctypes.c_int,
+    ndpointer(ctypes.c_int, flags='C_CONTIGUOUS'),
+]
+
+
+c_find_word_boxes = lib.findWordBoxes
+c_find_word_boxes.argtypes = [
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ctypes.c_int,
+    ctypes.c_int,
+    ndpointer(ctypes.c_int, flags='C_CONTIGUOUS'),
+    ctypes.c_int,
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ctypes.c_float,
+    ctypes.c_int,
+    ctypes.c_int,
+    ndpointer(ctypes.c_float, flags='C_CONTIGUOUS'),
+    ndpointer(ctypes.c_int, flags='C_CONTIGUOUS'),
+]
+
+c_get_max_num_components = lib.getMaxNumComponents
+
+
+def find_char_boxes(markers, num_classes):
+    # Allocate numpy buffer to store the results
+    out_boxes = np.zeros((num_classes+1)*4, dtype=np.int32)
+    c_find_char_boxes(markers, markers.shape[1], markers.shape[0],
+            out_boxes.shape[0], out_boxes)
+
+    char_boxes = []
+    for idx in range(2,num_classes+1):
+        l,t = out_boxes[idx*4+0], out_boxes[idx*4+1]
+        r,b = out_boxes[idx*4+2], out_boxes[idx*4+3]
+        w, h = r-l, b-t
+        box = np.array([[l, t], [l + w, t], [l + w, t + h], [l, t + h]], dtype=np.float32)
+        char_boxes.append(box)
+
+    return char_boxes
+
+def find_word_boxes(textmap, labels, nLabels, stats, text_threshold,
+        fast_mode=False, rotated_box=True):
+    """
+    This function mostly mimics the hot loop from craft_utils.py . However it has two
+    new parameters:
+        fast_mode (bool): uses a faster post processing algorithm. The results should
+            stay mostly the same.
+        rotated_box (bool): indicates if the return boxes should be rotated boxes (the
+            original) or a simple bounding box. The latter is much faster, however it
+            does not works well for text on an angle.
+    """
+    # Allocate numpy buffer to store the results, 4 points and a label
+    out_boxes = np.zeros(nLabels*8, dtype=np.float32)
+    out_labels = np.zeros(nLabels, dtype=np.int32)
+
+    stat_left   = np.array([stats[k,cv2.CC_STAT_LEFT]   for k in range(nLabels)], dtype=np.float32)
+    stat_top    = np.array([stats[k,cv2.CC_STAT_TOP]    for k in range(nLabels)], dtype=np.float32)
+    stat_width  = np.array([stats[k,cv2.CC_STAT_WIDTH]  for k in range(nLabels)], dtype=np.float32)
+    stat_height = np.array([stats[k,cv2.CC_STAT_HEIGHT] for k in range(nLabels)], dtype=np.float32)
+    stat_area   = np.array([stats[k,cv2.CC_STAT_AREA]   for k in range(nLabels)], dtype=np.float32)
+
+    num_boxes = c_find_word_boxes(textmap, textmap.shape[1], textmap.shape[0],
+            labels, nLabels, stat_left, stat_top, stat_width, stat_height,
+            stat_area, text_threshold, int(fast_mode), int(rotated_box), out_boxes,
+            out_labels)
+    out_boxes = [np.array(out_boxes[x*8:x*8+8]).reshape(4,2) for x in range(num_boxes)]
+    return out_boxes, out_labels

cpp_bindings/src/convex_hull.cpp

@@ -0,0 +1,43 @@
+#include "convex_hull.hpp"
+
+
+bool cmp(Point a, Point b) {
+    return a.x < b.x || (a.x == b.x && a.y < b.y);
+}
+
+bool cw(Point a, Point b, Point c) {
+    return a.x*(b.y-c.y)+b.x*(c.y-a.y)+c.x*(a.y-b.y) < 0;
+}
+
+bool ccw(Point a, Point b, Point c) {
+    return a.x*(b.y-c.y)+b.x*(c.y-a.y)+c.x*(a.y-b.y) > 0;
+}
+
+void convexHull(std::vector<Point>& a) {
+    if (a.size() == 1)
+        return;
+
+    sort(a.begin(), a.end(), &cmp);
+    Point p1 = a[0], p2 = a.back();
+    std::vector<Point> up, down;
+    up.push_back(p1);
+    down.push_back(p1);
+    for (int i = 1; i < (int)a.size(); i++) {
+        if (i == (int)a.size() - 1 || cw(p1, a.at(i), p2)) {
+            while (up.size() >= 2 && !cw(up.at(up.size()-2), up.at(up.size()-1), a.at(i)))
+                up.pop_back();
+            up.push_back(a.at(i));
+        }
+        if (i == (int)a.size() - 1 || ccw(p1, a.at(i), p2)) {
+            while(down.size() >= 2 && !ccw(down.at(down.size()-2), down.at(down.size()-1), a.at(i)))
+                down.pop_back();
+            down.push_back(a.at(i));
+        }
+    }
+
+    a.clear();
+    for (int i = 0; i < (int)up.size(); i++)
+        a.push_back(up.at(i));
+    for (int i = down.size() - 2; i > 0; i--)
+        a.push_back(down.at(i));
+}

cpp_bindings/src/convex_hull.hpp

@@ -0,0 +1,27 @@
+// Original code: https://cp-algorithms.com/geometry/grahams-scan-convex-hull.html
+#ifndef CONVEX_HULL_H
+#define CONVEX_HULL_H
+
+#include <algorithm>
+#include <vector>
+
+
+struct Point {
+    float x, y;
+    Point(int x, int y) {
+        this->x = x;
+        this->y = y;
+    };
+    Point() {
+        this->x = 0;
+        this->y = 0;
+    };
+};
+
+bool cmp(Point a, Point b);
+bool cw(Point a, Point b, Point c);
+bool ccw(Point a, Point b, Point c);
+
+void convexHull(std::vector<Point>& a);
+
+#endif

cpp_bindings/src/dilate.cpp

@@ -0,0 +1,86 @@
+#include "dilate.hpp"
+
+
+// https://github.com/opencv/opencv/blob/198b5096aaf8f5d855b98337e9de2fc45485c5a7/modules/ts/src/ts_func.cpp#L642
+void dilate(const unsigned char *src, int srcW, int srcH,
+        unsigned char *dst, int dstW, int dstH,
+        const unsigned char *kernel, int kerW, int kerH,
+        int roiLeft, int roiTop, int roiRight, int roiBottom)
+{
+    if (dstW < srcW || dstH < srcH)
+        return;
+    if (src == dst)
+        return;
+
+    int new_src_w = srcW + kerW/2 + kerW - kerW/2 - 1;
+    unsigned char *new_src = copyMakeBorder(src, srcW, srcH, kerH/2, kerH - kerH/2 - 1,
+               kerW/2, kerW - kerW/2 - 1);
+
+    int *ofs = new int[kerW*kerH];
+    int ofs_size = 0;
+    for( int i = 0; i < kerH; i++ )
+        for( int j = 0; j < kerW; j++ )
+            if( kernel[i*kerW + j] != 0 ) {
+                ofs[ofs_size++] = (i*new_src_w + j);
+            }
+
+    if( ofs_size == 0 )
+        ofs[ofs_size++] = kerH/2*new_src_w + kerW;
+
+    for( int y = roiTop; y < roiBottom; y++ )
+    {
+        const unsigned char *sptr = &new_src[y*new_src_w];
+        unsigned char *dptr = &dst[y*dstW];
+
+        for( int x = roiLeft; x < roiRight; x++ )
+        {
+            unsigned char result = sptr[x + ofs[0]];
+            for( int i = 1; i < ofs_size; i++ ) {
+                result = std::max(result, sptr[x + ofs[i]]);
+            }
+            dptr[x] = result;
+        }
+    }
+
+    delete [] ofs;
+    delete [] new_src;
+}
+
+unsigned char* copyMakeBorder(const unsigned char *src, int srcW, int srcH, int top, int bottom, int left, int right)
+{
+    unsigned char *dst = new unsigned char[(srcH + top + bottom) * (srcW + left + right)];
+    int dstW = (srcW + left + right);
+    int i, j, esz = sizeof(unsigned char);
+    int width = srcW*esz;
+
+    left *= esz;
+    right *= esz;
+    for( i = 0; i < srcH; i++ )
+    {
+        const unsigned char* sptr = &src[i*srcW];
+        unsigned char* dptr = &dst[(i + top)*dstW + left];
+        for( j = 0; j < left; j++ )
+            dptr[j - left] = 0;
+        if( dptr != sptr )
+            for( j = 0; j < width; j++ )
+                dptr[j] = sptr[j];
+        for( j = 0; j < right; j++ )
+            dptr[j + width] = 0;
+    }
+
+    for( i = 0; i < top; i++ )
+    {
+        unsigned char* dptr = &dst[i*dstW];
+        for( j = 0; j < dstW; j++ )
+            dptr[j] = 0;
+    }
+
+    for( i = 0; i < bottom; i++ )
+    {
+        unsigned char* dptr = &dst[(i + top + srcH)*dstW];
+        for( j = 0; j < dstW; j++ )
+            dptr[j] = 0;
+    }
+
+    return dst;
+}

cpp_bindings/src/dilate.hpp

@@ -0,0 +1,16 @@
+#ifndef DILATE_H
+#define DILATE_H
+
+#include <algorithm>
+#include <cmath>
+
+// WARNING (gfickel): src and dst cannot be the same on dilate!
+void dilate(const unsigned char *src, int srcW, int srcH,
+        unsigned char *dst, int dstW, int dstH,
+        const unsigned char *kernel, int kerW, int kerH,
+        int roiLeft, int roiTop, int roiRight, int roiBottom);
+
+unsigned char* copyMakeBorder(const unsigned char *src, int srcW,
+        int srcH, int top, int bottom, int left, int right);
+
+#endif