source/PaddleOCR/deploy/cpp_infer/src/pipelines/ocr/result.cc

// Copyright (c) 2025 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

#include "result.h"

#include <algorithm>
#include <codecvt>
#include <fstream>
#include <locale>
#include <random>
#include <string>

#include "src/utils/utility.h"
#include "third_party/nlohmann/json.hpp"

using json = nlohmann::json;

void OCRResult::SaveToImg(const std::string &save_path) {
  cv::Mat image = pipeline_result_.doc_preprocessor_res.output_image;
  auto texts = pipeline_result_.rec_texts;
  std::vector<std::vector<cv::Point>> boxes;
  std::vector<std::vector<cv::Point2f>> boxes_float =
      pipeline_result_.rec_polys;
  for (const auto &floatPolygon : pipeline_result_.rec_polys) {
    std::vector<cv::Point> intPolygon;
    for (const auto &point : floatPolygon) {
      intPolygon.push_back(cv::Point(cvRound(point.x), cvRound(point.y)));
    }
    boxes.push_back(intPolygon);
  }

  if (image.empty()) {
    INFOE("Input image is empty.");
    exit(-1);
  }

  int h = image.rows;
  int w = image.cols;

  cv::Mat img_left = image.clone();

  cv::Mat img_right(h, w, CV_8UC3, cv::Scalar(255, 255, 255));

  std::random_device rd;
  std::mt19937 gen(rd());
  std::uniform_int_distribution<> dis(0, 255);

  for (size_t i = 0; i < boxes.size(); ++i) {
    auto &box = boxes[i];
    const auto &box_float = boxes_float[i];
    const auto &text = texts[i];

    cv::Scalar color(dis(gen), dis(gen), dis(gen));

    if (box.size() > 4) {
      const std::vector<std::vector<cv::Point>> polygons{box};
      cv::fillPoly(img_left, polygons, color);
      cv::polylines(img_left, polygons, true, color, 8);
      box = GetMinareaRect(box);
      std::vector<int> ys;
      for (const auto &pt : box)
        ys.push_back(pt.y);
      int min_y = *std::min_element(ys.begin(), ys.end());
      int max_y = *std::max_element(ys.begin(), ys.end());
      int height = static_cast<int>(0.5 * (max_y - min_y));
      double mean_y = std::accumulate(ys.begin(), ys.end(), 0.0) / ys.size();
      if (box.size() >= 4) {
        box[0].y = static_cast<int>(mean_y);
        box[1].y = static_cast<int>(mean_y);
        box[2].y = static_cast<int>(mean_y + std::min(20, height));
        box[3].y = static_cast<int>(mean_y + std::min(20, height));
      }
    } else {
      cv::fillPoly(img_left, std::vector<std::vector<cv::Point>>{box}, color);
    }
#ifdef USE_FREETYPE
    cv::Mat img_right_text = DrawBoxTextFine(cv::Size(w, h), box_float, text,
                                             pipeline_result_.vis_fonts);
    cv::polylines(img_right_text, box, true, color, 1);
    cv::bitwise_and(img_right, img_right_text, img_right);
#endif
  }
  cv::Mat blended;
  cv::addWeighted(image, 0.5, img_left, 0.5, 0, blended);
#ifdef USE_FREETYPE
  cv::Mat ocr_res_image(h, w * 2, CV_8UC3, cv::Scalar(255, 255, 255));
  blended.copyTo(ocr_res_image(cv::Rect(0, 0, w, h)));
  img_right.copyTo(ocr_res_image(cv::Rect(w, 0, w, h)));
#else
  cv::Mat ocr_res_image = blended;
#endif

  auto model_settings = pipeline_result_.model_settings;
  std::unordered_map<std::string, cv::Mat> res_img_dict;
  res_img_dict["ocr_res_img"] = ocr_res_image;

  auto ocr_path = Utility::SmartCreateDirectoryForImage(
      save_path, pipeline_result_.input_path, "_ocr_res_img");
  if (!ocr_path.ok()) {
    INFOE(ocr_path.status().ToString().c_str());
    exit(-1);
  }
  auto doc_pre_path = Utility::SmartCreateDirectoryForImage(
      save_path, pipeline_result_.input_path, "_doc_preprocessor_res");
  if (!doc_pre_path.ok()) {
    INFOE(doc_pre_path.status().ToString().c_str());
    exit(-1);
  }
  cv::imwrite(ocr_path.value(), ocr_res_image);
  if (model_settings["use_doc_preprocessor"]) {
    int h1 = pipeline_result_.doc_preprocessor_res.input_image.size[0];
    int w1 = pipeline_result_.doc_preprocessor_res.input_image.size[1];
    int h2 = pipeline_result_.doc_preprocessor_res.rotate_image.size[0];
    int w2 = pipeline_result_.doc_preprocessor_res.rotate_image.size[1];
    int h3 = pipeline_result_.doc_preprocessor_res.output_image.size[0];
    int w3 = pipeline_result_.doc_preprocessor_res.output_image.size[1];
    int h_all = std::max(h1, std::max(h2, h3));
    int total_w = w1 + w2 + w3;

    cv::Mat doc_pre_res_image(h_all, total_w, CV_8UC3,
                              cv::Scalar(255, 255, 255));

    pipeline_result_.doc_preprocessor_res.input_image.copyTo(
        doc_pre_res_image(cv::Rect(0, 0, w1, h1)));
    pipeline_result_.doc_preprocessor_res.rotate_image.copyTo(
        doc_pre_res_image(cv::Rect(w1, 0, w2, h2)));
    pipeline_result_.doc_preprocessor_res.output_image.copyTo(
        doc_pre_res_image(cv::Rect(w1 + w2, 0, w3, h3)));
    cv::imwrite(doc_pre_path.value(), doc_pre_res_image);
    res_img_dict["doc_preprocessor_res"] = doc_pre_res_image;
  }
}

#ifdef USE_FREETYPE
cv::Mat OCRResult::DrawBoxTextFine(const cv::Size &img_size,
                                   const std::vector<cv::Point2f> &box,
                                   const std::string &txt,
                                   const std::string &vis_font) {
  int box_height = cv::norm(box[0] - box[3]);
  int box_width = cv::norm(box[0] - box[1]);
  auto ft2 = cv::freetype::createFreeType2();
  ft2->loadFontData(vis_font, 0);

  bool vertical_mode = box_height > 2 * box_width && box_height > 30;
  int n = std::max(int(txt.size()), 1);

  int font_height = 10;
  if (!txt.empty()) {
    if (vertical_mode) {
      font_height = CreateFontVertical(ft2, txt, box_height, box_width);
    } else {
      font_height = CreateFont(ft2, txt, box_height, box_width);
    }
  }
  cv::Mat img_text(box_height, box_width, CV_8UC3, cv::Scalar(255, 255, 255));
  int x = 0, y = 0;

  if (!txt.empty()) {
    if (vertical_mode) {
      DrawVerticalText(ft2, img_text, txt, x, y, font_height,
                       cv::Scalar(0, 0, 0));
    } else {
      int baseline = 0;
      cv::Size textsize = ft2->getTextSize(txt, font_height, -1, &baseline);
      x = (box_width - textsize.width) / 2;
      y = (box_height + textsize.height) / 2 - baseline;
      ft2->putText(img_text, txt, cv::Point(x, y), font_height,
                   cv::Scalar(0, 0, 0), -1, cv::LINE_AA, true);
    }
  }
  std::vector<cv::Point2f> src_pts = {{0, 0},
                                      {float(box_width), 0},
                                      {float(box_width), float(box_height)},
                                      {0, float(box_height)}};
  cv::Mat M = cv::getPerspectiveTransform(src_pts, box);

  cv::Mat dst(img_size, CV_8UC3, cv::Scalar(255, 255, 255));
  cv::warpPerspective(img_text, dst, M, img_size, cv::INTER_NEAREST,
                      cv::BORDER_CONSTANT, cv::Scalar(255, 255, 255));
  return dst;
}
cv::Size OCRResult::getActualCharSize(cv::Ptr<cv::freetype::FreeType2> &ft2,
                                      const std::string &utf8_char,
                                      int font_height) {
  cv::Mat temp = cv::Mat::zeros(300, 300, CV_8UC1);

  cv::Point pos(100, 150);

  ft2->putText(temp, utf8_char, pos, font_height, cv::Scalar(255), -1,
               cv::LINE_AA, false);

  std::vector<cv::Point> nonZeroPoints;
  cv::findNonZero(temp, nonZeroPoints);

  if (nonZeroPoints.empty()) {
    return cv::Size(0, 0);
  }

  cv::Rect boundingRect = cv::boundingRect(nonZeroPoints);
  return cv::Size(boundingRect.width, boundingRect.height);
}
void OCRResult::DrawVerticalText(cv::Ptr<cv::freetype::FreeType2> &ft2,
                                 cv::Mat &img, const std::string &text, int x,
                                 int y, int font_height, cv::Scalar color,
                                 float line_spacing) {
  std::wstring wtext =
      std::wstring_convert<std::codecvt_utf8<wchar_t>>().from_bytes(text);
  for (size_t i = 0; i < wtext.size(); ++i) {
    std::wstring single_char(1, wtext[i]);
    std::string utf8_char =
        std::wstring_convert<std::codecvt_utf8<wchar_t>>().to_bytes(
            single_char);
    ft2->putText(img, utf8_char, cv::Point(x, y), font_height, color, -1,
                 cv::LINE_AA, true);
    int baseline = 0;
    cv::Size size = ft2->getTextSize(utf8_char, font_height, -1, &baseline);
    size.height += baseline;
    y += size.height * 1.1 + line_spacing;
  }
}
int OCRResult::CreateFont(cv::Ptr<cv::freetype::FreeType2> &ft2,
                          const std::string &text, int region_height,
                          int region_width) {
  int font_height = std::max(int(region_height * 0.8), 10);
  int baseline = 0;
  cv::Size text_size = ft2->getTextSize(text, font_height, -1, &baseline);
  if (text_size.width > region_width) {
    font_height =
        static_cast<int>(font_height * region_width / text_size.width);
    text_size = ft2->getTextSize(text, font_height, -1, &baseline);
  }
  return font_height;
}
int OCRResult::CreateFontVertical(cv::Ptr<cv::freetype::FreeType2> &ft2,
                                  const std::string &text, int region_height,
                                  int region_width, float scale) {
  std::wstring_convert<std::codecvt_utf8<wchar_t>> conv;
  std::wstring wtext = conv.from_bytes(text);
  int n = static_cast<int>(wtext.length());
  int baseFontSize = static_cast<int>(region_height / n * 0.8 * scale);
  baseFontSize = std::max(baseFontSize, 10);

  int maxCharWidth = 0;
  for (size_t i = 0; i < wtext.length(); ++i) {
    std::wstring singleChar(1, wtext[i]);
    std::string utf8Char =
        std::wstring_convert<std::codecvt_utf8<wchar_t>>().to_bytes(singleChar);
    cv::Size textSize = getActualCharSize(ft2, utf8Char, baseFontSize);
    maxCharWidth = std::max(maxCharWidth, textSize.width);
  }

  int finalFontSize = baseFontSize;
  if (maxCharWidth > region_width) {
    finalFontSize =
        static_cast<int>(baseFontSize * region_width / maxCharWidth);
    finalFontSize = std::max(finalFontSize, 10);
  }

  return finalFontSize;
}
#endif

std::vector<cv::Point>
OCRResult::GetMinareaRect(const std::vector<cv::Point> &points) {
  cv::RotatedRect bounding_box = cv::minAreaRect(points);

  cv::Point2f boxPts[4];
  bounding_box.points(boxPts);
  std::vector<cv::Point2f> ptsVec(boxPts, boxPts + 4);

  std::sort(
      ptsVec.begin(), ptsVec.end(),
      [](const cv::Point2f &a, const cv::Point2f &b) { return a.x < b.x; });
  int index_a, index_b, index_c, index_d;
  if (ptsVec[1].y > ptsVec[0].y) {
    index_a = 0;
    index_d = 1;
  } else {
    index_a = 1;
    index_d = 0;
  }
  if (ptsVec[3].y > ptsVec[2].y) {
    index_b = 2;
    index_c = 3;
  } else {
    index_b = 3;
    index_c = 2;
  }

  std::vector<cv::Point> box = {ptsVec[index_a], ptsVec[index_b],
                                ptsVec[index_c], ptsVec[index_d]};

  for (auto &pt : box) {
    pt.x = static_cast<int>(std::round(pt.x));
    pt.y = static_cast<int>(std::round(pt.y));
  }

  return box;
}

void OCRResult::SaveToJson(const std::string &save_path) const {
  nlohmann::ordered_json j;
  j["input_path"] = pipeline_result_.input_path;

  j["page_index"] = nullptr;

  j["model_settings"] = pipeline_result_.model_settings;

  auto it = pipeline_result_.model_settings.find("use_doc_preprocessor");
  if (it != pipeline_result_.model_settings.end() && it->second) {
    nlohmann::ordered_json j_doc_pre;
    j_doc_pre["model_settings"] =
        pipeline_result_.doc_preprocessor_res.model_settings;
    j_doc_pre["angle"] = pipeline_result_.doc_preprocessor_res.angle;
    j["doc_preprocessor_res"] = j_doc_pre;
  }
  json polys_json = json::array();
  for (const auto &polygon : pipeline_result_.dt_polys) {
    json poly_json = json::array();
    for (const auto &point : polygon) {
      poly_json.push_back(
          {static_cast<int>(point.x), static_cast<int>(point.y)});
    }
    polys_json.push_back(poly_json);
  }
  j["dt_polys"] = polys_json;
  nlohmann::ordered_json j_text_det_params;
  j_text_det_params["limit_side_len"] =
      pipeline_result_.text_det_params.text_det_limit_side_len;
  j_text_det_params["limit_type"] =
      pipeline_result_.text_det_params.text_det_limit_type;
  j_text_det_params["thresh"] =
      pipeline_result_.text_det_params.text_det_thresh;
  j_text_det_params["max_side_limit"] =
      pipeline_result_.text_det_params.text_det_max_side_limit;
  j_text_det_params["box_thresh"] =
      pipeline_result_.text_det_params.text_det_box_thresh;
  j_text_det_params["unclip_ratio"] =
      pipeline_result_.text_det_params.text_det_unclip_ratio;
  j["text_det_params"] = j_text_det_params;
  j["text_type"] = pipeline_result_.text_type;

  if (!pipeline_result_.textline_orientation_angles.empty()) {
    j["textline_orientation_angles"] =
        pipeline_result_.textline_orientation_angles;
  }
  j["text_rec_score_thresh"] = pipeline_result_.text_rec_score_thresh;
  j["rec_texts"] = pipeline_result_.rec_texts;
  j["rec_scores"] = pipeline_result_.rec_scores;
  json rec_polys_json = json::array();
  for (const auto &polygon : pipeline_result_.rec_polys) {
    json poly_json = json::array();
    for (const auto &point : polygon) {
      poly_json.push_back(
          {static_cast<int>(point.x), static_cast<int>(point.y)});
    }
    rec_polys_json.push_back(poly_json);
  }
  j["rec_polys"] = rec_polys_json;

  std::vector<std::array<int, 4>> int_vec;
  int_vec.reserve(pipeline_result_.rec_boxes.size());

  std::transform(pipeline_result_.rec_boxes.begin(),
                 pipeline_result_.rec_boxes.end(), std::back_inserter(int_vec),
                 [](const std::array<float, 4> &arr) {
                   std::array<int, 4> res;
                   for (size_t i = 0; i < 4; ++i) {
                     res[i] = static_cast<int>(arr[i]);
                   }
                   return res;
                 });
  j["rec_boxes"] = int_vec;

  absl::StatusOr<std::string> full_path;
  if (pipeline_result_.input_path.empty()) {
    INFOW("Input path is empty, will use output_res.json instead!");
    full_path = Utility::SmartCreateDirectoryForJson(save_path, "output");
  } else {
    full_path = Utility::SmartCreateDirectoryForJson(
        save_path, pipeline_result_.input_path);
  }
  if (!full_path.ok()) {
    INFOE(full_path.status().ToString().c_str());
    exit(-1);
  }
  std::ofstream file(full_path.value());
  if (file.is_open()) {
    file << j.dump(4);
    file.close();
  } else {
    INFOE("Could not open file for writing: %s", save_path.c_str());
    exit(-1);
  }
}

void PrintDocPreprocessorPipelineResult(
    const DocPreprocessorPipelineResult &doc) {
  std::cout << "{\n";
  std::cout << "    \"model_settings\": {";
  bool first = true;
  for (const auto &kv : doc.model_settings) {
    if (!first)
      std::cout << ", ";
    std::cout << "\"" << kv.first << "\": " << (kv.second ? "true" : "false");
    first = false;
  }
  std::cout << "},\n";
  std::cout << "    \"angle\": " << doc.angle << "\n";
  std::cout << "  }";
}

void PrintPolys(const std::vector<std::vector<cv::Point2f>> &polys) {
  std::cout << "[";
  for (size_t i = 0; i < polys.size(); ++i) {
    if (i != 0)
      std::cout << ",\n ";
    std::cout << "[";
    for (size_t j = 0; j < polys[i].size(); ++j) {
      if (j != 0)
        std::cout << ", ";
      std::cout << "[" << polys[i][j].x << ", " << polys[i][j].y << "]";
    }
    std::cout << "]";
  }
  std::cout << "]";
}

void PrintModelSettings(const std::unordered_map<std::string, bool> &ms) {
  std::cout << "{";
  bool first = true;
  for (const auto &kv : ms) {
    if (!first)
      std::cout << ", ";
    std::cout << "\"" << kv.first << "\": " << (kv.second ? "true" : "false");
    first = false;
  }
  std::cout << "}";
}

void PrintArray(const std::vector<float> &arr) {
  std::cout << "[";
  for (size_t i = 0; i < arr.size(); ++i) {
    if (i != 0)
      std::cout << ", ";
    std::cout << arr[i];
  }
  std::cout << "]";
}

void PrintStringArray(const std::vector<std::string> &arr) {
  std::cout << "[";
  for (size_t i = 0; i < arr.size(); ++i) {
    if (i != 0)
      std::cout << ", ";
    std::cout << "\"" << arr[i] << "\"";
  }
  std::cout << "]";
}

void PrintIntArray(const std::vector<int> &arr) {
  std::cout << "[";
  for (size_t i = 0; i < arr.size(); ++i) {
    if (i != 0)
      std::cout << ", ";
    std::cout << arr[i];
  }
  std::cout << "]";
}

void PrintRecBoxes(const std::vector<std::array<float, 4>> &arr) {
  std::cout << "[";
  for (size_t i = 0; i < arr.size(); ++i) {
    if (i != 0)
      std::cout << ", ";
    std::cout << "[" << arr[i][0] << ", " << arr[i][1] << ", " << arr[i][2]
              << ", " << arr[i][3] << "]";
  }
  std::cout << "],";
}

void PrintTextDetParams(const TextDetParams &p) {
  std::cout << "{";
  std::cout << "\"limit_side_len\": " << p.text_det_limit_side_len << ", ";
  std::cout << "\"limit_type\": \"" << p.text_det_limit_type << "\", ";
  std::cout << "\"thresh\": " << p.text_det_thresh << ", ";
  std::cout << "\"max_side_limit\": " << p.text_det_max_side_limit << ", ";
  std::cout << "\"box_thresh\": " << p.text_det_box_thresh << ", ";
  std::cout << "\"unclip_ratio\": " << p.text_det_unclip_ratio;
  std::cout << "}";
}

void OCRResult::Print() const {
  std::cout << "{\n";
  std::cout << "  \"input_path\": \"" << pipeline_result_.input_path << "\",\n";
  if (pipeline_result_.model_settings.at("use_doc_preprocessor")) {
    std::cout << "  \"doc_preprocessor_res\": ";
    PrintDocPreprocessorPipelineResult(pipeline_result_.doc_preprocessor_res);
    std::cout << ",\n";
  }
  std::cout << "  \"dt_polys\": ";
  PrintPolys(pipeline_result_.dt_polys);
  std::cout << ",\n";
  std::cout << "  \"model_settings\": ";
  PrintModelSettings(pipeline_result_.model_settings);
  std::cout << ",\n";
  std::cout << "  \"text_det_params\": ";
  PrintTextDetParams(pipeline_result_.text_det_params);
  std::cout << ",\n";
  std::cout << "  \"text_type\": \"" << pipeline_result_.text_type << "\",\n";
  std::cout << "  \"text_rec_score_thresh\": "
            << pipeline_result_.text_rec_score_thresh << ",\n";
  std::cout << "  \"rec_texts\": ";
  PrintStringArray(pipeline_result_.rec_texts);
  std::cout << ",\n";
  std::cout << "  \"rec_scores\": ";
  PrintArray(pipeline_result_.rec_scores);
  std::cout << ",\n";
  std::cout << "  \"textline_orientation_angles\": ";
  PrintIntArray(pipeline_result_.textline_orientation_angles);
  std::cout << ",\n";
  std::cout << "  \"rec_polys\": ";
  PrintPolys(pipeline_result_.rec_polys);
  std::cout << ",\n";
  std::cout << "  \"rec_boxes\": ";
  PrintRecBoxes(pipeline_result_.rec_boxes);
  std::cout << "\n}" << std::endl;
}