create

.idea/Cpp4App_test.iml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <excludeFolder url="file://$MODULE_DIR$/venv" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.10 (Cpp4App_test)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,16 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="CssUnknownProperty" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="myCustomPropertiesEnabled" value="true" />
+      <option name="myIgnoreVendorSpecificProperties" value="false" />
+      <option name="myCustomPropertiesList">
+        <value>
+          <list size="1">
+            <item index="0" class="java.lang.String" itemvalue="transform" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

.idea/inspectionProfiles/profiles_settings.xml

@@ -0,0 +1,6 @@
+<component name="InspectionProjectProfileManager">
+  <settings>
+    <option name="USE_PROJECT_PROFILE" value="false" />
+    <version value="1.0" />
+  </settings>
+</component>

.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.10 (Cpp4App_test)" project-jdk-type="Python SDK" />
+</project>

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/Cpp4App_test.iml" filepath="$PROJECT_DIR$/.idea/Cpp4App_test.iml" />
+    </modules>
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

.idea/workspace.xml

@@ -0,0 +1,59 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="AutoImportSettings">
+    <option name="autoReloadType" value="SELECTIVE" />
+  </component>
+  <component name="ChangeListManager">
+    <list default="true" id="5e4481c0-7ba2-42e4-bbe6-4c36a0d36baa" name="Changes" comment="">
+      <change afterPath="$PROJECT_DIR$/app.py" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/main" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/packages.txt" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/requirements.txt" afterDir="false" />
+      <change afterPath="$PROJECT_DIR$/run_sem_test.py" afterDir="false" />
+      <change beforePath="$PROJECT_DIR$/README.md" beforeDir="false" afterPath="$PROJECT_DIR$/README.md" afterDir="false" />
+    </list>
+    <option name="SHOW_DIALOG" value="false" />
+    <option name="HIGHLIGHT_CONFLICTS" value="true" />
+    <option name="HIGHLIGHT_NON_ACTIVE_CHANGELIST" value="false" />
+    <option name="LAST_RESOLUTION" value="IGNORE" />
+  </component>
+  <component name="Git.Settings">
+    <option name="RECENT_GIT_ROOT_PATH" value="$PROJECT_DIR$" />
+  </component>
+  <component name="MarkdownSettingsMigration">
+    <option name="stateVersion" value="1" />
+  </component>
+  <component name="ProjectId" id="2kW6Fg72Z5pXQX4LLQVp4SZZKLB" />
+  <component name="ProjectViewState">
+    <option name="hideEmptyMiddlePackages" value="true" />
+    <option name="showLibraryContents" value="true" />
+  </component>
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.OpenProjectViewOnStart": "true",
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "WebServerToolWindowFactoryState": "false",
+    "node.js.detected.package.eslint": "true",
+    "node.js.detected.package.tslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "node.js.selected.package.tslint": "(autodetect)",
+    "vue.rearranger.settings.migration": "true"
+  }
+}]]></component>
+  <component name="SpellCheckerSettings" RuntimeDictionaries="0" Folders="0" CustomDictionaries="0" DefaultDictionary="application-level" UseSingleDictionary="true" transferred="true" />
+  <component name="TaskManager">
+    <task active="true" id="Default" summary="Default task">
+      <changelist id="5e4481c0-7ba2-42e4-bbe6-4c36a0d36baa" name="Changes" comment="" />
+      <created>1723386888312</created>
+      <option name="number" value="Default" />
+      <option name="presentableId" value="Default" />
+      <updated>1723386888312</updated>
+      <workItem from="1723386893769" duration="12000" />
+      <workItem from="1723386970626" duration="451000" />
+    </task>
+    <servers />
+  </component>
+  <component name="TypeScriptGeneratedFilesManager">
+    <option name="version" value="3" />
+  </component>
+</project>

CDM/.idea/.gitignore

@@ -0,0 +1,3 @@
+# Default ignored files
+/shelf/
+/workspace.xml

CDM/.idea/UIED.iml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/resnet" isTestSource="false" />
+    </content>
+    <orderEntry type="jdk" jdkName="Python 3.7" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+  <component name="TestRunnerService">
+    <option name="PROJECT_TEST_RUNNER" value="py.test" />
+  </component>
+</module>

CDM/.idea/inspectionProfiles/Project_Default.xml

@@ -0,0 +1,29 @@
+<component name="InspectionProjectProfileManager">
+  <profile version="1.0">
+    <option name="myName" value="Project Default" />
+    <inspection_tool class="DuplicatedCode" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <Languages>
+        <language minSize="54" name="Python" />
+      </Languages>
+    </inspection_tool>
+    <inspection_tool class="PyInterpreterInspection" enabled="false" level="WARNING" enabled_by_default="false" />
+    <inspection_tool class="PyPackageRequirementsInspection" enabled="true" level="WARNING" enabled_by_default="true">
+      <option name="ignoredPackages">
+        <value>
+          <list size="3">
+            <item index="0" class="java.lang.String" itemvalue="Tensorflow" />
+            <item index="1" class="java.lang.String" itemvalue="Sklearn" />
+            <item index="2" class="java.lang.String" itemvalue="Opencv" />
+          </list>
+        </value>
+      </option>
+    </inspection_tool>
+    <inspection_tool class="PyPep8Inspection" enabled="true" level="WEAK WARNING" enabled_by_default="true">
+      <option name="ignoredErrors">
+        <list>
+          <option value="E501" />
+        </list>
+      </option>
+    </inspection_tool>
+  </profile>
+</component>

CDM/.idea/misc.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectRootManager" version="2" project-jdk-name="Python 3.7" project-jdk-type="Python SDK" />
+</project>

CDM/.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/UIED.iml" filepath="$PROJECT_DIR$/.idea/UIED.iml" />
+    </modules>
+  </component>
+</project>

CDM/.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

CDM/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [2021] [UIED mulong.xie@anu.edu.au]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

CDM/README.md

@@ -0,0 +1,80 @@
+# UIED - UI element detection, detecting UI elements from UI screenshots or drawnings
+
+This project is still ongoing and this repo may be updated irregularly, I developed a web app for the UIED in http://uied.online
+
+## Related Publications: 
+[1. UIED: a hybrid tool for GUI element detection](https://dl.acm.org/doi/10.1145/3368089.3417940)
+
+[2. Object Detection for Graphical User Interface: Old Fashioned or Deep Learning or a Combination?](https://arxiv.org/abs/2008.05132)
+
+>The repo has been **upgraded with Google OCR** for GUI text detection, to use the original version in our paper (using [EAST](https://github.com/argman/EAST) as text detector), check the relase [v2.3](https://github.com/MulongXie/UIED/releases/tag/v2.3) and download the pre-trained model in [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing).
+
+## What is it?
+
+UI Element Detection (UIED) is an old-fashioned computer vision (CV) based element detection approach for graphic user interface. 
+
+The input of UIED could be various UI image, such as mobile app or web page screenshot, UI design drawn by Photoshop or Sketch, and even some hand-drawn UI design. Then the approach detects and classifies text and graphic UI elements, and exports the detection result as JSON file for future application. 
+
+UIED comprises two parts to detect UI text and graphic elements, such as button, image and input bar. 
+* For text, it leverages [Google OCR](https://cloud.google.com/vision/docs/ocr) to perfrom detection. 
+
+* For graphical elements, it uses old-fashioned CV approaches to locate the elements and a CNN classifier to achieve classification. 
+
+> UIED is highly customizable, you can replace both parts by your choice (e.g. other text detection approaches). Unlike black-box end-to-end deep learning approach, you can revise the algorithms in the non-text detection and merging (partially or entirely) easily to fit your task.
+
+![UIED Approach](https://github.com/MulongXie/UIED/blob/master/data/demo/approach.png)
+
+## How to use?
+
+### Dependency
+* **Python 3.5**
+* **Opencv 3.4.2**
+* **Pandas**
+<!-- * **Tensorflow 1.10.0**
+* **Keras 2.2.4**
+* **Sklearn 0.22.2** -->
+
+### Installation
+<!-- Install the mentioned dependencies, and download two pre-trained models from [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing) for EAST text detection and GUI element classification. -->
+
+<!-- Change ``CNN_PATH`` and ``EAST_PATH`` in *config/CONFIG.py* to your locations. -->
+
+The new version of UIED equipped with Google OCR is easy to deploy and no pre-trained model is needed. Simply donwload the repo along with the dependencies.
+
+> Please replace the Google OCR key at `detect_text/ocr.py line 28` with your own (apply in [Google website](https://cloud.google.com/vision)).
+
+### Usage
+To test your own image(s):
+* To test single image, change *input_path_img* in ``run_single.py`` to your input image and the results will be output to *output_root*.
+* To test mutiple images, change *input_img_root* in ``run_batch.py`` to your input directory and the results will be output to *output_root*.
+* To adjust the parameters lively, using ``run_testing.py`` 
+
+> Note: The best set of parameters vary for different types of GUI image (Mobile App, Web, PC). I highly recommend to first play with the ``run_testing.py`` to pick a good set of parameters for your data.
+   
+## Folder structure
+``cnn/``
+* Used to train classifier for graphic UI elements
+* Set path of the CNN classification model
+
+``config/``
+* Set data paths 
+* Set parameters for graphic elements detection
+
+``data/``
+* Input UI images and output detection results
+
+``detect_compo/``
+* Non-text GUI component detection
+
+``detect_text/``
+* GUI text detection using Google OCR
+
+``detect_merge/``
+* Merge the detection results of non-text and text GUI elements
+
+The major detection algorithms are in ``detect_compo/``, ``detect_text/`` and ``detect_merge/``
+
+## Demo
+GUI element detection result for web screenshot
+ 
+![UI Components detection result](https://github.com/MulongXie/UIED/blob/master/data/demo/demo.png)

CDM/cnn/CNN.py

@@ -0,0 +1,114 @@
+import keras
+from keras.applications.resnet50 import ResNet50
+from keras.models import Model,load_model
+from keras.layers import Dense, Activation, Flatten, Dropout
+from sklearn.metrics import confusion_matrix
+import numpy as np
+import cv2
+
+from config.CONFIG import Config
+cfg = Config()
+
+
+class CNN:
+    def __init__(self, classifier_type, is_load=True):
+        '''
+        :param classifier_type: 'Text' or 'Noise' or 'Elements'
+        '''
+        self.data = None
+        self.model = None
+
+        self.classifier_type = classifier_type
+
+        self.image_shape = (32,32,3)
+        self.class_number = None
+        self.class_map = None
+        self.model_path = None
+        self.classifier_type = classifier_type
+        if is_load:
+            self.load(classifier_type)
+
+    def build_model(self, epoch_num, is_compile=True):
+        base_model = ResNet50(include_top=False, weights='imagenet', input_shape=self.image_shape)
+        for layer in base_model.layers:
+            layer.trainable = False
+        self.model = Flatten()(base_model.output)
+        self.model = Dense(128, activation='relu')(self.model)
+        self.model = Dropout(0.5)(self.model)
+        self.model = Dense(15, activation='softmax')(self.model)
+
+        self.model = Model(inputs=base_model.input, outputs=self.model)
+        if is_compile:
+            self.model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy'])
+            self.model.fit(self.data.X_train, self.data.Y_train, batch_size=64, epochs=epoch_num, verbose=1,
+                           validation_data=(self.data.X_test, self.data.Y_test))
+
+    def train(self, data, epoch_num=30):
+        self.data = data
+        self.build_model(epoch_num)
+        self.model.save(self.model_path)
+        print("Trained model is saved to", self.model_path)
+
+    def load(self, classifier_type):
+        if classifier_type == 'Text':
+            self.model_path = 'E:/Mulong/Model/rico_compos/cnn-textview-2.h5'
+            self.class_map = ['Text', 'Non-Text']
+        elif classifier_type == 'Noise':
+            self.model_path = 'E:/Mulong/Model/rico_compos/cnn-noise-1.h5'
+            self.class_map = ['Noise', 'Non-Noise']
+        elif classifier_type == 'Elements':
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-19.h5'
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-28.h5'
+            # self.model_path = 'E:/Mulong/Model/rico_compos/resnet-ele14-45.h5'
+            self.model_path = cfg.CNN_PATH
+            self.class_map = cfg.element_class
+            self.image_shape = (64, 64, 3)
+        elif classifier_type == 'Image':
+            self.model_path = 'E:/Mulong/Model/rico_compos/cnn-image-1.h5'
+            self.class_map = ['Image', 'Non-Image']
+        self.class_number = len(self.class_map)
+        self.model = load_model(self.model_path)
+        print('Model Loaded From', self.model_path)
+
+    def preprocess_img(self, image):
+        image = cv2.resize(image, self.image_shape[:2])
+        x = (image / 255).astype('float32')
+        x = np.array([x])
+        return x
+
+    def predict(self, imgs, compos, load=False, show=False):
+        """
+        :type img_path: list of img path
+        """
+        if load:
+            self.load(self.classifier_type)
+        if self.model is None:
+            print("*** No model loaded ***")
+            return
+        for i in range(len(imgs)):
+            X = self.preprocess_img(imgs[i])
+            Y = self.class_map[np.argmax(self.model.predict(X))]
+            compos[i].category = Y
+            if show:
+                print(Y)
+                cv2.imshow('element', imgs[i])
+                cv2.waitKey()
+
+    def evaluate(self, data, load=True):
+        if load:
+            self.load(self.classifier_type)
+        X_test = data.X_test
+        Y_test = [np.argmax(y) for y in data.Y_test]
+        Y_pre = [np.argmax(y_pre) for y_pre in self.model.predict(X_test, verbose=1)]
+
+        matrix = confusion_matrix(Y_test, Y_pre)
+        print(matrix)
+
+        TP, FP, FN = 0, 0, 0
+        for i in range(len(matrix)):
+            TP += matrix[i][i]
+            FP += sum(matrix[i][:]) - matrix[i][i]
+            FN += sum(matrix[:][i]) - matrix[i][i]
+        precision = TP/(TP+FP)
+        recall = TP / (TP+FN)
+        print("Precision:%.3f, Recall:%.3f" % (precision, recall))

CDM/cnn/Config.py

@@ -0,0 +1,21 @@
+
+class Config:
+    def __init__(self):
+        # cnn 4 classes
+        # self.MODEL_PATH = 'E:/Mulong/Model/ui_compos/cnn6_icon.h5'   # cnn 4 classes
+        # self.class_map = ['Image', 'Icon', 'Button', 'Input']
+
+        # resnet 14 classes
+        # self.DATA_PATH = "E:/Mulong/Datasets/rico/elements-14-2"
+        # self.MODEL_PATH = 'E:/Mulong/Model/rico_compos/resnet-ele14.h5'
+        # self.class_map = ['Button', 'CheckBox', 'Chronometer', 'EditText', 'ImageButton', 'ImageView',
+        #                   'ProgressBar', 'RadioButton', 'RatingBar', 'SeekBar', 'Spinner', 'Switch',
+        #                   'ToggleButton', 'VideoView', 'TextView']  # ele-14
+
+        self.DATA_PATH = "E:\Mulong\Datasets\dataset_webpage\Components3"
+
+        self.MODEL_PATH = 'E:/Mulong/Model/rico_compos/cnn2-textview.h5'
+        self.class_map = ['Text', 'Non-Text']
+
+        self.image_shape = (32, 32, 3)
+        self.class_number = len(self.class_map)

CDM/cnn/Data.py

@@ -0,0 +1,69 @@
+import cv2
+import numpy as np
+from os.path import join as pjoin
+import glob
+from tqdm import tqdm
+from Config import Config
+
+cfg = Config()
+
+
+class Data:
+    def __init__(self):
+        self.data_num = 0
+        self.images = []
+        self.labels = []
+        self.X_train, self.Y_train = None, None
+        self.X_test, self.Y_test = None, None
+
+        self.image_shape = cfg.image_shape
+        self.class_number = cfg.class_number
+        self.class_map = cfg.class_map
+        self.DATA_PATH = cfg.DATA_PATH
+
+    def load_data(self, resize=True, shape=None, max_number=1000000):
+        # if customize shape
+        if shape is not None:
+            self.image_shape = shape
+        else:
+            shape = self.image_shape
+
+        # load data
+        for p in glob.glob(pjoin(self.DATA_PATH, '*')):
+            print("*** Loading components of %s: %d ***" %(p.split('\\')[-1], int(len(glob.glob(pjoin(p, '*.png'))))))
+            label = self.class_map.index(p.split('\\')[-1])  # map to index of classes
+            for i, image_path in enumerate(tqdm(glob.glob(pjoin(p, '*.png'))[:max_number])):
+                image = cv2.imread(image_path)
+                if resize:
+                    image = cv2.resize(image, shape[:2])
+                self.images.append(image)
+                self.labels.append(label)
+
+        assert len(self.images) == len(self.labels)
+        self.data_num = len(self.images)
+        print('%d Data Loaded' % self.data_num)
+
+    def generate_training_data(self, train_data_ratio=0.8):
+        # transfer int into c dimensions one-hot array
+        def expand(label, class_number):
+            # return y : (num_class, num_samples)
+            y = np.eye(class_number)[label]
+            y = np.squeeze(y)
+            return y
+
+        # reshuffle
+        np.random.seed(0)
+        self.images = np.random.permutation(self.images)
+        np.random.seed(0)
+        self.labels = np.random.permutation(self.labels)
+        Y = expand(self.labels, self.class_number)
+
+        # separate dataset
+        cut = int(train_data_ratio * self.data_num)
+        self.X_train = (self.images[:cut] / 255).astype('float32')
+        self.X_test = (self.images[cut:] / 255).astype('float32')
+        self.Y_train = Y[:cut]
+        self.Y_test = Y[cut:]
+
+        print('X_train:%d, Y_train:%d' % (len(self.X_train), len(self.Y_train)))
+        print('X_test:%d, Y_test:%d' % (len(self.X_test), len(self.Y_test)))

CDM/config/CONFIG.py

@@ -0,0 +1,45 @@
+from os.path import join as pjoin
+import os
+
+
+class Config:
+
+    def __init__(self):
+        # setting CNN (graphic elements) model
+        self.image_shape = (64, 64, 3)
+        # self.MODEL_PATH = 'E:\\Mulong\\Model\\UI2CODE\\cnn6_icon.h5'
+        # self.class_map = ['button', 'input', 'icon', 'img', 'text']
+        self.CNN_PATH = 'E:/Mulong/Model/rico_compos/cnn-rico-1.h5'
+        self.element_class = ['Button', 'CheckBox', 'Chronometer', 'EditText', 'ImageButton', 'ImageView',
+                              'ProgressBar', 'RadioButton', 'RatingBar', 'SeekBar', 'Spinner', 'Switch',
+                              'ToggleButton', 'VideoView', 'TextView']
+        self.class_number = len(self.element_class)
+
+        # setting EAST (ocr) model
+        self.EAST_PATH = 'E:/Mulong/Model/East/east_icdar2015_resnet_v1_50_rbox'
+
+        self.COLOR = {'Button': (0, 255, 0), 'CheckBox': (0, 0, 255), 'Chronometer': (255, 166, 166),
+                      'EditText': (255, 166, 0),
+                      'ImageButton': (77, 77, 255), 'ImageView': (255, 0, 166), 'ProgressBar': (166, 0, 255),
+                      'RadioButton': (166, 166, 166),
+                      'RatingBar': (0, 166, 255), 'SeekBar': (0, 166, 10), 'Spinner': (50, 21, 255),
+                      'Switch': (80, 166, 66), 'ToggleButton': (0, 66, 80), 'VideoView': (88, 66, 0),
+                      'TextView': (169, 255, 0), 'NonText': (0,0,255),
+                      'Compo':(0, 0, 255), 'Text':(169, 255, 0), 'Block':(80, 166, 66)}
+
+    def build_output_folders(self):
+        # setting data flow paths
+        self.ROOT_INPUT = "E:\\Mulong\\Datasets\\rico\\combined"
+        self.ROOT_OUTPUT = "E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_v3"
+
+        self.ROOT_IMG_ORG = pjoin(self.ROOT_INPUT, "org")
+        self.ROOT_IP = pjoin(self.ROOT_OUTPUT, "ip")
+        self.ROOT_OCR = pjoin(self.ROOT_OUTPUT, "ocr")
+        self.ROOT_MERGE = pjoin(self.ROOT_OUTPUT, "merge")
+        self.ROOT_IMG_COMPONENT = pjoin(self.ROOT_OUTPUT, "components")
+        if not os.path.exists(self.ROOT_IP):
+            os.mkdir(self.ROOT_IP)
+        if not os.path.exists(self.ROOT_OCR):
+            os.mkdir(self.ROOT_OCR)
+        if not os.path.exists(self.ROOT_MERGE):
+            os.mkdir(self.ROOT_MERGE)

CDM/config/CONFIG_UIED.py

@@ -0,0 +1,49 @@
+class Config:
+
+    def __init__(self):
+        # Adjustable
+        # self.THRESHOLD_PRE_GRADIENT = 4             # dribbble:4 rico:4 web:1
+        # self.THRESHOLD_OBJ_MIN_AREA = 55            # bottom line 55 of small circle
+        # self.THRESHOLD_BLOCK_GRADIENT = 5
+
+        # *** Frozen ***
+        self.THRESHOLD_REC_MIN_EVENNESS = 0.7
+        self.THRESHOLD_REC_MAX_DENT_RATIO = 0.25
+        self.THRESHOLD_LINE_THICKNESS = 8
+        self.THRESHOLD_LINE_MIN_LENGTH = 0.95
+        self.THRESHOLD_COMPO_MAX_SCALE = (0.25, 0.98)  # (120/800, 422.5/450) maximum height and width ratio for a atomic compo (button)
+        self.THRESHOLD_TEXT_MAX_WORD_GAP = 10
+        self.THRESHOLD_TEXT_MAX_HEIGHT = 0.04  # 40/800 maximum height of text
+        self.THRESHOLD_TOP_BOTTOM_BAR = (0.045, 0.94)  # (36/800, 752/800) height ratio of top and bottom bar
+        self.THRESHOLD_BLOCK_MIN_HEIGHT = 0.03  # 24/800
+
+        # deprecated
+        # self.THRESHOLD_OBJ_MIN_PERIMETER = 0
+        # self.THRESHOLD_BLOCK_MAX_BORDER_THICKNESS = 8
+        # self.THRESHOLD_BLOCK_MAX_CROSS_POINT = 0.1
+        # self.THRESHOLD_UICOMPO_MIN_W_H_RATIO = 0.4
+        # self.THRESHOLD_TEXT_MAX_WIDTH = 150
+        # self.THRESHOLD_LINE_MIN_LENGTH_H = 50
+        # self.THRESHOLD_LINE_MIN_LENGTH_V = 50
+        # self.OCR_PADDING = 5
+        # self.OCR_MIN_WORD_AREA = 0.45
+        # self.THRESHOLD_MIN_IOU = 0.1              # dribbble:0.003 rico:0.1 web:0.1
+        # self.THRESHOLD_BLOCK_MIN_EDGE_LENGTH = 210   # dribbble:68 rico:210 web:70
+        # self.THRESHOLD_UICOMPO_MAX_W_H_RATIO = 10   # dribbble:10 rico:10 web:22
+
+        self.CLASS_MAP = {'0':'Button', '1':'CheckBox', '2':'Chronometer', '3':'EditText', '4':'ImageButton', '5':'ImageView',
+               '6':'ProgressBar', '7':'RadioButton', '8':'RatingBar', '9':'SeekBar', '10':'Spinner', '11':'Switch',
+               '12':'ToggleButton', '13':'VideoView', '14':'TextView'}
+        self.COLOR = {'Button': (0, 255, 0), 'CheckBox': (0, 0, 255), 'Chronometer': (255, 166, 166),
+                      'EditText': (255, 166, 0),
+                      'ImageButton': (77, 77, 255), 'ImageView': (255, 0, 166), 'ProgressBar': (166, 0, 255),
+                      'RadioButton': (166, 166, 166),
+                      'RatingBar': (0, 166, 255), 'SeekBar': (0, 166, 10), 'Spinner': (50, 21, 255),
+                      'Switch': (80, 166, 66), 'ToggleButton': (0, 66, 80), 'VideoView': (88, 66, 0),
+                      'TextView': (169, 255, 0),
+
+                      'Text':(169, 255, 0), 'Non-Text':(255, 0, 166),
+
+                      'Noise':(6,6,255), 'Non-Noise': (6,255,6),
+
+                      'Image':(255,6,6), 'Non-Image':(6,6,255)}

CDM/detect_classify/classification.py

@@ -0,0 +1,380 @@
+from CDM.detect_merge.Element import Element
+import CDM.detect_compo.lib_ip.ip_preprocessing as pre
+import time
+import cv2
+import torch
+import numpy as np
+from torchvision import models
+from torch import nn
+import pandas as pd
+import re
+import openai
+import random
+import os
+from CDM.detect_merge.merge import reassign_ids
+import CDM.detect_merge.merge as merge
+from os.path import join as pjoin, exists
+
+label_dic ={'72':'Location', '42':'Photos', '77':'Social media', '91':'Voices', '6':'Email', '89':'Social media', '40':'Location', '43':'Phone', '82':'Photos',
+                                                                        '3':'Contacts', '68':'Contacts', '49':'Profile', '56':'Photos'}
+
+keyword_list = {'Name':['name', 'first name', 'last name', 'full name', 'real name', 'surname', 'family name', 'given name'],
+                        'Birthday':['birthday', 'date of birth', 'birth date', 'DOB', 'dob full birthday', 'birth year'],
+                        'Address':['mailing address', 'physical address', 'postal address', 'billing address', 'shipping address', 'delivery address', 'residence', 'collect address', 'personal address', 'residential address'],
+                        'Phone':['phone', 'phone number', 'mobile', 'mobile phone', 'mobile number', 'telephone', 'telephone number', 'call'],
+                        'Email':['email', 'e-mail', 'email address', 'e-mail address'],
+                        'Contacts':['contacts', 'phone-book', 'phone book', 'phonebook', 'contact list', 'phone contacts', 'address book'],
+                        'Location':['location', 'locate', 'geography', 'geo', 'geo-location', 'precision location', 'nearby'],
+                        'Photos':['camera', 'photo', 'scan', 'album', 'picture', 'gallery', 'photo library', 'storage', 'image', 'video', 'scanner', 'photograph'],
+                        'Voices':['microphone', 'voice', 'mic', 'speech', 'talk'],
+                        'Financial info':['credit card', 'pay', 'payment', 'debit card', 'mastercard', 'wallet'],
+                        'IP':['IP', 'Internet Protocol', 'IP address', 'internet protocol address'],
+                        'Cookies':['cookies', 'cookie'],
+                        'Social media':['facebook', 'twitter', 'socialmedia', 'social media'],
+                        'Profile':['profile', 'account'],
+                        'Gender':['gender']}
+
+def get_data_type(sentence, keywords, use_gpt=True):
+
+    sent_data_type = "others"
+
+    if use_gpt:
+        openai.api_key = os.environ["OPENAI_API_KEY"]
+
+        prompt = f"Is this piece of texts \"{sentence}\" related to any following privacy information data types? Or not relevant to any of them? ONLY answer the data type or \"not relevant\". ONLY use following data type list. Data types and their Description:\n" \
+                 f"Name: How a user refers to themselves," \
+                 f" Birthday: A user’s birthday," \
+                 f" Address: A user’s address," \
+                 f" Phone: A user’s phone number," \
+                 f" Email: A user’s email address," \
+                 f" Contacts: A user’s contact information, or the access to the contact permission," \
+                 f" Location: A user’s location information, or the access to the location permission," \
+                 f" Photos: A user’s photos, videos, or the access to the camera permission," \
+                 f" Voices: A user’s voices, recordings, or the access to the microphone permission," \
+                 f" Financial Info: Information about a user’s financial accounts, purchases, or transactions," \
+                 f" Profile: A user’s account information," \
+                 f"Social Media: A user's social media information, or the access to social media accounts"
+
+        response = openai.ChatCompletion.create(
+            # engine="text-davinci-002",
+            model="gpt-3.5-turbo",
+            messages=[
+                # {"role": "system", "content": "You are a helpful assistant."},
+                {"role": "user", "content": prompt}
+            ],
+            max_tokens=100,
+            n=1,
+            stop=None,
+            temperature=0,
+        )
+
+        # response_full_text = response.choices[0].text.strip()
+        response_full_text = response.choices[0].message['content']
+        for k in keywords.keys():
+            if k == "Financial info" or k == "Social media":
+                if k.lower() in response_full_text.lower():
+                    sent_data_type = k
+                    break
+            else:
+                words = re.split(r'\W+', response_full_text.lower())
+                if k.lower() in words:
+                    sent_data_type = k
+                    break
+
+        # print("----------------------")
+        # print("sentence: ", sentence)
+        # print("prompt: ", prompt)
+        # print("response: ", response_full_text)
+        # print("sent_data_type: ", sent_data_type)
+
+    else:
+        for k in keywords.keys():
+            for w in keywords[k]:
+                words = re.split(r'\W+', sentence.lower())
+                if w.lower() in words:
+                    sent_data_type = k
+                    break
+            if sent_data_type != "others":
+                break
+
+    return sent_data_type
+
+# def get_clf_model(use_resnet18=True, use_gpu=False):
+#
+#     device = 'cpu'
+#     if use_gpu:
+#         device = 'cuda:0'
+#
+#     if use_resnet18:
+#         model = models.resnet18().to(device)
+#         in_feature_num = model.fc.in_features
+#         model.fc = nn.Linear(in_feature_num, 99)
+#         model.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(5, 5), padding=(3, 3), stride=(2, 2),
+#                                 bias=False)
+#
+#         PATH = "./CDM/model/model-99-resnet18.pkl"
+#         model.load_state_dict(torch.load(PATH, map_location=torch.device(device)))
+#
+#         model.eval()
+#     else:
+#         # replace with your own model
+#         None
+#
+#     return model
+
+def get_clf_model(clf_model="ResNet18", use_gpu=False):
+
+    device = 'cpu'
+    if use_gpu:
+        device = 'cuda:0'
+
+    if clf_model == "ResNet18":
+        model = models.resnet18().to(device)
+        in_feature_num = model.fc.in_features
+        model.fc = nn.Linear(in_feature_num, 99)
+        model.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(5, 5), padding=(3, 3), stride=(2, 2),
+                                bias=False)
+
+        PATH = "./CDM/model/model-99-resnet18.pkl"
+        model.load_state_dict(torch.load(PATH, map_location=torch.device(device)))
+
+        model.eval()
+    elif clf_model == "ViT":
+        model = torch.load('./CDM/model/model-99-ViT-entire.pkl', map_location=torch.device(device))
+        model = model.to(device)
+        model.eval()
+
+    else:
+        # replace with your own model
+        None
+
+    return model
+
+def compo_classification(input_img, output_root, segment_root, merge_json, output_data, resize_by_height=800, clf_model="ResNet18"):
+    # load text and non-text compo
+    ele_id = 0
+    compos = []
+    texts = []
+    elements = []
+
+    for compo in merge_json['compos']:
+        if compo['class'] == 'Text':
+            element = Element(ele_id,
+                              (compo["position"]['column_min'], compo["position"]['row_min'],
+                               compo["position"]['column_max'], compo["position"]['row_max']),
+                              'Text', text_content=compo['text_content'])
+            texts.append(element)
+            ele_id += 1
+        else:
+            element = Element(ele_id,
+                              (compo["position"]['column_min'], compo["position"]['row_min'],
+                               compo["position"]['column_max'], compo["position"]['row_max']),
+                              compo['class'])
+            compos.append(element)
+            ele_id += 1
+
+    org, grey = pre.read_img(input_img, resize_by_height)
+
+    grey = grey.astype('float32')
+    grey = grey / 255
+
+    # grey = (grey - grey.mean()) / grey.std()
+
+    # --------- classification ----------
+
+    classification_start_time = time.process_time()
+
+    for compo in compos:
+
+        # comp_grey = grey[compo.row_min:compo.row_max, compo.col_min:compo.col_max]
+        #
+        # comp_crop = cv2.resize(comp_grey, (32, 32))
+        #
+        # comp_crop = comp_crop.reshape(1, 1, 32, 32)
+        #
+        # comp_tensor = torch.tensor(comp_crop)
+        # comp_tensor = comp_tensor.permute(0, 1, 3, 2)
+        #
+        # model = get_clf_model()
+        # pred_label = model(comp_tensor)
+        #
+        # if str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0]) in label_dic.keys():
+        #     compo.label = label_dic[str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0])]
+        #     elements.append(compo)
+        # else:
+        #     compo.label = str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0])
+
+        if clf_model == "ResNet18":
+
+            comp_grey = grey[compo.row_min:compo.row_max, compo.col_min:compo.col_max]
+
+            comp_crop = cv2.resize(comp_grey, (32, 32))
+
+            comp_crop = comp_crop.reshape(1, 1, 32, 32)
+
+            comp_tensor = torch.tensor(comp_crop)
+            comp_tensor = comp_tensor.permute(0, 1, 3, 2)
+
+            model = get_clf_model(clf_model)
+            pred_label = model(comp_tensor)
+
+            if str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0]) in label_dic.keys():
+                compo.label = label_dic[str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0])]
+                elements.append(compo)
+            else:
+                compo.label = str(np.argmax(pred_label.cpu().data.numpy(), axis=1)[0])
+
+        elif clf_model == "ViT":
+
+            comp_grey = grey[compo.row_min:compo.row_max, compo.col_min:compo.col_max]
+
+            comp_crop = cv2.resize(comp_grey, (224, 224))
+
+            # Convert the image to tensor
+            comp_tensor = torch.from_numpy(comp_crop)
+
+            # Reshape and repeat along the channel dimension to convert to RGB
+            comp_tensor = comp_tensor.view(1, 224, 224).repeat(3, 1, 1)
+
+            # comp_tensor = comp_tensor.permute(0, 2, 1)
+
+            comp_tensor = comp_tensor.unsqueeze(0)  # add a batch dimension
+
+            model = get_clf_model(clf_model)
+            # pred_label = model(comp_tensor)
+
+            # Forward pass through the model
+            with torch.no_grad():
+                output = model(comp_tensor)
+
+            # Get the predicted label
+            _, predicted = torch.max(output.logits, 1)
+
+            # print("predicted_label: ", predicted.cpu().numpy())
+
+            if str(predicted.cpu().numpy()[0]) in label_dic.keys():
+                compo.label = label_dic[str(predicted.cpu().numpy()[0])]
+                elements.append(compo)
+            else:
+                compo.label = str(predicted.cpu().numpy()[0])
+
+        else:
+            print("clf_model has to be ResNet18 or ViT")
+
+    time_cost_ic = time.process_time() - classification_start_time
+    print("time cost for icon classification: %2.2f s" % time_cost_ic)
+    # ic_time_cost_all.append(time_cost_ic)
+
+    # --------- end classification ----------
+
+    text_selection_time = time.process_time()
+
+    for this_text in texts:
+        # found_flag = 0
+        #
+        # for key in keyword_list:
+        #     for w in keyword_list[key]:
+        #         words = re.split(r'\W+', this_text.text_content.lower())
+        #         if w.lower() in words:
+        #             this_text.label = key
+        #             elements.append(this_text)
+        #             found_flag = 1
+        #             break
+        #
+        # if found_flag == 0:
+        #     this_text.label = 'others'
+
+        retries = 10
+        for i in range(retries):
+            try:
+                text_label = get_data_type(this_text.text_content.lower(), keyword_list, use_gpt=False)
+                break
+            except openai.error.RateLimitError as e:
+                if "overloaded" in str(e):
+                    # Exponential backoff with jitter
+                    sleep_time = 2 * (2 ** i) + random.uniform(0, 0.1)
+                    time.sleep(sleep_time)
+                else:
+                    raise
+            except Exception as e:
+                raise
+
+        this_text.label = text_label
+
+        if this_text.label != "others":
+            elements.append(this_text)
+
+    time_cost_ts = time.process_time() - text_selection_time
+    print("time cost for text selection: %2.2f s" % time_cost_ts)
+    # ts_time_cost_all.append(time_cost_ts)
+
+    # ---------- end -------------------------------
+
+    full_size_org, full_size_grey = pre.read_img(input_img)
+    ratio = full_size_org.shape[0]/org.shape[0]
+
+    show = False
+    wait_key = 0
+
+    reassign_ids(elements)
+    board = merge.show_elements(full_size_org, elements, ratio, show=show, win_name='elements after merging', wait_key=wait_key, line=3)
+    board_one_element = merge.show_one_element(full_size_org, elements, ratio, show=show, win_name='elements after merging', wait_key=wait_key, line=3)
+
+    classification_root = pjoin(output_root, 'classification')
+
+    # save all merged elements, clips and blank background
+    name = input_img.replace('\\', '/').split('/')[-1][:-4]
+    components = merge.save_elements(pjoin(classification_root, name + '.json'), elements, full_size_org.shape, ratio)
+    cv2.imwrite(pjoin(classification_root, name + '.jpg'), board)
+
+    print("len(board_one_element): ", len(board_one_element))
+
+    for i in range(len(elements)):
+        e_name = str(int(elements[i].id) + 1)
+        cv2.imwrite(pjoin(classification_root + '/GUI', name + '-' + e_name + '.jpg'), board_one_element[i])
+
+    print('[Classification Completed] Input: %s Output: %s' % (input_img, pjoin(classification_root, name + '.jpg')))
+
+    # ---------- matching result -----------
+
+    index = input_img.split('/')[-1][:-4]
+    app_id = str(index).split('-')[0]
+
+    index_path = pjoin(segment_root, app_id, 'classified_sentences/keyword_index.txt')
+    dict_index = {}
+    if exists(index_path):
+        with open(index_path, 'r') as g:
+            for line in g:
+                key, value = line.strip().split(':', 1)
+                dict_index[key] = value
+
+    for item in elements:
+        complete_path = pjoin(segment_root, app_id, 'classified_sentences', item.label + '.txt')
+        print("complete_path: ", complete_path)
+
+        if exists(complete_path):
+
+            with open(complete_path, 'r', encoding='utf-8') as file:
+                content = file.read()
+
+            # Replace line breaks with spaces and strip any extra whitespace
+            this_text = ' '.join(content.splitlines()).strip()
+
+            lines = content.splitlines()
+            non_empty_lines = [line for line in lines if line.strip() != ""]
+            for i in range(len(non_empty_lines)):
+                if non_empty_lines[i][0].isalpha():
+                    non_empty_lines[i] = non_empty_lines[i][0].upper() + non_empty_lines[i][1:]
+
+            # output_data = output_data.append({'screenshot': 's' + str(index), 'id': item.id + 1, 'label': item.label, 'index': dict_index[item.label], 'text': this_text, 'sentences': non_empty_lines}, ignore_index=True)
+            output_data = pd.concat([output_data, pd.DataFrame([{'screenshot': 's' + str(index), 'id': item.id + 1,
+                                                                 'label': item.label, 'index': dict_index[item.label],
+                                                                 'text': this_text, 'sentences': non_empty_lines}])])
+
+        else:
+            # output_data = output_data.append({'screenshot': 's' + str(index), 'id': item.id + 1, 'label': item.label, 'index': "None", 'text': "No information!", 'sentences': "None"},
+            #                                  ignore_index=True)
+            output_data = pd.concat([output_data, pd.DataFrame([{'screenshot': 's' + str(index), 'id': item.id + 1,
+                                                                 'label': item.label, 'index': "None",
+                                                                 'text': "No information!", 'sentences': "None"}])])
+    return time_cost_ic, time_cost_ts, output_data, board

CDM/detect_compo/deprecated/Block.py

@@ -0,0 +1,56 @@
+import cv2
+from os.path import join as pjoin
+import time
+import numpy as np
+
+from CDM.detect_compo.lib_ip.Component import Component
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+class Block(Component):
+    def __init__(self, region, image_shape):
+        super().__init__(region, image_shape)
+        self.category = 'Block'
+        self.parent = None
+        self.children = []
+        self.uicompo_ = None
+        self.top_or_botm = None
+        self.redundant = False
+
+    def block_is_uicompo(self, image_shape, max_compo_scale):
+        '''
+        Check the if the block is a ui component according to its relative size
+        '''
+        row, column = image_shape[:2]
+        # print(height, height / row, max_compo_scale[0], height / row > max_compo_scale[0])
+        # draw.draw_bounding_box(org, [corner], show=True)
+        # ignore atomic components
+        if self.bbox.height / row > max_compo_scale[0] or self.bbox.width / column > max_compo_scale[1]:
+            return False
+        return True
+
+    def block_is_top_or_bottom_bar(self, image_shape, top_bottom_height):
+        '''
+        Check if the block is top bar or bottom bar
+        '''
+        height, width = image_shape[:2]
+        (column_min, row_min, column_max, row_max) = self.bbox.put_bbox()
+        if column_min < 5 and row_min < 5 and \
+                width - column_max < 5 and row_max < height * top_bottom_height[0]:
+            self.uicompo_ = True
+            return True
+        if column_min < 5 and row_min > height * top_bottom_height[1] and \
+                width - column_max < 5 and height - row_max < 5:
+            self.uicompo_ = True
+            return True
+        return False
+
+    def block_erase_from_bin(self, binary, pad):
+        (column_min, row_min, column_max, row_max) = self.put_bbox()
+        column_min = max(column_min - pad, 0)
+        column_max = min(column_max + pad, binary.shape[1])
+        row_min = max(row_min - pad, 0)
+        row_max = min(row_max + pad, binary.shape[0])
+        cv2.rectangle(binary, (column_min, row_min), (column_max, row_max), (0), -1)
+

CDM/detect_compo/deprecated/block_division.py

@@ -0,0 +1,108 @@
+import cv2
+import numpy as np
+from random import randint as rint
+import time
+
+import CDM.detect_compo.lib_ip.ip_preprocessing as pre
+import CDM.detect_compo.lib_ip.ip_detection as det
+import CDM.detect_compo.lib_ip.ip_draw as draw
+import CDM.detect_compo.lib_ip.ip_segment as seg
+from CDM.detect_compo.lib_ip.Block import Block
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+def block_hierarchy(blocks):
+    for i in range(len(blocks) - 1):
+        for j in range(i + 1, len(blocks)):
+            relation = blocks[i].compo_relation(blocks[j])
+            if relation == -1:
+                blocks[j].children.append(i)
+            if relation == 1:
+                blocks[i].children.append(j)
+    return
+
+
+def block_bin_erase_all_blk(binary, blocks, pad=0, show=False):
+    '''
+    erase the block parts from the binary map
+    :param binary: binary map of original image
+    :param blocks_corner: corners of detected layout block
+    :param show: show or not
+    :param pad: expand the bounding boxes of blocks
+    :return: binary map without block parts
+    '''
+
+    bin_org = binary.copy()
+    for block in blocks:
+        block.block_erase_from_bin(binary, pad)
+    if show:
+        cv2.imshow('before', bin_org)
+        cv2.imshow('after', binary)
+        cv2.waitKey()
+
+
+def block_division(grey, org, grad_thresh,
+                   show=False, write_path=None,
+                   step_h=10, step_v=10,
+                   line_thickness=C.THRESHOLD_LINE_THICKNESS,
+                   min_rec_evenness=C.THRESHOLD_REC_MIN_EVENNESS,
+                   max_dent_ratio=C.THRESHOLD_REC_MAX_DENT_RATIO,
+                   min_block_height_ratio=C.THRESHOLD_BLOCK_MIN_HEIGHT):
+    '''
+    :param grey: grey-scale of original image
+    :return: corners: list of [(top_left, bottom_right)]
+                        -> top_left: (column_min, row_min)
+                        -> bottom_right: (column_max, row_max)
+    '''
+    blocks = []
+    mask = np.zeros((grey.shape[0]+2, grey.shape[1]+2), dtype=np.uint8)
+    broad = np.zeros((grey.shape[0], grey.shape[1], 3), dtype=np.uint8)
+    broad_all = broad.copy()
+
+    row, column = grey.shape[0], grey.shape[1]
+    for x in range(0, row, step_h):
+        for y in range(0, column, step_v):
+            if mask[x, y] == 0:
+                # region = flood_fill_bfs(grey, x, y, mask)
+
+                # flood fill algorithm to get background (layout block)
+                mask_copy = mask.copy()
+                ff = cv2.floodFill(grey, mask, (y, x), None, grad_thresh, grad_thresh, cv2.FLOODFILL_MASK_ONLY)
+                # ignore small regions
+                if ff[0] < 500: continue
+                mask_copy = mask - mask_copy
+                region = np.reshape(cv2.findNonZero(mask_copy[1:-1, 1:-1]), (-1, 2))
+                region = [(p[1], p[0]) for p in region]
+
+                block = Block(region, grey.shape)
+                # draw.draw_region(region, broad_all)
+                # if block.height < 40 and block.width < 40:
+                #     continue
+                if block.height < 30:
+                    continue
+
+                # print(block.area / (row * column))
+                if block.area / (row * column) > 0.9:
+                    continue
+                elif block.area / (row * column) > 0.7:
+                    block.redundant = True
+
+                # get the boundary of this region
+                # ignore lines
+                if block.compo_is_line(line_thickness):
+                    continue
+                # ignore non-rectangle as blocks must be rectangular
+                if not block.compo_is_rectangle(min_rec_evenness, max_dent_ratio):
+                    continue
+                # if block.height/row < min_block_height_ratio:
+                #     continue
+                blocks.append(block)
+                # draw.draw_region(region, broad)
+    if show:
+        cv2.imshow('flood-fill all', broad_all)
+        cv2.imshow('block', broad)
+        cv2.waitKey()
+    if write_path is not None:
+        cv2.imwrite(write_path, broad)
+    return blocks

CDM/detect_compo/deprecated/ip_detection_utils.py

@@ -0,0 +1,461 @@
+import numpy as np
+import cv2
+from collections import Counter
+
+import lib_ip.ip_draw as draw
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+# detect object(connected region)
+# def boundary_bfs_connected_area(img, x, y, mark):
+#     def neighbor(img, x, y, mark, stack):
+#         for i in range(x - 1, x + 2):
+#             if i < 0 or i >= img.shape[0]: continue
+#             for j in range(y - 1, y + 2):
+#                 if j < 0 or j >= img.shape[1]: continue
+#                 if img[i, j] == 255 and mark[i, j] == 0:
+#                     stack.append([i, j])
+#                     mark[i, j] = 255
+#
+#     stack = [[x, y]]  # points waiting for inspection
+#     area = [[x, y]]  # points of this area
+#     mark[x, y] = 255  # drawing broad
+#
+#     while len(stack) > 0:
+#         point = stack.pop()
+#         area.append(point)
+#         neighbor(img, point[0], point[1], mark, stack)
+#     return area
+
+
+# def line_check_perpendicular(lines_h, lines_v, max_thickness):
+#     """
+#     lines: [line_h, line_v]
+#         -> line_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#         -> line_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     """
+#     is_per_h = np.full(len(lines_h), False)
+#     is_per_v = np.full(len(lines_v), False)
+#     for i in range(len(lines_h)):
+#         # save the intersection point of h
+#         lines_h[i]['inter_point'] = set()
+#         h = lines_h[i]
+#
+#         for j in range(len(lines_v)):
+#             # save the intersection point of v
+#             if 'inter_point' not in lines_v[j]: lines_v[j]['inter_point'] = set()
+#             v = lines_v[j]
+#
+#             # if h is perpendicular to v in head of v
+#             if abs(h['head'][1]-v['head'][1]) <= max_thickness:
+#                 if abs(h['head'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('head')
+#                     lines_v[j]['inter_point'].add('head')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#                 elif abs(h['end'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('end')
+#                     lines_v[j]['inter_point'].add('head')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#
+#             # if h is perpendicular to v in end of v
+#             elif abs(h['head'][1]-v['end'][1]) <= max_thickness:
+#                 if abs(h['head'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('head')
+#                     lines_v[j]['inter_point'].add('end')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#                 elif abs(h['end'][0] - v['head'][0]) <= max_thickness:
+#                     lines_h[i]['inter_point'].add('end')
+#                     lines_v[j]['inter_point'].add('end')
+#                     is_per_h[i] = True
+#                     is_per_v[j] = True
+#     per_h = []
+#     per_v = []
+#     for i in range(len(is_per_h)):
+#         if is_per_h[i]:
+#             lines_h[i]['inter_point'] = list(lines_h[i]['inter_point'])
+#             per_h.append(lines_h[i])
+#     for i in range(len(is_per_v)):
+#         if is_per_v[i]:
+#             lines_v[i]['inter_point'] = list(lines_v[i]['inter_point'])
+#             per_v.append(lines_v[i])
+#     return per_h, per_v
+
+
+# def line_shrink_corners(corner, lines_h, lines_v):
+#     """
+#     shrink the corner according to lines:
+#              col_min_shrink: shrink right (increase)
+#              col_max_shrink: shrink left  (decrease)
+#              row_min_shrink: shrink down  (increase)
+#              row_max_shrink: shrink up    (decrease)
+#     :param lines_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#     :param lines_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     :return: shrunken corner: (top_left, bottom_right)
+#     """
+#     (col_min, row_min), (col_max, row_max) = corner
+#     col_min_shrink, row_min_shrink = col_min, row_min
+#     col_max_shrink, row_max_shrink = col_max, row_max
+#     valid_frame = False
+#
+#     for h in lines_h:
+#         # ignore outer border
+#         if len(h['inter_point']) == 2:
+#             valid_frame = True
+#             continue
+#         # shrink right -> col_min move to end
+#         if h['inter_point'][0] == 'head':
+#             col_min_shrink = max(h['end'][0], col_min_shrink)
+#         # shrink left -> col_max move to head
+#         elif h['inter_point'][0] == 'end':
+#             col_max_shrink = min(h['head'][0], col_max_shrink)
+#
+#     for v in lines_v:
+#         # ignore outer border
+#         if len(v['inter_point']) == 2:
+#             valid_frame = True
+#             continue
+#         # shrink down -> row_min move to end
+#         if v['inter_point'][0] == 'head':
+#             row_min_shrink = max(v['end'][1], row_min_shrink)
+#         # shrink up -> row_max move to head
+#         elif v['inter_point'][0] == 'end':
+#             row_max_shrink = min(v['head'][1], row_max_shrink)
+#
+#     # return the shrunken corner if only there is line intersecting with two other lines
+#     if valid_frame:
+#         return (col_min_shrink, row_min_shrink), (col_max_shrink, row_max_shrink)
+#     return corner
+
+
+# def line_cvt_relative_position(col_min, row_min, lines_h, lines_v):
+#     """
+#     convert the relative position of lines in the entire image
+#     :param col_min: based column the img lines belong to
+#     :param row_min: based row the img lines belong to
+#     :param lines_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+#     :param lines_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+#     :return: lines_h_cvt, lines_v_cvt
+#     """
+#     for h in lines_h:
+#         h['head'][0] += col_min
+#         h['head'][1] += row_min
+#         h['end'][0] += col_min
+#         h['end'][1] += row_min
+#     for v in lines_v:
+#         v['head'][0] += col_min
+#         v['head'][1] += row_min
+#         v['end'][0] += col_min
+#         v['end'][1] += row_min
+#
+#     return lines_h, lines_v
+
+
+# check if an object is so slim
+# @boundary: [border_up, border_bottom, border_left, border_right]
+# -> up, bottom: (column_index, min/max row border)
+# -> left, right: (row_index, min/max column border) detect range of each row
+def clipping_by_line(boundary, boundary_rec, lines):
+    boundary = boundary.copy()
+    for orient in lines:
+        # horizontal
+        if orient == 'h':
+            # column range of sub area
+            r1, r2 = 0, 0
+            for line in lines[orient]:
+                if line[0] == 0:
+                    r1 = line[1]
+                    continue
+                r2 = line[0]
+                b_top = []
+                b_bottom = []
+                for i in range(len(boundary[0])):
+                    if r2 > boundary[0][i][0] >= r1:
+                        b_top.append(boundary[0][i])
+                for i in range(len(boundary[1])):
+                    if r2 > boundary[1][i][0] >= r1:
+                        b_bottom.append(boundary[1][i])
+
+                b_left = [x for x in boundary[2]]  # (row_index, min column border)
+                for i in range(len(b_left)):
+                    if b_left[i][1] < r1:
+                        b_left[i][1] = r1
+                b_right = [x for x in boundary[3]]  # (row_index, max column border)
+                for i in range(len(b_right)):
+                    if b_right[i][1] > r2:
+                        b_right[i][1] = r2
+
+                boundary_rec.append([b_top, b_bottom, b_left, b_right])
+                r1 = line[1]
+
+
+# remove imgs that contain text
+# def rm_text(org, corners, compo_class,
+#             max_text_height=C.THRESHOLD_TEXT_MAX_HEIGHT, max_text_width=C.THRESHOLD_TEXT_MAX_WIDTH,
+#             ocr_padding=C.OCR_PADDING, ocr_min_word_area=C.OCR_MIN_WORD_AREA, show=False):
+#     """
+#     Remove area that full of text
+#     :param org: original image
+#     :param corners: [(top_left, bottom_right)]
+#                     -> top_left: (column_min, row_min)
+#                     -> bottom_right: (column_max, row_max)
+#     :param compo_class: classes of corners
+#     :param max_text_height: Too large to be text
+#     :param max_text_width: Too large to be text
+#     :param ocr_padding: Padding for clipping
+#     :param ocr_min_word_area: If too text area ratio is too large
+#     :param show: Show or not
+#     :return: corners without text objects
+#     """
+#     new_corners = []
+#     new_class = []
+#     for i in range(len(corners)):
+#         corner = corners[i]
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         height = row_max - row_min
+#         width = col_max - col_min
+#         # highly likely to be block or img if too large
+#         if height > max_text_height and width > max_text_width:
+#             new_corners.append(corner)
+#             new_class.append(compo_class[i])
+#         else:
+#             row_min = row_min - ocr_padding if row_min - ocr_padding >= 0 else 0
+#             row_max = row_max + ocr_padding if row_max + ocr_padding < org.shape[0] else org.shape[0]
+#             col_min = col_min - ocr_padding if col_min - ocr_padding >= 0 else 0
+#             col_max = col_max + ocr_padding if col_max + ocr_padding < org.shape[1] else org.shape[1]
+#             # check if this area is text
+#             clip = org[row_min: row_max, col_min: col_max]
+#             if not ocr.is_text(clip, ocr_min_word_area, show=show):
+#                 new_corners.append(corner)
+#                 new_class.append(compo_class[i])
+#     return new_corners, new_class
+
+
+# def rm_img_in_compo(corners_img, corners_compo):
+#     """
+#     Remove imgs in component
+#     """
+#     corners_img_new = []
+#     for img in corners_img:
+#         is_nested = False
+#         for compo in corners_compo:
+#             if util.corner_relation(img, compo) == -1:
+#                 is_nested = True
+#                 break
+#         if not is_nested:
+#             corners_img_new.append(img)
+#     return corners_img_new
+
+
+# def block_or_compo(org, binary, corners,
+#                    max_thickness=C.THRESHOLD_BLOCK_MAX_BORDER_THICKNESS, max_block_cross_points=C.THRESHOLD_BLOCK_MAX_CROSS_POINT,
+#                    min_compo_w_h_ratio=C.THRESHOLD_UICOMPO_MIN_W_H_RATIO, max_compo_w_h_ratio=C.THRESHOLD_UICOMPO_MAX_W_H_RATIO,
+#                    min_block_edge=C.THRESHOLD_BLOCK_MIN_EDGE_LENGTH):
+#     """
+#     Check if the objects are img components or just block
+#     :param org: Original image
+#     :param binary:  Binary image from pre-processing
+#     :param corners: [(top_left, bottom_right)]
+#                     -> top_left: (column_min, row_min)
+#                     -> bottom_right: (column_max, row_max)
+#     :param max_thickness: The max thickness of border of blocks
+#     :param max_block_cross_points: Ratio of point of interaction
+#     :return: corners of blocks and imgs
+#     """
+#     blocks = []
+#     imgs = []
+#     compos = []
+#     for corner in corners:
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         height = row_max - row_min
+#         width = col_max - col_min
+#
+#         block = False
+#         vacancy = [0, 0, 0, 0]
+#         for i in range(1, max_thickness):
+#             try:
+#                 # top to bottom
+#                 if vacancy[0] == 0 and (col_max - col_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i, col_min + i: col_max - i]) / 255) / (col_max - col_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[0] = 1
+#                 # bottom to top
+#                 if vacancy[1] == 0 and (col_max - col_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_max - i, col_min + i: col_max - i]) / 255) / (col_max - col_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[1] = 1
+#                 # left to right
+#                 if vacancy[2] == 0 and (row_max - row_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i: row_max - i, col_min + i]) / 255) / (row_max - row_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[2] = 1
+#                 # right to left
+#                 if vacancy[3] == 0 and (row_max - row_min - 2 * i) is not 0 and (
+#                         np.sum(binary[row_min + i: row_max - i, col_max - i]) / 255) / (row_max - row_min - 2 * i) <= max_block_cross_points:
+#                     vacancy[3] = 1
+#                 if np.sum(vacancy) == 4:
+#                     block = True
+#             except:
+#                 pass
+#
+#         # too big to be UI components
+#         if block:
+#             if height > min_block_edge and width > min_block_edge:
+#                 blocks.append(corner)
+#             else:
+#                 if min_compo_w_h_ratio < width / height < max_compo_w_h_ratio:
+#                     compos.append(corner)
+#         # filter out small objects
+#         else:
+#             if height > min_block_edge:
+#                 imgs.append(corner)
+#             else:
+#                 if min_compo_w_h_ratio < width / height < max_compo_w_h_ratio:
+#                     compos.append(corner)
+#     return blocks, imgs, compos
+
+
+# def compo_on_img(processing, org, binary, clf,
+#                  compos_corner, compos_class):
+#     """
+#     Detect potential UI components inner img;
+#     Only leave non-img
+#     """
+#     pad = 2
+#     for i in range(len(compos_corner)):
+#         if compos_class[i] != 'img':
+#             continue
+#         ((col_min, row_min), (col_max, row_max)) = compos_corner[i]
+#         col_min = max(col_min - pad, 0)
+#         col_max = min(col_max + pad, org.shape[1])
+#         row_min = max(row_min - pad, 0)
+#         row_max = min(row_max + pad, org.shape[0])
+#         area = (col_max - col_min) * (row_max - row_min)
+#         if area < 600:
+#             continue
+#
+#         clip_org = org[row_min:row_max, col_min:col_max]
+#         clip_bin_inv = pre.reverse_binary(binary[row_min:row_max, col_min:col_max])
+#
+#         compos_boundary_new, compos_corner_new, compos_class_new = processing(clip_org, clip_bin_inv, clf)
+#         compos_corner_new = util.corner_cvt_relative_position(compos_corner_new, col_min, row_min)
+#
+#         assert len(compos_corner_new) == len(compos_class_new)
+#
+#         # only leave non-img elements
+#         for i in range(len(compos_corner_new)):
+#             ((col_min_new, row_min_new), (col_max_new, row_max_new)) = compos_corner_new[i]
+#             area_new = (col_max_new - col_min_new) * (row_max_new - row_min_new)
+#             if compos_class_new[i] != 'img' and area_new / area < 0.8:
+#                 compos_corner.append(compos_corner_new[i])
+#                 compos_class.append(compos_class_new[i])
+#
+#     return compos_corner, compos_class
+
+
+# def strip_img(corners_compo, compos_class, corners_img):
+#     """
+#     Separate img from other compos
+#     :return: compos without img
+#     """
+#     corners_compo_withuot_img = []
+#     compo_class_withuot_img = []
+#     for i in range(len(compos_class)):
+#         if compos_class[i] == 'img':
+#             corners_img.append(corners_compo[i])
+#         else:
+#             corners_compo_withuot_img.append(corners_compo[i])
+#             compo_class_withuot_img.append(compos_class[i])
+#     return corners_compo_withuot_img, compo_class_withuot_img
+
+
+# def merge_corner(corners, compos_class, min_selected_IoU=C.THRESHOLD_MIN_IOU, is_merge_nested_same=True):
+#     """
+#     Calculate the Intersection over Overlap (IoU) and merge corners according to the value of IoU
+#     :param is_merge_nested_same: if true, merge the nested corners with same class whatever the IoU is
+#     :param corners: corners: [(top_left, bottom_right)]
+#                             -> top_left: (column_min, row_min)
+#                             -> bottom_right: (column_max, row_max)
+#     :return: new corners
+#     """
+#     new_corners = []
+#     new_class = []
+#     for i in range(len(corners)):
+#         is_intersected = False
+#         for j in range(len(new_corners)):
+#             r = util.corner_relation_nms(corners[i], new_corners[j], min_selected_IoU)
+#             # r = util.corner_relation(corners[i], new_corners[j])
+#             if is_merge_nested_same:
+#                 if compos_class[i] == new_class[j]:
+#                     # if corners[i] is in new_corners[j], ignore corners[i]
+#                     if r == -1:
+#                         is_intersected = True
+#                         break
+#                     # if new_corners[j] is in corners[i], replace new_corners[j] with corners[i]
+#                     elif r == 1:
+#                         is_intersected = True
+#                         new_corners[j] = corners[i]
+#
+#             # if above IoU threshold, and corners[i] is in new_corners[j], ignore corners[i]
+#             if r == -2:
+#                 is_intersected = True
+#                 break
+#             # if above IoU threshold, and new_corners[j] is in corners[i], replace new_corners[j] with corners[i]
+#             elif r == 2:
+#                 is_intersected = True
+#                 new_corners[j] = corners[i]
+#                 new_class[j] = compos_class[i]
+#
+#             # containing and too small
+#             elif r == -3:
+#                 is_intersected = True
+#                 break
+#             elif r == 3:
+#                 is_intersected = True
+#                 new_corners[j] = corners[i]
+#
+#             # if [i] and [j] are overlapped but no containing relation, merge corners when same class
+#             elif r == 4:
+#                 is_intersected = True
+#                 if compos_class[i] == new_class[j]:
+#                     new_corners[j] = util.corner_merge_two_corners(corners[i], new_corners[j])
+#
+#         if not is_intersected:
+#             new_corners.append(corners[i])
+#             new_class.append(compos_class[i])
+#     return new_corners, new_class
+
+
+# def select_corner(corners, compos_class, class_name):
+#     """
+#     Select corners in given compo type
+#     """
+#     corners_wanted = []
+#     for i in range(len(compos_class)):
+#         if compos_class[i] == class_name:
+#             corners_wanted.append(corners[i])
+#     return corners_wanted
+
+
+# def flood_fill_bfs(img, x_start, y_start, mark, grad_thresh):
+#     def neighbor(x, y):
+#         for i in range(x - 1, x + 2):
+#             if i < 0 or i >= img.shape[0]: continue
+#             for j in range(y - 1, y + 2):
+#                 if j < 0 or j >= img.shape[1]: continue
+#                 if mark[i, j] == 0 and abs(img[i, j] - img[x, y]) < grad_thresh:
+#                     stack.append([i, j])
+#                     mark[i, j] = 255
+#
+#     stack = [[x_start, y_start]]  # points waiting for inspection
+#     region = [[x_start, y_start]]  # points of this connected region
+#     mark[x_start, y_start] = 255  # drawing broad
+#     while len(stack) > 0:
+#         point = stack.pop()
+#         region.append(point)
+#         neighbor(point[0], point[1])
+#     return region

CDM/detect_compo/deprecated/ip_segment.py

@@ -0,0 +1,123 @@
+import cv2
+import numpy as np
+import shutil
+import os
+from os.path import join as pjoin
+
+
+def segment_img(org, segment_size, output_path, overlap=100):
+    if not os.path.exists(output_path):
+        os.mkdir(output_path)
+
+    height, width = np.shape(org)[0], np.shape(org)[1]
+    top = 0
+    bottom = segment_size
+    segment_no = 0
+    while top < height and bottom < height:
+        segment = org[top:bottom]
+        cv2.imwrite(os.path.join(output_path, str(segment_no) + '.png'), segment)
+        segment_no += 1
+        top += segment_size - overlap
+        bottom = bottom + segment_size - overlap if bottom + segment_size - overlap <= height else height
+
+
+def clipping(img, components, pad=0, show=False):
+    """
+    :param adjust: shrink(negative) or expand(positive) the bounding box
+    :param img: original image
+    :param corners: ((column_min, row_min),(column_max, row_max))
+    :return: list of clipping images
+    """
+    clips = []
+    for component in components:
+        clip = component.compo_clipping(img, pad=pad)
+        clips.append(clip)
+        if show:
+            cv2.imshow('clipping', clip)
+            cv2.waitKey()
+    return clips
+
+
+def dissemble_clip_img_hollow(clip_root, org, compos):
+    if os.path.exists(clip_root):
+        shutil.rmtree(clip_root)
+    os.mkdir(clip_root)
+    cls_dirs = []
+
+    bkg = org.copy()
+    hollow_out = np.ones(bkg.shape[:2], dtype=np.uint8) * 255
+    for compo in compos:
+        cls = compo.category
+        c_root = pjoin(clip_root, cls)
+        c_path = pjoin(c_root, str(compo.id) + '.jpg')
+        if cls not in cls_dirs:
+            os.mkdir(c_root)
+            cls_dirs.append(cls)
+        clip = compo.compo_clipping(org)
+        cv2.imwrite(c_path, clip)
+
+        col_min, row_min, col_max, row_max = compo.put_bbox()
+        hollow_out[row_min: row_max, col_min: col_max] = 0
+
+    bkg = cv2.merge((bkg, hollow_out))
+    cv2.imwrite(os.path.join(clip_root, 'bkg.png'), bkg)
+
+
+def dissemble_clip_img_fill(clip_root, org, compos, flag='most'):
+
+    def average_pix_around(pad=6, offset=3):
+        up = row_min - pad if row_min - pad >= 0 else 0
+        left = col_min - pad if col_min - pad >= 0 else 0
+        bottom = row_max + pad if row_max + pad < org.shape[0] - 1 else org.shape[0] - 1
+        right = col_max + pad if col_max + pad < org.shape[1] - 1 else org.shape[1] - 1
+
+        average = []
+        for i in range(3):
+            avg_up = np.average(org[up:row_min - offset, left:right, i])
+            avg_bot = np.average(org[row_max + offset:bottom, left:right, i])
+            avg_left = np.average(org[up:bottom, left:col_min - offset, i])
+            avg_right = np.average(org[up:bottom, col_max + offset:right, i])
+            average.append(int((avg_up + avg_bot + avg_left + avg_right)/4))
+        return average
+
+    def most_pix_around(pad=6, offset=2):
+        up = row_min - pad if row_min - pad >= 0 else 0
+        left = col_min - pad if col_min - pad >= 0 else 0
+        bottom = row_max + pad if row_max + pad < org.shape[0] - 1 else org.shape[0] - 1
+        right = col_max + pad if col_max + pad < org.shape[1] - 1 else org.shape[1] - 1
+
+        most = []
+        for i in range(3):
+            val = np.concatenate((org[up:row_min - offset, left:right, i].flatten(),
+                            org[row_max + offset:bottom, left:right, i].flatten(),
+                            org[up:bottom, left:col_min - offset, i].flatten(),
+                            org[up:bottom, col_max + offset:right, i].flatten()))
+            # print(val)
+            # print(np.argmax(np.bincount(val)))
+            most.append(int(np.argmax(np.bincount(val))))
+        return most
+
+    if os.path.exists(clip_root):
+        shutil.rmtree(clip_root)
+    os.mkdir(clip_root)
+    cls_dirs = []
+
+    bkg = org.copy()
+    for compo in compos:
+        cls = compo.category
+        c_root = pjoin(clip_root, cls)
+        c_path = pjoin(c_root, str(compo.id) + '.jpg')
+        if cls not in cls_dirs:
+            os.mkdir(c_root)
+            cls_dirs.append(cls)
+        clip = compo.compo_clipping(org)
+        cv2.imwrite(c_path, clip)
+
+        col_min, row_min, col_max, row_max = compo.put_bbox()
+        if flag == 'average':
+            color = average_pix_around()
+        elif flag == 'most':
+            color = most_pix_around()
+        cv2.rectangle(bkg, (col_min, row_min), (col_max, row_max), color, -1)
+
+    cv2.imwrite(os.path.join(clip_root, 'bkg.png'), bkg)

CDM/detect_compo/deprecated/ocr_classify_text.py

@@ -0,0 +1,113 @@
+import pytesseract as pyt
+import cv2
+
+import lib_ip.ip_draw as draw
+from config.CONFIG_UIED import Config
+
+C = Config()
+
+
+def is_text(img, min_word_area, show=False):
+    broad = img.copy()
+    area_word = 0
+    area_total = img.shape[0] * img.shape[1]
+
+    try:
+        # ocr text detection
+        data = pyt.image_to_data(img).split('\n')
+    except:
+        print(img.shape)
+        return -1
+    word = []
+    for d in data[1:]:
+        d = d.split()
+        if d[-1] != '-1':
+            if d[-1] != '-' and d[-1] != '—' and int(d[-3]) < 50 and int(d[-4]) < 100:
+                word.append(d)
+                t_l = (int(d[-6]), int(d[-5]))
+                b_r = (int(d[-6]) + int(d[-4]), int(d[-5]) + int(d[-3]))
+                area_word += int(d[-4]) * int(d[-3])
+                cv2.rectangle(broad, t_l, b_r, (0,0,255), 1)
+
+    if show:
+        for d in word: print(d)
+        print(area_word/area_total)
+        cv2.imshow('a', broad)
+        cv2.waitKey(0)
+        cv2.destroyAllWindows()
+    # no text in this clip or relatively small text area
+    if len(word) == 0 or area_word/area_total < min_word_area:
+        return False
+    return True
+
+
+def text_detection(org, img_clean):
+    try:
+        data = pyt.image_to_data(img_clean).split('\n')
+    except:
+        return org, None
+    corners_word = []
+    for d in data[1:]:
+        d = d.split()
+        if d[-1] != '-1':
+            if d[-1] != '-' and d[-1] != '—' and 5 < int(d[-3]) < 40 and 5 < int(d[-4]) < 100:
+                t_l = (int(d[-6]), int(d[-5]))
+                b_r = (int(d[-6]) + int(d[-4]), int(d[-5]) + int(d[-3]))
+                corners_word.append((t_l, b_r))
+    return corners_word
+
+
+# def text_merge_word_into_line(org, corners_word, max_words_gap=C.THRESHOLD_TEXT_MAX_WORD_GAP):
+#
+#     def is_in_line(word):
+#         for i in range(len(lines)):
+#             line = lines[i]
+#             # at the same row
+#             if abs(line['center'][1] - word['center'][1]) < max_words_gap:
+#                 # small gap between words
+#                 if (abs(line['center'][0] - word['center'][0]) - abs(line['width']/2 + word['width']/2)) < max_words_gap:
+#                     return i
+#         return -1
+#
+#     def merge_line(word, index):
+#         line = lines[index]
+#         # on the left
+#         if word['center'][0] < line['center'][0]:
+#             line['col_min'] = word['col_min']
+#         # on the right
+#         else:
+#             line['col_max'] = word['col_max']
+#         line['row_min'] = min(line['row_min'], word['row_min'])
+#         line['row_max'] = max(line['row_max'], word['row_max'])
+#         line['width'] = line['col_max'] - line['col_min']
+#         line['height'] = line['row_max'] - line['row_min']
+#         line['center'] = ((line['col_max'] + line['col_min'])/2, (line['row_max'] + line['row_min'])/2)
+#
+#     words = []
+#     for corner in corners_word:
+#         word = {}
+#         (top_left, bottom_right) = corner
+#         (col_min, row_min) = top_left
+#         (col_max, row_max) = bottom_right
+#         word['col_min'], word['col_max'], word['row_min'], word['row_max'] = col_min, col_max, row_min, row_max
+#         word['height'] = row_max - row_min
+#         word['width'] = col_max - col_min
+#         word['center'] = ((col_max + col_min)/2, (row_max + row_min)/2)
+#         words.append(word)
+#
+#     lines = []
+#     for word in words:
+#         line_index = is_in_line(word)
+#         # word is in current line
+#         if line_index != -1:
+#             merge_line(word, line_index)
+#         # word is not in current line
+#         else:
+#             # this single word as a new line
+#             lines.append(word)
+#
+#     corners_line = []
+#     for l in lines:
+#         corners_line.append(((l['col_min'], l['row_min']), (l['col_max'], l['row_max'])))
+#     return corners_line
+

CDM/detect_compo/ip_region_proposal.py

@@ -0,0 +1,200 @@
+import cv2
+from os.path import join as pjoin
+import time
+
+import CDM.detect_compo.lib_ip.ip_preprocessing as pre
+import CDM.detect_compo.lib_ip.ip_draw as draw
+import CDM.detect_compo.lib_ip.ip_detection as det
+import CDM.detect_compo.lib_ip.file_utils as file
+import CDM.detect_compo.lib_ip.Component as Compo
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+def nesting_inspection(org, grey, compos, ffl_block):
+    '''
+    Inspect all big compos through block division by flood-fill
+    :param ffl_block: gradient threshold for flood-fill
+    :return: nesting compos
+    '''
+    nesting_compos = []
+    for i, compo in enumerate(compos):
+        if compo.height > 50:
+            replace = False
+            clip_grey = compo.compo_clipping(grey)
+            n_compos = det.nested_components_detection(clip_grey, org, grad_thresh=ffl_block, show=False)
+            Compo.cvt_compos_relative_pos(n_compos, compo.bbox.col_min, compo.bbox.row_min)
+
+            for n_compo in n_compos:
+                if n_compo.redundant:
+                    compos[i] = n_compo
+                    replace = True
+                    break
+            if not replace:
+                nesting_compos += n_compos
+    return nesting_compos
+
+
+def compo_detection(input_img_path, output_root, uied_params,
+                    resize_by_height=800, classifier=None, show=False, wai_key=0):
+
+    start = time.process_time()
+    name = input_img_path.split('/')[-1][:-4] if '/' in input_img_path else input_img_path.split('\\')[-1][:-4]
+    ip_root = file.build_directory(pjoin(output_root, "ip"))
+
+    # *** Step 1 *** pre-processing: read img -> get binary map
+    org, grey = pre.read_img(input_img_path, resize_by_height)
+    binary = pre.binarization(org, grad_min=int(uied_params['min-grad']))
+
+    full_size_org, full_size_grey = pre.read_img(input_img_path)
+    ratio = full_size_org.shape[0] / org.shape[0]
+
+    # *** Step 2 *** element detection
+    det.rm_line(binary, show=show, wait_key=wai_key)
+    uicompos = det.component_detection(binary, min_obj_area=int(uied_params['min-ele-area']))
+
+    # *** Step 3 *** results refinement
+    uicompos = det.compo_filter(uicompos, min_area=int(uied_params['min-ele-area']), img_shape=binary.shape)
+    uicompos = det.merge_intersected_compos(uicompos)
+    det.compo_block_recognition(binary, uicompos)
+    if uied_params['merge-contained-ele']:
+        uicompos = det.rm_contained_compos_not_in_block(uicompos)
+    Compo.compos_update(uicompos, org.shape)
+    Compo.compos_containment(uicompos)
+
+    # *** Step 4 ** nesting inspection: check if big compos have nesting element
+    uicompos += nesting_inspection(org, grey, uicompos, ffl_block=uied_params['ffl-block'])
+    Compo.compos_update(uicompos, org.shape)
+    draw.draw_bounding_box(full_size_org, ratio, uicompos, show=show, name='merged compo', write_path=pjoin(ip_root, name + '.jpg'), wait_key=wai_key)
+
+    # # classify icons
+    # model = models.resnet18().to('cpu')
+    # in_feature_num = model.fc.in_features
+    # model.fc = nn.Linear(in_feature_num, 99)
+    # # model.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(3,3), padding=(3,3), stride=(2,2), bias=False)
+    # model.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(5, 5), padding=(3, 3), stride=(2, 2),
+    #                         bias=False)
+    # # PATH = "C:/ANU/2022 s2/honours project/code/UIED-master/model/model-99-resnet18.pkl"
+    # PATH = "./model/model-99-resnet18.pkl"
+    # # trained_model = model()
+    # model.load_state_dict(torch.load(PATH, map_location=torch.device('cpu')))
+    #
+    # model.eval()
+    #
+    # # ----------------- try on semantics dataset---------------------
+    #
+    # # sample_data = np.load('C:/ANU/2022 s2/honours project/code/semantic-icon-classifier-master/data/training_x.npy')
+    # #
+    # # array = np.reshape(sample_data[0, :, :, :], [32, 32])
+    # #
+    # # print("array: ", array)
+    # #
+    # # cv2.imshow("array", array)
+    # # cv2.waitKey(0)
+    # #
+    # # array = array.astype('float32')
+    # # array = array / 255
+    # # array = (array - array.mean()) / array.std()
+    # #
+    # # print("array mean: ", array.mean())
+    # # print("array std: ", array.std())
+    # #
+    # # array = array.reshape(1, 1, 32, 32)
+    # #
+    # # array = torch.tensor(array)
+    # # print("array_tensor: ", array)
+    # # array_pred_label = model(array)
+    # # print("output: ", array_pred_label)
+    #
+    # # ----------------- end trying ---------------------
+    #
+    # grey = grey.astype('float32')
+    # grey = grey / 255
+    # # grey = grey / np.linalg.norm(grey)
+    #
+    # grey = (grey-grey.mean())/grey.std()
+    # print("grey mean: ", grey.mean())
+    # print("grey std: ", grey.std())
+    #
+    # # grey = grey.to(torch.float32)
+    #
+    # # plt.imshow(Image.fromarray(binary))
+    # # plt.show()
+    # # cv2.imshow("grey", grey)
+    #
+    # privacy_compos = []
+    # for comp in uicompos:
+    #
+    #     # cv2.imshow("comp", grey[comp.bbox.row_min:comp.bbox.row_max, comp.bbox.col_min:comp.bbox.col_max])
+    #     # cv2.waitKey(0)
+    #
+    #     # col_mid = int((comp.bbox.col_min+comp.bbox.col_max)/2)
+    #     # row_mid = int((comp.bbox.row_min+comp.bbox.row_max)/2)
+    #     # comp_crop = grey[max(0, row_mid-16):min(grey.shape[1], row_mid+16), max(0, col_mid-16):min(grey.shape[0], col_mid+16)]
+    #     #
+    #     # if comp_crop.shape[0] != 32 or comp_crop.shape[1] != 32:
+    #     #     print("A component is not classified, size: ", comp_crop.shape)
+    #     #     print("col_mid: ", col_mid)
+    #     #     print("row_mid: ", row_mid)
+    #     #     print("shape[0]: ", comp_crop.shape[0])
+    #     #     print("shape[1]: ", comp_crop.shape[1])
+    #     #     print("max(0, row_mid-16) and min(binary.shape[1], row_mid+16): ", max(0, row_mid-16), min(grey.shape[1], row_mid+16))
+    #
+    #     comp_grey = grey[comp.bbox.row_min:comp.bbox.row_max, comp.bbox.col_min:comp.bbox.col_max]
+    #
+    #     # cv2.imshow("comp_grey", comp_grey)
+    #     # cv2.waitKey(0)
+    #
+    #     # print("comp_crop: ", comp_crop)
+    #     # comp_crop = comp_grey.reshape(1, 1, 32, 32)
+    #     comp_crop = cv2.resize(comp_grey, (32, 32))
+    #     print("comp_crop: ", comp_crop)
+    #
+    #     # cv2.imshow("comp_crop", comp_crop)
+    #     # cv2.waitKey(0)
+    #
+    #     comp_crop = comp_crop.reshape(1, 1, 32, 32)
+    #
+    #     comp_tensor = torch.tensor(comp_crop)
+    #     comp_tensor = comp_tensor.permute(0, 1, 3, 2)
+    #     print("comp_tensor: ", comp_tensor)
+    #     # comp_float = comp_tensor.to(torch.float32)
+    #     # print("comp_float: ", comp_float)
+    #     # pred_label = model(comp_float)
+    #     pred_label = model(comp_tensor)
+    #     print("output: ", pred_label)
+    #     print("label: ", np.argmax(pred_label.cpu().data.numpy(), axis=1))
+    #     if np.argmax(pred_label.cpu().data.numpy(), axis=1) in [72.0, 42.0, 77.0, 91.0, 6.0, 89.0, 40.0, 43.0, 82.0, 3.0, 68.0,
+    #                                                             49.0, 56.0, 89.0]:
+    #         privacy_compos.append(comp)
+    #
+    # draw.draw_bounding_box(org, privacy_compos, show=show, name='merged compo', write_path=pjoin(ip_root, name + '.jpg'), wait_key=wai_key)
+
+    # *** Step 5 *** image inspection: recognize image -> remove noise in image -> binarize with larger threshold and reverse -> rectangular compo detection
+    # if classifier is not None:
+    #     classifier['Image'].predict(seg.clipping(org, uicompos), uicompos)
+    #     draw.draw_bounding_box_class(org, uicompos, show=show)
+    #     uicompos = det.rm_noise_in_large_img(uicompos, org)
+    #     draw.draw_bounding_box_class(org, uicompos, show=show)
+    #     det.detect_compos_in_img(uicompos, binary_org, org)
+    #     draw.draw_bounding_box(org, uicompos, show=show)
+    # if classifier is not None:
+    #     classifier['Noise'].predict(seg.clipping(org, uicompos), uicompos)
+    #     draw.draw_bounding_box_class(org, uicompos, show=show)
+    #     uicompos = det.rm_noise_compos(uicompos)
+
+    # *** Step 6 *** element classification: all category classification
+    # if classifier is not None:
+    #     classifier['Elements'].predict([compo.compo_clipping(org) for compo in uicompos], uicompos)
+    #     draw.draw_bounding_box_class(org, uicompos, show=show, name='cls', write_path=pjoin(ip_root, 'result.jpg'))
+    #     draw.draw_bounding_box_class(org, uicompos, write_path=pjoin(output_root, 'result.jpg'))
+
+    # *** Step 7 *** save detection result
+
+    Compo.compos_update(uicompos, org.shape)
+    file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos)
+    # file.save_corners_json(pjoin(ip_root, name + '.json'), uicompos, full_size_org, ratio)
+
+    cd_time = time.process_time() - start
+    print("[Compo Detection Completed in %.3f s] Input: %s Output: %s" % (cd_time, input_img_path, pjoin(ip_root, name + '.json')))
+    return cd_time

CDM/detect_compo/lib_ip/Bbox.py

@@ -0,0 +1,122 @@
+import numpy as np
+import CDM.detect_compo.lib_ip.ip_draw as draw
+
+
+class Bbox:
+    def __init__(self, col_min, row_min, col_max, row_max):
+        self.col_min = col_min
+        self.row_min = row_min
+        self.col_max = col_max
+        self.row_max = row_max
+
+        self.width = col_max - col_min
+        self.height = row_max - row_min
+        self.box_area = self.width * self.height
+
+    def put_bbox(self):
+        return self.col_min, self.row_min, self.col_max, self.row_max
+
+    def bbox_cal_area(self):
+        self.box_area = self.width * self.height
+        return self.box_area
+
+    def bbox_relation(self, bbox_b):
+        """
+        :return: -1 : a in b
+                 0  : a, b are not intersected
+                 1  : b in a
+                 2  : a, b are identical or intersected
+        """
+        col_min_a, row_min_a, col_max_a, row_max_a = self.put_bbox()
+        col_min_b, row_min_b, col_max_b, row_max_b = bbox_b.put_bbox()
+
+        # if a is in b
+        if col_min_a > col_min_b and row_min_a > row_min_b and col_max_a < col_max_b and row_max_a < row_max_b:
+            return -1
+        # if b is in a
+        elif col_min_a < col_min_b and row_min_a < row_min_b and col_max_a > col_max_b and row_max_a > row_max_b:
+            return 1
+        # a and b are non-intersect
+        elif (col_min_a > col_max_b or row_min_a > row_max_b) or (col_min_b > col_max_a or row_min_b > row_max_a):
+            return 0
+        # intersection
+        else:
+            return 2
+
+    def bbox_relation_nms(self, bbox_b, bias=(0, 0)):
+        '''
+        Calculate the relation between two rectangles by nms
+       :return: -1 : a in b
+         0  : a, b are not intersected
+         1  : b in a
+         2  : a, b are intersected
+       '''
+        col_min_a, row_min_a, col_max_a, row_max_a = self.put_bbox()
+        col_min_b, row_min_b, col_max_b, row_max_b = bbox_b.put_bbox()
+
+        bias_col, bias_row = bias
+        # get the intersected area
+        col_min_s = max(col_min_a - bias_col, col_min_b - bias_col)
+        row_min_s = max(row_min_a - bias_row, row_min_b - bias_row)
+        col_max_s = min(col_max_a + bias_col, col_max_b + bias_col)
+        row_max_s = min(row_max_a + bias_row, row_max_b + bias_row)
+        w = np.maximum(0, col_max_s - col_min_s)
+        h = np.maximum(0, row_max_s - row_min_s)
+        inter = w * h
+        area_a = (col_max_a - col_min_a) * (row_max_a - row_min_a)
+        area_b = (col_max_b - col_min_b) * (row_max_b - row_min_b)
+        iou = inter / (area_a + area_b - inter)
+        ioa = inter / self.box_area
+        iob = inter / bbox_b.box_area
+
+        if iou == 0 and ioa == 0 and iob == 0:
+            return 0
+
+        # import lib_ip.ip_preprocessing as pre
+        # org_iou, _ = pre.read_img('uied/data/input/7.jpg', 800)
+        # print(iou, ioa, iob)
+        # board = draw.draw_bounding_box(org_iou, [self], color=(255,0,0))
+        # draw.draw_bounding_box(board, [bbox_b], color=(0,255,0), show=True)
+
+        # contained by b
+        if ioa >= 1:
+            return -1
+        # contains b
+        if iob >= 1:
+            return 1
+        # not intersected with each other
+        # intersected
+        if iou >= 0.02 or iob > 0.2 or ioa > 0.2:
+            return 2
+        # if iou == 0:
+        # print('ioa:%.5f; iob:%.5f; iou:%.5f' % (ioa, iob, iou))
+        return 0
+
+    def bbox_cvt_relative_position(self, col_min_base, row_min_base):
+        '''
+        Convert to relative position based on base coordinator
+        '''
+        self.col_min += col_min_base
+        self.col_max += col_min_base
+        self.row_min += row_min_base
+        self.row_max += row_min_base
+
+    def bbox_merge(self, bbox_b):
+        '''
+        Merge two intersected bboxes
+        '''
+        col_min_a, row_min_a, col_max_a, row_max_a = self.put_bbox()
+        col_min_b, row_min_b, col_max_b, row_max_b = bbox_b.put_bbox()
+        col_min = min(col_min_a, col_min_b)
+        col_max = max(col_max_a, col_max_b)
+        row_min = min(row_min_a, row_min_b)
+        row_max = max(row_max_a, row_max_b)
+        new_bbox = Bbox(col_min, row_min, col_max, row_max)
+        return new_bbox
+
+    def bbox_padding(self, image_shape, pad):
+        row, col = image_shape[:2]
+        self.col_min = max(self.col_min - pad, 0)
+        self.col_max = min(self.col_max + pad, col)
+        self.row_min = max(self.row_min - pad, 0)
+        self.row_max = min(self.row_max + pad, row)

CDM/detect_compo/lib_ip/Component.py

@@ -0,0 +1,238 @@
+from CDM.detect_compo.lib_ip.Bbox import Bbox
+import CDM.detect_compo.lib_ip.ip_draw as draw
+
+import cv2
+
+
+def cvt_compos_relative_pos(compos, col_min_base, row_min_base):
+    for compo in compos:
+        compo.compo_relative_position(col_min_base, row_min_base)
+
+
+def compos_containment(compos):
+    for i in range(len(compos) - 1):
+        for j in range(i + 1, len(compos)):
+            relation = compos[i].compo_relation(compos[j])
+            if relation == -1:
+                compos[j].contain.append(i)
+            if relation == 1:
+                compos[i].contain.append(j)
+
+
+def compos_update(compos, org_shape):
+    for i, compo in enumerate(compos):
+        # start from 1, id 0 is background
+        compo.compo_update(i + 1, org_shape)
+
+
+class Component:
+    def __init__(self, region, image_shape):
+        self.id = None
+        self.region = region
+        self.boundary = self.compo_get_boundary()
+        self.bbox = self.compo_get_bbox()
+        self.bbox_area = self.bbox.box_area
+
+        self.region_area = len(region)
+        self.width = len(self.boundary[0])
+        self.height = len(self.boundary[2])
+        self.image_shape = image_shape
+        self.area = self.width * self.height
+
+        self.category = 'Compo'
+        self.contain = []
+
+        self.rect_ = None
+        self.line_ = None
+        self.redundant = False
+
+    def compo_update(self, id, org_shape):
+        self.id = id
+        self.image_shape = org_shape
+        self.width = self.bbox.width
+        self.height = self.bbox.height
+        self.bbox_area = self.bbox.box_area
+        self.area = self.width * self.height
+
+    def put_bbox(self):
+        return self.bbox.put_bbox()
+
+    def compo_update_bbox_area(self):
+        self.bbox_area = self.bbox.bbox_cal_area()
+
+    def compo_get_boundary(self):
+        '''
+        get the bounding boundary of an object(region)
+        boundary: [top, bottom, left, right]
+        -> up, bottom: (column_index, min/max row border)
+        -> left, right: (row_index, min/max column border) detect range of each row
+        '''
+        border_up, border_bottom, border_left, border_right = {}, {}, {}, {}
+        for point in self.region:
+            # point: (row_index, column_index)
+            # up, bottom: (column_index, min/max row border) detect range of each column
+            if point[1] not in border_up or border_up[point[1]] > point[0]:
+                border_up[point[1]] = point[0]
+            if point[1] not in border_bottom or border_bottom[point[1]] < point[0]:
+                border_bottom[point[1]] = point[0]
+            # left, right: (row_index, min/max column border) detect range of each row
+            if point[0] not in border_left or border_left[point[0]] > point[1]:
+                border_left[point[0]] = point[1]
+            if point[0] not in border_right or border_right[point[0]] < point[1]:
+                border_right[point[0]] = point[1]
+
+        boundary = [border_up, border_bottom, border_left, border_right]
+        # descending sort
+        for i in range(len(boundary)):
+            boundary[i] = [[k, boundary[i][k]] for k in boundary[i].keys()]
+            boundary[i] = sorted(boundary[i], key=lambda x: x[0])
+        return boundary
+
+    def compo_get_bbox(self):
+        """
+        Get the top left and bottom right points of boundary
+        :param boundaries: boundary: [top, bottom, left, right]
+                            -> up, bottom: (column_index, min/max row border)
+                            -> left, right: (row_index, min/max column border) detect range of each row
+        :return: corners: [(top_left, bottom_right)]
+                            -> top_left: (column_min, row_min)
+                            -> bottom_right: (column_max, row_max)
+        """
+        col_min, row_min = (int(min(self.boundary[0][0][0], self.boundary[1][-1][0])), int(min(self.boundary[2][0][0], self.boundary[3][-1][0])))
+        col_max, row_max = (int(max(self.boundary[0][0][0], self.boundary[1][-1][0])), int(max(self.boundary[2][0][0], self.boundary[3][-1][0])))
+        bbox = Bbox(col_min, row_min, col_max, row_max)
+        return bbox
+
+    def compo_is_rectangle(self, min_rec_evenness, max_dent_ratio, test=False):
+        '''
+        detect if an object is rectangle by evenness and dent of each border
+        '''
+        dent_direction = [1, -1, 1, -1]  # direction for convex
+
+        flat = 0
+        parameter = 0
+        for n, border in enumerate(self.boundary):
+            parameter += len(border)
+            # dent detection
+            pit = 0  # length of pit
+            depth = 0  # the degree of surface changing
+            if n <= 1:
+                adj_side = max(len(self.boundary[2]), len(self.boundary[3]))  # get maximum length of adjacent side
+            else:
+                adj_side = max(len(self.boundary[0]), len(self.boundary[1]))
+
+            # -> up, bottom: (column_index, min/max row border)
+            # -> left, right: (row_index, min/max column border) detect range of each row
+            abnm = 0
+            for i in range(int(3 + len(border) * 0.02), len(border) - 1):
+                # calculate gradient
+                difference = border[i][1] - border[i + 1][1]
+                # the degree of surface changing
+                depth += difference
+                # ignore noise at the start of each direction
+                if i / len(border) < 0.08 and (dent_direction[n] * difference) / adj_side > 0.5:
+                    depth = 0  # reset
+
+                # print(border[i][1], i / len(border), depth, (dent_direction[n] * difference) / adj_side)
+                # if the change of the surface is too large, count it as part of abnormal change
+                if abs(depth) / adj_side > 0.3:
+                    abnm += 1  # count the size of the abnm
+                    # if the abnm is too big, the shape should not be a rectangle
+                    if abnm / len(border) > 0.1:
+                        if test:
+                            print('abnms', abnm, abnm / len(border))
+                            draw.draw_boundary([self], self.image_shape, show=True)
+                        self.rect_ = False
+                        return False
+                    continue
+                else:
+                    # reset the abnm if the depth back to normal
+                    abnm = 0
+
+                # if sunken and the surface changing is large, then counted as pit
+                if dent_direction[n] * depth < 0 and abs(depth) / adj_side > 0.15:
+                    pit += 1
+                    continue
+
+                # if the surface is not changing to a pit and the gradient is zero, then count it as flat
+                if abs(depth) < 1 + adj_side * 0.015:
+                    flat += 1
+                if test:
+                    print(depth, adj_side, flat)
+            # if the pit is too big, the shape should not be a rectangle
+            if pit / len(border) > max_dent_ratio:
+                if test:
+                    print('pit', pit, pit / len(border))
+                    draw.draw_boundary([self], self.image_shape, show=True)
+                self.rect_ = False
+                return False
+        if test:
+            print(flat / parameter, '\n')
+            draw.draw_boundary([self], self.image_shape, show=True)
+        # ignore text and irregular shape
+        if self.height / self.image_shape[0] > 0.3:
+            min_rec_evenness = 0.85
+        if (flat / parameter) < min_rec_evenness:
+            self.rect_ = False
+            return False
+        self.rect_ = True
+        return True
+
+    def compo_is_line(self, min_line_thickness):
+        """
+        Check this object is line by checking its boundary
+        :param boundary: boundary: [border_top, border_bottom, border_left, border_right]
+                                    -> top, bottom: list of (column_index, min/max row border)
+                                    -> left, right: list of (row_index, min/max column border) detect range of each row
+        :param min_line_thickness:
+        :return: Boolean
+        """
+        # horizontally
+        slim = 0
+        for i in range(self.width):
+            if abs(self.boundary[1][i][1] - self.boundary[0][i][1]) <= min_line_thickness:
+                slim += 1
+        if slim / len(self.boundary[0]) > 0.93:
+            self.line_ = True
+            return True
+        # vertically
+        slim = 0
+        for i in range(self.height):
+            if abs(self.boundary[2][i][1] - self.boundary[3][i][1]) <= min_line_thickness:
+                slim += 1
+        if slim / len(self.boundary[2]) > 0.93:
+            self.line_ = True
+            return True
+        self.line_ = False
+        return False
+
+    def compo_relation(self, compo_b, bias=(0, 0)):
+        """
+        :return: -1 : a in b
+                 0  : a, b are not intersected
+                 1  : b in a
+                 2  : a, b are identical or intersected
+        """
+        return self.bbox.bbox_relation_nms(compo_b.bbox, bias)
+
+    def compo_relative_position(self, col_min_base, row_min_base):
+        '''
+        Convert to relative position based on base coordinator
+        '''
+        self.bbox.bbox_cvt_relative_position(col_min_base, row_min_base)
+
+    def compo_merge(self, compo_b):
+        self.bbox = self.bbox.bbox_merge(compo_b.bbox)
+        self.compo_update(self.id, self.image_shape)
+
+    def compo_clipping(self, img, pad=0, show=False):
+        (column_min, row_min, column_max, row_max) = self.put_bbox()
+        column_min = max(column_min - pad, 0)
+        column_max = min(column_max + pad, img.shape[1])
+        row_min = max(row_min - pad, 0)
+        row_max = min(row_max + pad, img.shape[0])
+        clip = img[row_min:row_max, column_min:column_max]
+        if show:
+            cv2.imshow('clipping', clip)
+            cv2.waitKey()
+        return clip

CDM/detect_compo/lib_ip/file_utils.py

@@ -0,0 +1,80 @@
+import os
+import pandas as pd
+import json
+from os.path import join as pjoin
+import time
+import cv2
+
+
+def save_corners(file_path, corners, compo_name, clear=True):
+    try:
+        df = pd.read_csv(file_path, index_col=0)
+    except:
+        df = pd.DataFrame(columns=['component', 'x_max', 'x_min', 'y_max', 'y_min', 'height', 'width'])
+
+    if clear:
+        df = df.drop(df.index)
+    for corner in corners:
+        (up_left, bottom_right) = corner
+        c = {'component': compo_name}
+        (c['y_min'], c['x_min']) = up_left
+        (c['y_max'], c['x_max']) = bottom_right
+        c['width'] = c['y_max'] - c['y_min']
+        c['height'] = c['x_max'] - c['x_min']
+        df = df.append(c, True)
+    df.to_csv(file_path)
+
+
+def save_corners_json(file_path, compos):
+    # img_shape = [int(x * ratio) for x in compos[0].image_shape]
+    # w_h_ratio = org.shape[1] / org.shape[0]
+    # img_shape = org.shape
+
+    img_shape = compos[0].image_shape
+    output = {'img_shape': img_shape, 'compos': []}
+    f_out = open(file_path, 'w')
+
+    for compo in compos:
+        bbox = compo.put_bbox()
+        # bbox = [int(x * ratio) for x in bbox]
+        c = {'id': compo.id, 'class': compo.category}
+        (c['column_min'], c['row_min'], c['column_max'], c['row_max']) = bbox
+        c['width'] = compo.width
+        c['height'] = compo.height
+        # c['width'] = int(compo.width * ratio)
+        # c['height'] = int(compo.height * ratio)
+        output['compos'].append(c)
+
+    json.dump(output, f_out, indent=4)
+
+
+def save_clipping(org, output_root, corners, compo_classes, compo_index):
+    if not os.path.exists(output_root):
+        os.mkdir(output_root)
+    pad = 2
+    for i in range(len(corners)):
+        compo = compo_classes[i]
+        (up_left, bottom_right) = corners[i]
+        (col_min, row_min) = up_left
+        (col_max, row_max) = bottom_right
+        col_min = max(col_min - pad, 0)
+        col_max = min(col_max + pad, org.shape[1])
+        row_min = max(row_min - pad, 0)
+        row_max = min(row_max + pad, org.shape[0])
+
+        # if component type already exists, index increase by 1, otherwise add this type
+        compo_path = pjoin(output_root, compo)
+        if compo_classes[i] not in compo_index:
+            compo_index[compo_classes[i]] = 0
+            if not os.path.exists(compo_path):
+                os.mkdir(compo_path)
+        else:
+            compo_index[compo_classes[i]] += 1
+        clip = org[row_min:row_max, col_min:col_max]
+        cv2.imwrite(pjoin(compo_path, str(compo_index[compo_classes[i]]) + '.png'), clip)
+
+
+def build_directory(directory):
+    if not os.path.exists(directory):
+        os.mkdir(directory)
+    return directory

CDM/detect_compo/lib_ip/ip_detection.py

@@ -0,0 +1,574 @@
+import cv2
+import numpy as np
+
+import CDM.detect_compo.lib_ip.ip_draw as draw
+import CDM.detect_compo.lib_ip.ip_preprocessing as pre
+from CDM.detect_compo.lib_ip.Component import Component
+import CDM.detect_compo.lib_ip.Component as Compo
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+def merge_intersected_corner(compos, org, is_merge_contained_ele, max_gap=(0, 0), max_ele_height=25):
+    '''
+    :param is_merge_contained_ele: if true, merge compos nested in others
+    :param max_gap: (horizontal_distance, vertical_distance) to be merge into one line/column
+    :param max_ele_height: if higher than it, recognize the compo as text
+    :return:
+    '''
+    changed = False
+    new_compos = []
+    Compo.compos_update(compos, org.shape)
+    for i in range(len(compos)):
+        merged = False
+        cur_compo = compos[i]
+        for j in range(len(new_compos)):
+            relation = cur_compo.compo_relation(new_compos[j], max_gap)
+            # print(relation)
+            # draw.draw_bounding_box(org, [cur_compo, new_compos[j]], name='b-merge', show=True)
+            # merge compo[i] to compo[j] if
+            # 1. compo[j] contains compo[i]
+            # 2. compo[j] intersects with compo[i] with certain iou
+            # 3. is_merge_contained_ele and compo[j] is contained in compo[i]
+            if relation == 1 or \
+                    relation == 2 or \
+                    (is_merge_contained_ele and relation == -1):
+                # (relation == 2 and new_compos[j].height < max_ele_height and cur_compo.height < max_ele_height) or\
+
+                new_compos[j].compo_merge(cur_compo)
+                cur_compo = new_compos[j]
+                # draw.draw_bounding_box(org, [new_compos[j]], name='a-merge', show=True)
+                merged = True
+                changed = True
+                # break
+        if not merged:
+            new_compos.append(compos[i])
+
+    if not changed:
+        return compos
+    else:
+        return merge_intersected_corner(new_compos, org, is_merge_contained_ele, max_gap, max_ele_height)
+
+
+def merge_intersected_compos(compos):
+    changed = True
+    while changed:
+        changed = False
+        temp_set = []
+        for compo_a in compos:
+            merged = False
+            for compo_b in temp_set:
+                if compo_a.compo_relation(compo_b) == 2:
+                    compo_b.compo_merge(compo_a)
+                    merged = True
+                    changed = True
+                    break
+            if not merged:
+                temp_set.append(compo_a)
+        compos = temp_set.copy()
+    return compos
+
+
+def rm_contained_compos_not_in_block(compos):
+    '''
+    remove all components contained by others that are not Block
+    '''
+    marked = np.full(len(compos), False)
+    for i in range(len(compos) - 1):
+        for j in range(i + 1, len(compos)):
+            relation = compos[i].compo_relation(compos[j])
+            if relation == -1 and compos[j].category != 'Block':
+                marked[i] = True
+            if relation == 1 and compos[i].category != 'Block':
+                marked[j] = True
+    new_compos = []
+    for i in range(len(marked)):
+        if not marked[i]:
+            new_compos.append(compos[i])
+    return new_compos
+
+
+def merge_text(compos, org_shape, max_word_gad=4, max_word_height=20):
+    def is_text_line(compo_a, compo_b):
+        (col_min_a, row_min_a, col_max_a, row_max_a) = compo_a.put_bbox()
+        (col_min_b, row_min_b, col_max_b, row_max_b) = compo_b.put_bbox()
+
+        col_min_s = max(col_min_a, col_min_b)
+        col_max_s = min(col_max_a, col_max_b)
+        row_min_s = max(row_min_a, row_min_b)
+        row_max_s = min(row_max_a, row_max_b)
+
+        # on the same line
+        # if abs(row_min_a - row_min_b) < max_word_gad and abs(row_max_a - row_max_b) < max_word_gad:
+        if row_min_s < row_max_s:
+            # close distance
+            if col_min_s < col_max_s or \
+                    (0 < col_min_b - col_max_a < max_word_gad) or (0 < col_min_a - col_max_b < max_word_gad):
+                return True
+        return False
+
+    changed = False
+    new_compos = []
+    row, col = org_shape[:2]
+    for i in range(len(compos)):
+        merged = False
+        height = compos[i].height
+        # ignore non-text
+        # if height / row > max_word_height_ratio\
+        #         or compos[i].category != 'Text':
+        if height > max_word_height:
+            new_compos.append(compos[i])
+            continue
+        for j in range(len(new_compos)):
+            # if compos[j].category != 'Text':
+            #     continue
+            if is_text_line(compos[i], new_compos[j]):
+                new_compos[j].compo_merge(compos[i])
+                merged = True
+                changed = True
+                break
+        if not merged:
+            new_compos.append(compos[i])
+
+    if not changed:
+        return compos
+    else:
+        return merge_text(new_compos, org_shape)
+
+
+def rm_top_or_bottom_corners(components, org_shape, top_bottom_height=C.THRESHOLD_TOP_BOTTOM_BAR):
+    new_compos = []
+    height, width = org_shape[:2]
+    for compo in components:
+        (column_min, row_min, column_max, row_max) = compo.put_bbox()
+        # remove big ones
+        # if (row_max - row_min) / height > 0.65 and (column_max - column_min) / width > 0.8:
+        #     continue
+        if not (row_max < height * top_bottom_height[0] or row_min > height * top_bottom_height[1]):
+            new_compos.append(compo)
+    return new_compos
+
+
+def rm_line_v_h(binary, show=False, max_line_thickness=C.THRESHOLD_LINE_THICKNESS):
+    def check_continuous_line(line, edge):
+        continuous_length = 0
+        line_start = -1
+        for j, p in enumerate(line):
+            if p > 0:
+                if line_start == -1:
+                    line_start = j
+                continuous_length += 1
+            elif continuous_length > 0:
+                if continuous_length / edge > 0.6:
+                    return [line_start, j]
+                continuous_length = 0
+                line_start = -1
+
+        if continuous_length / edge > 0.6:
+            return [line_start, len(line)]
+        else:
+            return None
+
+    def extract_line_area(line, start_idx, flag='v'):
+        for e, l in enumerate(line):
+            if flag == 'v':
+                map_line[start_idx + e, l[0]:l[1]] = binary[start_idx + e, l[0]:l[1]]
+
+    map_line = np.zeros(binary.shape[:2], dtype=np.uint8)
+    cv2.imshow('binary', binary)
+
+    width = binary.shape[1]
+    start_row = -1
+    line_area = []
+    for i, row in enumerate(binary):
+        line_v = check_continuous_line(row, width)
+        if line_v is not None:
+            # new line
+            if start_row == -1:
+                start_row = i
+                line_area = []
+            line_area.append(line_v)
+        else:
+            # checking line
+            if start_row != -1:
+                if i - start_row < max_line_thickness:
+                    # binary[start_row: i] = 0
+                    # map_line[start_row: i] = binary[start_row: i]
+                    print(line_area, start_row, i)
+                    extract_line_area(line_area, start_row)
+                start_row = -1
+
+    height = binary.shape[0]
+    start_col = -1
+    for i in range(width):
+        col = binary[:, i]
+        line_h = check_continuous_line(col, height)
+        if line_h is not None:
+            # new line
+            if start_col == -1:
+                start_col = i
+        else:
+            # checking line
+            if start_col != -1:
+                if i - start_col < max_line_thickness:
+                    # binary[:, start_col: i] = 0
+                    map_line[:, start_col: i] = binary[:, start_col: i]
+                start_col = -1
+
+    binary -= map_line
+
+    if show:
+        cv2.imshow('no-line', binary)
+        cv2.imshow('lines', map_line)
+        cv2.waitKey()
+
+
+def rm_line(binary,
+            max_line_thickness=C.THRESHOLD_LINE_THICKNESS,
+            min_line_length_ratio=C.THRESHOLD_LINE_MIN_LENGTH,
+            show=False, wait_key=0):
+    def is_valid_line(line):
+        line_length = 0
+        line_gap = 0
+        for j in line:
+            if j > 0:
+                if line_gap > 5:
+                    return False
+                line_length += 1
+                line_gap = 0
+            elif line_length > 0:
+                line_gap += 1
+        if line_length / width > 0.95:
+            return True
+        return False
+
+    height, width = binary.shape[:2]
+    board = np.zeros(binary.shape[:2], dtype=np.uint8)
+
+    start_row, end_row = -1, -1
+    check_line = False
+    check_gap = False
+    for i, row in enumerate(binary):
+        # line_ratio = (sum(row) / 255) / width
+        # if line_ratio > 0.9:
+        if is_valid_line(row):
+            # new start: if it is checking a new line, mark this row as start
+            if not check_line:
+                start_row = i
+                check_line = True
+        else:
+            # end the line
+            if check_line:
+                # thin enough to be a line, then start checking gap
+                if i - start_row < max_line_thickness:
+                    end_row = i
+                    check_gap = True
+                else:
+                    start_row, end_row = -1, -1
+                check_line = False
+        # check gap
+        if check_gap and i - end_row > max_line_thickness:
+            binary[start_row: end_row] = 0
+            start_row, end_row = -1, -1
+            check_line = False
+            check_gap = False
+
+    if (check_line and (height - start_row) < max_line_thickness) or check_gap:
+        binary[start_row: end_row] = 0
+
+    if show:
+        cv2.imshow('no-line binary', binary)
+        if wait_key is not None:
+            cv2.waitKey(wait_key)
+        if wait_key == 0:
+            cv2.destroyWindow('no-line binary')
+
+
+def rm_noise_compos(compos):
+    compos_new = []
+    for compo in compos:
+        if compo.category == 'Noise':
+            continue
+        compos_new.append(compo)
+    return compos_new
+
+
+def rm_noise_in_large_img(compos, org,
+                      max_compo_scale=C.THRESHOLD_COMPO_MAX_SCALE):
+    row, column = org.shape[:2]
+    remain = np.full(len(compos), True)
+    new_compos = []
+    for compo in compos:
+        if compo.category == 'Image':
+            for i in compo.contain:
+                remain[i] = False
+    for i in range(len(remain)):
+        if remain[i]:
+            new_compos.append(compos[i])
+    return new_compos
+
+
+def detect_compos_in_img(compos, binary, org, max_compo_scale=C.THRESHOLD_COMPO_MAX_SCALE, show=False):
+    compos_new = []
+    row, column = binary.shape[:2]
+    for compo in compos:
+        if compo.category == 'Image':
+            compo.compo_update_bbox_area()
+            # org_clip = compo.compo_clipping(org)
+            # bin_clip = pre.binarization(org_clip, show=show)
+            bin_clip = compo.compo_clipping(binary)
+            bin_clip = pre.reverse_binary(bin_clip, show=show)
+
+            compos_rec, compos_nonrec = component_detection(bin_clip, test=False, step_h=10, step_v=10, rec_detect=True)
+            for compo_rec in compos_rec:
+                compo_rec.compo_relative_position(compo.bbox.col_min, compo.bbox.row_min)
+                if compo_rec.bbox_area / compo.bbox_area < 0.8 and compo_rec.bbox.height > 20 and compo_rec.bbox.width > 20:
+                    compos_new.append(compo_rec)
+                    # draw.draw_bounding_box(org, [compo_rec], show=True)
+
+            # compos_inner = component_detection(bin_clip, rec_detect=False)
+            # for compo_inner in compos_inner:
+            #     compo_inner.compo_relative_position(compo.bbox.col_min, compo.bbox.row_min)
+            #     draw.draw_bounding_box(org, [compo_inner], show=True)
+            #     if compo_inner.bbox_area / compo.bbox_area < 0.8:
+            #         compos_new.append(compo_inner)
+    compos += compos_new
+
+
+def compo_filter(compos, min_area, img_shape):
+    # max_height = img_shape[0] * 0.8
+    # compos_new = []
+    # for compo in compos:
+    #     if compo.area < min_area:
+    #         continue
+    #     if compo.height > max_height:
+    #         continue
+    #     ratio_h = compo.width / compo.height
+    #     ratio_w = compo.height / compo.width
+    #     if ratio_h > 50 or ratio_w > 40 or \
+    #             (min(compo.height, compo.width) < 8 and max(ratio_h, ratio_w) > 10):
+    #         continue
+    #     compos_new.append(compo)
+    # return compos_new
+
+    # mobile semantics filter
+    # compos_new = []
+    #
+    # for compo in compos:
+    #
+    #     if compo.area >= 0.05 * (img_shape[0] * img_shape[1]):
+    #         continue
+    #
+    #     smaller_dimension = min(compo.width, compo.height)
+    #     larger_dimension = max(compo.width, compo.height)
+    #
+    #     if smaller_dimension/larger_dimension <= 0.75:
+    #         continue
+    #
+    #     compos_new.append(compo)
+    #
+    # return compos_new
+
+    # my own filter
+    compos_new = []
+
+    for compo in compos:
+
+        if compo.area >= 0.1 * (img_shape[0] * img_shape[1]):
+            continue
+
+        if compo.area <= 0.0005 * (img_shape[0] * img_shape[1]):
+            continue
+
+        smaller_dimension = min(compo.width, compo.height)
+        larger_dimension = max(compo.width, compo.height)
+
+        if smaller_dimension / larger_dimension <= 0.6:
+            continue
+
+        compos_new.append(compo)
+
+    return compos_new
+
+
+def is_block(clip, thread=0.15):
+    '''
+    Block is a rectangle border enclosing a group of compos (consider it as a wireframe)
+    Check if a compo is block by checking if the inner side of its border is blank
+    '''
+    side = 4  # scan 4 lines inner forward each border
+    # top border - scan top down
+    blank_count = 0
+    for i in range(1, 5):
+        if sum(clip[side + i]) / 255 > thread * clip.shape[1]:
+            blank_count += 1
+    if blank_count > 2: return False
+    # left border - scan left to right
+    blank_count = 0
+    for i in range(1, 5):
+        if sum(clip[:, side + i]) / 255 > thread * clip.shape[0]:
+            blank_count += 1
+    if blank_count > 2: return False
+
+    side = -4
+    # bottom border - scan bottom up
+    blank_count = 0
+    for i in range(-1, -5, -1):
+        if sum(clip[side + i]) / 255 > thread * clip.shape[1]:
+            blank_count += 1
+    if blank_count > 2: return False
+    # right border - scan right to left
+    blank_count = 0
+    for i in range(-1, -5, -1):
+        if sum(clip[:, side + i]) / 255 > thread * clip.shape[0]:
+            blank_count += 1
+    if blank_count > 2: return False
+    return True
+
+
+def compo_block_recognition(binary, compos, block_side_length=0.15):
+    height, width = binary.shape
+    for compo in compos:
+        if compo.height / height > block_side_length and compo.width / width > block_side_length:
+            clip = compo.compo_clipping(binary)
+            if is_block(clip):
+                compo.category = 'Block'
+
+
+# take the binary image as input
+# calculate the connected regions -> get the bounding boundaries of them -> check if those regions are rectangles
+# return all boundaries and boundaries of rectangles
+def component_detection(binary, min_obj_area,
+                        line_thickness=C.THRESHOLD_LINE_THICKNESS,
+                        min_rec_evenness=C.THRESHOLD_REC_MIN_EVENNESS,
+                        max_dent_ratio=C.THRESHOLD_REC_MAX_DENT_RATIO,
+                        step_h = 5, step_v = 2,
+                        rec_detect=False, show=False, test=False):
+    """
+    :param binary: Binary image from pre-processing
+    :param min_obj_area: If not pass then ignore the small object
+    :param min_obj_perimeter: If not pass then ignore the small object
+    :param line_thickness: If not pass then ignore the slim object
+    :param min_rec_evenness: If not pass then this object cannot be rectangular
+    :param max_dent_ratio: If not pass then this object cannot be rectangular
+    :return: boundary: [top, bottom, left, right]
+                        -> up, bottom: list of (column_index, min/max row border)
+                        -> left, right: list of (row_index, min/max column border) detect range of each row
+    """
+    mask = np.zeros((binary.shape[0] + 2, binary.shape[1] + 2), dtype=np.uint8)
+    compos_all = []
+    compos_rec = []
+    compos_nonrec = []
+    row, column = binary.shape[0], binary.shape[1]
+    for i in range(0, row, step_h):
+        for j in range(i % 2, column, step_v):
+            if binary[i, j] == 255 and mask[i, j] == 0:
+                # get connected area
+                # region = util.boundary_bfs_connected_area(binary, i, j, mask)
+
+                mask_copy = mask.copy()
+                ff = cv2.floodFill(binary, mask, (j, i), None, 0, 0, cv2.FLOODFILL_MASK_ONLY)
+                if ff[0] < min_obj_area: continue
+                mask_copy = mask - mask_copy
+                region = np.reshape(cv2.findNonZero(mask_copy[1:-1, 1:-1]), (-1, 2))
+                region = [(p[1], p[0]) for p in region]
+
+                # filter out some compos
+                component = Component(region, binary.shape)
+                # calculate the boundary of the connected area
+                # ignore small area
+                if component.width <= 3 or component.height <= 3:
+                    continue
+                # check if it is line by checking the length of edges
+                # if component.compo_is_line(line_thickness):
+                #     continue
+
+                if test:
+                    print('Area:%d' % (len(region)))
+                    draw.draw_boundary([component], binary.shape, show=True)
+
+                compos_all.append(component)
+
+                if rec_detect:
+                    # rectangle check
+                    if component.compo_is_rectangle(min_rec_evenness, max_dent_ratio):
+                        component.rect_ = True
+                        compos_rec.append(component)
+                    else:
+                        component.rect_ = False
+                        compos_nonrec.append(component)
+
+                if show:
+                    print('Area:%d' % (len(region)))
+                    draw.draw_boundary(compos_all, binary.shape, show=True)
+
+    # draw.draw_boundary(compos_all, binary.shape, show=True)
+    if rec_detect:
+        return compos_rec, compos_nonrec
+    else:
+        return compos_all
+
+
+def nested_components_detection(grey, org, grad_thresh,
+                   show=False, write_path=None,
+                   step_h=10, step_v=10,
+                   line_thickness=C.THRESHOLD_LINE_THICKNESS,
+                   min_rec_evenness=C.THRESHOLD_REC_MIN_EVENNESS,
+                   max_dent_ratio=C.THRESHOLD_REC_MAX_DENT_RATIO):
+    '''
+    :param grey: grey-scale of original image
+    :return: corners: list of [(top_left, bottom_right)]
+                        -> top_left: (column_min, row_min)
+                        -> bottom_right: (column_max, row_max)
+    '''
+    compos = []
+    mask = np.zeros((grey.shape[0]+2, grey.shape[1]+2), dtype=np.uint8)
+    broad = np.zeros((grey.shape[0], grey.shape[1], 3), dtype=np.uint8)
+    broad_all = broad.copy()
+
+    row, column = grey.shape[0], grey.shape[1]
+    for x in range(0, row, step_h):
+        for y in range(0, column, step_v):
+            if mask[x, y] == 0:
+                # region = flood_fill_bfs(grey, x, y, mask)
+
+                # flood fill algorithm to get background (layout block)
+                mask_copy = mask.copy()
+                ff = cv2.floodFill(grey, mask, (y, x), None, grad_thresh, grad_thresh, cv2.FLOODFILL_MASK_ONLY)
+                # ignore small regions
+                if ff[0] < 500: continue
+                mask_copy = mask - mask_copy
+                region = np.reshape(cv2.findNonZero(mask_copy[1:-1, 1:-1]), (-1, 2))
+                region = [(p[1], p[0]) for p in region]
+
+                compo = Component(region, grey.shape)
+                # draw.draw_region(region, broad_all)
+                # if block.height < 40 and block.width < 40:
+                #     continue
+                if compo.height < 30:
+                    continue
+
+                # print(block.area / (row * column))
+                if compo.area / (row * column) > 0.9:
+                    continue
+                elif compo.area / (row * column) > 0.7:
+                    compo.redundant = True
+
+                # get the boundary of this region
+                # ignore lines
+                if compo.compo_is_line(line_thickness):
+                    continue
+                # ignore non-rectangle as blocks must be rectangular
+                if not compo.compo_is_rectangle(min_rec_evenness, max_dent_ratio):
+                    continue
+                # if block.height/row < min_block_height_ratio:
+                #     continue
+                compos.append(compo)
+                # draw.draw_region(region, broad)
+    if show:
+        cv2.imshow('flood-fill all', broad_all)
+        cv2.imshow('block', broad)
+        cv2.waitKey()
+    if write_path is not None:
+        cv2.imwrite(write_path, broad)
+    return compos

CDM/detect_compo/lib_ip/ip_draw.py

@@ -0,0 +1,139 @@
+import cv2
+import numpy as np
+from random import randint as rint
+from CDM.config.CONFIG_UIED import Config
+
+
+C = Config()
+
+
+def draw_bounding_box_class(org, components, color_map=C.COLOR, line=2, show=False, write_path=None, name='board'):
+    """
+    Draw bounding box of components with their classes on the original image
+    :param org: original image
+    :param components: bbox [(column_min, row_min, column_max, row_max)]
+                    -> top_left: (column_min, row_min)
+                    -> bottom_right: (column_max, row_max)
+    :param color_map: colors mapping to different components
+    :param line: line thickness
+    :param compo_class: classes matching the corners of components
+    :param show: show or not
+    :return: labeled image
+    """
+    board = org.copy()
+    for compo in components:
+        bbox = compo.put_bbox()
+        board = cv2.rectangle(board, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color_map[compo.category], line)
+        # board = cv2.putText(board, compo.category, (bbox[0]+5, bbox[1]+20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color_map[compo.category], 2)
+    if show:
+        cv2.imshow(name, board)
+        cv2.waitKey(0)
+    if write_path is not None:
+        cv2.imwrite(write_path, board)
+    return board
+
+
+def draw_bounding_box(org, ratio, components, color=(0, 255, 0), line=2,
+                      show=False, write_path=None, name='board', is_return=False, wait_key=0):
+    """
+    Draw bounding box of components on the original image
+    :param org: original image
+    :param components: bbox [(column_min, row_min, column_max, row_max)]
+                    -> top_left: (column_min, row_min)
+                    -> bottom_right: (column_max, row_max)
+    :param color: line color
+    :param line: line thickness
+    :param show: show or not
+    :return: labeled image
+    """
+    if not show and write_path is None and not is_return: return
+    board = org.copy()
+    # board = cv2.imread(img_path)
+    # ratio = board.shape[0]/org.shape[0]
+
+    for compo in components:
+        bbox = compo.put_bbox()
+
+        # bounding box on full size image
+        # bbox = int(ratio * bbox)
+        bbox = [int(x * ratio) for x in bbox]
+        board = cv2.rectangle(board, (bbox[0], bbox[1]), (bbox[2], bbox[3]), color, line)
+    if show:
+        cv2.imshow(name, board)
+        if wait_key is not None:
+            cv2.waitKey(wait_key)
+        if wait_key == 0:
+            cv2.destroyWindow(name)
+    if write_path is not None:
+        # board = cv2.resize(board, (1080, 1920))
+        # board = board[100:-110]
+        cv2.imwrite(write_path, board)
+    return board
+
+
+def draw_line(org, lines, color=(0, 255, 0), show=False):
+    """
+    Draw detected lines on the original image
+    :param org: original image
+    :param lines: [line_h, line_v]
+            -> line_h: horizontal {'head':(column_min, row), 'end':(column_max, row), 'thickness':int)
+            -> line_v: vertical {'head':(column, row_min), 'end':(column, row_max), 'thickness':int}
+    :param color: drawn color
+    :param show: show or not
+    :return: image with lines drawn
+    """
+    board = org.copy()
+    line_h, line_v = lines
+    for line in line_h:
+        cv2.line(board, tuple(line['head']), tuple(line['end']), color, line['thickness'])
+    for line in line_v:
+        cv2.line(board, tuple(line['head']), tuple(line['end']), color, line['thickness'])
+    if show:
+        cv2.imshow('img', board)
+        cv2.waitKey(0)
+    return board
+
+
+def draw_boundary(components, shape, show=False):
+    """
+    Draw boundary of objects on the black withe
+    :param components: boundary: [top, bottom, left, right]
+                        -> up, bottom: (column_index, min/max row border)
+                        -> left, right: (row_index, min/max column border) detect range of each row
+    :param shape: shape or original image
+    :param show: show or not
+    :return: drawn board
+    """
+    board = np.zeros(shape[:2], dtype=np.uint8)  # binary board
+    for component in components:
+        # up and bottom: (column_index, min/max row border)
+        for point in component.boundary[0] + component.boundary[1]:
+            board[point[1], point[0]] = 255
+        # left, right: (row_index, min/max column border)
+        for point in component.boundary[2] + component.boundary[3]:
+            board[point[0], point[1]] = 255
+    if show:
+        cv2.imshow('rec', board)
+        cv2.waitKey(0)
+    return board
+
+
+def draw_region(region, broad, show=False):
+    color = (rint(0,255), rint(0,255), rint(0,255))
+    for point in region:
+        broad[point[0], point[1]] = color
+
+    if show:
+        cv2.imshow('region', broad)
+        cv2.waitKey()
+    return broad
+
+
+def draw_region_bin(region, broad, show=False):
+    for point in region:
+        broad[point[0], point[1]] = 255
+
+    if show:
+        cv2.imshow('region', broad)
+        cv2.waitKey()
+    return broad

CDM/detect_compo/lib_ip/ip_preprocessing.py

@@ -0,0 +1,69 @@
+import cv2
+import numpy as np
+from CDM.config.CONFIG_UIED import Config
+C = Config()
+
+
+def read_img(path, resize_height=None, kernel_size=None):
+
+    def resize_by_height(org):
+        w_h_ratio = org.shape[1] / org.shape[0]
+        resize_w = resize_height * w_h_ratio
+        re = cv2.resize(org, (int(resize_w), int(resize_height)))
+        return re
+
+    try:
+        img = cv2.imread(path)
+        if kernel_size is not None:
+            img = cv2.medianBlur(img, kernel_size)
+        if img is None:
+            print("*** Image does not exist ***")
+            return None, None
+        if resize_height is not None:
+            img = resize_by_height(img)
+        gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+        return img, gray
+
+    except Exception as e:
+        print(e)
+        print("*** Img Reading Failed ***\n")
+        return None, None
+
+
+def gray_to_gradient(img):
+    if len(img.shape) == 3:
+        img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+    img_f = np.copy(img)
+    img_f = img_f.astype("float")
+
+    kernel_h = np.array([[0,0,0], [0,-1.,1.], [0,0,0]])
+    kernel_v = np.array([[0,0,0], [0,-1.,0], [0,1.,0]])
+    dst1 = abs(cv2.filter2D(img_f, -1, kernel_h))
+    dst2 = abs(cv2.filter2D(img_f, -1, kernel_v))
+    gradient = (dst1 + dst2).astype('uint8')
+    return gradient
+
+
+def reverse_binary(bin, show=False):
+    """
+    Reverse the input binary image
+    """
+    r, bin = cv2.threshold(bin, 1, 255, cv2.THRESH_BINARY_INV)
+    if show:
+        cv2.imshow('binary_rev', bin)
+        cv2.waitKey()
+    return bin
+
+
+def binarization(org, grad_min, show=False, write_path=None, wait_key=0):
+    grey = cv2.cvtColor(org, cv2.COLOR_BGR2GRAY)
+    grad = gray_to_gradient(grey)        # get RoI with high gradient
+    rec, binary = cv2.threshold(grad, grad_min, 255, cv2.THRESH_BINARY)    # enhance the RoI
+    morph = cv2.morphologyEx(binary, cv2.MORPH_CLOSE, (3, 3))  # remove noises
+    if write_path is not None:
+        cv2.imwrite(write_path, morph)
+    if show:
+        cv2.imshow('binary', morph)
+        if wait_key is not None:
+            cv2.waitKey(wait_key)
+    return morph

CDM/detect_compo/model/model-99-resnet18.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b31df5d3ed9c743990fb7a27baf71626cf7766df36d1f414496c89d34a854f2
+size 44957605

CDM/detect_merge/Element.py

@@ -0,0 +1,113 @@
+import numpy as np
+import cv2
+
+
+class Element:
+    def __init__(self, id, corner, category, text_content=None):
+        self.id = id
+        self.category = category
+        self.col_min, self.row_min, self.col_max, self.row_max = corner
+        self.width = self.col_max - self.col_min
+        self.height = self.row_max - self.row_min
+        self.area = self.width * self.height
+
+        self.text_content = text_content
+        self.parent_id = None
+        self.children = []  # list of elements
+        self.label = None
+
+    def init_bound(self):
+        self.width = self.col_max - self.col_min
+        self.height = self.row_max - self.row_min
+        self.area = self.width * self.height
+
+    def put_bbox(self):
+        return self.col_min, self.row_min, self.col_max, self.row_max
+
+    def wrap_info(self):
+        info = {'id':self.id, 'class': self.category, 'height': self.height, 'width': self.width,
+                'position': {'column_min': self.col_min, 'row_min': self.row_min, 'column_max': self.col_max,
+                             'row_max': self.row_max}, 'label': self.label}
+        if self.text_content is not None:
+            info['text_content'] = self.text_content
+        if len(self.children) > 0:
+            info['children'] = []
+            for child in self.children:
+                info['children'].append(child.id)
+        if self.parent_id is not None:
+            info['parent'] = self.parent_id
+        return info
+
+    def resize(self, resize_ratio):
+        self.col_min = int(self.col_min * resize_ratio)
+        self.row_min = int(self.row_min * resize_ratio)
+        self.col_max = int(self.col_max * resize_ratio)
+        self.row_max = int(self.row_max * resize_ratio)
+        self.init_bound()
+
+    def element_merge(self, element_b, new_element=False, new_category=None, new_id=None):
+        col_min_a, row_min_a, col_max_a, row_max_a = self.put_bbox()
+        col_min_b, row_min_b, col_max_b, row_max_b = element_b.put_bbox()
+        new_corner = (min(col_min_a, col_min_b), min(row_min_a, row_min_b), max(col_max_a, col_max_b), max(row_max_a, row_max_b))
+        if element_b.text_content is not None:
+            self.text_content = element_b.text_content if self.text_content is None else self.text_content + '\n' + element_b.text_content
+        if new_element:
+            return Element(new_id, new_corner, new_category)
+        else:
+            self.col_min, self.row_min, self.col_max, self.row_max = new_corner
+            self.init_bound()
+
+    def calc_intersection_area(self, element_b, bias=(0, 0)):
+        a = self.put_bbox()
+        b = element_b.put_bbox()
+        col_min_s = max(a[0], b[0]) - bias[0]
+        row_min_s = max(a[1], b[1]) - bias[1]
+        col_max_s = min(a[2], b[2])
+        row_max_s = min(a[3], b[3])
+        w = np.maximum(0, col_max_s - col_min_s)
+        h = np.maximum(0, row_max_s - row_min_s)
+        inter = w * h
+
+        iou = inter / (self.area + element_b.area - inter)
+        ioa = inter / self.area
+        iob = inter / element_b.area
+
+        return inter, iou, ioa, iob
+
+    def element_relation(self, element_b, bias=(0, 0)):
+        """
+        @bias: (horizontal bias, vertical bias)
+        :return: -1 : a in b
+                 0  : a, b are not intersected
+                 1  : b in a
+                 2  : a, b are identical or intersected
+        """
+        inter, iou, ioa, iob = self.calc_intersection_area(element_b, bias)
+
+        # area of intersection is 0
+        if ioa == 0:
+            return 0
+        # a in b
+        if ioa >= 1:
+            return -1
+        # b in a
+        if iob >= 1:
+            return 1
+        return 2
+
+    def visualize_element(self, img, color=(0, 255, 0), line=1, show=False, ratio=1):
+        loc = self.put_bbox()
+
+        if ratio != 1:
+            loc = [int(x * ratio) for x in loc]
+
+        # cv2.rectangle(img, loc[:2], loc[2:], color, line)
+        cv2.rectangle(img, (loc[0], loc[1]), (loc[2], loc[3]), color, line)
+        cv2.putText(img, str(int(self.id) + 1), (int(ratio*(self.col_min - 10)), int(ratio*(self.row_max + 10))), cv2.FONT_HERSHEY_SIMPLEX, 1,
+                    color, line)
+        # for child in self.children:
+        #     child.visualize_element(img, color=(255, 0, 255), line=line)
+        if show:
+            cv2.imshow('element', img)
+            cv2.waitKey(0)
+            cv2.destroyWindow('element')

CDM/detect_merge/merge.py

@@ -0,0 +1,361 @@
+import json
+import cv2
+import numpy as np
+from os.path import join as pjoin
+import os
+import time
+import shutil
+
+from CDM.detect_merge.Element import Element
+from torchvision import models
+from torch import nn
+import torch
+
+import CDM.detect_compo.lib_ip.ip_preprocessing as pre
+
+# ----------------- load pre-trained classification model ----------------
+
+# model = models.resnet18().to('cpu')
+# in_feature_num = model.fc.in_features
+# model.fc = nn.Linear(in_feature_num, 99)
+# model.conv1 = nn.Conv2d(in_channels=1, out_channels=64, kernel_size=(5, 5), padding=(3, 3), stride=(2, 2),
+#                         bias=False)
+#
+# PATH = "./model/model-99-resnet18.pkl"
+# model.load_state_dict(torch.load(PATH, map_location=torch.device('cpu')))
+#
+# model.eval()
+
+# ----------------- end loading ------------------------------------------
+
+# information_type = {'Name':['name', 'first name', 'last name', 'full name', 'real name', 'surname', 'family name', 'given name'],
+#                         'Birthday':['birthday', 'date of birth', 'birth date', 'DOB', 'dob full birthday'],
+#                         'Address':['address', 'mailing address', 'physical address', 'postal address', 'billing address', 'shipping address'],
+#                         'Phone':['phone', 'phone number', 'mobile', 'mobile phone', 'mobile number', 'telephone', 'telephone number', 'call'],
+#                         'Email':['email', 'e-mail', 'email address', 'e-mail address'],
+#                         'Contacts':['contacts', 'phone-book', 'phone book'],
+#                         'Location':['location', 'locate', 'place', 'geography', 'geo', 'geo-location', 'precision location'],
+#                         'Camera':['camera', 'photo', 'scan', 'album', 'picture', 'gallery', 'photo library', 'storage', 'image', 'video'],
+#                         'Microphone':['microphone', 'voice, mic', 'speech', 'talk'],
+#                         'Financial':['credit card', 'pay', 'payment', 'debit card', 'mastercard', 'wallet'],
+#                         'IP':['IP', 'Internet Protocol', 'IP address', 'internet protocol address'],
+#                         'Cookies':['cookies', 'cookie'],
+#                         'Social':['facebook', 'twitter']}
+
+def show_elements(org_img, eles, ratio, show=False, win_name='element', wait_key=0, shown_resize=None, line=2):
+    color_map = {'Text':(0, 0, 255), 'Compo':(0, 255, 0), 'Block':(0, 255, 0), 'Text Content':(255, 0, 255)}
+    img = org_img.copy()
+    for ele in eles:
+        color = color_map[ele.category]
+        ele.visualize_element(img=img, color=color, line=line, ratio=ratio)
+    img_resize = img
+    if shown_resize is not None:
+        img_resize = cv2.resize(img, shown_resize)
+    if show:
+        cv2.imshow(win_name, img_resize)
+        cv2.waitKey(wait_key)
+        if wait_key == 0:
+            cv2.destroyWindow(win_name)
+    return img_resize
+
+def show_one_element(org_img, eles, ratio, show=False, win_name='element', wait_key=0, shown_resize=None, line=2):
+    color_map = {'Text': (0, 0, 255), 'Compo': (0, 255, 0), 'Block': (0, 255, 0), 'Text Content': (255, 0, 255)}
+    all_img = []
+    for ele in eles:
+        img = org_img.copy()
+        color = color_map[ele.category]
+        ele.visualize_element(img=img, color=color, line=line, ratio=ratio)
+        img_resize = img
+        all_img.append(img_resize)
+        if shown_resize is not None:
+            img_resize = cv2.resize(img, shown_resize)
+        if show:
+            cv2.imshow(win_name, img_resize)
+            cv2.waitKey(wait_key)
+            if wait_key == 0:
+                cv2.destroyWindow(win_name)
+    return all_img
+
+
+def save_elements(output_file, elements, img_shape, ratio=1):
+    components = {'compos': [], 'img_shape': img_shape}
+    for i, ele in enumerate(elements):
+
+        if ratio != 1:
+            ele.resize(ratio)
+            ele.width = ele.col_max - ele.col_min
+            ele.height = ele.row_max - ele.row_min
+
+        c = ele.wrap_info()
+        # c['id'] = i
+        components['compos'].append(c)
+    json.dump(components, open(output_file, 'w'), indent=4)
+    return components
+
+
+def reassign_ids(elements):
+    for i, element in enumerate(elements):
+        element.id = i
+
+
+def refine_texts(texts, img_shape):
+    refined_texts = []
+    # for text in texts:
+    #     # remove potential noise
+    #     if len(text.text_content) > 1 and text.height / img_shape[0] < 0.075:
+    #         refined_texts.append(text)
+
+    for text in texts:
+        # remove potential noise
+        if text.height / img_shape[0] < 0.075:
+            refined_texts.append(text)
+
+    return refined_texts
+
+
+def merge_text_line_to_paragraph(elements, max_line_gap=5):
+    texts = []
+    non_texts = []
+    for ele in elements:
+        if ele.category == 'Text':
+            texts.append(ele)
+        else:
+            non_texts.append(ele)
+
+    changed = True
+    while changed:
+        changed = False
+        temp_set = []
+        for text_a in texts:
+            merged = False
+            for text_b in temp_set:
+                inter_area, _, _, _ = text_a.calc_intersection_area(text_b, bias=(0, max_line_gap))
+                if inter_area > 0:
+                    text_b.element_merge(text_a)
+                    merged = True
+                    changed = True
+                    break
+            if not merged:
+                temp_set.append(text_a)
+        texts = temp_set.copy()
+    return non_texts + texts
+
+
+def refine_elements(compos, texts, input_img_path, intersection_bias=(2, 2), containment_ratio=0.8, ):
+    '''
+    1. remove compos contained in text
+    2. remove compos containing text area that's too large
+    3. store text in a compo if it's contained by the compo as the compo's text child element
+    '''
+
+    # resize_by_height = 800
+    # org, grey = pre.read_img(input_img_path, resize_by_height)
+    #
+    # grey = grey.astype('float32')
+    # grey = grey / 255
+    #
+    # grey = (grey - grey.mean()) / grey.std()
+
+    elements = []
+    contained_texts = []
+
+    # classification_start_time = time.time()
+
+    for compo in compos:
+        is_valid = True
+        text_area = 0
+        for text in texts:
+            inter, iou, ioa, iob = compo.calc_intersection_area(text, bias=intersection_bias)
+            if inter > 0:
+                # the non-text is contained in the text compo
+                if ioa >= containment_ratio:
+                    is_valid = False
+                    break
+                text_area += inter
+                # the text is contained in the non-text compo
+                if iob >= containment_ratio and compo.category != 'Block':
+                    contained_texts.append(text)
+            # print("id: ", compo.id)
+            # print("text.text_content: ", text.text_content)
+            # print("is_valid: ", is_valid)
+            # print("inter: ", inter)
+            # print("iou: ", iou)
+            # print("ioa: ", ioa)
+            # print("iob: ", iob)
+            # print("text_area: ", text_area)
+            # print("compo.area: ", compo.area)
+        if is_valid and text_area / compo.area < containment_ratio:
+            # for t in contained_texts:
+            #     t.parent_id = compo.id
+            # compo.children += contained_texts
+
+            # --------- classification ----------
+
+            # comp_grey = grey[compo.row_min:compo.row_max, compo.col_min:compo.col_max]
+            #
+            # comp_crop = cv2.resize(comp_grey, (32, 32))
+            #
+            # comp_crop = comp_crop.reshape(1, 1, 32, 32)
+            #
+            # comp_tensor = torch.tensor(comp_crop)
+            # comp_tensor = comp_tensor.permute(0, 1, 3, 2)
+            #
+            # pred_label = model(comp_tensor)
+            #
+            # if np.argmax(pred_label.cpu().data.numpy(), axis=1) in [72.0, 42.0, 77.0, 91.0, 6.0, 89.0, 40.0, 43.0, 82.0,
+            #                                                         3.0, 68.0, 49.0, 56.0, 89.0]:
+            #     elements.append(compo)
+
+            # --------- end classification ----------
+
+            elements.append(compo)
+    # time_cost_ic = time.time() - classification_start_time
+    # print("time cost for icon classification: %2.2f s" % time_cost_ic)
+
+    # text_selection_time = time.time()
+
+    # elements += texts
+    for text in texts:
+        if text not in contained_texts:
+            elements.append(text)
+
+            # ---------- Simulate keyword search -----------
+
+            # for key in keyword_list:
+            #     for w in keyword_list[key]:
+            #         if w in text.text_content.lower():
+            #             elements.append(text)
+
+            # ---------- end -------------------------------
+
+    # time_cost_ts = time.time() - text_selection_time
+    # print("time cost for text selection: %2.2f s" % time_cost_ts)
+
+    # return elements, time_cost_ic, time_cost_ts
+    return elements
+
+
+def check_containment(elements):
+    for i in range(len(elements) - 1):
+        for j in range(i + 1, len(elements)):
+            relation = elements[i].element_relation(elements[j], bias=(2, 2))
+            if relation == -1:
+                elements[j].children.append(elements[i])
+                elements[i].parent_id = elements[j].id
+            if relation == 1:
+                elements[i].children.append(elements[j])
+                elements[j].parent_id = elements[i].id
+
+
+def remove_top_bar(elements, img_height):
+    new_elements = []
+    max_height = img_height * 0.04
+    for ele in elements:
+        if ele.row_min < 10 and ele.height < max_height:
+            continue
+        new_elements.append(ele)
+    return new_elements
+
+
+def remove_bottom_bar(elements, img_height):
+    new_elements = []
+    for ele in elements:
+        # parameters for 800-height GUI
+        if ele.row_min > 750 and 20 <= ele.height <= 30 and 20 <= ele.width <= 30:
+            continue
+        new_elements.append(ele)
+    return new_elements
+
+
+def compos_clip_and_fill(clip_root, org, compos):
+    def most_pix_around(pad=6, offset=2):
+        '''
+        determine the filled background color according to the most surrounding pixel
+        '''
+        up = row_min - pad if row_min - pad >= 0 else 0
+        left = col_min - pad if col_min - pad >= 0 else 0
+        bottom = row_max + pad if row_max + pad < org.shape[0] - 1 else org.shape[0] - 1
+        right = col_max + pad if col_max + pad < org.shape[1] - 1 else org.shape[1] - 1
+        most = []
+        for i in range(3):
+            val = np.concatenate((org[up:row_min - offset, left:right, i].flatten(),
+                            org[row_max + offset:bottom, left:right, i].flatten(),
+                            org[up:bottom, left:col_min - offset, i].flatten(),
+                            org[up:bottom, col_max + offset:right, i].flatten()))
+            most.append(int(np.argmax(np.bincount(val))))
+        return most
+
+    if os.path.exists(clip_root):
+        shutil.rmtree(clip_root)
+    os.mkdir(clip_root)
+
+    bkg = org.copy()
+    cls_dirs = []
+    for compo in compos:
+        cls = compo['class']
+        if cls == 'Background':
+            compo['path'] = pjoin(clip_root, 'bkg.png')
+            continue
+        c_root = pjoin(clip_root, cls)
+        c_path = pjoin(c_root, str(compo['id']) + '.jpg')
+        compo['path'] = c_path
+        if cls not in cls_dirs:
+            os.mkdir(c_root)
+            cls_dirs.append(cls)
+
+        position = compo['position']
+        col_min, row_min, col_max, row_max = position['column_min'], position['row_min'], position['column_max'], position['row_max']
+        cv2.imwrite(c_path, org[row_min:row_max, col_min:col_max])
+        # Fill up the background area
+        cv2.rectangle(bkg, (col_min, row_min), (col_max, row_max), most_pix_around(), -1)
+    cv2.imwrite(pjoin(clip_root, 'bkg.png'), bkg)
+
+
+def merge(img_path, compo_path, text_path, merge_root=None, is_paragraph=False, is_remove_top_bar=False, is_remove_bottom_bar=False, show=False, wait_key=0):
+    compo_json = json.load(open(compo_path, 'r'))
+    text_json = json.load(open(text_path, 'r'))
+
+    # load text and non-text compo
+    ele_id = 0
+    compos = []
+    for compo in compo_json['compos']:
+        element = Element(ele_id, (compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max']), compo['class'])
+        compos.append(element)
+        ele_id += 1
+    texts = []
+    for text in text_json['texts']:
+        element = Element(ele_id, (text['column_min'], text['row_min'], text['column_max'], text['row_max']), 'Text', text_content=text['content'])
+        texts.append(element)
+        ele_id += 1
+    if compo_json['img_shape'] != text_json['img_shape']:
+        resize_ratio = compo_json['img_shape'][0] / text_json['img_shape'][0]
+        for text in texts:
+            text.resize(resize_ratio)
+
+    # check the original detected elements
+    img = cv2.imread(img_path)
+    img_resize = cv2.resize(img, (compo_json['img_shape'][1], compo_json['img_shape'][0]))
+    ratio = img.shape[0] / img_resize.shape[0]
+
+    show_elements(img, texts + compos, ratio, show=show, win_name='all elements before merging', wait_key=wait_key, line=3)
+
+    # refine elements
+    texts = refine_texts(texts, compo_json['img_shape'])
+    elements = refine_elements(compos, texts, img_path)
+    if is_remove_top_bar:
+        elements = remove_top_bar(elements, img_height=compo_json['img_shape'][0])
+    if is_remove_bottom_bar:
+        elements = remove_bottom_bar(elements, img_height=compo_json['img_shape'][0])
+    if is_paragraph:
+        elements = merge_text_line_to_paragraph(elements, max_line_gap=7)
+    reassign_ids(elements)
+    check_containment(elements)
+    board = show_elements(img, elements, ratio, show=show, win_name='elements after merging', wait_key=wait_key, line=3)
+
+    # save all merged elements, clips and blank background
+    name = img_path.replace('\\', '/').split('/')[-1][:-4]
+    components = save_elements(pjoin(merge_root, name + '.json'), elements, img_resize.shape)
+    cv2.imwrite(pjoin(merge_root, name + '.jpg'), board)
+    print('[Merge Completed] Input: %s Output: %s' % (img_path, pjoin(merge_root, name + '.jpg')))
+    return board, components
+    # return this_ic_time, this_ts_time

CDM/detect_text/Text.py

@@ -0,0 +1,181 @@
+import cv2
+import numpy as np
+
+
+class Text:
+    def __init__(self, id, content, location):
+        self.id = id
+        self.content = content
+        self.location = location
+
+        self.width = self.location['right'] - self.location['left']
+        self.height = self.location['bottom'] - self.location['top']
+        self.area = self.width * self.height
+        self.word_width = self.width / len(self.content)
+
+    '''
+    ********************************
+    *** Relation with Other text ***
+    ********************************
+    '''
+    def is_justified(self, ele_b, direction='h', max_bias_justify=4):
+        '''
+        Check if the element is justified
+        :param max_bias_justify: maximum bias if two elements to be justified
+        :param direction:
+             - 'v': vertical up-down connection
+             - 'h': horizontal left-right connection
+        '''
+        l_a = self.location
+        l_b = ele_b.location
+        # connected vertically - up and below
+        if direction == 'v':
+            # left and right should be justified
+            if abs(l_a['left'] - l_b['left']) < max_bias_justify and abs(l_a['right'] - l_b['right']) < max_bias_justify:
+                return True
+            return False
+        elif direction == 'h':
+            # top and bottom should be justified
+            if abs(l_a['top'] - l_b['top']) < max_bias_justify and abs(l_a['bottom'] - l_b['bottom']) < max_bias_justify:
+                return True
+            return False
+
+    def is_on_same_line(self, text_b, direction='h', bias_gap=4, bias_justify=4):
+        '''
+        Check if the element is on the same row(direction='h') or column(direction='v') with ele_b
+        :param direction:
+             - 'v': vertical up-down connection
+             - 'h': horizontal left-right connection
+        :return:
+        '''
+        l_a = self.location
+        l_b = text_b.location
+        # connected vertically - up and below
+        if direction == 'v':
+            # left and right should be justified
+            if self.is_justified(text_b, direction='v', max_bias_justify=bias_justify):
+                # top and bottom should be connected (small gap)
+                if abs(l_a['bottom'] - l_b['top']) < bias_gap or abs(l_a['top'] - l_b['bottom']) < bias_gap:
+                    return True
+            return False
+        elif direction == 'h':
+            # top and bottom should be justified
+            if self.is_justified(text_b, direction='h', max_bias_justify=bias_justify):
+                # top and bottom should be connected (small gap)
+                if abs(l_a['right'] - l_b['left']) < bias_gap or abs(l_a['left'] - l_b['right']) < bias_gap:
+                    return True
+            return False
+
+    def is_intersected(self, text_b, bias):
+        l_a = self.location
+        l_b = text_b.location
+        left_in = max(l_a['left'], l_b['left']) + bias
+        top_in = max(l_a['top'], l_b['top']) + bias
+        right_in = min(l_a['right'], l_b['right'])
+        bottom_in = min(l_a['bottom'], l_b['bottom'])
+
+        w_in = max(0, right_in - left_in)
+        h_in = max(0, bottom_in - top_in)
+        area_in = w_in * h_in
+        if area_in > 0:
+            return True
+
+    '''
+    ***********************
+    *** Revise the Text ***
+    ***********************
+    '''
+    def merge_text(self, text_b):
+        text_a = self
+        top = min(text_a.location['top'], text_b.location['top'])
+        left = min(text_a.location['left'], text_b.location['left'])
+        right = max(text_a.location['right'], text_b.location['right'])
+        bottom = max(text_a.location['bottom'], text_b.location['bottom'])
+        self.location = {'left': left, 'top': top, 'right': right, 'bottom': bottom}
+        self.width = self.location['right'] - self.location['left']
+        self.height = self.location['bottom'] - self.location['top']
+        self.area = self.width * self.height
+
+        left_element = text_a
+        right_element = text_b
+        if text_a.location['left'] > text_b.location['left']:
+            left_element = text_b
+            right_element = text_a
+        self.content = left_element.content + ' ' + right_element.content
+        self.word_width = self.width / len(self.content)
+
+    def shrink_bound(self, binary_map):
+        bin_clip = binary_map[self.location['top']:self.location['bottom'], self.location['left']:self.location['right']]
+        height, width = np.shape(bin_clip)
+
+        shrink_top = 0
+        shrink_bottom = 0
+        for i in range(height):
+            # top
+            if shrink_top == 0:
+                if sum(bin_clip[i]) == 0:
+                    shrink_top = 1
+                else:
+                    shrink_top = -1
+            elif shrink_top == 1:
+                if sum(bin_clip[i]) != 0:
+                    self.location['top'] += i
+                    shrink_top = -1
+            # bottom
+            if shrink_bottom == 0:
+                if sum(bin_clip[height-i-1]) == 0:
+                    shrink_bottom = 1
+                else:
+                    shrink_bottom = -1
+            elif shrink_bottom == 1:
+                if sum(bin_clip[height-i-1]) != 0:
+                    self.location['bottom'] -= i
+                    shrink_bottom = -1
+
+            if shrink_top == -1 and shrink_bottom == -1:
+                break
+
+        shrink_left = 0
+        shrink_right = 0
+        for j in range(width):
+            # left
+            if shrink_left == 0:
+                if sum(bin_clip[:, j]) == 0:
+                    shrink_left = 1
+                else:
+                    shrink_left = -1
+            elif shrink_left == 1:
+                if sum(bin_clip[:, j]) != 0:
+                    self.location['left'] += j
+                    shrink_left = -1
+            # right
+            if shrink_right == 0:
+                if sum(bin_clip[:, width-j-1]) == 0:
+                    shrink_right = 1
+                else:
+                    shrink_right = -1
+            elif shrink_right == 1:
+                if sum(bin_clip[:, width-j-1]) != 0:
+                    self.location['right'] -= j
+                    shrink_right = -1
+
+            if shrink_left == -1 and shrink_right == -1:
+                break
+        self.width = self.location['right'] - self.location['left']
+        self.height = self.location['bottom'] - self.location['top']
+        self.area = self.width * self.height
+        self.word_width = self.width / len(self.content)
+
+    '''
+    *********************
+    *** Visualization ***
+    *********************
+    '''
+    def visualize_element(self, img, color=(0, 0, 255), line=1, show=False):
+        loc = self.location
+        cv2.rectangle(img, (loc['left'], loc['top']), (loc['right'], loc['bottom']), color, line)
+        if show:
+            print(self.content)
+            cv2.imshow('text', img)
+            cv2.waitKey()
+            cv2.destroyWindow('text')

CDM/detect_text/ocr.py

@@ -0,0 +1,43 @@
+import cv2
+import os
+import requests
+import json
+from base64 import b64encode
+import time
+
+
+def Google_OCR_makeImageData(imgpath):
+    with open(imgpath, 'rb') as f:
+        ctxt = b64encode(f.read()).decode()
+        img_req = {
+            'image': {
+                'content': ctxt
+            },
+            'features': [{
+                'type': 'DOCUMENT_TEXT_DETECTION',
+                # 'type': 'TEXT_DETECTION',
+                'maxResults': 1
+            }]
+        }
+    return json.dumps({"requests": img_req}).encode()
+
+
+def ocr_detection_google(imgpath):
+    # start = time.clock()
+    url = 'https://vision.googleapis.com/v1/images:annotate'
+
+    api_key = os.environ.get('google_ocr')
+
+    imgdata = Google_OCR_makeImageData(imgpath)
+    response = requests.post(url,
+                             data=imgdata,
+                             params={'key': api_key},
+                             headers={'Content_Type': 'application/json'})
+    # print('*** Text Detection Time Taken:%.3fs ***' % (time.clock() - start))
+    print("*** Please replace the Google OCR key at detect_text/ocr.py line 28 with your own (apply in https://cloud.google.com/vision) ***")
+    # print('response.json(): ', response.json())
+    if response.json()['responses'] == [{}]:
+        # No Text
+        return None
+    else:
+        return response.json()['responses'][0]['textAnnotations'][1:]

CDM/detect_text/text_detection.py

@@ -0,0 +1,289 @@
+import CDM.detect_text.ocr as ocr
+from CDM.detect_text.Text import Text
+import numpy as np
+import cv2
+import json
+import time
+import os
+from os.path import join as pjoin
+# from paddleocr import PaddleOCR
+import pytesseract
+
+# paddle_model = PaddleOCR(use_angle_cls=True, lang="en") #'ch' for chinese and english, 'en' for english
+
+
+def save_detection_json(file_path, texts, img_shape):
+    f_out = open(file_path, 'w')
+    output = {'img_shape': img_shape, 'texts': []}
+    for text in texts:
+        c = {'id': text.id, 'content': text.content}
+        loc = text.location
+        c['column_min'], c['row_min'], c['column_max'], c['row_max'] = loc['left'], loc['top'], loc['right'], loc['bottom']
+        c['width'] = text.width
+        c['height'] = text.height
+        output['texts'].append(c)
+    json.dump(output, f_out, indent=4)
+
+
+def visualize_texts(org_img, texts, shown_resize_height=None, show=False, write_path=None):
+    img = org_img.copy()
+    for text in texts:
+        text.visualize_element(img, line=2)
+
+    img_resize = img
+    if shown_resize_height is not None:
+        img_resize = cv2.resize(img, (int(shown_resize_height * (img.shape[1]/img.shape[0])), shown_resize_height))
+
+    if show:
+        cv2.imshow('texts', img_resize)
+        cv2.waitKey(0)
+        cv2.destroyWindow('texts')
+    if write_path is not None:
+        cv2.imwrite(write_path, img)
+
+
+def text_sentences_recognition(texts):
+    '''
+    Merge separate words detected by Google ocr into a sentence
+    '''
+    changed = True
+    while changed:
+        changed = False
+        temp_set = []
+        for text_a in texts:
+            merged = False
+            for text_b in temp_set:
+                if text_a.is_on_same_line(text_b, 'h', bias_justify=0.2 * min(text_a.height, text_b.height), bias_gap=2 * max(text_a.word_width, text_b.word_width)):
+                    text_b.merge_text(text_a)
+                    merged = True
+                    changed = True
+                    break
+            if not merged:
+                temp_set.append(text_a)
+        texts = temp_set.copy()
+
+    for i, text in enumerate(texts):
+        text.id = i
+    return texts
+
+
+def merge_intersected_texts(texts):
+    '''
+    Merge intersected texts (sentences or words)
+    '''
+    changed = True
+    while changed:
+        changed = False
+        temp_set = []
+        for text_a in texts:
+            merged = False
+            for text_b in temp_set:
+                if text_a.is_intersected(text_b, bias=2):
+                    text_b.merge_text(text_a)
+                    merged = True
+                    changed = True
+                    break
+            if not merged:
+                temp_set.append(text_a)
+        texts = temp_set.copy()
+    return texts
+
+
+def text_cvt_orc_format(ocr_result):
+    texts = []
+    if ocr_result is not None:
+        for i, result in enumerate(ocr_result):
+            error = False
+            x_coordinates = []
+            y_coordinates = []
+            text_location = result['boundingPoly']['vertices']
+            content = result['description']
+            for loc in text_location:
+                if 'x' not in loc or 'y' not in loc:
+                    error = True
+                    break
+                x_coordinates.append(loc['x'])
+                y_coordinates.append(loc['y'])
+            if error: continue
+            location = {'left': min(x_coordinates), 'top': min(y_coordinates),
+                        'right': max(x_coordinates), 'bottom': max(y_coordinates)}
+            texts.append(Text(i, content, location))
+    return texts
+
+
+def text_cvt_orc_format_paddle(paddle_result):
+    texts = []
+    for i, line in enumerate(paddle_result):
+        points = np.array(line[0])
+        # points = points * 5
+        location = {'left': int(min(points[:, 0])), 'top': int(min(points[:, 1])), 'right': int(max(points[:, 0])),
+                    'bottom': int(max(points[:, 1]))}
+        content = line[1][0]
+        texts.append(Text(i, content, location))
+    return texts
+
+
+def text_cvt_orc_format_tesseract(tesseract_result):
+    # texts = []
+    # i_real = 0
+    # for i, line in enumerate(tesseract_result['text']):
+    #     content = line.strip()
+    #     location = {
+    #         'left': int(tesseract_result['left'][i]),
+    #         'top': int(tesseract_result['top'][i]),
+    #         'right': int(tesseract_result['left'][i]) + int(tesseract_result['width'][i]),
+    #         'bottom': int(tesseract_result['top'][i]) + int(tesseract_result['height'][i])
+    #     }
+    #     if len(content) > 0:
+    #         texts.append(Text(i_real, content, location))
+    #         i_real = i_real + 1
+
+    # Extract line boxes
+    texts = []
+    i_real = 0
+    line_boxes = []
+    n_boxes = len(tesseract_result['level'])
+    for i in range(n_boxes):
+        if tesseract_result['level'][i] == 4 and len(tesseract_result['text'][i].strip()) > 0:
+            # (x, y, w, h) = (tesseract_result['left'][i], tesseract_result['top'][i], tesseract_result['width'][i], tesseract_result['height'][i])
+            content = tesseract_result['text'][i].strip()
+            location = {
+                'left': int(tesseract_result['left'][i]),
+                'top': int(tesseract_result['top'][i]),
+                'right': int(tesseract_result['left'][i]) + int(tesseract_result['width'][i]),
+                'bottom': int(tesseract_result['top'][i]) + int(tesseract_result['height'][i])
+            }
+            texts.append(Text(i_real, content, location))
+            i_real = i_real + 1
+    # print("ocr result: ", texts)
+
+    return texts
+
+def text_cvt_orc_format_tesseract_by_line(data):
+
+    # line_data = []
+    line_num = None
+    line_text = []
+    line_box = [0, 0, 0, 0]
+    texts = []
+    i_real = 0
+
+    for i in range(len(data['level'])):
+        # check if the level is word
+        if data['level'][i] == 5:
+            if line_num != data['line_num'][i]:
+                if line_num is not None:  # append the previous line data to line_data
+                    content = ' '.join(line_text)
+                    location = {
+                        'left': line_box[0],
+                        'top': line_box[1],
+                        'right': line_box[2],
+                        'bottom': line_box[3]
+                    }
+                    texts.append(Text(i_real, content, location))
+                    i_real = i_real + 1
+
+                # start a new line
+                line_num = data['line_num'][i]
+                line_text = [data['text'][i]]
+                line_box = [
+                    data['left'][i],
+                    data['top'][i],
+                    data['left'][i] + data['width'][i],
+                    data['top'][i] + data['height'][i],
+                ]
+            else:  # add a word to the current line
+                line_text.append(data['text'][i])
+                line_box[2] = max(line_box[2], data['left'][i] + data['width'][i])
+                line_box[3] = max(line_box[3], data['top'][i] + data['height'][i])
+
+        # append the last line data to line_data
+    if line_text:
+        content = ' '.join(line_text)
+        location = {
+            'left': line_box[0],
+            'top': line_box[1],
+            'right': line_box[2],
+            'bottom': line_box[3]
+        }
+        texts.append(Text(i_real, content, location))
+        i_real = i_real + 1
+
+    return texts
+
+
+def text_filter_noise(texts):
+    valid_texts = []
+    for text in texts:
+        if len(text.content) <= 1 and text.content.lower() not in ['a', ',', '.', '!', '?', '$', '%', ':', '&', '+']:
+            continue
+        valid_texts.append(text)
+    return valid_texts
+
+
+def text_detection(input_file='../data/input/30800.jpg', output_file='../data/output', show=False, method='google', paddle_model=None):
+    '''
+    :param method: google or paddle
+    :param paddle_model: the preload paddle model for paddle ocr
+    '''
+    start = time.process_time()
+    name = input_file.split('/')[-1][:-4]
+    ocr_root = pjoin(output_file, 'ocr')
+    img = cv2.imread(input_file)
+    if img is None:
+        print("imread nothing!")
+
+    # resize the img to speed up the ocr
+    # img = cv2.resize(img, (int(img.shape[1]/5), int(img.shape[0]/5)))
+    # cv2.imshow("img", img)
+    # cv2.waitKey(0)
+
+    if method == 'google':
+        print('*** Detect Text through Google OCR ***')
+        ocr_result = ocr.ocr_detection_google(input_file)
+        texts = text_cvt_orc_format(ocr_result)
+        texts = merge_intersected_texts(texts)
+        texts = text_filter_noise(texts)
+        texts = text_sentences_recognition(texts)
+        ocr_time_cost = time.process_time() - start
+    elif method == 'paddle':
+        # The import of the paddle ocr can be separate to the beginning of the program if you decide to use this method
+        # from paddleocr import PaddleOCR
+        print('*** Detect Text through Paddle OCR ***')
+        # if paddle_model is None:
+            # paddle_model = PaddleOCR(use_angle_cls=True, lang="en") #'ch' for chinese and english, 'en' for english
+            # None
+        result = paddle_model.ocr(input_file, cls=True)
+        ocr_time_cost = time.process_time() - start
+        texts = text_cvt_orc_format_paddle(result)
+
+    elif method == 'pytesseract':
+
+        img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
+
+        # Perform OCR using Tesseract
+        result = pytesseract.image_to_data(img_rgb, output_type=pytesseract.Output.DICT)
+        print("ocr result: ", result)
+
+        ocr_time_cost = time.process_time() - start
+
+        # Convert the Tesseract result to the desired format
+        texts = text_cvt_orc_format_tesseract_by_line(result)
+        print("texts: ", texts)
+    else:
+        raise ValueError('Method has to be "google" or "paddle" or "pytesseract"')
+
+    visualize_texts(img, texts, shown_resize_height=800, show=show, write_path=pjoin(ocr_root, name+'.png'))
+    save_detection_json(pjoin(ocr_root, name+'.json'), texts, img.shape)
+    # ocr_time_cost = time.process_time() - start
+    print("[Text Detection Completed in %.3f s] Input: %s Output: %s" % (ocr_time_cost, input_file, pjoin(ocr_root, name+'.json')))
+
+    # print("!!! detected content !!!")
+    # for text in texts:
+    #     print(text.content)
+
+    return ocr_time_cost
+
+
+# text_detection()
+

CDM/input_examples/README.md

@@ -0,0 +1,80 @@
+# UIED - UI element detection, detecting UI elements from UI screenshots or drawnings
+
+This project is still ongoing and this repo may be updated irregularly, I developed a web app for the UIED in http://uied.online
+
+## Related Publications: 
+[1. UIED: a hybrid tool for GUI element detection](https://dl.acm.org/doi/10.1145/3368089.3417940)
+
+[2. Object Detection for Graphical User Interface: Old Fashioned or Deep Learning or a Combination?](https://arxiv.org/abs/2008.05132)
+
+>The repo has been **upgraded with Google OCR** for GUI text detection, to use the original version in our paper (using [EAST](https://github.com/argman/EAST) as text detector), check the relase [v2.3](https://github.com/MulongXie/UIED/releases/tag/v2.3) and download the pre-trained model in [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing).
+
+## What is it?
+
+UI Element Detection (UIED) is an old-fashioned computer vision (CV) based element detection approach for graphic user interface. 
+
+The input of UIED could be various UI image, such as mobile app or web page screenshot, UI design drawn by Photoshop or Sketch, and even some hand-drawn UI design. Then the approach detects and classifies text and graphic UI elements, and exports the detection result as JSON file for future application. 
+
+UIED comprises two parts to detect UI text and graphic elements, such as button, image and input bar. 
+* For text, it leverages [Google OCR](https://cloud.google.com/vision/docs/ocr) to perfrom detection. 
+
+* For graphical elements, it uses old-fashioned CV approaches to locate the elements and a CNN classifier to achieve classification. 
+
+> UIED is highly customizable, you can replace both parts by your choice (e.g. other text detection approaches). Unlike black-box end-to-end deep learning approach, you can revise the algorithms in the non-text detection and merging (partially or entirely) easily to fit your task.
+
+![UIED Approach](https://github.com/MulongXie/UIED/blob/master/data/demo/approach.png)
+
+## How to use?
+
+### Dependency
+* **Python 3.5**
+* **Opencv 3.4.2**
+* **Pandas**
+<!-- * **Tensorflow 1.10.0**
+* **Keras 2.2.4**
+* **Sklearn 0.22.2** -->
+
+### Installation
+<!-- Install the mentioned dependencies, and download two pre-trained models from [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing) for EAST text detection and GUI element classification. -->
+
+<!-- Change ``CNN_PATH`` and ``EAST_PATH`` in *config/CONFIG.py* to your locations. -->
+
+The new version of UIED equipped with Google OCR is easy to deploy and no pre-trained model is needed. Simply donwload the repo along with the dependencies.
+
+> Please replace the Google OCR key at `detect_text/ocr.py line 28` with your own (apply in [Google website](https://cloud.google.com/vision)).
+
+### Usage
+To test your own image(s):
+* To test single image, change *input_path_img* in ``run_single.py`` to your input image and the results will be output to *output_root*.
+* To test mutiple images, change *input_img_root* in ``run_batch.py`` to your input directory and the results will be output to *output_root*.
+* To adjust the parameters lively, using ``run_testing.py`` 
+
+> Note: The best set of parameters vary for different types of GUI image (Mobile App, Web, PC). I highly recommend to first play with the ``run_testing.py`` to pick a good set of parameters for your data.
+   
+## Folder structure
+``cnn/``
+* Used to train classifier for graphic UI elements
+* Set path of the CNN classification model
+
+``config/``
+* Set data paths 
+* Set parameters for graphic elements detection
+
+``data/``
+* Input UI images and output detection results
+
+``detect_compo/``
+* Non-text GUI component detection
+
+``detect_text/``
+* GUI text detection using Google OCR
+
+``detect_merge/``
+* Merge the detection results of non-text and text GUI elements
+
+The major detection algorithms are in ``detect_compo/``, ``detect_text/`` and ``detect_merge/``
+
+## Demo
+GUI element detection result for web screenshot
+ 
+![UI Components detection result](https://github.com/MulongXie/UIED/blob/master/data/demo/demo.png)

CDM/logs/cfg-for-web.txt

@@ -0,0 +1,19 @@
+Testing image: data/input/9.png
+
+1. detect_compo/ip_region_proposal.py
+# smaller minarea 50 -> 25
+line 70: uied_params = {'param-grad':5, 'param-block':5, 'param-minarea':25}
+
+2. detect_compo/lib_ip/ip_detection.py
+line 289-290 comment: # remove filter of aspect ratio
+line 342-344 comment: # remove is_line check
+
+3. detect_text_east/lib_east/eval.py
+# smaller max_word_gap 10 -> 5
+line 52: def merge_text(corners, max_word_gad=5) #
+
+4. merge.py
+# smaller horizontal max gap to merge lines (6,0) -> (4,0)
+line 199 max_gap=(4,0)
+# smaller vertical max gap to merge paragraph (0,6) -> (0,4)
+line 202 max_gap=(0,6)

CDM/logs/log.txt

@@ -0,0 +1,22 @@
+16:10 8/7/2020
+- Synchronized with Webapp.
+- Add image inspection.
+- Used No-line v1.
+- No-line v2 requires bug fix with consideration of gap. 
+
+
+11：00 23/7/2020
+- Synchronized with Webapp.
+
+10：53 4/8/2020
+- Synchronized with Webapp.
+
+7/10/2020
+- Extract parameters as configurable
+
+30/10/2020
+- Speed optimization (500% boost)
+
+11/11/2020
+- Revise rm_line
+- Add adjustable track bar testing

CDM/logs/speed-improvement.txt

@@ -0,0 +1,12 @@
+Optimization:
+1. ip_preprocessing.py / gray_to_gradient : 0.5s -> 0.02s
+
+2. ip_draw.py / draw_bounding_box : if not show and write_path is None: return : 0.005s -> 0s
+
+3. ip_detection.py / component_detection : if ff[0] < min_obj_area: continue : 2.5s -> 0.3s
+
+4. ip_detection.py / component_detection : cv2.findNonZero : 0.65s -> 0.33s
+
+5. block_division.py / block_division : if ff[0] < 500 : continue: 1.97s -> 1s
+
+6. block_division.py / block_division : Turn off draw : 1s -> 0.65s

CDM/model/model-99-ViT-entire.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc13133f12a561224c075dac2633af6dbe6036e6c6603c266efc0e6536727ca6
+size 343682793

CDM/model/model-99-resnet18.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b31df5d3ed9c743990fb7a27baf71626cf7766df36d1f414496c89d34a854f2
+size 44957605

CDM/result_classification/README.md

@@ -0,0 +1,80 @@
+# UIED - UI element detection, detecting UI elements from UI screenshots or drawnings
+
+This project is still ongoing and this repo may be updated irregularly, I developed a web app for the UIED in http://uied.online
+
+## Related Publications: 
+[1. UIED: a hybrid tool for GUI element detection](https://dl.acm.org/doi/10.1145/3368089.3417940)
+
+[2. Object Detection for Graphical User Interface: Old Fashioned or Deep Learning or a Combination?](https://arxiv.org/abs/2008.05132)
+
+>The repo has been **upgraded with Google OCR** for GUI text detection, to use the original version in our paper (using [EAST](https://github.com/argman/EAST) as text detector), check the relase [v2.3](https://github.com/MulongXie/UIED/releases/tag/v2.3) and download the pre-trained model in [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing).
+
+## What is it?
+
+UI Element Detection (UIED) is an old-fashioned computer vision (CV) based element detection approach for graphic user interface. 
+
+The input of UIED could be various UI image, such as mobile app or web page screenshot, UI design drawn by Photoshop or Sketch, and even some hand-drawn UI design. Then the approach detects and classifies text and graphic UI elements, and exports the detection result as JSON file for future application. 
+
+UIED comprises two parts to detect UI text and graphic elements, such as button, image and input bar. 
+* For text, it leverages [Google OCR](https://cloud.google.com/vision/docs/ocr) to perfrom detection. 
+
+* For graphical elements, it uses old-fashioned CV approaches to locate the elements and a CNN classifier to achieve classification. 
+
+> UIED is highly customizable, you can replace both parts by your choice (e.g. other text detection approaches). Unlike black-box end-to-end deep learning approach, you can revise the algorithms in the non-text detection and merging (partially or entirely) easily to fit your task.
+
+![UIED Approach](https://github.com/MulongXie/UIED/blob/master/data/demo/approach.png)
+
+## How to use?
+
+### Dependency
+* **Python 3.5**
+* **Opencv 3.4.2**
+* **Pandas**
+<!-- * **Tensorflow 1.10.0**
+* **Keras 2.2.4**
+* **Sklearn 0.22.2** -->
+
+### Installation
+<!-- Install the mentioned dependencies, and download two pre-trained models from [this link](https://drive.google.com/drive/folders/1MK0Om7Lx0wRXGDfNcyj21B0FL1T461v5?usp=sharing) for EAST text detection and GUI element classification. -->
+
+<!-- Change ``CNN_PATH`` and ``EAST_PATH`` in *config/CONFIG.py* to your locations. -->
+
+The new version of UIED equipped with Google OCR is easy to deploy and no pre-trained model is needed. Simply donwload the repo along with the dependencies.
+
+> Please replace the Google OCR key at `detect_text/ocr.py line 28` with your own (apply in [Google website](https://cloud.google.com/vision)).
+
+### Usage
+To test your own image(s):
+* To test single image, change *input_path_img* in ``run_single.py`` to your input image and the results will be output to *output_root*.
+* To test mutiple images, change *input_img_root* in ``run_batch.py`` to your input directory and the results will be output to *output_root*.
+* To adjust the parameters lively, using ``run_testing.py`` 
+
+> Note: The best set of parameters vary for different types of GUI image (Mobile App, Web, PC). I highly recommend to first play with the ``run_testing.py`` to pick a good set of parameters for your data.
+   
+## Folder structure
+``cnn/``
+* Used to train classifier for graphic UI elements
+* Set path of the CNN classification model
+
+``config/``
+* Set data paths 
+* Set parameters for graphic elements detection
+
+``data/``
+* Input UI images and output detection results
+
+``detect_compo/``
+* Non-text GUI component detection
+
+``detect_text/``
+* GUI text detection using Google OCR
+
+``detect_merge/``
+* Merge the detection results of non-text and text GUI elements
+
+The major detection algorithms are in ``detect_compo/``, ``detect_text/`` and ``detect_merge/``
+
+## Demo
+GUI element detection result for web screenshot
+ 
+![UI Components detection result](https://github.com/MulongXie/UIED/blob/master/data/demo/demo.png)

CDM/result_processing/Untitled.ipynb

@@ -0,0 +1,937 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "import numpy as np\n",
+    "import cv2\n",
+    "from glob import glob\n",
+    "from os.path import join as pjoin\n",
+    "from tqdm import tqdm\n",
+    "\n",
+    "\n",
+    "def resize_label(bboxes, d_height, gt_height, bias=0):\n",
+    "    bboxes_new = []\n",
+    "    scale = gt_height / d_height\n",
+    "    for bbox in bboxes:\n",
+    "        bbox = [int(b * scale + bias) for b in bbox]\n",
+    "        bboxes_new.append(bbox)\n",
+    "    return bboxes_new\n",
+    "\n",
+    "\n",
+    "def draw_bounding_box(org, corners, color=(0, 255, 0), line=2, show=False):\n",
+    "    board = org.copy()\n",
+    "    for i in range(len(corners)):\n",
+    "        board = cv2.rectangle(board, (corners[i][0], corners[i][1]), (corners[i][2], corners[i][3]), color, line)\n",
+    "    if show:\n",
+    "        cv2.imshow('a', cv2.resize(board, (500, 1000)))\n",
+    "        cv2.waitKey(0)\n",
+    "    return board\n",
+    "\n",
+    "\n",
+    "def load_detect_result_json(reslut_file_root, shrink=0):\n",
+    "    def is_bottom_or_top(corner):\n",
+    "        column_min, row_min, column_max, row_max = corner\n",
+    "        if row_max < 36 or row_min > 725:\n",
+    "            return True\n",
+    "        return False\n",
+    "\n",
+    "    result_files = glob(pjoin(reslut_file_root, '*.json'))\n",
+    "    compos_reform = {}\n",
+    "    print('Loading %d detection results' % len(result_files))\n",
+    "    for reslut_file in tqdm(result_files):\n",
+    "        img_name = reslut_file.split('\\\\')[-1].split('.')[0]\n",
+    "        compos = json.load(open(reslut_file, 'r'))['compos']\n",
+    "        for compo in compos:\n",
+    "            if is_bottom_or_top((compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max'])):\n",
+    "                continue\n",
+    "            if img_name not in compos_reform:\n",
+    "                compos_reform[img_name] = {'bboxes': [[compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink]],\n",
+    "                                           'categories': [compo['category']]}\n",
+    "            else:\n",
+    "                compos_reform[img_name]['bboxes'].append([compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink])\n",
+    "                compos_reform[img_name]['categories'].append(compo['category'])\n",
+    "    return compos_reform\n",
+    "\n",
+    "\n",
+    "def load_ground_truth_json(gt_file):\n",
+    "    def get_img_by_id(img_id):\n",
+    "        for image in images:\n",
+    "            if image['id'] == img_id:\n",
+    "                return image['file_name'].split('/')[-1][:-4], (image['height'], image['width'])\n",
+    "\n",
+    "    def cvt_bbox(bbox):\n",
+    "        '''\n",
+    "        :param bbox: [x,y,width,height]\n",
+    "        :return: [col_min, row_min, col_max, row_max]\n",
+    "        '''\n",
+    "        bbox = [int(b) for b in bbox]\n",
+    "        return [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]\n",
+    "\n",
+    "    data = json.load(open(gt_file, 'r'))\n",
+    "    images = data['images']\n",
+    "    annots = data['annotations']\n",
+    "    compos = {}\n",
+    "    print('Loading %d ground truth' % len(annots))\n",
+    "    for annot in tqdm(annots):\n",
+    "        img_name, size = get_img_by_id(annot['image_id'])\n",
+    "        if img_name not in compos:\n",
+    "            compos[img_name] = {'bboxes': [cvt_bbox(annot['bbox'])], 'categories': [annot['category_id']], 'size': size}\n",
+    "        else:\n",
+    "            compos[img_name]['bboxes'].append(cvt_bbox(annot['bbox']))\n",
+    "            compos[img_name]['categories'].append(annot['category_id'])\n",
+    "    return compos\n",
+    "\n",
+    "\n",
+    "def eval(detection, ground_truth, img_root, show=True, no_text=False, only_text=False):\n",
+    "    def compo_filter(compos, flag):\n",
+    "        if not no_text and not only_text:\n",
+    "            return compos\n",
+    "        compos_new = {'bboxes': [], 'categories': []}\n",
+    "        for k, category in enumerate(compos['categories']):\n",
+    "            if only_text:\n",
+    "                if flag == 'det' and category != 'TextView':\n",
+    "                    continue\n",
+    "                if flag == 'gt' and int(category) != 14:\n",
+    "                    continue\n",
+    "            elif no_text:\n",
+    "                if flag == 'det' and category == 'TextView':\n",
+    "                    continue\n",
+    "                if flag == 'gt' and int(category) == 14:\n",
+    "                    continue\n",
+    "\n",
+    "            compos_new['bboxes'].append(compos['bboxes'][k])\n",
+    "            compos_new['categories'].append(category)\n",
+    "        return compos_new\n",
+    "\n",
+    "    def match(org, d_bbox, gt_bboxes, matched):\n",
+    "        '''\n",
+    "        :param matched: mark if the ground truth component is matched\n",
+    "        :param d_bbox: [col_min, row_min, col_max, row_max]\n",
+    "        :param gt_bboxes: list of ground truth [[col_min, row_min, col_max, row_max]]\n",
+    "        :return: Boolean: if IOU large enough or detected box is contained by ground truth\n",
+    "        '''\n",
+    "        area_d = (d_bbox[2] - d_bbox[0]) * (d_bbox[3] - d_bbox[1])\n",
+    "        for i, gt_bbox in enumerate(gt_bboxes):\n",
+    "            if matched[i] == 0:\n",
+    "                continue\n",
+    "            area_gt = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])\n",
+    "            col_min = max(d_bbox[0], gt_bbox[0])\n",
+    "            row_min = max(d_bbox[1], gt_bbox[1])\n",
+    "            col_max = min(d_bbox[2], gt_bbox[2])\n",
+    "            row_max = min(d_bbox[3], gt_bbox[3])\n",
+    "            # if not intersected, area intersection should be 0\n",
+    "            w = max(0, col_max - col_min)\n",
+    "            h = max(0, row_max - row_min)\n",
+    "            area_inter = w * h\n",
+    "            if area_inter == 0:\n",
+    "                continue\n",
+    "            iod = area_inter / area_d\n",
+    "            iou = area_inter / (area_d + area_gt - area_inter)\n",
+    "            # if show:\n",
+    "            #     cv2.putText(org, (str(round(iou, 2)) + ',' + str(round(iod, 2))), (d_bbox[0], d_bbox[1]),\n",
+    "            #                 cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)\n",
+    "\n",
+    "            if iou > 0.9 or iod == 1:\n",
+    "                matched[i] = 0\n",
+    "                return True\n",
+    "        return False\n",
+    "\n",
+    "    amount = len(detection)\n",
+    "    TP, FP, FN = 0, 0, 0\n",
+    "    pres, recalls, f1s = [], [], []\n",
+    "    for i, image_id in enumerate(detection):\n",
+    "        TP_this, FP_this, FN_this = 0, 0, 0\n",
+    "        img = cv2.imread(pjoin(img_root, image_id + '.jpg'))\n",
+    "        d_compos = detection[image_id]\n",
+    "        gt_compos = ground_truth[image_id]\n",
+    "\n",
+    "        org_height = gt_compos['size'][0]\n",
+    "\n",
+    "        d_compos = compo_filter(d_compos, 'det')\n",
+    "        gt_compos = compo_filter(gt_compos, 'gt')\n",
+    "\n",
+    "        d_compos['bboxes'] = resize_label(d_compos['bboxes'], 800, org_height)\n",
+    "        matched = np.ones(len(gt_compos['bboxes']), dtype=int)\n",
+    "        for d_bbox in d_compos['bboxes']:\n",
+    "            if match(img, d_bbox, gt_compos['bboxes'], matched):\n",
+    "                TP += 1\n",
+    "                TP_this += 1\n",
+    "            else:\n",
+    "                FP += 1\n",
+    "                FP_this += 1\n",
+    "        FN += sum(matched)\n",
+    "        FN_this = sum(matched)\n",
+    "\n",
+    "        try:\n",
+    "            pre_this = TP_this / (TP_this + FP_this)\n",
+    "            recall_this = TP_this / (TP_this + FN_this)\n",
+    "            f1_this = 2 * (pre_this * recall_this) / (pre_this + recall_this)\n",
+    "        except:\n",
+    "            print('empty')\n",
+    "            continue\n",
+    "\n",
+    "        pres.append(pre_this)\n",
+    "        recalls.append(recall_this)\n",
+    "        f1s.append(f1_this)\n",
+    "        if show:\n",
+    "            print(image_id + '.jpg')\n",
+    "            print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (\n",
+    "                i, amount, TP_this, FP_this, FN_this, pre_this, recall_this))\n",
+    "            cv2.imshow('org', cv2.resize(img, (500, 1000)))\n",
+    "            broad = draw_bounding_box(img, d_compos['bboxes'], color=(255, 0, 0), line=3)\n",
+    "            draw_bounding_box(broad, gt_compos['bboxes'], color=(0, 0, 255), show=True, line=2)\n",
+    "\n",
+    "        if i % 200 == 0:\n",
+    "            precision = TP / (TP + FP)\n",
+    "            recall = TP / (TP + FN)\n",
+    "            f1 = 2 * (precision * recall) / (precision + recall)\n",
+    "            print(\n",
+    "                '[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))\n",
+    "\n",
+    "    precision = TP / (TP + FP)\n",
+    "    recall = TP / (TP + FN)\n",
+    "    print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))\n",
+    "    # print(\"Average precision:%.4f; Average recall:%.3f\" % (sum(pres)/len(pres), sum(recalls)/len(recalls)))\n",
+    "\n",
+    "    return pres, recalls, f1s"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 71,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import numpy as np\n",
+    "import math\n",
+    "\n",
+    "def draw_plot(data, title='Score for our approach'):\n",
+    "    for i in range(len(data)):\n",
+    "        data[i] = [d for d in data[i] if not math.isnan(d)]\n",
+    "#     plt.title(title)\n",
+    "    labels = ['Precision', 'Recall', 'F1']\n",
+    "    bplot = plt.boxplot(data, patch_artist=True, labels=labels)  # 设置箱型图可填充\n",
+    "    colors = ['pink', 'lightblue', 'lightgreen']\n",
+    "    for patch, color in zip(bplot['boxes'], colors):\n",
+    "        patch.set_facecolor(color) \n",
+    "    plt.grid(axis='y')\n",
+    "    plt.xticks(fontsize=16)\n",
+    "    plt.yticks(fontsize=16)\n",
+    "    plt.savefig(title + '.png')\n",
+    "    plt.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "scrolled": true
+   },
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  9%|███████▏                                                                     | 442/4708 [00:00<00:01, 4173.66it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading 4708 detection results\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|████████████████████████████████████████████████████████████████████████████| 4708/4708 [00:01<00:00, 4404.67it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "detect = load_detect_result_json('E:\\\\Mulong\\\\Result\\\\rico\\\\rico_uied\\\\rico_new_uied_cls\\\\merge')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "  8%|█████▉                                                                    | 6915/86646 [00:00<00:01, 68670.52it/s]"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Loading 86646 ground truth\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "100%|██████████████████████████████████████████████████████████████████████████| 86646/86646 [00:11<00:00, 7576.11it/s]\n"
+     ]
+    }
+   ],
+   "source": [
+    "gt = load_ground_truth_json('E:\\\\Mulong\\\\Datasets\\\\rico\\\\instances_test.json')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0/4707] TP:16, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[200/4707] TP:2222, FP:2920, FN:1705, Precesion:0.432, Recall:0.566, F1:0.490\n",
+      "[400/4707] TP:4616, FP:5737, FN:3346, Precesion:0.446, Recall:0.580, F1:0.504\n",
+      "[600/4707] TP:6963, FP:8682, FN:4812, Precesion:0.445, Recall:0.591, F1:0.508\n",
+      "[800/4707] TP:9367, FP:11432, FN:6305, Precesion:0.450, Recall:0.598, F1:0.514\n",
+      "[1000/4707] TP:11222, FP:14346, FN:7511, Precesion:0.439, Recall:0.599, F1:0.507\n",
+      "[1200/4707] TP:13680, FP:17278, FN:8901, Precesion:0.442, Recall:0.606, F1:0.511\n",
+      "[1400/4707] TP:16274, FP:20664, FN:10379, Precesion:0.441, Recall:0.611, F1:0.512\n",
+      "[1600/4707] TP:18431, FP:23002, FN:11556, Precesion:0.445, Recall:0.615, F1:0.516\n",
+      "[1800/4707] TP:20718, FP:25600, FN:13049, Precesion:0.447, Recall:0.614, F1:0.517\n",
+      "[2000/4707] TP:23009, FP:28626, FN:14588, Precesion:0.446, Recall:0.612, F1:0.516\n",
+      "[2200/4707] TP:25424, FP:31555, FN:16191, Precesion:0.446, Recall:0.611, F1:0.516\n",
+      "[2400/4707] TP:27559, FP:34176, FN:17388, Precesion:0.446, Recall:0.613, F1:0.517\n",
+      "[2600/4707] TP:29820, FP:37065, FN:18617, Precesion:0.446, Recall:0.616, F1:0.517\n",
+      "[2800/4707] TP:32108, FP:39846, FN:20018, Precesion:0.446, Recall:0.616, F1:0.518\n",
+      "[3000/4707] TP:34188, FP:43112, FN:21399, Precesion:0.442, Recall:0.615, F1:0.515\n",
+      "[3200/4707] TP:36558, FP:46011, FN:23002, Precesion:0.443, Recall:0.614, F1:0.514\n",
+      "[3400/4707] TP:38783, FP:48918, FN:24365, Precesion:0.442, Recall:0.614, F1:0.514\n",
+      "[3600/4707] TP:40958, FP:51829, FN:25605, Precesion:0.441, Recall:0.615, F1:0.514\n",
+      "[3800/4707] TP:43270, FP:54963, FN:26841, Precesion:0.440, Recall:0.617, F1:0.514\n",
+      "[4000/4707] TP:45512, FP:57838, FN:28141, Precesion:0.440, Recall:0.618, F1:0.514\n",
+      "[4200/4707] TP:47544, FP:60789, FN:29420, Precesion:0.439, Recall:0.618, F1:0.513\n",
+      "[4400/4707] TP:49907, FP:64407, FN:30897, Precesion:0.437, Recall:0.618, F1:0.512\n",
+      "[4600/4707] TP:52181, FP:67592, FN:32399, Precesion:0.436, Recall:0.617, F1:0.511\n",
+      "[4706/4707] TP:53393, FP:69230, FN:33248, Precesion:0.435, Recall:0.616, F1:0.511\n"
+     ]
+    }
+   ],
+   "source": [
+    "no_text = False\n",
+    "only_text = False\n",
+    "pres_all, recalls_all, f1_all = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0/4707] TP:1, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[200/4707] TP:973, FP:2022, FN:891, Precesion:0.325, Recall:0.522, F1:0.400\n",
+      "empty\n",
+      "[400/4707] TP:1921, FP:3905, FN:1788, Precesion:0.330, Recall:0.518, F1:0.403\n",
+      "[600/4707] TP:2847, FP:6079, FN:2717, Precesion:0.319, Recall:0.512, F1:0.393\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[800/4707] TP:3774, FP:7895, FN:3574, Precesion:0.323, Recall:0.514, F1:0.397\n",
+      "empty\n",
+      "[1000/4707] TP:4478, FP:9951, FN:4229, Precesion:0.310, Recall:0.514, F1:0.387\n",
+      "empty\n",
+      "empty\n",
+      "[1200/4707] TP:5451, FP:12055, FN:4960, Precesion:0.311, Recall:0.524, F1:0.391\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1400/4707] TP:6493, FP:14405, FN:5804, Precesion:0.311, Recall:0.528, F1:0.391\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1600/4707] TP:7372, FP:15980, FN:6375, Precesion:0.316, Recall:0.536, F1:0.397\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1800/4707] TP:8273, FP:17814, FN:7156, Precesion:0.317, Recall:0.536, F1:0.399\n",
+      "empty\n",
+      "empty\n",
+      "[2000/4707] TP:9273, FP:19993, FN:8051, Precesion:0.317, Recall:0.535, F1:0.398\n",
+      "empty\n",
+      "[2200/4707] TP:10293, FP:22055, FN:8869, Precesion:0.318, Recall:0.537, F1:0.400\n",
+      "[2400/4707] TP:11207, FP:23944, FN:9524, Precesion:0.319, Recall:0.541, F1:0.401\n",
+      "empty\n",
+      "empty\n",
+      "[2600/4707] TP:12103, FP:25932, FN:10276, Precesion:0.318, Recall:0.541, F1:0.401\n",
+      "[2800/4707] TP:12994, FP:27792, FN:11122, Precesion:0.319, Recall:0.539, F1:0.400\n",
+      "empty\n",
+      "empty\n",
+      "[3000/4707] TP:13839, FP:30256, FN:11943, Precesion:0.314, Recall:0.537, F1:0.396\n",
+      "[3200/4707] TP:14758, FP:32276, FN:12851, Precesion:0.314, Recall:0.535, F1:0.395\n",
+      "empty\n",
+      "[3400/4707] TP:15718, FP:34337, FN:13627, Precesion:0.314, Recall:0.536, F1:0.396\n",
+      "[3600/4707] TP:16695, FP:36424, FN:14263, Precesion:0.314, Recall:0.539, F1:0.397\n",
+      "[3800/4707] TP:17641, FP:38693, FN:14932, Precesion:0.313, Recall:0.542, F1:0.397\n",
+      "empty\n",
+      "empty\n",
+      "[4000/4707] TP:18651, FP:40641, FN:15653, Precesion:0.315, Recall:0.544, F1:0.399\n",
+      "empty\n",
+      "[4200/4707] TP:19554, FP:42631, FN:16305, Precesion:0.314, Recall:0.545, F1:0.399\n",
+      "empty\n",
+      "empty\n",
+      "[4400/4707] TP:20584, FP:45335, FN:17197, Precesion:0.312, Recall:0.545, F1:0.397\n",
+      "[4600/4707] TP:21416, FP:47595, FN:17950, Precesion:0.310, Recall:0.544, F1:0.395\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4706/4707] TP:21870, FP:48657, FN:18391, Precesion:0.310, Recall:0.543, F1:0.395\n"
+     ]
+    }
+   ],
+   "source": [
+    "no_text = True\n",
+    "only_text = False\n",
+    "pres_non_text, recalls_non_text, f1_non_text = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "[0/4707] TP:15, FP:0, FN:0, Precesion:1.000, Recall:1.000, F1:1.000\n",
+      "empty\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Anaconda\\lib\\site-packages\\ipykernel_launcher.py:165: RuntimeWarning: invalid value encountered in double_scalars\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[200/4707] TP:1041, FP:1106, FN:1022, Precesion:0.485, Recall:0.505, F1:0.495\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[400/4707] TP:2185, FP:2342, FN:2068, Precesion:0.483, Recall:0.514, F1:0.498\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[600/4707] TP:3272, FP:3447, FN:2939, Precesion:0.487, Recall:0.527, F1:0.506\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[800/4707] TP:4505, FP:4625, FN:3819, Precesion:0.493, Recall:0.541, F1:0.516\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1000/4707] TP:5426, FP:5713, FN:4600, Precesion:0.487, Recall:0.541, F1:0.513\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1200/4707] TP:6649, FP:6803, FN:5521, Precesion:0.494, Recall:0.546, F1:0.519\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1400/4707] TP:7890, FP:8150, FN:6466, Precesion:0.492, Recall:0.550, F1:0.519\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1600/4707] TP:8964, FP:9117, FN:7276, Precesion:0.496, Recall:0.552, F1:0.522\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[1800/4707] TP:10052, FP:10179, FN:8286, Precesion:0.497, Recall:0.548, F1:0.521\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[2000/4707] TP:11126, FP:11243, FN:9147, Precesion:0.497, Recall:0.549, F1:0.522\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[2200/4707] TP:12213, FP:12418, FN:10240, Precesion:0.496, Recall:0.544, F1:0.519\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[2400/4707] TP:13243, FP:13341, FN:10973, Precesion:0.498, Recall:0.547, F1:0.521\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[2600/4707] TP:14377, FP:14473, FN:11681, Precesion:0.498, Recall:0.552, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[2800/4707] TP:15494, FP:15674, FN:12516, Precesion:0.497, Recall:0.553, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[3000/4707] TP:16471, FP:16734, FN:13334, Precesion:0.496, Recall:0.553, F1:0.523\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[3200/4707] TP:17644, FP:17891, FN:14307, Precesion:0.497, Recall:0.552, F1:0.523\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[3400/4707] TP:18711, FP:18935, FN:15092, Precesion:0.497, Recall:0.554, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[3600/4707] TP:19710, FP:19958, FN:15895, Precesion:0.497, Recall:0.554, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[3800/4707] TP:20845, FP:21054, FN:16693, Precesion:0.498, Recall:0.555, F1:0.525\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4000/4707] TP:21881, FP:22177, FN:17468, Precesion:0.497, Recall:0.556, F1:0.525\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4200/4707] TP:22842, FP:23306, FN:18263, Precesion:0.495, Recall:0.556, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4400/4707] TP:23930, FP:24465, FN:19093, Precesion:0.494, Recall:0.556, F1:0.524\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4600/4707] TP:25015, FP:25747, FN:20199, Precesion:0.493, Recall:0.553, F1:0.521\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "empty\n",
+      "[4706/4707] TP:25638, FP:26458, FN:20742, Precesion:0.492, Recall:0.553, F1:0.521\n"
+     ]
+    }
+   ],
+   "source": [
+    "no_text = False\n",
+    "only_text = True\n",
+    "pres_text, recalls_text, f1_text = eval(detect, gt, 'E:\\\\Mulong\\\\Datasets\\\\rico\\\\combined', show=False, no_text=no_text, only_text=only_text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Anaconda\\lib\\site-packages\\matplotlib\\figure.py:448: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
+      "  % get_backend())\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAD/lJREFUeJzt3XuMXGd9xvHvE4dwycWJa7qlScARmIKLCqFLAqSUdaEioVXcqoQmKrcqYKlqoBRaGkQVTPoHhapFvRha00ZQoISAIHUjQ0DUq4SAwevciBMiuSbUblDDxQQBhRDy6x9zDMNk1zO7nt11Xn8/0sjn8s45vxm/fvb43ZnzpqqQJLXlmOUuQJI0foa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUHHLteJV69eXWvWrFmu00vSQ9KuXbu+XlWPHtZu2cJ9zZo1zMzMLNfpJekhKclXRmnnsIwkNchwl6QGGe6S1CDDXZIaZLhLUoOGhnuSK5Lck+S2OfYnyd8l2ZPk1iRPH3+ZkqT5GOXK/T3AuYfYfx6wtntsBN51+GVJkg7H0HCvquuAbx6iyQbgX6tnB3BykseMq0BJ0vyN40tMpwL7+tb3d9u+OtgwyUZ6V/dMTEwwPT09htMfPdavX7+g523fvn3MlUgPtpD+ad9cPOMI98yybdZZt6tqC7AFYHJysqampsZw+qPHoSYzT3LI/dJim6v/2TeXxzg+LbMfOL1v/TTg7jEcV5K0QOMI963Ay7pPzTwTuLeqHjQkI0laOkOHZZJ8EJgCVifZD7wZeBhAVf0jsA14IbAH+B7w+4tVrCRpNEPDvaouGrK/gD8cW0WSpMPmN1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S5mXVqlUkGfkBzKt9ElatWrXMr/KhbxyTdUg6ihw4cGDRJ984+ENBC+eVuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVopHBPcm6SO5PsSXLpLPsfm2R7kpuS3JrkheMvVZI0qqHhnmQFsBk4D1gHXJRk3UCzPweuqqozgQuBd467UEnS6Ea5cj8L2FNVe6vqPuBKYMNAmwJO6pZXAnePr0RJ0nyNMkH2qcC+vvX9wNkDbTYBn0zyauB44PljqU6StCCjhPts05APTn1+EfCeqvrrJM8C3pfkKVX1wE8dKNkIbASYmJhgenp6ASVrLr6fWipL0dfsz4cnVYM5PdCgF9abquoF3fobAarqrX1tdgPnVtW+bn0v8Myqumeu405OTtbMzMzhvwIBkIRhf5fSOCxFX7M/zy3JrqqaHNZulDH3ncDaJGckOY7eL0y3DrT5b+B53YmfDDwC+Nr8SpYkjcvQcK+q+4FLgGuBO+h9KmZ3ksuTnN81ez3wqiS3AB8EXlH+2JWkZTPKmDtVtQ3YNrDtsr7l24FzxluaJGmh/IaqJDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUEjfYlJkg6qN58Em1Yu/jl0WAx3SfOSt3x7aW4ctmlRT9E8h2UkqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhvsRaNWqVSSZ1wOYV/tVq1Yt86uUtJi8/cAR6MCBA0vy9W5J7fLKXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDRgr3JOcmuTPJniSXztHmxUluT7I7yb+Nt0xJ0nwMvbdMkhXAZuDXgf3AziRbq+r2vjZrgTcC51TVgSQ/u1gFS5KGG+XK/SxgT1Xtrar7gCuBDQNtXgVsrqoDAFV1z3jLlCTNxyh3hTwV2Ne3vh84e6DNEwGS3ACsADZV1ScGD5RkI7ARYGJigunp6QWUfHRYivfG918LZf888mXYrWWTXAC8oKpe2a2/FDirql7d1+Ya4IfAi4HTgOuBp1TVt+Y67uTkZM3MzBz+K2hQkiW55e9in0Ntsn8uryS7qmpyWLtRhmX2A6f3rZ8G3D1Lm3+vqh9W1ZeBO4G1oxYrSRqvUcJ9J7A2yRlJjgMuBLYOtLkaWA+QZDW9YZq94yxUkjS6oeFeVfcDlwDXAncAV1XV7iSXJzm/a3Yt8I0ktwPbgT+tqm8sVtGSpEMbOua+WBxzn5tjmjqS2T+X1zjH3CVJDzGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoNGmUNVS6zefBJsWrn455AWKMmiHv+UU05Z1OMfDQz3I1De8u2luV/2pkU9hRo1377pvdmXh8MyktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBI4V7knOT3JlkT5JLD9HuRUkqyeT4SpQkzdfQcE+yAtgMnAesAy5Ksm6WdicCrwE+P+4iJUnzM8qV+1nAnqraW1X3AVcCG2Zp9xfA24Hvj7E+SdICjBLupwL7+tb3d9t+LMmZwOlVdc0Ya5MkLdAo0+zNNlnij+fMSnIM8A7gFUMPlGwENgJMTEwwPT09UpFHo6V4b3z/tVTsa0svw+Y2TPIsYFNVvaBbfyNAVb21W18J/Bfwne4pPwd8Ezi/qmbmOu7k5GTNzMy5+6i2FHNOOq+llop9bbyS7KqqoR9aGWVYZiewNskZSY4DLgS2HtxZVfdW1eqqWlNVa4AdDAl2SdLiGhruVXU/cAlwLXAHcFVV7U5yeZLzF7tASdL8jTLmTlVtA7YNbLtsjrZTh1+WJOlw+A1VSWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aKQJsrX0kizq8U855ZRFPb6k5WW4H4Gqat7PSbKg50lqk8MyktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoJHCPcm5Se5MsifJpbPsf12S25PcmuTTSR43/lIlSaMaGu5JVgCbgfOAdcBFSdYNNLsJmKyqXwI+Arx93IVKkkY3ypX7WcCeqtpbVfcBVwIb+htU1faq+l63ugM4bbxlSpLmY5Qbh50K7Otb3w+cfYj2FwMfn21Hko3ARoCJiQmmp6dHq1Ij8f3Ukcq+ufRGCffZ7j076+0Hk7wEmASeO9v+qtoCbAGYnJysqamp0arUSHw/daSyby69UcJ9P3B63/ppwN2DjZI8H3gT8Nyq+sF4ypMkLcQoY+47gbVJzkhyHHAhsLW/QZIzgX8Czq+qe8ZfpiRpPoaGe1XdD1wCXAvcAVxVVbuTXJ7k/K7ZXwEnAB9OcnOSrXMcTpK0BEaaiamqtgHbBrZd1rf8/DHXJUk6DH5DVZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDVopHBPcm6SO5PsSXLpLPsfnuRD3f7PJ1kz7kIlSaMbGu5JVgCbgfOAdcBFSdYNNLsYOFBVTwDeAbxt3IVKkkY3ypX7WcCeqtpbVfcBVwIbBtpsAN7bLX8EeF6SjK9MSdJ8jBLupwL7+tb3d9tmbVNV9wP3Aj8zjgIlSfN37AhtZrsCrwW0IclGYCPAxMQE09PTI5xeB61fv/6Q++f6z9L27dsXoxzppxyqf9o3l94o4b4fOL1v/TTg7jna7E9yLLAS+ObggapqC7AFYHJysqamphZQ8tGr6kE/L6Ujhv3zyDLKsMxOYG2SM5IcB1wIbB1osxV4ebf8IuA/y79pSVo2Q6/cq+r+JJcA1wIrgCuqaneSy4GZqtoK/AvwviR76F2xX7iYRUuSDm2UYRmqahuwbWDbZX3L3wcuGG9pkqSF8huqktQgw12SGmS4S1KDDHdJapDhLkkNynJ9HD3J14CvLMvJ27Qa+PpyFyHNwr45Xo+rqkcPa7Rs4a7xSjJTVZPLXYc0yL65PByWkaQGGe6S1CDDvR1blrsAaQ72zWXgmLskNcgrd0lqkOH+EJbks0P2b0ty8lLVI81HkjVJbuuWp5Jcs9w1tWSku0Jq8SVZUVU/ms9zqurZQ/a/8PCqkh6smx85VfXActeiuXnlvgS6K5QvJXlvkluTfCTJo5LcleSyJJ8BLkjy+CSfSLIryfVJntQ9fyLJx5Lc0j2e3W3/TvfnY5Jcl+TmJLcleU63/a4kq7vl13X7bkvy2r667kjy7iS7k3wyySO7fa9JcntX75XL8LbpCNLXV94J3Ai8NMnnktyY5MNJTujaPSPJZ7t++oUkJ3bPvb5re+PB/nuIcz2368s3J7kpyYlL8RqbU1U+FvkBrKE3p+w53foVwJ8AdwFv6Gv3aWBtt3w2vRmtAD4EvLZbXgGs7Ja/0/35euBNfftP7JbvovftwF8GvggcD5wA7AbO7Oq6H3ha1/4q4CXd8t3Aw7vlk5f7PfRxRPThB4Bndn3qOuD4bt+fAZcBxwF7gWd020+iNzrwKOAR3ba19Cb5OXjM27rlKeCabvk/+v6tnAAcu9yv/6H4cFhm6eyrqhu65fcDr+mWPwTQXfk8G/hw32TCD+/+/DXgZQDVG7q5d+DYO4ErkjwMuLqqbh7Y/yvAx6rqu925Pgo8h970iF/ua7+L3j84gFuBDyS5Grh6IS9YzflKVe1I8pvAOuCGrq8eB3wO+AXgq1W1E6Cqvg2Q5HjgH5I8DfgR8MQh57kB+JskHwA+WlX7F+XVNM5hmaUz+JnTg+vf7f48BvhWVT2t7/HkkQ5cdR3wq8D/0Jvu8GUDTWafer7nB33LP+Inv4f5DWAzvav+Xd3E5zq6HeyrAT7V10/XVdXF3fbZPlv9x8D/Ak8FJun9MJhTVf0l8ErgkcCOg8OTmh/Dfek8NsmzuuWLgM/07+yucr6c5ALo/dIqyVO73Z8G/qDbviLJSf3PTfI44J6qeje9+WyfPnDu64Df6sb5jwd+G7h+rkKTHAOcXlXbgTcAJ9P777EEsAM4J8kTALp+9UTgS8DPJ3lGt/3E7qJgJb0r+geAl9IbOpxTksdX1Rer6m3ADGC4L4DhvnTuAF6e5FZgFfCuWdr8HnBxklvojYtv6Lb/EbA+yRfpDZ384sDzpoCbk9wE/A7wt/07q+pG4D3AF4DPA/9cVTcdotYVwPu7890EvKOqvjXi61TjquprwCuAD3b9eQfwpKq6D/hd4O+7Pvwp4BHAO+n1/R30hmS+O+uBf+K13S/+bwH+D/j44ryStvkN1SWQZA29XxY9ZZlLkXSU8MpdkhrklbskNcgrd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSg/wdEbKhcLyCIwQAAAABJRU5ErkJggg==\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "draw_plot([pres_all, recalls_all])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "D:\\Anaconda\\lib\\site-packages\\matplotlib\\figure.py:448: UserWarning: Matplotlib is currently using module://ipykernel.pylab.backend_inline, which is a non-GUI backend, so cannot show the figure.\n",
+      "  % get_backend())\n"
+     ]
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAD+BJREFUeJzt3X+QXWV9x/H3hyD+gBBMY7cW0DAaq6lTxa6gUuum2hFsJ2mnYsnUXx00M52itdpaHDsY6R9WO63TH9EWW0arVkRHacpE0bHZAdFoNvySgMykEZsUp/gDEbWKyLd/3INer7u5dzd3d8OT92vmzp5znuee8703Tz45efaee1JVSJLacsxyFyBJGj/DXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktSgY5frwGvWrKm1a9cu1+El6UFpz549X6uqRw3rt2zhvnbtWmZmZpbr8JL0oJTky6P0c1pGkhpkuEtSgwx3SWqQ4S5JDTLcJalBQ8M9yaVJ7kxy8xztSfJ3SfYluSnJ08ZfpiRpPkY5c383cPYh2s8B1nWPLcA7D78sSdLhGBruVXU18I1DdNkE/Gv17AJOSvLocRUoSZq/cVzEdDJwoG/9YLftK4Mdk2yhd3bPxMQE09PTYzj80WPDhg0Let7OnTvHXImOZhs3buSee+5Z1GOsXLmS7du3L+oxWjeOcM8s22a963ZVXQJcAjA5OVlTU1NjOPzR41A3M09yyHZpXO65555FH2tJMB8Ozzg+LXMQOLVv/RTgjjHsV5K0QOMI9+3AS7tPzTwDuLuqfmpKRpK0dIZOyyT5ADAFrElyEHgT8BCAqvpHYAfwAmAf8F3g9xerWEnSaIaGe1VtHtJewB+OrSJJ0mHzClVJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUoJHCPcnZSW5Lsi/JhbO0PybJziTXJ7kpyQvGX6okaVRDwz3JCmAbcA6wHticZP1Atz8HLq+q04HzgHeMu1BJ0uhGOXM/A9hXVfur6l7gMmDTQJ8CTuyWVwF3jK9ESdJ8HTtCn5OBA33rB4EzB/psBT6R5FXA8cDzxlKdJGlBRgn3zLKtBtY3A++uqr9O8kzgvUmeXFX3/8SOki3AFoCJiQmmp6cXULLm4vuppbIUY83xfHhSNZjTAx16Yb21qp7frb8BoKre0tdnL3B2VR3o1vcDz6iqO+fa7+TkZM3MzBz+KxAASRj2ZymNw1KMNcfz3JLsqarJYf1GmXPfDaxLclqS4+j9wnT7QJ//Bp7bHfhJwMOAr86vZEnSuAwN96q6D7gAuAq4ld6nYvYmuTjJxq7b64BXJrkR+ADw8vKfXUlaNqPMuVNVO4AdA9su6lu+BThrvKVJkhbKK1QlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuB+BVq9eTZJ5PYB59V+9evUyv0pJi2mkj0Jqad11111LcgWgpHZ55i5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQX79gKR5qTedCFtXLf4xdFgMd0nzkjd/a0m++6i2Luohmue0jCQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaNFK4Jzk7yW1J9iW5cI4+L0pyS5K9Sf5tvGVKkuZj6LdCJlkBbAN+HTgI7E6yvapu6euzDngDcFZV3ZXkZxerYEnScKOcuZ8B7Kuq/VV1L3AZsGmgzyuBbVV1F0BV3TneMiVJ8zHK97mfDBzoWz8InDnQ5wkASa4FVgBbq+rjgztKsgXYAjAxMcH09PQCSj46LMV74/uvhXJ8Hvky7Ev3k5wLPL+qXtGtvwQ4o6pe1dfnSuAHwIuAU4BrgCdX1Tfn2u/k5GTNzMwc/itoUJKluRnCIh9DbXJ8Lq8ke6pqcli/UaZlDgKn9q2fAtwxS59/r6ofVNWXgNuAdaMWK0kar1HCfTewLslpSY4DzgO2D/S5AtgAkGQNvWma/eMsVJI0uqHhXlX3ARcAVwG3ApdX1d4kFyfZ2HW7Cvh6kluAncCfVtXXF6toSdKhDZ1zXyzOuc/NOU0dyRyfy2ucc+6SpAcZw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBo9xDVUus3nQibF21+MeQ1CzD/QiUN39rab4ve+uiHkLSMnJaRpIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1aKRwT3J2ktuS7Ety4SH6vTBJJZkcX4mSpPkaGu5JVgDbgHOA9cDmJOtn6bcSeDXwuXEXKUman1HO3M8A9lXV/qq6F7gM2DRLv78A3gZ8b4z1SZIWYJRwPxk40Ld+sNv2I0lOB06tqivHWJskaYFGuc1eZtn2o3vAJTkGeDvw8qE7SrYAWwAmJiaYnp4eqcij0VK8N77/WqhktlgYn5UrVzo+D1OG3aszyTOBrVX1/G79DQBV9ZZufRXwX8C3u6f8HPANYGNVzcy138nJyZqZmbP5qJZkae6husjHkMCxNm5J9lTV0A+tjDItsxtYl+S0JMcB5wHbH2isqrurak1Vra2qtcAuhgS7JGlxDQ33qroPuAC4CrgVuLyq9ia5OMnGxS5QkjR/o8y5U1U7gB0D2y6ao+/U4ZclSTocXqEqSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lq0LHLXYBml2RR9//IRz5yUfcvaXkZ7kegqpr3c5Is6HmS2uS0jCQ1yHCXpAYZ7pLUIMNdkhpkuEtSg0YK9yRnJ7ktyb4kF87S/toktyS5Kcmnkjx2/KVKkkY1NNyTrAC2AecA64HNSdYPdLsemKyqXwI+DLxt3IVKkkY3ypn7GcC+qtpfVfcClwGb+jtU1c6q+m63ugs4ZbxlSpLmY5SLmE4GDvStHwTOPET/84GPzdaQZAuwBWBiYoLp6enRqtRIfD91pHJsLr1Rwn226+BnvRQyyYuBSeA5s7VX1SXAJQCTk5M1NTU1WpUaie+njlSOzaU3SrgfBE7tWz8FuGOwU5LnAW8EnlNV3x9PeZKkhRhlzn03sC7JaUmOA84Dtvd3SHI68E/Axqq6c/xlSpLmY2i4V9V9wAXAVcCtwOVVtTfJxUk2dt3+CjgB+FCSG5Jsn2N3kqQlMNK3QlbVDmDHwLaL+pafN+a6JEmHwStUJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoMMd0lqkOEuSQ0y3CWpQYa7JDXIcJekBhnuktQgw12SGmS4S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAYZ7pLUIMNdkhpkuEtSgwx3SWqQ4S5JDTLcJalBhrskNchwl6QGGe6S1CDDXZIaZLhLUoNGCvckZye5Lcm+JBfO0v7QJB/s2j+XZO24C5UkjW5ouCdZAWwDzgHWA5uTrB/odj5wV1U9Hng78NZxFypJGt0oZ+5nAPuqan9V3QtcBmwa6LMJeE+3/GHguUkyvjIlSfMxSrifDBzoWz/YbZu1T1XdB9wN/Mw4CpQkzd+xI/SZ7Qy8FtCHJFuALQATExNMT0+PcHg9YMOGDYdsn+s/Szt37lyMcqSfcKjx6dhceqOE+0Hg1L71U4A75uhzMMmxwCrgG4M7qqpLgEsAJicna2pqagElH72qfurfS+mI4fg8sowyLbMbWJfktCTHAecB2wf6bAde1i2/EPjP8k9akpbN0DP3qrovyQXAVcAK4NKq2pvkYmCmqrYD/wK8N8k+emfs5y1m0ZKkQxtlWoaq2gHsGNh2Ud/y94Bzx1uaJGmhvEJVkhpkuEtSgwx3SWqQ4S5JDTLcJalBWa6Poyf5KvDlZTl4m9YAX1vuIqRZODbH67FV9ahhnZYt3DVeSWaqanK565AGOTaXh9MyktQgw12SGmS4t+OS5S5AmoNjcxk45y5JDfLMXZIaZLg/iCX5zJD2HUlOWqp6pPlIsjbJzd3yVJIrl7umloz0rZBafElWVNUP5/OcqnrWkPYXHF5V0k/r7o+cqrp/uWvR3DxzXwLdGcoXk7wnyU1JPpzkEUluT3JRkk8D5yZ5XJKPJ9mT5JokT+yeP5Hko0lu7B7P6rZ/u/v56CRXJ7khyc1Jnt1tvz3Jmm75tV3bzUle01fXrUnelWRvkk8keXjX9uokt3T1XrYMb5uOIH1j5R3AdcBLknw2yXVJPpTkhK7f05N8phunn0+ysnvuNV3f6x4Yv4c41nO6sXxDkuuTrFyK19icqvKxyA9gLb17yp7VrV8K/AlwO/D6vn6fAtZ1y2fSu6MVwAeB13TLK4BV3fK3u5+vA97Y176yW76d3tWBvwx8ATgeOAHYC5ze1XUf8NSu/+XAi7vlO4CHdssnLfd76OOIGMP3A8/oxtTVwPFd258BFwHHAfuBp3fbT6Q3O/AI4GHdtnX0bvLzwD5v7pangCu75f/o+7tyAnDscr/+B+PDaZmlc6Cqru2W3we8ulv+IEB35vMs4EN9NxN+aPfz14CXAlRv6ubugX3vBi5N8hDgiqq6YaD9V4CPVtV3umN9BHg2vdsjfqmv/x56f+EAbgLen+QK4IqFvGA158tVtSvJbwLrgWu7sXoc8FngF4CvVNVugKr6FkCS44F/SPJU4IfAE4Yc51rgb5K8H/hIVR1clFfTOKdlls7gZ04fWP9O9/MY4JtV9dS+x5NG2nHV1cCvAv9D73aHLx3oMvut53u+37f8Q378e5jfALbRO+vf0934XEe3B8ZqgE/2jdP1VXV+t322z1b/MfC/wFOASXr/GMypqv4SeAXwcGDXA9OTmh/Dfek8Jskzu+XNwKf7G7uznC8lORd6v7RK8pSu+VPAH3TbVyQ5sf+5SR4L3FlV76J3P9unDRz7auC3unn+44HfBq6Zq9AkxwCnVtVO4PXASfT+eywB7ALOSvJ4gG5cPQH4IvDzSZ7ebV/ZnRSsondGfz/wEnpTh3NK8riq+kJVvRWYAQz3BTDcl86twMuS3ASsBt45S5/fA85PciO9efFN3fY/AjYk+QK9qZNfHHjeFHBDkuuB3wH+tr+xqq4D3g18Hvgc8M9Vdf0hal0BvK873vXA26vqmyO+TjWuqr4KvBz4QDeedwFPrKp7gd8F/r4bw58EHga8g97Y30VvSuY7s+74x17T/eL/RuD/gI8tzitpm1eoLoEka+n9sujJy1yKpKOEZ+6S1CDP3CWpQZ65S1KDDHdJapDhLkkNMtwlqUGGuyQ1yHCXpAb9P9Yjn2/XeLGfAAAAAElFTkSuQmCC\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "draw_plot([pres_non_text, recalls_non_text])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import matplotlib.pyplot as plt\n",
+    "import seaborn as sns\n",
+    "import pandas as pd\n",
+    "\n",
+    "pres1 = pd.DataFrame({'score_type':'Precision', 'score': pres_non_text, 'class':'Non_text'})\n",
+    "pres2 = pd.DataFrame({'score_type':'Precision', 'score': pres_all, 'class':'All_element'})\n",
+    "\n",
+    "recalls1 = pd.DataFrame({'score_type':'Recall', 'score':recalls_non_text, 'class':'Non_text'})\n",
+    "recalls2 = pd.DataFrame({'score_type':'Recall', 'score':recalls_all, 'class':'All_element'})\n",
+    "\n",
+    "f1s1 = pd.DataFrame({'score_type':'F1', 'score':f1_non_text, 'class':'Non_text'})\n",
+    "f1s2 = pd.DataFrame({'score_type':'F1', 'score':f1_all, 'class':'All_element'})\n",
+    "\n",
+    "data=pd.concat([pres1, pres2, recalls1, recalls2, f1s1, f1s2])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 36,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "<matplotlib.axes._subplots.AxesSubplot at 0x179cafcdac8>"
+      ]
+     },
+     "execution_count": 36,
+     "metadata": {},
+     "output_type": "execute_result"
+    },
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xl8VNXdx/HPL4EkCAiytCprrKBQJAFZIorWgop1RwX3gj7leSwWl0pr61KL1dKK+hK0bg+1WnEBi4pKCz5a64JgEMPuggoSsYqgVNYk5Pf8MZPrJGaZTHJnsnzfr1denXvumXt+M7fym3PvPeeYuyMiIgKQluoARESk4VBSEBGRgJKCiIgElBRERCSgpCAiIgElBRERCSgpiIhIQElBREQCSgoiIhJokeoAaqtTp07es2fPVIchItKovPXWW1+4e+ea6jW6pNCzZ0+WLl2a6jBERBoVM9sQTz1dPhIRkYCSgoiIBJQUREQk0OjuKYhI41BcXExhYSG7d+9OdSjNSlZWFl27dqVly5YJvV9JQURCUVhYSNu2benZsydmlupwmgV3Z8uWLRQWFpKdnZ3QMUK7fGRmfzazz81sVRX7zcymm9k6M1thZgPDikVEkm/37t107NhRCSGJzIyOHTvWqXcW5j2FvwCjqtl/ItAr+jcBuCfEWEQkBZQQkq+u33loScHdXwG2VlPlNOBhj1gMtDezA8KKR0REapbKewpdgI0x24XRsk8rVjSzCUR6E3Tv3j0pwSXimXdWVFp+2qH9Q223uPgjAFq2TOwaYnP1yIo3a/2eC/oPSbi93bsX1fo9WVnDEm6vudm2e1e57XZZraqtf+ONN9KmTRuuvvrqhNssLd0DQFpaZsLHaGhS+UhqZX0cr6yiu9/v7oPcfVDnzjWO0hYRkQSlMikUAt1itrsCm1IUi4g0cQ8//DD9+/cnJyeHCy+8sNy+Bx54gMGDB5OTk8OZZ57Jzp07AZgzZw79+vUjJyeHo48+GoDVq1czZMgQcnNzyc0dxPvvr0v6ZwlTKpPCPOCi6FNIecA2d//WpSMRkbpavXo1N998My+99BLLly/nzjvvLLd/9OjR5Ofns3z5cvr06cPMmTMBmDJlCgsWLGD58uXMmzcPgHvvvZfLL7+cgoIC3nxzEV27dkn65wlTaPcUzOwx4AdAJzMrBH4DtARw93uB+cCPgHXATmB8WLGISPP20ksvcdZZZ9GpUycAOnToUG7/qlWruO666/jqq6/Yvn07J5xwAgBHHnkk48aNY8yYMYwePRqAI444gptvvpnCwkJOP/1kevU6OLkfJmShJQV3P7eG/Q5MDKt9EZEy7l7to5rjxo3j6aefJicnh7/85S+8/PLLQKRXsGTJEp5//nlyc3MpKCjgvPPOY+jQoTz//POceOLJ3H//PYwcWd3T942L5j4SkSZvxIgRzJ49my1btgCwdWv5p+W//vprDjjgAIqLi5k1a1ZQ/sEHHzB06FCmTJlCp06d2LhxIx9++CEHHXQQkyZN4pRTTmblykrH5zZamuZCRJq873//+1x77bUcc8wxpKenM2DAAGIX67rpppsYOnQoPXr04LDDDuPrr78GYPLkybz//vu4OyNGjCAnJ4epU6fyyCOP0LJlS7773e9w/fW/TtGnCodFruI0HoMGDfKGushOKsYpFBWtKbedkdE3tLaaGo1TCNfatWvp06dP0tqr7TiF+tBQxylU9t2b2VvuPqim9+rykYiIBJQUREQSUFq6CygFSqOvmwYlBRERCSgpiIhIQElBREQCSgoiIhLQOAURSYo5857hyx3b6+14+7Vuw9mnngbAl7t2fmt/Wdl+rfaptzabAyUFkXq0a9eLSTteq1Yj6rWtsH25Yzt7D+5Wc8V4j7duY411OuzTmquuuorbbrsNgGnTprF9+3ZuvPHGeolh/foNLFq0mPPOG5vQ+wsKCti0aRM/+tGP6iWe+qDLRyLSZGVmZjJ37ly++OKLUI6/fv0GHnvsiYTfX1BQwPz58+sxorpTUhCRJqtFixZMmDCBO+6441v7NmzYwIgRI+jfvz8jRozg448/BiKT402aNIlhw4Zx0EEH8eSTT1Z5/F//+npee20RAwcO5Y477mDv3r1MnjyZwYMH079/f+677z4AnnrqKUaOHIm78+mnn9K7d28+/vhjbrjhBp544glyc3N54onEk0t9UlIQkSZt4sSJzJo1i23btpUrv+yyy7joootYsWIF559/PpMmTQr2ffrpp7z22ms899xzXHPNNVUe+5ZbbuKoo4axbNkSrrzySmbOnEm7du3Iz88nPz+fBx54gI8++ogzzjiD/fffn7vvvpuf/OQn/Pa3v6V79+5MmTKFsWPHUlBQwNixiV2Cqm+6pyAiTdq+++7LRRddxPTp02nV6pv5kN544w3mzp0LwIUXXsgvfvGLYN/pp59OWloaffv25bPPPvvWMUtLd5S9Cv63tHQHCxcuZMWKFUHvYtu2bbz//vtkZ2czY8YM+vXrR15eHueeW+3KAimlpNBI7dmzrMryzMyBSY5GpGG74oorGDhwIOPHV72WV+x6C5mZ30xwV5tJQ92dGTNmBIv0xPrkk09IS0vjs88+o7S0lLS0hnmhpmFGJSJNzn6t25C+bmO9/e3Xuk3cbXfo0IExY8YEy2wCDBs2jMcffxyAWbNmcdRRR9X6M7Vt25bt278Otk844QTuueceiouLAXjvvffYsWMHJSUljB8/nkcffZQ+ffpw++23B+8vm6a7oVBPQUSSomxMQar8/Oc/56677gq2p0+fzsUXX8ytt95K586defDBB2t9zP79+9GiRQsGDMjjxz++gCuumMz69esZOHAg7k7nzp15+umnue222xg+fDjDhw8nNzeXwYMHc9JJJ3HssccydepUcnNz+dWvftUg7itoPYV6lMz1FKq6fATo8lGMe/JfSVpblw4+ut7HKVSnoY9TSOZ6CpUNXisTxuC1b+4plJeW1rre20qE1lMQEZF6octHIiI1WLlyJRdeeGFMSSmZmZm88cbLqQopNEoKIiI1OOywwygoKAi2q7p81BQoKdRBvGv8VqxXl3V+RUTCpHsKIiISUFIQEZGALh+JSFL84x9PU1RUf+spZGS0YdSo0+vteBKhpCAiSVFUtJ3jjjug3o73wgufxlXvqaeeYvTo0axdu5ZDDz2U9evXc/LJJ7Nq1Spefvllpk2bxnPPPVertl9++RVuu206zz5b9QyqYVm/fj2LFi3ivPPOC+X4unwkIk3aY489xlFHHRVMadHYrV+/nkcffTS04yspiEiTtX37dl5//XVmzpyZUFLYsWMHF198MYMHH86AATk89dTjlJaWXQJzwNmxYzuXXPI/DB16dLk6f/7zvZx++umccsopZGdnc9ddd3H77bczYMAA8vLy2Lp1KwAffPABo0aN4vDDD2f48OG88847QNXrOlxzzTW8+uqr5ObmVrpORF0pKYhIkzX/2WcZNWoUvXv3pkOHDixbVvX0MJW5+eab+eEPf8iSJf/ixRef55e/vI4dO8qPUbjllls59thjKq2zatUqHn30Ud58802uvfZa9tlnH95++22OOOIIHn74YQAmTJjAjBkzeOutt5g2bRo//elPg2NXtq7D1KlTGT58OAUFBVx55ZV1+XoqFeo9BTMbBdwJpAP/6+5TK+zvDjwEtI/WucbdG9badCLSIBWX7uWLndXfuH5y9mz+e+JEvti5nZNHn8Fjjz3GxIkT425j4cKFzJs3j2nT/gjA7t17+Pjj8mtDv/DCSzz77Hxuv336t+oce+yxtG3blrZt29KuXTtOOeUUIDIYbsWKFWzfvp1FixZx9tlnB8fbs2dP8LqmdR3CEFpSMLN04G7gOKAQyDezee6+JqbadcBsd7/HzPoC84GeYcUkIs3H1i1beO1fr/DOmrWYGXv37iU9La3cL/GauDt/+9vf6NWrS7nyzz7bXK7OnDmPcMghvcvVWbJkabl1GdLS0oLttLQ0SkpKKC0tpX379uVGS8dKdF2HugizpzAEWOfuHwKY2ePAaUBsUnBg3+jrdsCmEONp9HbvXpRQvaysYWGEI1IrGRlt4n5iqCbFpXtJb1H9jKTPPv0MY847l9tmTA/KzjzxJAoLC+Nu54QTTmDGjBnceefvMTPefns5AwbklKtz/PEjuOuu+5g+fVqVdaqy7777kp2dzZw5czj77LNxd1asWEFOTtXvD3sNhjCTQhcgtp9VCAytUOdGYKGZ/QxoDYys7EBmNgGYANC9e/d6D1REwlefYwpqumwEMHfOHCZddVW5sjPPPJNbbrkl7nauv/56rrjiCnJz83B3evTo/q3HUK+77pdceeUvq61TnVmzZnHppZfyu9/9juLiYs4555xqk0L//v1p0aIFOTk5jBs3rt7vK4S2noKZnQ2c4O7/Fd2+EBji7j+LqXNVNIbbzOwIYCbQz91LKz0oDWs9hXjnPqoo0bmP4u0pVNScewpaTyF1wlxPIZ6kUJlO+8S/Wlusb544qp20tMTaq6uGup5CIdAtZrsr3748dAkwG8Dd3wCygE4hxiQiItUIMynkA73MLNvMMoBzgHkV6nwMjAAwsz5EksJmRESS6MEHHyQ3N7fcX22eUmpKQrun4O4lZnYZsIDI46Z/dvfVZjYFWOru84CfAw+Y2ZVEbjqP88a2PqiIVMndMbNUh1Gj8ePHM378+FSHUS/q+k9oqOMUomMO5lcouyHm9RrgyDBjEJHUyMrKYsuWLXTs2LFRJIamwN3ZsmULWVlZCR9DE+KJSCi6du1KYWEhmzfX/xXh7UV7aq5Uic0ZmTVXqoR7Yu2ZJdZeXWRlZdG1a9eE36+kICKhaNmyJdnZ2aEc++HlSxJ630V9chN6X+JP/iXWXipp7iMREQkoKYiISEBJQUREAkoKIiIS0I1mkXqyePFi/v3v5E3Bsv/+rcnLy0tae9I8qKcgIiIB9RRE6kleXh67du2ouWI9adVKvQSpf0oKIiIQygy31R2zoc5yq8tHIiISUFIQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAIavNbAJHMATUMdPCMiqaOegoiIBNRTqME9+a8k9Zjj+tV7cyKNkv7bSw31FEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiEtAjqdJkLV68mHfz85PX3t4McnKS1pxIKNRTEBGRgHoK0mTl5eXxdnpR8tobnBfKNCUiyRRqT8HMRpnZu2a2zsyuqaLOGDNbY2arzezRMOMREZHqhdZTMLN04G7gOKAQyDezee6+JqZOL+BXwJHu/qWZfSeseEREpGZh9hSGAOvc/UN3LwIeB06rUOcnwN3u/iWAu38eYjwiIlKDMJNCF2BjzHZhtCxWb6C3mb1uZovNbFRlBzKzCWa21MyWbt68OaRwRUQkzKRglZR5he0WQC/gB8C5wP+aWftvvcn9fncf5O6DOnfuXO+BiohIRJhJoRDoFrPdFdhUSZ1n3L3Y3T8C3iWSJEREJAXCTAr5QC8zyzazDOAcYF6FOk8DxwKYWScil5M+DDEmERGpRmhJwd1LgMuABcBaYLa7rzazKWZ2arTaAmCLma0B/glMdvctYcUkIiLVC3XwmrvPB+ZXKLsh5rUDV0X/REQkxTTNhYiIBJQUREQkoKQgIiIBTYgnIs3exjXvsOCTL5Pa5v77tyYvLy+pbcYj7p6CmR1lZuOjrzubWXZ4YYmISCrE1VMws98Ag4BDgAeBlsAjwJHhhSYikhzd+h7KCf2Kk9pmq1YNr5cA8fcUzgBOBXYAuPsmoG1YQYmISGrEmxSKomMKHMDMWocXkoiIpEq8SWG2md0HtDeznwD/BzwQXlgiIpIKcd1TcPdpZnYc8B8i9xVucPcXQo1MRESSrsakEF1BbYG7jwSUCEREmrAaLx+5+15gp5m1S0I8IiKSQvEOXtsNrDSzF4g+gQTg7pNCiUpERFIi3qTwfPRPQpTsUZUNdUSliKROvDeaH4oulNM7WvSuuyd3pIdII9Cq1Ygq9+3evajWx8vKGlaXcERqLd4RzT8AHgLWE1l7uZuZ/djdXwkvtOYn2aMqG+qIShFJnXgvH90GHO/u7wKYWW/gMeDwsAITEZHki3fwWsuyhADg7u8Rmf9IRESakHh7CkvNbCbw1+j2+cBb4YQkIs3d4sWLeTc/P6lt5u9qxeDBhyS1zYYo3qRwKTARmETknsIrwJ/CCkpERFIj3qTQArjT3W+HYJRzZmhRiUizlpeXx9vpRUltc3CSp85uqOK9p/Ai0CpmuxWRSfFERKQJiTcpZLn79rKN6Ot9wglJRERSJd6ksMPMBpZtmNkgYFc4IYmISKrEe0/hcmCOmW0istDOgcDY0KISEZGUiDcpZAMDgO5ElubMI7oKm4iINB3xXj663t3/A7QHjgPuB+4JLSoREUmJeJPC3uj/ngTc6+7PABnhhCQiIqkSb1L4JLpG8xhgvpll1uK9IiLSSMT7D/sYYAEwyt2/AjoAk0OLSkREUiKupODuO919rru/H93+1N0X1vQ+MxtlZu+a2Tozu6aaemeZmUcfdRURkRQJ7RJQdCqMu4ETgb7AuWbWt5J6bYnMqbQkrFhERCQ+8T6SmoghwDp3/xDAzB4HTgPWVKh3E/BH4OoQY0mIZmoUkeYmzJvFXYCNMduF0bKAmQ0Aurn7c9UdyMwmmNlSM1u6efPm+o9URESAcHsKVklZMODNzNKAO4BxNR3I3e8nMjaCQYMGJW3QnGZqFJHmJsyeQiHQLWa7K7ApZrst0A942czWExklPU83m0VEUifMpJAP9DKzbDPLAM4B5pXtdPdt7t7J3Xu6e09gMXCquy8NMSYREalGaEnB3UuAy4iMb1gLzHb31WY2xcxODatdERFJXJj3FHD3+cD8CmU3VFH3B2HGIs3TpYOPrnLfIyverPXxLug/pC7hiDR4mqpCREQCSgoiIhJQUhARkUCo9xRERBqLVq1GVLlv9+5FCR0zK2tYouGkjHoKIiISUFIQEZGAkoKIiAR0T0EkSWKvL+/Zs6zKepmZA5MRjkil1FMQEZGAkoKIiASUFEREJKCkICIiASUFEREJKCmIiEhASUFERAJKCiIiElBSEBGRgJKCiIgElBRERCSguY9EpEGqbn3th5cvSeiYF+UMTTScZkNJoYHRQh8ikkq6fCQiIgElBRERCSgpiIhIQElBREQCSgoiIhJQUhARkYCSgoiIBJQUREQkoMFrIiI1qDgAdM+eZZXWy8wcmIxwQhVqT8HMRpnZu2a2zsyuqWT/VWa2xsxWmNmLZtYjzHhERKR6ofUUzCwduBs4DigE8s1snruvian2NjDI3Xea2aXAH4GxYcUk0lCU/aIsKvrmP4eMjL6pCkckEGZPYQiwzt0/dPci4HHgtNgK7v5Pd98Z3VwMdA0xHhERqUGY9xS6ABtjtguB6qYovAT4e2U7zGwCMAGge/fu9RWfNHMX9B8SvH7mnRVV1jvt0P7JCEekQQizp2CVlHmlFc0uAAYBt1a2393vd/dB7j6oc+fO9RiiiIjECrOnUAh0i9nuCmyqWMnMRgLXAse4+54Q4xERkRqE2VPIB3qZWbaZZQDnAPNiK5jZAOA+4FR3/zzEWEREJA6hJQV3LwEuAxYAa4HZ7r7azKaY2anRarcCbYA5ZlZgZvOqOJyIiCRBqIPX3H0+ML9C2Q0xr0eG2b6IiNSORjTXoLp1Yh9Z8WZCx4x96kVEpCFRUhCRRueinPJPt89dW1Bl3dF9csMOp0nRhHgiIhJQUhARkYCSgoiIBHRPoRGJnb63qql7oWlM3ysiqaGegohILWVmDsQsq9xfU/kxpqQgIiIBJQUREQkoKYiISEBJQUREAkoKIiIS0COpItLoje6Ty8IP1pYrO/57fVIUTeOmnoKIiASUFEREJKCkIJJCGRl9MWtFRkbfVIcitVR27pra+VNSEBGRgJKCiIgE9PRRI1U2z0pR0Zpy5U2pGysiyaeegog0Ccd/rw+tMzJpnZGpx1HrQD2FOqi41vIz76yotN5ph/ZPRjgiInWmpCAikqCWLbNTHUK90+UjEWkyjux2EG0yMlMdRqOmpCAiIgElBRGROigp2ZTqEOqV7imISJOS890uSWtrz54VlJZuZu/eLWRmHpa0dsOknkIj11SH2oskasNXW5LSTklJEaWlmwEoLf2ckpKipLQbNiUFEWkyXtuwjvnvr+b1DetCb6u4eHG1242VkoKINAlFe/ey8vPI9f0Vn2+iaO/e0NoqKfk3UFyhtJiSks9CazNZlBREpEmYu3pZue2n1iyrombdFRe/V0X5u6G1mSyhJgUzG2Vm75rZOjO7ppL9mWb2RHT/EjPrGWY8ItI0FW77ki/37CpXtnX3Lj75z5ehtGfWsVbljUloScHM0oG7gROBvsC5ZlbxTuglwJfufjBwB/CHsOIRkabrtY8/qLT81Q2Vl9dVZubBtSpvTMLsKQwB1rn7h+5eBDwOnFahzmnAQ9HXTwIjzMxCjElEmqDhPb5Xq/K6MsvELLtCWTZmjX80dZhJoQuwMWa7MFpWaR13LwG2AY2//yUiSdVl3/3YL7NVubIOWa3osu9+obWZmdmTb/4JTYtuN35hDl6r7Be/J1AHM5sATADo3r173SMLSapmQ22Kk3IlWypnstX5qx+jvz+QmcteD7bP6Dsw1PbM0sjIyKWoaBkZGQMwaxrP7YT5KQqBbjHbXYGK48GDOmbWAmgHbK14IHe/390Hufugzp07hxSuiDRmGenpHPadAwHo/50DyUhPD73N9PT9yMo6kvT09qG3lSxh9hTygV4WufD2CXAOcF6FOvOAHwNvAGcBL7n7t3oKIiLxOKrHwXRrtx892ifvKrRZVtLaSobQkoK7l5jZZcACIB34s7uvNrMpwFJ3nwfMBP5qZuuI9BDOCSseEWkekpkQmqJQJ8Rz9/nA/AplN8S83g2cHWYMIiISv6ZxZ0REROqFkoKIiASUFEREJKCkICIiASUFEREJKCmIiEjAGttYMTPbDGxIdRwh6gR8keogJCE6d41bUz9/Pdy9xikhGl1SaOrMbKm7D0p1HFJ7OneNm85fhC4fiYhIQElBREQCSgoNz/2pDkASpnPXuOn8oXsKIiISQz0FEREJKCmIiEhASSFBZrbXzArMbJWZzTGzferhmIPMbHo1+w80syfr2o5Ur8K5fdbM6nVZLTMbZ2Z3RV/faGZX1+fxpWYx57jsr6eZdTSzf5rZ9rLz0xwpKSRul7vnuns/oAj4n9idFlGr79fdl7r7pGr2b3L3sxILV2oh9txuBSamOiCpd2XnuOxvPbAbuB5o1klaSaF+vAocHP21sdbM/gQsA7qZ2fFm9oaZLYv2KNoAmNlgM1tkZsvN7E0za2tmPzCz56L7j4n5FfN2dH9PM1sV3Z9lZg+a2cro/mOj5ePMbK6Z/cPM3jezP6boO2kq3gC6lG2Y2WQzyzezFWb225jyi6Jly83sr9GyU8xsSfT8/J+ZfTcF8Uuc3H2Hu79GJDk0W0oKdWRmLYATgZXRokOAh919ALADuA4Y6e4DgaXAVWaWATwBXO7uOcBIYFeFQ18NTHT3XGB4JfsnArj7YcC5wEP2zWKxucBY4DBgrJl1q6/P25yYWTowgsha4pjZ8UAvYAiR7/hwMzvazL4PXAv8MHo+L48e4jUgL/r/hceBXyT5I0jVWsX86Hoq1cE0JKEux9nEtTKzgujrV4msN30gsMHdF0fL84C+wOtmBpBB5JfnIcCn7p4P4O7/AYjWKfM6cLuZzQLmunthhf1HATOi73/HzDYAvaP7XnT3bdFjrgF6ABvr6XM3B2XntifwFvBCtPz46N/b0e02RJJEDvCku38B4O5bo/u7Ak+Y2QFEzv1HSYle4rEr+oNLKlBPIXGx1yR/5u5F0fIdMXUMeCGmXl93vyRaXu0AEXefCvwX0ApYbGaHVqhi335XYE/M670o+ddW2T8YPYj8Y152T8GA38ecz4PdfSZVn88ZwF3R3tx/A1mV1BFpUJQUwrUYONLMDgYws33MrDfwDnCgmQ2OlreNXoYKmNn33H2lu/+ByGWniknhFeD8aN3eQHfg3VA/TTMT7W1NAq42s5bAAuDimPtCXczsO8CLwBgz6xgt7xA9RDvgk+jrHyc1eJEE6RdkiNx9s5mNAx4zs8xo8XXu/p6ZjQVmmFkrIvcLRlZ4+xXRm8d7gTXA34EDYvb/CbjXzFYCJcA4d99T4RKT1JG7v21my4Fz3P2vZtYHeCP6PW8HLnD31WZ2M/AvM9tL5PLSOOBGYI6ZfULkB0J2Kj6DxM/M1gP7AhlmdjpwvLuvSW1UyaVpLkREJKDLRyIiElBSEBGRgJKCiIgElBRERCSgpCAiIgElBZEQmVl7M/tpquMQiZeSgkgtVBxkGIf2gJKCNBoavCZNnpm1BmYTmYsoHbgJ+BC4E2hNZFqQEUAxcA8wiMiAwKvc/Z/RAYgnEZmmojXwQzObDIwBMoGn3P03VTQ/FfhedC6lF4D9icyT9Ew0tllEJkfsAJwRPV428Ki7/zZa5wIiI6szgCXAT919b718OSIVKClIczAK2OTuJwGYWTsio47Hunu+me1LZFT55RCZeTY619TC6BQiAEcA/d19a4XZUg2YZ2ZHu/srlbR9DdCvbPI1MzsGuBJ4JhrHMCJTYFwQPV4/YCeQb2bPE5lLayxwpLsXR6dlPx94uD6/IJEySgrSHKwEppnZH4DngK+ofJba6maefSFm9tOqZkutLCmU4+7/MrO7o3MmjQb+5u4l0WkzXnD3LdFY5hKZCbcEOJxIkoDIBImfJ/pFiNRESUGavOhcU4cDPwJ+Dyyk8llNq5s4quLst7939/sSDOmvRH7tnwNcHBtqhXoebeshd/9Vgm2J1IpuNEuTZ2YHAjvd/RFgGpF1LiqbpTbemWermi21Ml8DbSuU/QW4AsDdV8eUH2dmHaKTJJ5OZE2NF4Gzyo4f3d+jNp9fpDbUU5Dm4DDgVjMrJXIz+VIiv8ArzlIb18yz7r6wstlSqeSyjrtvMbPXo8uo/t3dJ7v7Z2a2Fni6QvURy0twAAAAgElEQVTXiPQiDiZyo3kpgJldR+T+Rlo0/onAhjp/KyKV0CypIklmZvsQuc8xMGaFvHHAIHe/LJWxiejykUgSmdlIIosszShLCCINiXoKIvUguurai5XsGlH2RJFIY6CkICIiAV0+EhGRgJKCiIgElBRERCSgpCAiIgElBRERCfw/wQEZp/wRE5kAAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sns.boxenplot(x='score_type', y='score', hue='class', data=data, width=0.5, linewidth=1.0, palette=\"Set3\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 74,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEBCAYAAACe6Rn8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAF4NJREFUeJzt3XuQXGWdxvHvQ7gZFJ0IDIsmGWKyQFBh3UFThauT6AIumkBlBUtFECSwCBIUd6HQcIsoBQIrFJqwrCBQLiugyQpiEDIoSJSAgAly3UwSFpRLAiFcwu23f5zTVNvpTJ+e7p6emff5VHV1+vR5+/y630w/fd5zU0RgZmZp2qzdBZiZWfs4BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhDgEzs4Rt3u4Catluu+2iq6ur3WWYmQ0rd91119MRsX2t+YZ8CHR1dbF06dJ2l2FmNqxIWllkPg8HmZklzCFgZpYwh4CZWcIKhYCkd0u6UNIdkl6UFJK6CrbdTNLJkvokvSzpXkkzGynazMyao+iawETgIGAt8Js6l3EmcBpwEfAJYAnwE0n/VOfrmJlZkxXdO+jXEdEJIOlLwD5FGknaATgR+E5EnJtPXixpIvAd4IY66zUzsyYqtCYQEW8M8PX3BbYErqyYfiXwPkk7D/B1zcysCVq9YXh3YAPwSMX05fn95BYv38zM+tHqg8XGAM/GxhcyXlP2/EYkzQJmAXR2dtLb29uyAltp6tSpTXmdxYsXN+V1rD7N6D/3XXv4b6+4VoeAgGpXsld/jSJiPjAfoLu7O3p6eppf2SDYOPs2JqnQfDb4avWL+27o8t9eca0eDloDdEiq/NLvKHvezMzapNUhsBzYCnhPxfTStoD7W7x8MzPrR6tD4EbgFeBzFdM/DyyLiBUtXr6ZmfWj8DYBSf+c//Pv8/tPSHoKeCoibs3neQ24PCKOAIiIJyWdD5ws6XngbuBgYBowo0nvwczMBqieDcM/qXh8cX5/K9CT/3tUfit3CrAeOB7YEXgQOCgi/qeuSs3MrOkKh0BE9LtHz6bmiYjXgbn5zczMhhCfRdTMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOAUvSuPHjkdTQDWj4NcaNH9/mT8JS1+qLypgNSatXreLaBx5vdxnM3HWndpdgifOagJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSWsUAhIGivpGknPSVon6TpJ4wq2HSfpckmrJL0o6SFJcyVt01jpZmbWqJoXlZE0GrgF2AAcCgQwF1gs6f0R8UI/bbcBfgVsAXwTWAXsBZwOTAIObvQNmJnZwBW5stiRwARgl4h4BEDSfcDDwFHAef203Zvsy37fiFiUT1ssaQxwoqTREfHigKs3M7OGFBkOmg4sKQUAQESsAG4HZtRou2V+v65i+rP5slWwTjMza4EiIbA7sKzK9OXA5Bptf0W2xnC2pMmS3ippGnA88IP+hpLMzKz1ioTAGGBtlelrgI7+GkbEy8CH8+UsB54HbgZ+DhxbV6VmZtZ0RbYJQLYxuFLNoRxJWwNXAzsAh5BtGP4gMAd4DfiXTbSbBcwC6OzspLe3t2CZw9NIf3/WP/d/+/izB0VU+34vm0H6C/CziDiqYvrFwKcjYvt+2n4ZuAiYGBGPlk0/EpgP7BkR9/a3/O7u7li6dGnNNzJcSaJWH1jzSeLaBx5vdxnM3HUn93+bjPS/PUl3RUR3rfmKDActJ9suUGkycH+Ntu8D1pYHQO73+f1uBZZvZmYtUiQEFgJTJE0oTZDURbb758Iabf8MdEiaWDH9Q/n9/xUr08zMWqFICFwC9AELJM2QNB1YAKwG5pVmkjRe0muS5pS1vYxsY/ANkg6VNFXS14FzgbvIdjM1M7M2qRkC+W6c04CHgCuAq4AVwLSIWF82q4BR5a8ZEX3AFOAesqOMbyA7+Gw+8I8R8UZT3oWZmQ1Iob2DImIVMLPGPH1U2WMoIu4HDhpIcWZm1lo+i6iZWcIcAmZmCXMImJklzCFgZpYwh4CZWcIcAmZmCSt6AjmzESVO3RZ+vGu7y8jqMGsjh4AlSaevGzonkDut3VVYyjwcZGbDztiusUhq6AY01H5s19g2fwrN4TUBMxt2Hlv5GBesuaCtNcweM7uty28WrwmYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCDegaN77tB6x0jRvf5k/BzIYzHyzWgJWrVxG9d7a1BvXs1dblm9nw5jUBM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYYVCQNJYSddIek7SOknXSRpXdCGSdpP0E0lPS3pJ0oOSjh942WZm1gw1jxiWNBq4BdgAHAoEMBdYLOn9EfFCjfbdefte4EvAc8Ak4K0NVW5mZg0rctqII4EJwC4R8QiApPuAh4GjgPM21VDSZsDlwM0RcWDZU4sHXLGZmTVNkeGg6cCSUgAARMQK4HZgRo22PcBk+gkKMzNrnyIhsDuwrMr05WRf8P35cH6/taQlkl6V9KSk70l6Sz2FmplZ8xUZDhoDrK0yfQ3QUaPtTvn91cBFwElAN3AGMBY4sFojSbOAWQCdnZ309vYWKDNd/nyGN/ff8DUS+q7oqaSjyjQVaFda07gyIubk/+6VNAr4jqTJEXH/RguLmA/MB+ju7o6enp6CZabJn8/w5v4bvkZC3xUZDlpLtjZQqYPqawjlnsnvb6qYvii/37PA8s3MrEWKhMBysu0ClSYDG/2Kr9IWNl6TKK1FvFFg+WZm1iJFQmAhMEXShNIESV3A3vlz/fkF2fEF+1VM3ze/X1qoSjMza4kiIXAJ0AcskDRD0nRgAbAamFeaSdJ4Sa9JKo39ExHPAN8GjpZ0lqSPSzoJmANcXr7bqZmZDb6aG4Yj4gVJ04DzgSvIhnJuBmZHxPqyWQWMYuNgOQN4HjgGOBF4AjgHOLPh6s3MrCGF9g6KiFXAzBrz9FFlj6GICLKDxXzAmJnZEOOziJqZJcwhYGaWsKIHi5mZDRlx6rbw73Nqz9hCx5+6bVuX3ywOATMbdnT6Oi5Yc0Fba5g9ZjZxWltLaAoPB5mZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZgnzwWINiFO3hcUfa38NVrex48Yxc9edas84CHWYtZNDoAE6fR3Re2d7a+jZa0QctTjYVq1c2fBrSCI7Sa7Z8OXhIDOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGGFQkDSWEnXSHpO0jpJ10mq+/SHkk6WFJJuq79UMzNrtpohIGk0cAuwK3AocAgwCVgsaZuiC5I0ATgFeHJgpZqZWbMVOZX0kcAEYJeIeARA0n3Aw8BRwHkFl/V94Cpgl4LLNTOzFisyHDQdWFIKAICIWAHcDswoshBJnwU+AJw8kCLNzKw1ioTA7sCyKtOXA5NrNZbUAZwP/GtErKmvPDMza6UiITAGWFtl+hqgo0D7c4CHgMuKl2VmZoOh6Nh8tWvoqVYjSf8AfAH4QNRxHT5Js4BZAJ2dnfT29hZtmiR/Pu3jzz5tI6H/i4TAWrK1gUodVF9DKDcPuBR4TNI7ypY5Kn/8UkRsqGwUEfOB+QDd3d3R09NToMx0+fNpH3/2aRsJ/V8kBJaTbReoNBm4v0bb3fLb0VWeWwucAFxQoAYzM2uBIiGwEDhX0oSI+F8ASV3A3sBJNdpOrTLtAmAUcBzwSJXnzcxskBQJgUuAY4EFkr5Btn3gTGA12XAPAJLGA48CZ0TEGQAR0Vv5YpKeBTav9pyZmQ2umnsHRcQLwDSyPXyuIDvgawUwLSLWl80qsl/4Ph+RmdkwUWjvoIhYBcysMU8fBfYYioieIss0M7PW8692M7OEOQTMzBLmE7mZ2bDz7vHvZvaY2W2vYSRwCJjZsLO6b3XDryGJOk5kMGJ5OMjMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhPlU0g0YP3Yc6tmr7TWYmQ2UQ6ABfatWNvwaPqe5mbWTh4PMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYYVCQNJYSddIek7SOknXSap5lJKkbknzJT0g6UVJqyRdJWnnxks3M7NG1QwBSaOBW4BdgUOBQ4BJwGJJ29Ro/hlgd+B7wCeAk4APAEsljW2gbjMza4IiRwwfCUwAdomIRwAk3Qc8DBwFnNdP27Mj4qnyCZJuB1bkrztnIEWbmVlzFBkOmg4sKQUAQESsAG4HZvTXsDIA8mkrgaeAd9VXqpmZNVuRENgdWFZl+nJgcr0LlLQbsAPwp3rbmplZcxUZDhoDrK0yfQ3QUc/CJG0O/IBsTeDSfuabBcwC6OzspLe3t57FDDsj/f2NZO674c39B6p1BktJrwDfjYiTK6Z/C/i3iCh8JlJJPwCOAPaPiEVF2nR3d8fSpUuLLmLY8VlEhy/33fA20vtP0l0R0V1rviJf4GvJ1gYqdVB9DWFTBX2b7Nf9oUUDwMzMWqtICCwn2y5QaTJwf5GFSDqFbPfQr0TEFcXLMzOzViqyYXghMEXShNIESV3A3vlz/ZL0FWAucEpEXDiwMs3MrBWKhMAlQB+wQNIMSdOBBcBqYF5pJknjJb0maU7ZtM8AFwA3ArdImlJ2q3vPIjMza66aw0ER8YKkacD5wBWAgJuB2RGxvmxWAaP462DZL5++X34rdyvQM+DKzcysYYX27ImIVcDMGvP0kX3hl087DDhsYKWZmVmr+SyiZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSXMIWBmljCHgJlZwhwCZmYJcwiYmSWsUAhIGivpGknPSVon6TpJ4wq23VrSOZKekPSSpDskfaSxss3MrBlqhoCk0cAtwK7AocAhwCRgsaRtCizjUuBIYA7wSeAJ4JeS9hxo0WZm1hybF5jnSGACsEtEPAIg6T7gYeAo4LxNNZS0B/BZ4PCI+GE+7VZgOXAGML2h6s3MqpDUlPkiohnlDGlFhoOmA0tKAQAQESuA24EZBdq+Clxd1vY14L+AfSVtVXfFZmY1RERTbikoEgK7A8uqTF8OTC7QdkVEvFil7ZbAxALLNzOzFikSAmOAtVWmrwE6Gmhbet7MzNqkyDYBgGrrRUUG3TSQtpJmAbMAOjs76e3tLbCooWfq1KmF5qs1Lrl48eJmlGN1KtJ/7jsb7oqEwFqq/2LvoPqv/HJrgGq7knaUPb+RiJgPzAfo7u6Onp6eAmUOPamMKY5U7j9LQZHhoOVkY/uVJgP3F2i7c76baWXbV4BHNm5iZmaDpUgILASmSJpQmiCpC9g7f65W2y2AT5e13Rw4GFgUERvqrNfMzJqoSAhcAvQBCyTNkDQdWACsBuaVZpI0XtJrkuaUpkXEPWS7h14g6UuSPka2e+jOwKnNextmZjYQNUMgIl4ApgEPAVcAVwErgGkRsb5sVgGjqrzmF4EfAnOB64GxwH4RcXfD1ZuZWUMK7R0UEauAmTXm6aPKXj8R8RLw1fxmZmZDiM8iamaWMIeAmVnCHAJmZgnTUD8gRtJTwMp219FC2wFPt7sIGxD33fA20vtvfERsX2umIR8CI52kpRHR3e46rH7uu+HN/ZfxcJCZWcIcAmZmCXMItN/8dhdgA+a+G97cf3ibgJlZ0rwmYGaWsCRDQNJhkqLs9rykeyUdm5/ldLDqOE1SXatiknol9baopBGnSl+/IulRSWdJ2rrNtfVJuqzscanWrrYVNYJU6fvy28fzec6StEjSM/n0w9pc9qAbtC+8IerTwGPAtvm/LwR2AOb016iJ/gO4sc42x7SikASU+vptwIHAyfm/j2tnUTYoSn1frnQtlOOAe4CfA18YzKKGitRD4J6IKF3YZpGkicBsqoSAsusIbhERrzRr4RHxGBv/56zVptaFfKy68r6+SdIk4AhJx0fEG+0szFquvO8rvT0i3sj/9pMMgSSHg/pxJ/A2STvkq+pXSjpc0gNkV0LbH0DSaElnS1qRDy+skHSKpL/6PCVtL+liSaslbcjvr5C0Vf78RsNBko6X9CdJL0laK2mppAPLnt9oOEjSLpJ+KunZvN0SSftVzHNavro7SdL1ktZLWilpTmXdibgbeAvZUaMASNpZ0lWSnsr7657yz75svj3yz/uZ/PN+UNLJZc/vI+kGSU9IelHSMklfkzRqcN6aFeUfAF4TqLQz8DpQuk7CVGBP4HTgSaAv32bwS7JLZJ4J/BGYAnyT7FrMXwOQ1AH8Np82F7iPbKhpBrAlsNFV1SR9DvgucAbwG7IvqfdT/RrPpTY7AbcBzwPHAs8BXwaul/TJiPhFRZOfkl3f4XzgU/l7W51PS0kX2Wf1DICkscDvyPr5BOApsivgXSvpgIhYmM/3QaCX7NKoJ5CtyU0i66eSCcDNZMOLLwPdwGnA9sBJLX1XVs2oim19ERGvt62aoSYikrsBhwEB7EIWhB3AUWQB8LN8nj7gRWDHiraH5G0/UjH9FLK1hR3yx2fkr/d3/dRxWtYFbz6+CLi7Ru29QG/Z43OB14CJZdNGAQ+Wv1ZpWcAXK17vj2SX+mx7vwxiXx+ef2bHls13KdkX/zsr2t9ENpxQevxrstAcXXD5ypd7CrAW2KzsuT7gsiq1drX7cxsJt7LPs/J2W5V5J+bPHdbuugf7luIwQLkHgFeBNcDFZFdNO7zs+SUR8eeKNvuRndDut5I2L92ARWTXU56Sz7cPcGdE/KGOeu4E9pR0oaSPSxpdoM1H8jrfHPOM7FfOj/PX2rZi/usrHi8DxtVR43BV3teXAvMi4qKy5/cDbgCeq+jXXwJ7SNo274+9gasi4sVNLUjS30iaJ2kl2Q+DV8nWBt9BtjZog+tAYK+y2xHtLWdoSX046ECy1fnngZUR8XLF809UabMDMJ7sD7uad5bd31tnPT8Ctib7T3oM8KqkG4CvRnbltmrGANWC5s9kv0I7gHVl09dUzLchX+ZIV+rr7cmucneMpN9FxI/y53cg2zC4qY2D7yT7Qt+Mfjbm59tXFgI7ka19PQC8BBxAtjaQwmc91CyLTW8YTl7qIVDrP0e1ffifIbvG8kGbaNOX3z8NvKueYiJbL50HzMu3KexDto3gauBDm2i2BtixyvQdyeqv/NJP1Zt9LekWsm0050i6NrLraD9Dth3m7E20f5xsmO0N+u/X95BtAzgkIq4sTZT0qcbfglnzpT4cNBA3AmOB9RGxtMqtdH7yRcAHJe0xkIVExNqIuBr4b+C9/cx6KzCl/ACjfC+Ug4E/RMTzA1n+SBYRG4Cvk/36Lx13cSPZxt3lm+jXDfkQ0G3A5yW9ZRMvXxrCe3NNUdIWwOda8mbMGpT6msBAXAV8EbhZ0nfJhny2JPsFOB04IP+yOB/4LPArSXPJNsBuR7Z30NHVvpwlzScbmrqDbC+VvyXbEL2on3rOJ9sAdpOkU8mGfo7J2+7f6JsdqSJioaQ7gRMlXUR2bMjvgV/nj/vIhtLeC0yIiNK2ohPJgveOvP8fI9sbaM+IOA74E9k2o29Jep0sDE4YvHdm9ZD0UbIhwtLadLek9QARcU3bChtEDoE6RcSrkvYl29VvFtlupS8Aj5JtdH0ln+9ZSXuTbRA8iWxM+S/ALaV5qridLGAOAd5ONgRxJXBqP/U8LunDZMMY3we2IjsCcv+IqPdo5NR8g2zD79ERcb6k0q6cZ5F9MTxDtuH88lKDiLgz79czyHYB3YrsS/+H+fOvSDqAbE+vH5ENx/0nsAq4ZHDeltXhdOCjZY+/nN8g26Y24vksomZmCfM2ATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhP0//t3ZjeY2sNgAAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "draw_plot([pres_all, recalls_all, f1_all], title='Scores for All Elements')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 75,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEBCAYAAACe6Rn8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAF2hJREFUeJzt3X2QXFWdxvHvQ3gzKDoRGBadZIhhgaDCuoOmFtedRAVcNIHKCpaKIEpgESQobkGh4S2LUiBBodCEZQWBclkBTVYQAySDgkQJCJggQthMEhYUJIEQXgKB3/5xb1NtpzN9e6Z7embO86nq6vTpe/r+uk+mn77n3u6riMDMzNK0VasLMDOz1nEImJklzCFgZpYwh4CZWcIcAmZmCXMImJklzCFgZpYwh4CZWcIcAmZmCdu61QXUstNOO0VnZ2eryzAzG1buvffev0TEzrWWG/Ih0NnZydKlS1tdhpnZsCJpVZHlPB1kZpYwh4CZWcIcAmZmCSsUApLeKekSSXdLelFSSOos2HcrSadL6pX0sqQHJE0fSNFmZtYYRbcEJgCHA+uAX9W5jnOBs4BLgY8BS4AfS/rnOh/HzMwarOjRQb+MiHYASV8EDizSSdIuwKnAtyLiwrx5saQJwLeAm+us18zMGqjQlkBEvN7Pxz8I2Ba4pqL9GuA9knbv5+OamVkDNHvH8D7ARmBFRfvy/Hpik9dvZmZ9aPaXxcYAz8bmJzJeW3b/ZiTNAGYAtLe309PT07QCm2ny5MkNeZzFixc35HGsPo0YP49da/hvr7hmh4CAameyV1+dImIeMA+gq6sruru7G1/ZINg8+zYnqdByNvhqjYvHbujy315xzZ4OWgu0Sap8028ru9/MzFqk2SGwHNgOeFdFe2lfwENNXr+ZmfWh2SFwC/AK8JmK9s8CyyJiZZPXb2ZmfSi8T0DSv+T//Pv8+mOSngaejog78mU2AVdFxBcAIuIpSXOA0yU9D9wHHAFMAaY16DmYmVk/1bNj+McVty/Lr+8AuvN/j8ov5c4ANgAnA7sCfwQOj4j/qatSMzNruMIhEBF9HtGzpWUi4jVgdn4xM7MhxL8iamaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAJWnsuHFIGtAFGPBjjB03rsWvhKWunnMMm40Ya1av5oaHn2h1GUzfa7dWl2CJ85aAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJaxQCEjqkHS9pOckrZd0o6SxBfuOlXSVpNWSXpT0iKTZknYYWOlmZjZQNb8sJmk0sAjYCBwFBDAbWCzpvRHxQh99dwBuA7YBvgGsBvYHzgb2AI4Y6BMwM7P+K/KN4WOB8cCeEbECQNKDwKPAccBFffQ9gOzN/qCIWJi3LZY0BjhV0uiIeLHf1ZuZ2YAUmQ6aCiwpBQBARKwE7gKm1ei7bX69vqL92XzdKlinmZk1QZEQ2AdYVqV9OTCxRt/byLYYzpc0UdKbJU0BTga+39dUkpmZNV+REBgDrKvSvhZo66tjRLwMfDBfz3LgeeB24GfAiXVVamZmDVf0V0SjSlvNqRxJ2wPXAbsAR5LtGH4/MAvYBPzrFvrNAGYAtLe309PTU7DM4WmkPz/rm8e/dfzagyKqvb+XLSD9GfhpRBxX0X4Z8MmI2LmPvl8CLgUmRMRjZe3HAvOA/SLigb7W39XVFUuXLq35RIYrSdQaA2s8SUPmp6Q9/q0x0v/2JN0bEV21lisyHbScbL9ApYnAQzX6vgdYVx4Aud/m13sXWL+ZmTVJkRBYAEySNL7UIKmT7PDPBTX6/glokzShov0D+fX/FSvTzMyaoUgIXA70AvMlTZM0FZgPrAHmlhaSNE7SJkmzyvpeSbYz+GZJR0maLOlrwIXAvWSHmZqZWYvUDIH8MM4pwCPA1cC1wEpgSkRsKFtUwKjyx4yIXmAScD/Zt4xvJvvy2TzgoxHxekOehZmZ9Uuho4MiYjUwvcYyvVQ5YigiHgIO709xZmbWXP4VUTOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ8DMLGEOATOzhDkEzMwS5hAwM0uYQ2AAOseOQ9KALsCA+neOHdfiV8Fs8HV0drT8b6+js6PFr0JjFDrRvFW3as1qoueeltag7v1bun6zVnh81eNcvPbiltYwc8zMlq6/UbwlYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCCh0iKqkDmAN8FBBwGzAzIlYX7L83cA4wGdgBWA1cFhHf6U/RZgMVZ+4IP9qr1WVkdZi1UM0QkDQaWARsBI4CApgNLJb03oh4oUb/rrx/D/BF4DlgD+DNA6rcbAB09npuePiJVpfB9L12I85qdRWWsiJbAscC44E9I2IFgKQHgUeB44CLttRR0lbAVcDtEXFY2V2L+12xmZk1TJF9AlOBJaUAAIiIlcBdwLQafbuBifQRFGZm1jpFQmAfYFmV9uVkb/B9+WB+vb2kJZJelfSUpO9KelM9hZqZWeMVmQ4aA6yr0r4WaKvRd7f8+jrgUuA0oItsJ3EHcFi1TpJmADMA2tvb6enpKVBmuvz6DG8ev+FrJIxd0R+QiyptKtCvtKVxTUTMyv/dI2kU8C1JEyPioc1WFjEPmAfQ1dUV3d3dBctMk1+f4c3jN3yNhLErMh20jmxroFIb1bcQyj2TX99a0b4wv96vwPrNzKxJioTAcrL9ApUmApt9iq/SFzbfkihtRbxeYP1mZtYkRUJgATBJ0vhSg6RO4ID8vr78nOz7BQdXtB+UXy8tVKWZmTVFkRC4HOgF5kuaJmkqMB9YA8wtLSRpnKRNkkpz/0TEM8A3geMlnSfpI5JOA2YBV5UfdmpmZoOv5o7hiHhB0hSyn424mmwq53ayn43YULaogFFsHiznAM8DJwCnAk8CFwDnDrh6MzMbkEJHB+W/ETS9xjK9VDliKCKC7Mti/sKYmdkQ418RNTNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhhU4vaWY2lMSZO8J3ZrW0hpPP3LGl628Uh4CZDTs6ez0Xr724pTXMHDOTOKulJTSEp4PMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS1ihEJDUIel6Sc9JWi/pRklj612ZpNMlhaQ76y/VzMwarWYISBoNLAL2Ao4CjgT2ABZL2qHoiiSNB84AnupfqWZm1mhFvix2LDAe2DMiVgBIehB4FDgOuKjgur4HXAvsWXC9ZmbWZEWmg6YCS0oBABARK4G7gGlFViLp08D7gNP7U6SZmTVHkRDYB1hWpX05MLFWZ0ltwBzg3yJibX3lmZlZMxUJgTHAuirta4G2Av0vAB4BrixelpmZDYaic/NRpU21Okn6R+BzwPsiotpjbKnfDGAGQHt7Oz09PUW7Dqo4c0dY/OGW1zBUXx8rxuM3fI2EsSsSAuvItgYqtVF9C6HcXOAK4HFJbytb56j89ksRsbGyU0TMA+YBdHV1RXd3d4EyB58mryd67mltDd37E2d1t7QGG5ih+v/bahsJY1ckBJaT7ReoNBF4qEbfvfPL8VXuWwecArT292DNzBJWJAQWABdKGh8R/wsgqRM4ADitRt/JVdouBkYBJwErqtxvZmaDpEgIXA6cCMyX9HWy/QPnAmvIpnsAkDQOeAw4JyLOAYiInsoHk/QssHW1+8zMbHDVDIGIeEHSFLLDPK8m2yF8OzAzIjaULSqyT/j+PSIb8jrGjmX6Xru1ugw6xtb96ytmDVXo6KCIWA1Mr7FMLwWOGIqI7iLrNGum1atWDfgxJFHHQW9mQ5I/tZuZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJazQmcXMzIaSd457JzPHzGx5DSOBQ8DMhp01vWsG/Bg+PWjG00FmZglzCJiZJcwhYGaWMIeAmVnCHAJmZglzCJiZJcwhYGaWMIeAmVnCCoWApA5J10t6TtJ6STdKGlugX5ekeZIelvSipNWSrpW0+8BLNzOzgaoZApJGA4uAvYCjgCOBPYDFknao0f1TwD7Ad4GPAacB7wOWSuoYQN1mZtYARX424lhgPLBnRKwAkPQg8ChwHHBRH33Pj4inyxsk3QWszB93Vn+KNjOzxigyHTQVWFIKAICIWAncBUzrq2NlAORtq4CngXfUV6qZmTVakRDYB1hWpX05MLHeFUraG9gF+EO9fc3MrLGKTAeNAdZVaV8LtNWzMklbA98n2xK4oo/lZgAzANrb2+np6alnNcnx69M6fu2HN49f8Z+SrvZ7q+rH+i4F/gE4JCKqBUu2soh5wDyArq6u6O7u7seq0uHXp3X82g9vHr9iIbCObGugUhvVtxCqkvRNsk/3R0XEwqL9zMyseYqEwHKy/QKVJgIPFVmJpDPIDg/9ckRcXbw8MzNrpiI7hhcAkySNLzVI6gQOyO/rk6QvA7OBMyLikv6VaWZmzVAkBC4HeoH5kqZJmgrMB9YAc0sLSRonaZOkWWVtnwIuBm4BFkmaVHap+8giMzNrrJrTQRHxgqQpwBzgarIdwrcDMyNiQ9miAkbx18FycN5+cH4pdwfQ3e/Kh4BxHWNR9/4tr8HMrL801E+03NXVFUuXLm11GU3jk10PXx674W2kj5+keyOiq9Zy/hVRM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBLmEDAzS5hDwMwsYQ4BM7OEOQTMzBJWKAQkdUi6XtJzktZLulHS2IJ9t5d0gaQnJb0k6W5JHxpY2WZm1gg1Q0DSaGARsBdwFHAksAewWNIOBdZxBXAsMAv4OPAk8AtJ+/W3aDMza4ytCyxzLDAe2DMiVgBIehB4FDgOuGhLHSXtC3waOCYifpC33QEsB84Bpg6oejMzG5Ai00FTgSWlAACIiJXAXcC0An1fBa4r67sJ+C/gIEnb1V2xmZk1TJEQ2AdYVqV9OTCxQN+VEfFilb7bAhMKrN/MrC6Sal6KLJeCItNBY4B1VdrXAm0D6Fu638ysoSKi1SUMG0VCAKDaK1okJtWfvpJmADMA2tvb6enpKbCqoWfy5MmFlqv1iWPx4sWNKMfqVGT8PHY23BUJgXVU/8TeRvVP+eXWAtUOJW0ru38zETEPmAfQ1dUV3d3dBcocevxpZHjz+FkKiuwTWE42t19pIvBQgb6754eZVvZ9BVixeRczMxssRUJgATBJ0vhSg6RO4ID8vlp9twE+WdZ3a+AIYGFEbKyzXjMza6AiIXA50AvMlzRN0lRgPrAGmFtaSNI4SZskzSq1RcT9ZIeHXizpi5I+THZ46O7AmY17GmZm1h81QyAiXgCmAI8AVwPXAiuBKRGxoWxRAaOqPObngR8As4GbgA7g4Ii4b8DVm5nZgBQ6OigiVgPTayzTS5WjfiLiJeAr+cXMzIYQ/4qomVnCHAJmZglzCJiZJUxD/Qsxkp4GVrW6jibaCfhLq4uwfvHYDW8jffzGRcTOtRYa8iEw0klaGhFdra7D6uexG948fhlPB5mZJcwhYGaWMIdA681rdQHWbx674c3jh/cJmJklzVsCZmYJSzIEJB0tKcouz0t6QNKJ+a+cDlYdZ0mqa1NMUo+kniaVNOJUGetXJD0m6TxJ27e4tl5JV5bdLtXa2bKiRpAqY19++Ui+zHmSFkp6Jm8/usVlD7pBe8Mboj4JPA7smP/7EmAXYFZfnRroP4Bb6uxzQjMKSUBprN8CHAacnv/7pFYWZYOiNPblSudCOQm4H/gZ8LnBLGqoSD0E7o+I0oltFkqaAMykSggoO4/gNhHxSqNWHhGPs/l/zlp9ap3Ix6orH+tbJe0BfEHSyRHxeisLs6YrH/tKb42I1/O//SRDIMnpoD7cA7xF0i75pvo1ko6R9DDZmdAOAZA0WtL5klbm0wsrJZ0h6a9eT0k7S7pM0hpJG/PrqyVtl9+/2XSQpJMl/UHSS5LWSVoq6bCy+zebDpK0p6SfSHo277dE0sEVy5yVb+7uIekmSRskrZI0q7LuRNwHvInsW6MASNpd0rWSns7H6/7y175suX3z1/uZ/PX+o6TTy+4/UNLNkp6U9KKkZZK+KmnU4Dw1K8ofALwlUGl34DWgdJ6EycB+wNnAU0Bvvs/gF2SnyDwX+D0wCfgG2bmYvwogqQ34dd42G3iQbKppGrAtsNlZ1SR9Bvg2cA7wK7I3qfdS/RzPpT67AXcCzwMnAs8BXwJukvTxiPh5RZefkJ3fYQ7wify5rcnbUtJJ9lo9AyCpA/gN2TifAjxNdga8GyQdGhEL8uXeD/SQnRr1FLItuT3IxqlkPHA72fTiy0AXcBawM3BaU5+VVTOqYl9fRMRrLatmqImI5C7A0UAAe5IFYRtwHFkA/DRfphd4Edi1ou+Red8PVbSfQba1sEt++5z88f6ujzrOyobgjduXAvfVqL0H6Cm7fSGwCZhQ1jYK+GP5Y5XWBXy+4vF+T3aqz5aPyyCO9TH5a3Zi2XJXkL3xv72i/61k0wml278kC83RBdevfL1nAOuArcru6wWurFJrZ6tft5FwKXs9Ky93Vll2Qn7f0a2ue7AvKU4DlHsYeBVYC1xGdta0Y8ruXxIRf6roczDZD9r9WtLWpQuwkOx8ypPy5Q4E7omI39VRzz3AfpIukfQRSaML9PlQXucbc56Rfcr5Uf5YO1Ysf1PF7WXA2DpqHK7Kx/oKYG5EXFp2/8HAzcBzFeP6C2BfSTvm43EAcG1EvLilFUn6G0lzJa0i+2DwKtnW4NvItgZtcB0G7F92+UJryxlaUp8OOoxsc/55YFVEvFxx/5NV+uwCjCP7w67m7WXXD9RZzw+B7cn+k54AvCrpZuArkZ25rZoxQLWg+RPZp9A2YH1Z+9qK5Tbm6xzpSmO9M9lZ7k6Q9JuI+GF+/y5kOwa3tHPw7WRv6FvRx878fP/KAmA3sq2vh4GXgEPJtgZSeK2HmmWx5R3DyUs9BGr956h2DP8zZOdYPnwLfXrz678A76inmMi2S+cCc/N9CgeS7SO4DvjAFrqtBXat0r4rWf2Vb/qpemOsJS0i20dzgaQbIjuP9jNk+2HO30L/J8im2V6n73F9F9k+gCMj4ppSo6RPDPwpmDVe6tNB/XEL0AFsiIilVS6l3ydfCLxf0r79WUlErIuI64D/Bt7dx6J3AJPKv2CUH4VyBPC7iHi+P+sfySJiI/A1sk//pe9d3EK2c3f5FsZ1Yz4FdCfwWUlv2sLDl6bw3thSlLQN8JmmPBmzAUp9S6A/rgU+D9wu6dtkUz7bkn0CnAocmr9ZzAE+DdwmaTbZDtidyI4OOr7am7OkeWRTU3eTHaXyt2Q7ohf2Uc8csh1gt0o6k2zq54S87yEDfbIjVUQskHQPcKqkS8m+G/Jb4Jf57V6yqbR3A+MjorSv6FSy4L07H//HyY4G2i8iTgL+QLbP6N8lvUYWBqcM3jOzekj6J7IpwtLWdJekDQARcX3LChtEDoE6RcSrkg4iO9RvBtlhpS8Aj5HtdH0lX+5ZSQeQ7RA8jWxO+c/AotIyVdxFFjBHAm8lm4K4Bjizj3qekPRBsmmM7wHbkX0D8pCIqPfbyKn5OtmO3+MjYo6k0qGc55G9MTxDtuP8qlKHiLgnH9dzyA4B3Y7sTf8H+f2vSDqU7EivH5JNx/0nsBq4fHCeltXhbOCfym5/Kb9Atk9txPOviJqZJcz7BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEuYQMDNLmEPAzCxhDgEzs4Q5BMzMEvb/ucuWDpfEXmUAAAAASUVORK5CYII=\n",
+      "text/plain": [
+       "<Figure size 432x288 with 1 Axes>"
+      ]
+     },
+     "metadata": {
+      "needs_background": "light"
+     },
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "draw_plot([pres_non_text, recalls_non_text, f1_non_text], title='Score for Non-text Elements')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 51,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.5.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

CDM/result_processing/eval_classes.py

@@ -0,0 +1,215 @@
+import json
+import numpy as np
+import cv2
+from glob import glob
+from os.path import join as pjoin
+from tqdm import tqdm
+
+class_map = {'0':'Button', '1':'CheckBox', '2':'Chronometer', '3':'EditText', '4':'ImageButton', '5':'ImageView',
+               '6':'ProgressBar', '7':'RadioButton', '8':'RatingBar', '9':'SeekBar', '10':'Spinner', '11':'Switch',
+               '12':'ToggleButton', '13':'VideoView', '14':'TextView'}
+
+
+def resize_label(bboxes, d_height, gt_height, bias=0):
+    bboxes_new = []
+    scale = gt_height / d_height
+    for bbox in bboxes:
+        bbox = [int(b * scale + bias) for b in bbox]
+        bboxes_new.append(bbox)
+    return bboxes_new
+
+
+def draw_bounding_box(org, corners, color=(0, 255, 0), line=2, show=False):
+    board = org.copy()
+    for i in range(len(corners)):
+        board = cv2.rectangle(board, (corners[i][0], corners[i][1]), (corners[i][2], corners[i][3]), color, line)
+    if show:
+        cv2.imshow('a', cv2.resize(board, (500, 1000)))
+        cv2.waitKey(0)
+    return board
+
+
+def load_detect_result_json(reslut_file_root, shrink=4):
+    def is_bottom_or_top(corner):
+        column_min, row_min, column_max, row_max = corner
+        if row_max < 36 or row_min > 725:
+            return True
+        return False
+
+    result_files = glob(pjoin(reslut_file_root, '*.json'))
+    compos_reform = {}
+    print('Loading %d detection results' % len(result_files))
+    for reslut_file in tqdm(result_files):
+        img_name = reslut_file.split('\\')[-1].split('.')[0]
+        compos = json.load(open(reslut_file, 'r'))['compos']
+        for compo in compos:
+            if compo['column_max'] - compo['column_min'] < 10 or compo['row_max'] - compo['row_min'] < 10:
+                continue
+            if is_bottom_or_top((compo['column_min'], compo['row_min'], compo['column_max'], compo['row_max'])):
+                continue
+            if img_name not in compos_reform:
+                compos_reform[img_name] = {'bboxes': [[compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink]],
+                                           'categories': [compo['category']]}
+            else:
+                compos_reform[img_name]['bboxes'].append([compo['column_min'] + shrink, compo['row_min'] + shrink, compo['column_max'] - shrink, compo['row_max'] - shrink])
+                compos_reform[img_name]['categories'].append(compo['category'])
+    return compos_reform
+
+
+def load_ground_truth_json(gt_file):
+    def get_img_by_id(img_id):
+        for image in images:
+            if image['id'] == img_id:
+                return image['file_name'].split('/')[-1][:-4], (image['height'], image['width'])
+
+    def cvt_bbox(bbox):
+        '''
+        :param bbox: [x,y,width,height]
+        :return: [col_min, row_min, col_max, row_max]
+        '''
+        bbox = [int(b) for b in bbox]
+        return [bbox[0], bbox[1], bbox[0] + bbox[2], bbox[1] + bbox[3]]
+
+    data = json.load(open(gt_file, 'r'))
+    images = data['images']
+    annots = data['annotations']
+    compos = {}
+    print('Loading %d ground truth' % len(annots))
+    for annot in tqdm(annots):
+        img_name, size = get_img_by_id(annot['image_id'])
+        if img_name not in compos:
+            compos[img_name] = {'bboxes': [cvt_bbox(annot['bbox'])], 'categories': [class_map[str(annot['category_id'])]], 'size': size}
+        else:
+            compos[img_name]['bboxes'].append(cvt_bbox(annot['bbox']))
+            compos[img_name]['categories'].append(class_map[str(annot['category_id'])])
+    return compos
+
+
+def eval(detection, ground_truth, img_root, show=True, no_text=False, only_text=False):
+    def compo_filter(compos, flag):
+        if not no_text and not only_text:
+            return compos
+        compos_new = {'bboxes': [], 'categories': []}
+        for k, category in enumerate(compos['categories']):
+            if only_text:
+                if flag == 'det' and category != 'TextView':
+                    continue
+                if flag == 'gt' and category != 'TextView':
+                    continue
+            elif no_text:
+                if flag == 'det' and category == 'TextView':
+                    continue
+                if flag == 'gt' and category == 'TextView':
+                    continue
+
+            compos_new['bboxes'].append(compos['bboxes'][k])
+            compos_new['categories'].append(category)
+        return compos_new
+
+    def match(org, d_bbox, d_category, gt_compos, matched):
+        '''
+        :param matched: mark if the ground truth component is matched
+        :param d_bbox: [col_min, row_min, col_max, row_max]
+        :param gt_bboxes: list of ground truth [[col_min, row_min, col_max, row_max]]
+        :return: Boolean: if IOU large enough or detected box is contained by ground truth
+        '''
+        area_d = (d_bbox[2] - d_bbox[0]) * (d_bbox[3] - d_bbox[1])
+        gt_bboxes = gt_compos['bboxes']
+        gt_categories = gt_compos['categories']
+        for i, gt_bbox in enumerate(gt_bboxes):
+            if matched[i] == 0:
+                continue
+            area_gt = (gt_bbox[2] - gt_bbox[0]) * (gt_bbox[3] - gt_bbox[1])
+            col_min = max(d_bbox[0], gt_bbox[0])
+            row_min = max(d_bbox[1], gt_bbox[1])
+            col_max = min(d_bbox[2], gt_bbox[2])
+            row_max = min(d_bbox[3], gt_bbox[3])
+            # if not intersected, area intersection should be 0
+            w = max(0, col_max - col_min)
+            h = max(0, row_max - row_min)
+            area_inter = w * h
+            if area_inter == 0:
+                continue
+            iod = area_inter / area_d
+            iou = area_inter / (area_d + area_gt - area_inter)
+            # if show:
+            #     cv2.putText(org, (str(round(iou, 2)) + ',' + str(round(iod, 2))), (d_bbox[0], d_bbox[1]),
+            #                 cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2)
+
+            if iou > 0.9 or iod == 1:
+                if d_category == gt_categories[i]:
+                    matched[i] = 0
+                    return True
+        return False
+
+    amount = len(detection)
+    TP, FP, FN = 0, 0, 0
+    pres, recalls, f1s = [], [], []
+    for i, image_id in enumerate(detection):
+        TP_this, FP_this, FN_this = 0, 0, 0
+        img = cv2.imread(pjoin(img_root, image_id + '.jpg'))
+        d_compos = detection[image_id]
+        if image_id not in ground_truth:
+            continue
+        gt_compos = ground_truth[image_id]
+
+        org_height = gt_compos['size'][0]
+
+        d_compos = compo_filter(d_compos, 'det')
+        gt_compos = compo_filter(gt_compos, 'gt')
+
+        d_compos['bboxes'] = resize_label(d_compos['bboxes'], 800, org_height)
+        matched = np.ones(len(gt_compos['bboxes']), dtype=int)
+        for j, d_bbox in enumerate(d_compos['bboxes']):
+            if match(img, d_bbox, d_compos['categories'][j], gt_compos, matched):
+                TP += 1
+                TP_this += 1
+            else:
+                FP += 1
+                FP_this += 1
+        FN += sum(matched)
+        FN_this = sum(matched)
+
+        try:
+            pre_this = TP_this / (TP_this + FP_this)
+            recall_this = TP_this / (TP_this + FN_this)
+            f1_this = 2 * (pre_this * recall_this) / (pre_this + recall_this)
+        except:
+            print('empty')
+            continue
+
+        pres.append(pre_this)
+        recalls.append(recall_this)
+        f1s.append(f1_this)
+        if show:
+            print(image_id + '.jpg')
+            print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f' % (
+                i, amount, TP_this, FP_this, FN_this, pre_this, recall_this))
+            # cv2.imshow('org', cv2.resize(img, (500, 1000)))
+            broad = draw_bounding_box(img, d_compos['bboxes'], color=(255, 0, 0), line=3)
+            draw_bounding_box(broad, gt_compos['bboxes'], color=(0, 0, 255), show=True, line=2)
+
+        if i % 200 == 0:
+            precision = TP / (TP + FP)
+            recall = TP / (TP + FN)
+            f1 = 2 * (precision * recall) / (precision + recall)
+            print(
+                '[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))
+
+    precision = TP / (TP + FP)
+    recall = TP / (TP + FN)
+    print('[%d/%d] TP:%d, FP:%d, FN:%d, Precesion:%.3f, Recall:%.3f, F1:%.3f' % (i, amount, TP, FP, FN, precision, recall, f1))
+    # print("Average precision:%.4f; Average recall:%.3f" % (sum(pres)/len(pres), sum(recalls)/len(recalls)))
+
+    return pres, recalls, f1s
+
+
+no_text = True
+only_text = False
+
+# detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\ip')
+# detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_cls\\merge')
+detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_v3\\merge')
+# detect = load_detect_result_json('E:\\Mulong\\Result\\rico\\rico_uied\\rico_new_uied_v3\\ocr')
+gt = load_ground_truth_json('E:\\Mulong\\Datasets\\rico\\instances_test.json')
+eval(detect, gt, 'E:\\Mulong\\Datasets\\rico\\combined', show=False, no_text=no_text, only_text=only_text)