Update

app.py

@@ -12,11 +12,12 @@ import gradio as gr
 from datetime import datetime
 from modelscope.pipelines import pipeline
 from modelscope import snapshot_download
+from modelscope.utils.constant import Tasks
 from PIL import Image, ImageDraw, ImageFont
 
 from PCAgent.api import inference_chat
 from PCAgent.icon_localization import det
-from PCAgent.text_localization import ocr
+from PCAgent.text_localization_old import ocr
 from PCAgent.prompt_qwen import get_subtask_prompt as get_subtask_prompt
 from PCAgent.chat import init_action_chat, init_memory_chat, add_response
 from PCAgent.prompt_qwen import get_action_prompt, get_process_prompt, get_memory_prompt
@@ -26,8 +27,10 @@ vl_model_version = os.environ.get('vl_model_version')
 llm_model_version = os.environ.get('llm_model_version')
 API_url = os.environ.get('API_url')
 token = os.environ.get('token')
-os.environ["OCR_ACCESS_KEY_ID"] = os.environ.get('OCR_ACCESS_KEY_ID')
-os.environ["OCR_ACCESS_KEY_SECRET"] = os.environ.get('OCR_ACCESS_KEY_SECRET')
+# os.environ["OCR_ACCESS_KEY_ID"] = os.environ.get('OCR_ACCESS_KEY_ID')
+# os.environ["OCR_ACCESS_KEY_SECRET"] = os.environ.get('OCR_ACCESS_KEY_SECRET')
+ocr_detection = pipeline(Tasks.ocr_detection, model='damo/cv_resnet18_ocr-detection-line-level_damo')
+ocr_recognition = pipeline(Tasks.ocr_recognition, model='damo/cv_convnextTiny_ocr-recognition-document_damo')
 tff_file = os.environ.get('tff_file')
 radius = 100
 
@@ -127,7 +130,7 @@ def get_perception_infos(screenshot_file, screenshot_som_file, font_path):
 
     for i, img in enumerate(img_list):
         width, height = Image.open(img).size
-        sub_text, sub_coordinates = ocr(img) # for api
+        sub_text, sub_coordinates = ocr(img, ocr_detection, ocr_recognition) # for api
         for coordinate in sub_coordinates:
             coordinate[0] = int(max(0, img_x_list[i] + coordinate[0] - padding))
             coordinate[2] = int(min(total_width, img_x_list[i] + coordinate[2] + padding))

requirements.txt

@@ -11,4 +11,11 @@ transformers
 torchvision
 pycocotools
 timm
-termcolor
+termcolor
+TensorFlow==2.9.1
+keras==2.9.0
+SentencePiece
+tf_slim
+tf_keras==2.15.0
+pyclipper
+numpy==1.26.4