update

app.py

@@ -2,6 +2,9 @@ import gradio as gr
 from langchain_core.messages import HumanMessage, AIMessage
 from llm import DeepSeekLLM, OpenRouterLLM, TongYiLLM
 from config import settings
+from modelscope.outputs import OutputKeys
+from modelscope.pipelines import pipeline
+from modelscope.utils.constant import Tasks
 
 
 deep_seek_llm = DeepSeekLLM(api_key=settings.deep_seek_api_key)
@@ -39,6 +42,17 @@ def update_chat(_provider: str, _chat, _model: str, _temperature: float, _max_to
     return _chat
 
 
+def image_progress(_image):
+    _input = {
+        'img': _image['background'],
+        'mask': _image['layers'][0],
+    }
+    inpainting = pipeline(Tasks.image_inpainting, model='damo/cv_fft_inpainting_lama')
+    result = inpainting(_input)
+    vis_img = result[OutputKeys.OUTPUT_IMG]
+    return vis_img, _image['layers'][0]
+
+
 with gr.Blocks() as app:
     with gr.Tab('聊天'):
         chat_engine = gr.State(value=None)
@@ -176,7 +190,14 @@ with gr.Blocks() as app:
     with gr.Tab('画图'):
         with gr.Row():
             with gr.Column(scale=2, min_width=600):
-                gr.Image(label="Input Image")
+                image = gr.ImageMask(
+                    type='pil',
+                    brush=gr.Brush(colors=["rgba(0, 0, 0, 0.7)"]),
+                )
+                with gr.Row():
+                    image_preview = gr.Image(label='图片预览')
+                    mask_preview = gr.Image(label='蒙板预览')
+                image.change(image_progress, inputs=[image], outputs=[image_preview, mask_preview])
             with gr.Column(scale=1, min_width=300):
                 gr.Textbox(label="LoRA")
 

pyproject.toml

@@ -13,6 +13,12 @@ dependencies = [
     "pydantic-settings>=2.3.4",
     "langchain-openai>=0.1.16",
     "dashscope>=1.20.1",
+    "modelscope>=1.16.0",
+    "torch>=2.3.1",
+    "torchvision>=0.18.1",
+    "torchaudio>=2.3.1",
+    "setuptools>=71.0.0",
+    "oss2>=2.18.6",
 ]
 readme = "README.md"
 requires-python = ">= 3.8"

requirements-dev.lock

@@ -16,6 +16,11 @@ aiohttp==3.9.5
     # via langchain
 aiosignal==1.3.1
     # via aiohttp
+aliyun-python-sdk-core==2.15.1
+    # via aliyun-python-sdk-kms
+    # via oss2
+aliyun-python-sdk-kms==2.16.3
+    # via oss2
 altair==5.3.0
     # via gradio
 annotated-types==0.7.0
@@ -36,6 +41,8 @@ certifi==2024.7.4
     # via httpcore
     # via httpx
     # via requests
+cffi==1.16.0
+    # via cryptography
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
@@ -43,6 +50,10 @@ click==8.1.7
     # via uvicorn
 contourpy==1.2.1
     # via matplotlib
+crcmod==1.7
+    # via oss2
+cryptography==42.0.8
+    # via aliyun-python-sdk-core
 cycler==0.12.1
     # via matplotlib
 dashscope==1.20.1
@@ -66,14 +77,16 @@ ffmpy==0.3.2
 filelock==3.15.4
     # via diffusers
     # via huggingface-hub
+    # via torch
 fonttools==4.53.1
     # via matplotlib
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
-fsspec==2024.6.1
+fsspec==2024.5.0
     # via gradio-client
     # via huggingface-hub
+    # via torch
 gradio==4.38.1
     # via aitoolkits-webui
 gradio-client==1.1.0
@@ -108,6 +121,9 @@ jinja2==3.1.4
     # via altair
     # via fastapi
     # via gradio
+    # via torch
+jmespath==0.10.0
+    # via aliyun-python-sdk-core
 jsonpatch==1.33
     # via langchain-core
 jsonpointer==3.0.0
@@ -140,9 +156,15 @@ matplotlib==3.9.1
     # via gradio
 mdurl==0.1.2
     # via markdown-it-py
+modelscope==1.16.0
+    # via aitoolkits-webui
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
+networkx==3.3
+    # via torch
 numpy==1.26.4
     # via altair
     # via contourpy
@@ -151,12 +173,15 @@ numpy==1.26.4
     # via langchain
     # via matplotlib
     # via pandas
+    # via torchvision
 openai==1.35.13
     # via langchain-openai
 orjson==3.10.6
     # via fastapi
     # via gradio
     # via langsmith
+oss2==2.18.6
+    # via aitoolkits-webui
 packaging==24.1
     # via altair
     # via gradio
@@ -171,6 +196,11 @@ pillow==10.4.0
     # via diffusers
     # via gradio
     # via matplotlib
+    # via torchvision
+pycparser==2.22
+    # via cffi
+pycryptodome==3.20.0
+    # via oss2
 pydantic==2.8.2
     # via aitoolkits-webui
     # via fastapi
@@ -219,6 +249,8 @@ requests==2.32.3
     # via huggingface-hub
     # via langchain
     # via langsmith
+    # via modelscope
+    # via oss2
     # via tiktoken
 rich==13.7.1
     # via typer
@@ -231,9 +263,12 @@ safetensors==0.4.3
     # via diffusers
 semantic-version==2.10.0
     # via gradio
+setuptools==71.0.0
+    # via aitoolkits-webui
 shellingham==1.5.4
     # via typer
 six==1.16.0
+    # via oss2
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
@@ -243,6 +278,8 @@ sqlalchemy==2.0.31
     # via langchain
 starlette==0.37.2
     # via fastapi
+sympy==1.13.0
+    # via torch
 tenacity==8.5.0
     # via langchain
     # via langchain-core
@@ -252,8 +289,17 @@ tomlkit==0.12.0
     # via gradio
 toolz==0.12.1
     # via altair
+torch==2.3.1
+    # via aitoolkits-webui
+    # via torchaudio
+    # via torchvision
+torchaudio==2.3.1
+    # via aitoolkits-webui
+torchvision==0.18.1
+    # via aitoolkits-webui
 tqdm==4.66.4
     # via huggingface-hub
+    # via modelscope
     # via openai
 typer==0.12.3
     # via fastapi-cli
@@ -269,6 +315,7 @@ typing-extensions==4.12.2
     # via pydantic
     # via pydantic-core
     # via sqlalchemy
+    # via torch
     # via typer
     # via uvicorn
 tzdata==2024.1
@@ -277,6 +324,7 @@ ujson==5.10.0
     # via fastapi
 urllib3==2.2.2
     # via gradio
+    # via modelscope
     # via requests
 uvicorn==0.30.1
     # via fastapi

requirements.lock

@@ -16,6 +16,11 @@ aiohttp==3.9.5
     # via langchain
 aiosignal==1.3.1
     # via aiohttp
+aliyun-python-sdk-core==2.15.1
+    # via aliyun-python-sdk-kms
+    # via oss2
+aliyun-python-sdk-kms==2.16.3
+    # via oss2
 altair==5.3.0
     # via gradio
 annotated-types==0.7.0
@@ -36,6 +41,8 @@ certifi==2024.7.4
     # via httpcore
     # via httpx
     # via requests
+cffi==1.16.0
+    # via cryptography
 charset-normalizer==3.3.2
     # via requests
 click==8.1.7
@@ -43,6 +50,10 @@ click==8.1.7
     # via uvicorn
 contourpy==1.2.1
     # via matplotlib
+crcmod==1.7
+    # via oss2
+cryptography==42.0.8
+    # via aliyun-python-sdk-core
 cycler==0.12.1
     # via matplotlib
 dashscope==1.20.1
@@ -66,14 +77,16 @@ ffmpy==0.3.2
 filelock==3.15.4
     # via diffusers
     # via huggingface-hub
+    # via torch
 fonttools==4.53.1
     # via matplotlib
 frozenlist==1.4.1
     # via aiohttp
     # via aiosignal
-fsspec==2024.6.1
+fsspec==2024.5.0
     # via gradio-client
     # via huggingface-hub
+    # via torch
 gradio==4.38.1
     # via aitoolkits-webui
 gradio-client==1.1.0
@@ -108,6 +121,9 @@ jinja2==3.1.4
     # via altair
     # via fastapi
     # via gradio
+    # via torch
+jmespath==0.10.0
+    # via aliyun-python-sdk-core
 jsonpatch==1.33
     # via langchain-core
 jsonpointer==3.0.0
@@ -140,9 +156,15 @@ matplotlib==3.9.1
     # via gradio
 mdurl==0.1.2
     # via markdown-it-py
+modelscope==1.16.0
+    # via aitoolkits-webui
+mpmath==1.3.0
+    # via sympy
 multidict==6.0.5
     # via aiohttp
     # via yarl
+networkx==3.3
+    # via torch
 numpy==1.26.4
     # via altair
     # via contourpy
@@ -151,12 +173,15 @@ numpy==1.26.4
     # via langchain
     # via matplotlib
     # via pandas
+    # via torchvision
 openai==1.35.13
     # via langchain-openai
 orjson==3.10.6
     # via fastapi
     # via gradio
     # via langsmith
+oss2==2.18.6
+    # via aitoolkits-webui
 packaging==24.1
     # via altair
     # via gradio
@@ -171,6 +196,11 @@ pillow==10.4.0
     # via diffusers
     # via gradio
     # via matplotlib
+    # via torchvision
+pycparser==2.22
+    # via cffi
+pycryptodome==3.20.0
+    # via oss2
 pydantic==2.8.2
     # via aitoolkits-webui
     # via fastapi
@@ -219,6 +249,8 @@ requests==2.32.3
     # via huggingface-hub
     # via langchain
     # via langsmith
+    # via modelscope
+    # via oss2
     # via tiktoken
 rich==13.7.1
     # via typer
@@ -231,9 +263,12 @@ safetensors==0.4.3
     # via diffusers
 semantic-version==2.10.0
     # via gradio
+setuptools==71.0.0
+    # via aitoolkits-webui
 shellingham==1.5.4
     # via typer
 six==1.16.0
+    # via oss2
     # via python-dateutil
 sniffio==1.3.1
     # via anyio
@@ -243,6 +278,8 @@ sqlalchemy==2.0.31
     # via langchain
 starlette==0.37.2
     # via fastapi
+sympy==1.13.0
+    # via torch
 tenacity==8.5.0
     # via langchain
     # via langchain-core
@@ -252,8 +289,17 @@ tomlkit==0.12.0
     # via gradio
 toolz==0.12.1
     # via altair
+torch==2.3.1
+    # via aitoolkits-webui
+    # via torchaudio
+    # via torchvision
+torchaudio==2.3.1
+    # via aitoolkits-webui
+torchvision==0.18.1
+    # via aitoolkits-webui
 tqdm==4.66.4
     # via huggingface-hub
+    # via modelscope
     # via openai
 typer==0.12.3
     # via fastapi-cli
@@ -269,6 +315,7 @@ typing-extensions==4.12.2
     # via pydantic
     # via pydantic-core
     # via sqlalchemy
+    # via torch
     # via typer
     # via uvicorn
 tzdata==2024.1
@@ -277,6 +324,7 @@ ujson==5.10.0
     # via fastapi
 urllib3==2.2.2
     # via gradio
+    # via modelscope
     # via requests
 uvicorn==0.30.1
     # via fastapi