DYDYLAN commited on
Commit
b86b8a9
·
verified ·
1 Parent(s): 658b686

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +13 -31
app.py CHANGED
@@ -1,23 +1,21 @@
1
- import gradio as gr
2
  from transformers import pipeline
 
3
  from PIL import Image, ImageEnhance
4
  from pypdf import PdfReader
5
 
 
6
 
7
- # ===================== 1. 加载模型(适配 CPU Basic =====================
8
-
9
- # 文档 / 图表问答,用 Donut 做图表理解
10
- chart_qa_pipe = pipeline(
11
- "document-question-answering",
12
- model="naver-clova-ix/donut-base-finetuned-docvqa",
13
  device=-1
14
  )
15
 
16
- # 文本生成学术解释 / caption / 摘要
17
- # 如果觉得慢,可以把 large-ssm 换成 flan-t5-base
18
  text_pipe = pipeline(
19
  "text2text-generation",
20
- model="google/flan-t5-large-ssm",
21
  max_length=512,
22
  device=-1
23
  )
@@ -101,32 +99,16 @@ def summarise_context(snippet: str) -> str:
101
 
102
  # ===================== 3. 图表理解:用 Donut 做 QA =====================
103
 
104
- def describe_chart_with_qa(image: Image.Image) -> str:
105
  """
106
- document-question-answering 管线图表提问
107
- Donut 会尝试读坐标轴 / 图例 / 文本框,从而给出一句话描述。
108
  """
109
- if image is None:
110
- return ""
111
-
112
- question = (
113
- "This is a scientific bar chart. Describe what is on the horizontal axis "
114
- "and vertical axis, what groups or treatments are compared, and the overall trend."
115
- )
116
-
117
  try:
118
- result = chart_qa_pipe(image=image, question=question)
119
- # 有的实现返回 list,有的返回 dict,这里做个兼容
120
- if isinstance(result, list):
121
- answer = result[0].get("answer", "")
122
- else:
123
- answer = result.get("answer", "")
124
- return answer.strip()
125
- except Exception as e:
126
- print("Chart QA error:", e)
127
  return ""
128
 
129
-
130
  # ===================== 4. 核心工作流:ChartSmith v2 =====================
131
 
132
  def analyze_and_explain_v2(image, keywords, language, paper_file):
 
 
1
  from transformers import pipeline
2
+ import gradio as gr
3
  from PIL import Image, ImageEnhance
4
  from pypdf import PdfReader
5
 
6
+ # ===================== 1. 加载模型 =====================
7
 
8
+ # 图像理解:BLIP(不会报 sentencepiece 错,对 CPU 友好
9
+ vision_pipe = pipeline(
10
+ "image-to-text",
11
+ model="salesforce/blip-image-captioning-base",
 
 
12
  device=-1
13
  )
14
 
15
+ # 文本生成学术解释
 
16
  text_pipe = pipeline(
17
  "text2text-generation",
18
+ model="google/flan-t5-base", # 注意:换回 base,更轻量
19
  max_length=512,
20
  device=-1
21
  )
 
99
 
100
  # ===================== 3. 图表理解:用 Donut 做 QA =====================
101
 
102
+ def describe_chart_with_qa(image):
103
  """
104
+ 使BLIP 做图像描述作为图表结构的原始信息
 
105
  """
 
 
 
 
 
 
 
 
106
  try:
107
+ result = vision_pipe(image)[0]["generated_text"]
108
+ return result
109
+ except:
 
 
 
 
 
 
110
  return ""
111
 
 
112
  # ===================== 4. 核心工作流:ChartSmith v2 =====================
113
 
114
  def analyze_and_explain_v2(image, keywords, language, paper_file):