Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -5,134 +5,97 @@ from pypdf import PdfReader
|
|
| 5 |
|
| 6 |
# ===================== 1. 加载模型(CPU + flan-t5-base) =====================
|
| 7 |
|
| 8 |
-
# 图像 -> 文本说明:图像描述模型
|
| 9 |
vision_pipe = pipeline(
|
| 10 |
"image-to-text",
|
| 11 |
model="microsoft/git-base",
|
| 12 |
-
device=-1
|
| 13 |
)
|
| 14 |
|
| 15 |
-
# 文本改写 / 生成:flan-t5-base,带防重复参数
|
| 16 |
text_pipe = pipeline(
|
| 17 |
"text2text-generation",
|
| 18 |
model="google/flan-t5-base",
|
| 19 |
max_length=256,
|
| 20 |
num_beams=4,
|
| 21 |
-
no_repeat_ngram_size=3,
|
| 22 |
-
device=-1
|
| 23 |
)
|
| 24 |
|
| 25 |
-
|
| 26 |
-
# ===================== 2. 从论文 PDF 提取上下文 =====================
|
| 27 |
|
| 28 |
def extract_paper_context(paper_file):
|
| 29 |
-
"""
|
| 30 |
-
paper_file: gr.File 传进来的对象,一般是临时文件路径(str)
|
| 31 |
-
返回:截断后的论文文本字符串
|
| 32 |
-
"""
|
| 33 |
if paper_file is None:
|
| 34 |
return ""
|
| 35 |
-
|
| 36 |
try:
|
| 37 |
-
# gradio 通常会传一个路径字符串
|
| 38 |
path = paper_file if isinstance(paper_file, str) else paper_file.name
|
| 39 |
reader = PdfReader(path)
|
| 40 |
text = ""
|
| 41 |
-
# 只取前几页,避免太长
|
| 42 |
for i, page in enumerate(reader.pages[:3]):
|
| 43 |
page_text = page.extract_text() or ""
|
| 44 |
text += page_text + "\n"
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
return text
|
| 48 |
-
except Exception as e:
|
| 49 |
-
print("Error reading PDF:", e)
|
| 50 |
return ""
|
| 51 |
|
| 52 |
-
|
| 53 |
def summarize_paper_context(raw_text):
|
| 54 |
-
"""
|
| 55 |
-
用 flan-t5-base 把论文原文压缩成跟图相关的上下文总结
|
| 56 |
-
"""
|
| 57 |
if not raw_text:
|
| 58 |
return ""
|
| 59 |
-
|
| 60 |
prompt = f"""
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
Focus on:
|
| 64 |
-
- the research question or hypothesis,
|
| 65 |
-
- the main variables or conditions,
|
| 66 |
-
- what is being compared or measured.
|
| 67 |
|
| 68 |
Paper text:
|
| 69 |
{raw_text}
|
| 70 |
|
| 71 |
-
|
| 72 |
"""
|
| 73 |
-
|
| 74 |
-
return summary
|
| 75 |
|
| 76 |
|
| 77 |
-
# ===================== 3. 核心工作流
|
| 78 |
|
| 79 |
def analyze_and_enhance(image, style, language, paper_file):
|
| 80 |
if image is None:
|
| 81 |
return None, "请先上传图表。", "", None, "", ""
|
| 82 |
|
| 83 |
-
#
|
| 84 |
paper_raw = extract_paper_context(paper_file)
|
| 85 |
paper_context = summarize_paper_context(paper_raw) if paper_raw else ""
|
| 86 |
|
| 87 |
-
#
|
| 88 |
raw_caption = vision_pipe(image)[0]["generated_text"]
|
| 89 |
-
|
| 90 |
-
# 如果 vision 没认出是图表,就用一个安全的默认描述,避免 T5 乱写
|
| 91 |
rc = raw_caption.lower()
|
| 92 |
if not any(k in rc for k in ["bar", "chart", "graph", "plot", "curve", "line"]):
|
| 93 |
raw_caption = "a bar chart comparing multiple experimental groups under different conditions"
|
| 94 |
|
| 95 |
-
#
|
| 96 |
if paper_context:
|
| 97 |
-
context_part = f""
|
| 98 |
-
Here is the relevant context from the paper that may help you interpret the figure:
|
| 99 |
-
|
| 100 |
-
{paper_context}
|
| 101 |
-
"""
|
| 102 |
else:
|
| 103 |
-
context_part = "
|
| 104 |
|
| 105 |
prompt_academic = f"""
|
| 106 |
You are writing the Results section of a scientific paper.
|
| 107 |
|
| 108 |
{context_part}
|
| 109 |
|
| 110 |
-
|
| 111 |
-
description of the figure in 3–4 sentences. Follow this structure:
|
| 112 |
|
| 113 |
-
1. Identify the
|
| 114 |
-
2. Describe
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
4. Describe the overall trend (increase, decrease, or differences between groups).
|
| 119 |
-
5. Summarize the main comparison or key pattern, making sure it is consistent
|
| 120 |
-
with the paper context above.
|
| 121 |
|
| 122 |
-
Do NOT repeat
|
| 123 |
-
Keep it factual, generic, and neutral.
|
| 124 |
|
| 125 |
Figure description: {raw_caption}
|
| 126 |
"""
|
| 127 |
academic_desc = text_pipe(prompt_academic)[0]["generated_text"]
|
| 128 |
|
| 129 |
-
#
|
| 130 |
caption_prompt = f"""
|
| 131 |
-
Write a concise 1–2 sentence figure caption
|
| 132 |
-
|
| 133 |
-
The caption should mention the type of figure and highlight the main comparison
|
| 134 |
-
or trend, and it should be consistent with the paper context above.
|
| 135 |
-
Do not include fabricated numbers.
|
| 136 |
|
| 137 |
Description: {academic_desc}
|
| 138 |
|
|
@@ -140,12 +103,10 @@ Caption:
|
|
| 140 |
"""
|
| 141 |
caption_text = text_pipe(caption_prompt)[0]["generated_text"]
|
| 142 |
|
| 143 |
-
#
|
| 144 |
summary_prompt = f"""
|
| 145 |
-
Write a
|
| 146 |
-
|
| 147 |
-
the figure and rely only on the description below and the paper context.
|
| 148 |
-
Do not repeat sentences or invent exact numerical values.
|
| 149 |
|
| 150 |
Description: {academic_desc}
|
| 151 |
|
|
@@ -153,53 +114,68 @@ Paragraph:
|
|
| 153 |
"""
|
| 154 |
summary_text = text_pipe(summary_prompt)[0]["generated_text"]
|
| 155 |
|
| 156 |
-
#
|
| 157 |
if language == "中文":
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
)[0]["generated_text"]
|
| 161 |
-
caption_text = text_pipe(
|
| 162 |
-
f"Translate the following figure caption into Chinese:\n{caption_text}"
|
| 163 |
-
)[0]["generated_text"]
|
| 164 |
-
summary_text = text_pipe(
|
| 165 |
-
f"Translate the following paragraph into Chinese:\n{summary_text}"
|
| 166 |
-
)[0]["generated_text"]
|
| 167 |
-
else:
|
| 168 |
-
academic_desc_out = academic_desc
|
| 169 |
|
| 170 |
-
#
|
| 171 |
img_rgb = image.convert("RGB")
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
enhancer_b = ImageEnhance.Brightness(img_c)
|
| 175 |
-
enhanced_image = enhancer_b.enhance(1.1) # 略微提亮
|
| 176 |
|
| 177 |
-
return image, raw_caption,
|
| 178 |
|
| 179 |
|
| 180 |
-
# ===================== 4. Gradio
|
| 181 |
|
| 182 |
-
with gr.Blocks(
|
| 183 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 184 |
|
| 185 |
with gr.Row():
|
| 186 |
-
with gr.Column():
|
| 187 |
-
img_in = gr.Image(type="pil", label="上传你的学术图表
|
| 188 |
-
paper_file = gr.File(label="上传
|
| 189 |
-
style = gr.Dropdown(
|
| 190 |
-
["Formal academic", "Infographic", "Magazine-style"],
|
| 191 |
-
value="Formal academic",
|
| 192 |
-
label="重绘风格(当前版本仅用于说明,不改变生成)"
|
| 193 |
-
)
|
| 194 |
language = gr.Radio(["English", "中文"], value="English", label="输出语言")
|
| 195 |
-
btn = gr.Button("分析并美化图表")
|
| 196 |
-
|
|
|
|
| 197 |
orig_img = gr.Image(label="原始图表")
|
| 198 |
-
|
| 199 |
-
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 203 |
|
| 204 |
btn.click(
|
| 205 |
analyze_and_enhance,
|
|
|
|
| 5 |
|
| 6 |
# ===================== 1. 加载模型(CPU + flan-t5-base) =====================
|
| 7 |
|
|
|
|
| 8 |
vision_pipe = pipeline(
|
| 9 |
"image-to-text",
|
| 10 |
model="microsoft/git-base",
|
| 11 |
+
device=-1
|
| 12 |
)
|
| 13 |
|
|
|
|
| 14 |
text_pipe = pipeline(
|
| 15 |
"text2text-generation",
|
| 16 |
model="google/flan-t5-base",
|
| 17 |
max_length=256,
|
| 18 |
num_beams=4,
|
| 19 |
+
no_repeat_ngram_size=3,
|
| 20 |
+
device=-1
|
| 21 |
)
|
| 22 |
|
| 23 |
+
# ===================== 2. 论文 PDF 上下文处理 =====================
|
|
|
|
| 24 |
|
| 25 |
def extract_paper_context(paper_file):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
if paper_file is None:
|
| 27 |
return ""
|
|
|
|
| 28 |
try:
|
|
|
|
| 29 |
path = paper_file if isinstance(paper_file, str) else paper_file.name
|
| 30 |
reader = PdfReader(path)
|
| 31 |
text = ""
|
|
|
|
| 32 |
for i, page in enumerate(reader.pages[:3]):
|
| 33 |
page_text = page.extract_text() or ""
|
| 34 |
text += page_text + "\n"
|
| 35 |
+
return text.strip()[:4000]
|
| 36 |
+
except:
|
|
|
|
|
|
|
|
|
|
| 37 |
return ""
|
| 38 |
|
|
|
|
| 39 |
def summarize_paper_context(raw_text):
|
|
|
|
|
|
|
|
|
|
| 40 |
if not raw_text:
|
| 41 |
return ""
|
|
|
|
| 42 |
prompt = f"""
|
| 43 |
+
Summarize in 4-6 sentences the research background most relevant for understanding
|
| 44 |
+
a result figure. Focus on variables, comparisons, and experimental conditions.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 45 |
|
| 46 |
Paper text:
|
| 47 |
{raw_text}
|
| 48 |
|
| 49 |
+
Summary:
|
| 50 |
"""
|
| 51 |
+
return text_pipe(prompt)[0]["generated_text"]
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
+
# ===================== 3. 核心工作流 =====================
|
| 55 |
|
| 56 |
def analyze_and_enhance(image, style, language, paper_file):
|
| 57 |
if image is None:
|
| 58 |
return None, "请先上传图表。", "", None, "", ""
|
| 59 |
|
| 60 |
+
# 论文上下文
|
| 61 |
paper_raw = extract_paper_context(paper_file)
|
| 62 |
paper_context = summarize_paper_context(paper_raw) if paper_raw else ""
|
| 63 |
|
| 64 |
+
# 图像描述
|
| 65 |
raw_caption = vision_pipe(image)[0]["generated_text"]
|
|
|
|
|
|
|
| 66 |
rc = raw_caption.lower()
|
| 67 |
if not any(k in rc for k in ["bar", "chart", "graph", "plot", "curve", "line"]):
|
| 68 |
raw_caption = "a bar chart comparing multiple experimental groups under different conditions"
|
| 69 |
|
| 70 |
+
# Academic explanation
|
| 71 |
if paper_context:
|
| 72 |
+
context_part = f"Here is relevant context from the paper:\n{paper_context}\n"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 73 |
else:
|
| 74 |
+
context_part = "No additional paper context available.\n"
|
| 75 |
|
| 76 |
prompt_academic = f"""
|
| 77 |
You are writing the Results section of a scientific paper.
|
| 78 |
|
| 79 |
{context_part}
|
| 80 |
|
| 81 |
+
Write a structured academic explanation (3–4 sentences) of the figure:
|
|
|
|
| 82 |
|
| 83 |
+
1. Identify the figure type.
|
| 84 |
+
2. Describe generically what is on the horizontal axis.
|
| 85 |
+
3. Describe generically what is on the vertical axis.
|
| 86 |
+
4. Describe overall trends.
|
| 87 |
+
5. Summarize the main comparison, consistent with the paper context.
|
|
|
|
|
|
|
|
|
|
| 88 |
|
| 89 |
+
Do NOT repeat sentences or invent specific numbers.
|
|
|
|
| 90 |
|
| 91 |
Figure description: {raw_caption}
|
| 92 |
"""
|
| 93 |
academic_desc = text_pipe(prompt_academic)[0]["generated_text"]
|
| 94 |
|
| 95 |
+
# Caption
|
| 96 |
caption_prompt = f"""
|
| 97 |
+
Write a concise 1–2 sentence academic figure caption.
|
| 98 |
+
Keep it consistent with the paper context.
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
Description: {academic_desc}
|
| 101 |
|
|
|
|
| 103 |
"""
|
| 104 |
caption_text = text_pipe(caption_prompt)[0]["generated_text"]
|
| 105 |
|
| 106 |
+
# Summary paragraph
|
| 107 |
summary_prompt = f"""
|
| 108 |
+
Write a 3–4 sentence paragraph explaining the key message of this figure,
|
| 109 |
+
consistent with the paper context. Do not fabricate numbers.
|
|
|
|
|
|
|
| 110 |
|
| 111 |
Description: {academic_desc}
|
| 112 |
|
|
|
|
| 114 |
"""
|
| 115 |
summary_text = text_pipe(summary_prompt)[0]["generated_text"]
|
| 116 |
|
| 117 |
+
# Language translation
|
| 118 |
if language == "中文":
|
| 119 |
+
academic_desc = text_pipe(f"Translate into Chinese:\n{academic_desc}")[0]["generated_text"]
|
| 120 |
+
caption_text = text_pipe(f"Translate into Chinese:\n{caption_text}")[0]["generated_text"]
|
| 121 |
+
summary_text = text_pipe(f"Translate into Chinese:\n{summary_text}")[0]["generated_text"]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 122 |
|
| 123 |
+
# 图像增强
|
| 124 |
img_rgb = image.convert("RGB")
|
| 125 |
+
img_c = ImageEnhance.Contrast(img_rgb).enhance(1.4)
|
| 126 |
+
enhanced_image = ImageEnhance.Brightness(img_c).enhance(1.1)
|
|
|
|
|
|
|
| 127 |
|
| 128 |
+
return image, raw_caption, academic_desc, enhanced_image, caption_text, summary_text
|
| 129 |
|
| 130 |
|
| 131 |
+
# ===================== 4. Gradio UI(宽文本框 + 全宽布局) =====================
|
| 132 |
|
| 133 |
+
with gr.Blocks(css="""
|
| 134 |
+
.wide_textbox textarea {
|
| 135 |
+
font-size: 15px !important;
|
| 136 |
+
line-height: 1.5 !important;
|
| 137 |
+
width: 100% !important;
|
| 138 |
+
min-height: 140px !important;
|
| 139 |
+
}
|
| 140 |
+
""") as demo:
|
| 141 |
+
|
| 142 |
+
gr.Markdown("# **ChartSmith – AI 论文图表生成助手(支持上传论文 PDF)**")
|
| 143 |
|
| 144 |
with gr.Row():
|
| 145 |
+
with gr.Column(scale=1):
|
| 146 |
+
img_in = gr.Image(type="pil", label="上传你的学术图表")
|
| 147 |
+
paper_file = gr.File(label="上传论文 PDF(可选)", file_types=[".pdf"])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
language = gr.Radio(["English", "中文"], value="English", label="输出语言")
|
| 149 |
+
btn = gr.Button("分析并美化图表", variant="primary")
|
| 150 |
+
|
| 151 |
+
with gr.Column(scale=1.2):
|
| 152 |
orig_img = gr.Image(label="原始图表")
|
| 153 |
+
|
| 154 |
+
raw_caption_box = gr.Textbox(
|
| 155 |
+
label="Step 2: 初步自动描述(Vision-LLM)",
|
| 156 |
+
lines=4,
|
| 157 |
+
elem_classes=["wide_textbox"]
|
| 158 |
+
)
|
| 159 |
+
|
| 160 |
+
academic_box = gr.Textbox(
|
| 161 |
+
label="Step 3: 学术化解释(结合论文上下文)",
|
| 162 |
+
lines=6,
|
| 163 |
+
elem_classes=["wide_textbox"]
|
| 164 |
+
)
|
| 165 |
+
|
| 166 |
+
enhanced_img = gr.Image(label="Step 4: 增强后图示(对比度+亮度)")
|
| 167 |
+
|
| 168 |
+
caption_box = gr.Textbox(
|
| 169 |
+
label="Step 5: 自动生成图注(Caption)",
|
| 170 |
+
lines=4,
|
| 171 |
+
elem_classes=["wide_textbox"]
|
| 172 |
+
)
|
| 173 |
+
|
| 174 |
+
summary_box = gr.Textbox(
|
| 175 |
+
label="Step 5: 图表摘要 / 讨论",
|
| 176 |
+
lines=6,
|
| 177 |
+
elem_classes=["wide_textbox"]
|
| 178 |
+
)
|
| 179 |
|
| 180 |
btn.click(
|
| 181 |
analyze_and_enhance,
|