slm / app.py
primenumbersam's picture
feat: increase default max_new_tokens to 384 (max 1024) and enforce sentence completion
3ace22d
Raw History Blame Contribute Delete
16 kB
try:
import spaces
except ImportError:
class spaces:
@staticmethod
def GPU(duration=60):
def decorator(fn):
return fn
return decorator
import os
import time
import gradio as gr
from parsers import url_html, url_pdf, file_md, text_plain
from extractive import lexrank_summary, embedding_summary
from slm import (
ACTIVE_MODEL_KEYS,
CATALOG,
generate_summary,
generate_chat_stream
)
DEFAULT_FED_URL = "https://www.federalreserve.gov/monetarypolicy/fomcminutes20240131.htm"
DEFAULT_BOK_URL = "https://www.bok.or.kr/viewer/skin/doc.html"
# 소스 로드 핸들러
def load_source_text(source_type: str, url: str, file_obj) -> str:
if source_type == "url_html (웹 문서)":
target_url = url.strip() or DEFAULT_FED_URL
try:
return url_html(target_url)
except Exception as e:
return f"HTML 수집 실패 ({target_url}): {e}"
elif source_type == "url_pdf (온라인 PDF)":
target_url = url.strip() or DEFAULT_BOK_URL
try:
return url_pdf(target_url)
except Exception as e:
return f"PDF 수집 실패 ({target_url}): {e}"
elif source_type == "file_md (마크다운)":
if file_obj is None:
return "오류: 마크다운 파일을 업로드하라."
try:
path = file_obj.name if hasattr(file_obj, "name") else file_obj
return file_md(path)
except Exception as e:
return f"마크다운 파싱 실패: {e}"
return ""
# 개별 엔진 단계별 핸들러 (ZeroGPU 이벤트 직결)
def run_lexrank_step(text: str, sent_count: int) -> str:
clean = text.strip()
if not clean:
return "본문 텍스트가 비어 있음."
try:
res, t = lexrank_summary(clean, sentence_count=sent_count)
return f"[⏱️ {t:.1f}ms] LexRank\n\n{res}"
except Exception as e:
return f"[LexRank 오류]\n{e}"
def run_embedding_step(text: str, sent_count: int) -> str:
clean = text.strip()
if not clean:
return "본문 텍스트가 비어 있음."
try:
res, t = embedding_summary(clean, sentence_count=sent_count)
return f"[⏱️ {t:.1f}ms] multilingual-e5 + KMeans\n\n{res}"
except Exception as e:
return f"[Embedding 오류]\n{e}"
@spaces.GPU(duration=10)
def probe_zerogpu() -> str:
import torch
if not torch.cuda.is_available():
return f"❌ GPU 미인식 (torch {torch.__version__}, cuda: {torch.version.cuda})"
try:
dev_name = torch.cuda.get_device_name(0)
mem = torch.cuda.get_device_properties(0).total_memory / (1024**3)
return f"✅ ZeroGPU 정상: {dev_name} ({mem:.1f}GB) | torch {torch.__version__} | CUDA {torch.version.cuda}"
except Exception as e:
return f"⚠️ ZeroGPU 진단 예외: {e}"
@spaces.GPU(duration=60)
def _run_both_slms_gpu(clean: str, max_tokens: int):
m1_key = ACTIVE_MODEL_KEYS[0] if len(ACTIVE_MODEL_KEYS) > 0 else None
m2_key = ACTIVE_MODEL_KEYS[1] if len(ACTIVE_MODEL_KEYS) > 1 else None
m1_name = CATALOG.get(m1_key, {}).get("name", m1_key) if m1_key else "Slot A"
m2_name = CATALOG.get(m2_key, {}).get("name", m2_key) if m2_key else "Slot B"
# Slot A 요약
res1, met1 = generate_summary(m1_key, clean, max_new_tokens=max_tokens, force_cpu=False)
c1 = " (Cached)" if met1.get("cached") else ""
dev1 = met1.get("device", "GPU")
out1 = f"[⏱️ {met1['elapsed_ms']:.1f}ms | {met1['tok_s']:.1f} tok/s | 🖥️ {dev1}{c1}] {m1_name}\n\n{res1}"
# Slot B 요약
res2, met2 = generate_summary(m2_key, clean, max_new_tokens=max_tokens, force_cpu=False)
c2 = " (Cached)" if met2.get("cached") else ""
dev2 = met2.get("device", "GPU")
out2 = f"[⏱️ {met2['elapsed_ms']:.1f}ms | {met2['tok_s']:.1f} tok/s | 🖥️ {dev2}{c2}] {m2_name}\n\n{res2}"
return out1, out2
def _run_both_slms_cpu(clean: str, max_tokens: int, note: str = ""):
m1_key = ACTIVE_MODEL_KEYS[0] if len(ACTIVE_MODEL_KEYS) > 0 else None
m2_key = ACTIVE_MODEL_KEYS[1] if len(ACTIVE_MODEL_KEYS) > 1 else None
m1_name = CATALOG.get(m1_key, {}).get("name", m1_key) if m1_key else "Slot A"
m2_name = CATALOG.get(m2_key, {}).get("name", m2_key) if m2_key else "Slot B"
tag = f" ({note})" if note else ""
res1, met1 = generate_summary(m1_key, clean, max_new_tokens=max_tokens, force_cpu=True)
c1 = " (Cached)" if met1.get("cached") else ""
out1 = f"[⏱️ {met1['elapsed_ms']:.1f}ms | {met1['tok_s']:.1f} tok/s | 🖥️ CPU{tag}{c1}] {m1_name}\n\n{res1}"
res2, met2 = generate_summary(m2_key, clean, max_new_tokens=max_tokens, force_cpu=True)
c2 = " (Cached)" if met2.get("cached") else ""
out2 = f"[⏱️ {met2['elapsed_ms']:.1f}ms | {met2['tok_s']:.1f} tok/s | 🖥️ CPU{tag}{c2}] {m2_name}\n\n{res2}"
return out1, out2
def run_both_slms_step(text: str, max_tokens: int):
clean = text.strip()
if not clean:
return "본문 텍스트가 비어 있음.", "본문 텍스트가 비어 있음."
try:
return _run_both_slms_gpu(clean, max_tokens)
except Exception as e:
print(f"[ZeroGPU GPU 할당 실패 감지, CPU 즉시 폴백] {e}")
return _run_both_slms_cpu(clean, max_tokens, note="GPU 대기열/쿨다운 폴백")
# 채팅 핸들러
def user_post_chat(user_msg, history_a, history_b):
if not user_msg or not str(user_msg).strip():
return "", history_a, history_b
msg = str(user_msg).strip()
new_h_a = list(history_a or []) + [
{"role": "user", "content": msg},
{"role": "assistant", "content": "..."}
]
new_h_b = list(history_b or []) + [
{"role": "user", "content": msg},
{"role": "assistant", "content": "..."}
]
return "", new_h_a, new_h_b
@spaces.GPU(duration=60)
def _bot_stream_gpu(model_key, messages):
yield from generate_chat_stream(model_key, messages, force_cpu=False)
def bot_stream_chat(model_key, history):
if not history:
return
# 순수 대화 기록 전달
messages = []
for turn in history[:-1]:
messages.append({"role": turn["role"], "content": turn["content"]})
history[-1]["content"] = ""
try:
for token_chunk in _bot_stream_gpu(model_key, messages):
history[-1]["content"] = token_chunk
yield history
except Exception as e:
print(f"[채팅 GPU 할당 실패, CPU 폴백 스트리밍] {e}")
for token_chunk in generate_chat_stream(model_key, messages, force_cpu=True):
history[-1]["content"] = token_chunk
yield history
# 레이아웃 정의
custom_css = """
.metric-box { border-radius: 8px; border: 1px solid #e2e8f0; padding: 12px; }
.badge-tag { font-family: monospace; font-size: 0.85rem; }
"""
with gr.Blocks(title="Compare SLMs & Text Summarizers", theme=gr.themes.Monochrome(), css=custom_css) as demo:
gr.Markdown("# ⚡ Compare SLMs & Text Summarizers")
gr.Markdown(
"Hugging Face ZeroGPU 환경에서, Opensource SLMs, Rule-based Summarizers(Sumy), "
"SBERT Summarizer(multilingual-e5)의 요약 품질 및 응답속도를 비교한다."
)
with gr.Accordion("🔍 GPU 할당 및 VRAM 상태 점검", open=False):
with gr.Row():
probe_btn = gr.Button("점검 실행", variant="secondary")
probe_out = gr.Textbox(label="진단 결과", interactive=False)
probe_btn.click(fn=probe_zerogpu, outputs=[probe_out], api_name=False)
with gr.Tabs():
# Tab 1: Side-by-Side SLMs
with gr.Tab("⚔️ Side-by-Side SLMs"):
gr.Markdown("활성화된 오픈소스 SLM 2개에 동일한 질문을 전송하여 답변 품질과 응답 속도를 실시간 스트리밍으로 비교한다.")
with gr.Row():
arena_model_a = gr.Dropdown(
choices=ACTIVE_MODEL_KEYS,
value=ACTIVE_MODEL_KEYS[0] if ACTIVE_MODEL_KEYS else None,
label="Model A 선택"
)
arena_model_b = gr.Dropdown(
choices=ACTIVE_MODEL_KEYS,
value=ACTIVE_MODEL_KEYS[1] if len(ACTIVE_MODEL_KEYS) > 1 else (ACTIVE_MODEL_KEYS[0] if ACTIVE_MODEL_KEYS else None),
label="Model B 선택"
)
with gr.Row():
chatbot_a = gr.Chatbot(label="Model A Output", type="messages", height=450)
chatbot_b = gr.Chatbot(label="Model B Output", type="messages", height=450)
with gr.Row():
chat_input = gr.Textbox(scale=8, placeholder="두 모델에 질문을 입력하라...", show_label=False)
chat_send_btn = gr.Button("전송 ✉️", variant="primary", scale=2)
chat_send_btn.click(
fn=user_post_chat,
inputs=[chat_input, chatbot_a, chatbot_b],
outputs=[chat_input, chatbot_a, chatbot_b],
api_name=False
).then(
fn=bot_stream_chat,
inputs=[arena_model_a, chatbot_a],
outputs=[chatbot_a],
api_name=False
).then(
fn=bot_stream_chat,
inputs=[arena_model_b, chatbot_b],
outputs=[chatbot_b],
api_name=False
)
chat_input.submit(
fn=user_post_chat,
inputs=[chat_input, chatbot_a, chatbot_b],
outputs=[chat_input, chatbot_a, chatbot_b],
api_name=False
).then(
fn=bot_stream_chat,
inputs=[arena_model_a, chatbot_a],
outputs=[chatbot_a],
api_name=False
).then(
fn=bot_stream_chat,
inputs=[arena_model_b, chatbot_b],
outputs=[chatbot_b],
api_name=False
)
# Tab 2: Summarizers
with gr.Tab("📑 Summarizers"):
with gr.Row():
with gr.Column(scale=2):
source_select = gr.Radio(
choices=["text_plain (직접 입력)", "url_html (웹 문서)", "url_pdf (온라인 PDF)", "file_md (마크다운)"],
value="text_plain (직접 입력)",
label="입력 데이터 소스"
)
url_input = gr.Textbox(label="문서 URL", placeholder="https://...", visible=False)
file_input = gr.File(label="마크다운 파일 (.md)", file_types=[".md", ".txt"], visible=False)
load_btn = gr.Button("문서 가져오기 및 파싱", variant="secondary", visible=False)
input_editor = gr.Textbox(
lines=10,
label="본문 텍스트 (Input Text)",
placeholder="요약할 텍스트를 직접 입력하거나 위에서 원본 문서를 가져와라..."
)
with gr.Row():
slider_sent = gr.Slider(minimum=1, maximum=7, value=3, step=1, label="추출 문장 수 (CPU)")
slider_tok = gr.Slider(minimum=64, maximum=1024, value=384, step=32, label="최대 생성 토큰 (SLM)")
with gr.Row():
run_slm_btn = gr.Button("⚡ SLM 요약 단독 실행 (Slot A + B)", variant="primary", scale=2)
run_cpu_btn = gr.Button("🟢 CPU 요약 단독 실행 (LexRank + Embedding)", variant="secondary", scale=2)
run_all_btn = gr.Button("🚀 전체 비교 순차 실행", variant="secondary", scale=2)
def toggle_source_ui(choice):
if choice == "url_html (웹 문서)":
return (
gr.update(visible=True, value=DEFAULT_FED_URL, label="웹 문서 URL (예: FED Minutes HTML)"),
gr.update(visible=False),
gr.update(visible=True)
)
elif choice == "url_pdf (온라인 PDF)":
return (
gr.update(visible=True, value=DEFAULT_BOK_URL, label="온라인 PDF URL (예: 한국은행 의결문 PDF)"),
gr.update(visible=False),
gr.update(visible=True)
)
elif choice == "file_md (마크다운)":
return (
gr.update(visible=False),
gr.update(visible=True),
gr.update(visible=True)
)
return (
gr.update(visible=False),
gr.update(visible=False),
gr.update(visible=False)
)
source_select.change(
fn=toggle_source_ui,
inputs=[source_select],
outputs=[url_input, file_input, load_btn],
api_name=False
)
load_btn.click(
fn=load_source_text,
inputs=[source_select, url_input, file_input],
outputs=[input_editor],
api_name=False
)
gr.Markdown("### 📊 알고리즘별 답변 및 응답속독 결과")
with gr.Row():
with gr.Column():
gr.Markdown("#### 🟢 CPU Bound: Sumy (LexRank)")
box_lex = gr.Textbox(lines=8, label="LexRank 결과", show_label=False)
with gr.Column():
gr.Markdown("#### 🟢 CPU Bound: multilingual-e5 + KMeans")
box_emb = gr.Textbox(lines=8, label="Embedding 군집화 결과", show_label=False)
m1_lbl = CATALOG.get(ACTIVE_MODEL_KEYS[0], {}).get("name", "SLM Slot A") if len(ACTIVE_MODEL_KEYS) > 0 else "SLM Slot A"
m2_lbl = CATALOG.get(ACTIVE_MODEL_KEYS[1], {}).get("name", "SLM Slot B") if len(ACTIVE_MODEL_KEYS) > 1 else "SLM Slot B"
with gr.Row():
with gr.Column():
gr.Markdown(f"#### 🟣 @spaces.GPU Bound: {m1_lbl}")
box_slm1 = gr.Textbox(lines=9, label=m1_lbl, show_label=False)
with gr.Column():
gr.Markdown(f"#### 🟣 @spaces.GPU Bound: {m2_lbl}")
box_slm2 = gr.Textbox(lines=9, label=m2_lbl, show_label=False)
# 1) SLM 단독 실행 (ZeroGPU 쿨다운 최소화 및 단독 호출)
run_slm_btn.click(
fn=run_both_slms_step,
inputs=[input_editor, slider_tok],
outputs=[box_slm1, box_slm2],
api_name=False
)
# 2) CPU 요약 단독 실행
run_cpu_btn.click(
fn=run_lexrank_step,
inputs=[input_editor, slider_sent],
outputs=[box_lex],
api_name=False
).then(
fn=run_embedding_step,
inputs=[input_editor, slider_sent],
outputs=[box_emb],
api_name=False
)
# 3) 전체 순차 비교 실행
run_all_btn.click(
fn=run_lexrank_step,
inputs=[input_editor, slider_sent],
outputs=[box_lex],
api_name=False
).then(
fn=run_embedding_step,
inputs=[input_editor, slider_sent],
outputs=[box_emb],
api_name=False
).then(
fn=run_both_slms_step,
inputs=[input_editor, slider_tok],
outputs=[box_slm1, box_slm2],
api_name=False
)
if __name__ == "__main__":
demo.launch(show_api=False)