Create app.py

app.py

@@ -0,0 +1,83 @@
+import nltk
+nltk.download("punkt")
+
+import gradio as gr
+import trafilatura, requests, re
+from markdownify import markdownify as md
+from sumy.parsers.plaintext import PlaintextParser
+from sumy.nlp.tokenizers import Tokenizer
+from sumy.summarizers.text_rank import TextRankSummarizer
+from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline, AutoModelForVision2Seq
+
+MODEL_OPTIONS = {
+    "Qwen2.5-1.5B-Instruct": "Qwen/Qwen2.5-1.5B-Instruct",
+    "CLOVA-Donut-CORDv2": "naver-clova-ix/donut-base-finetuned-cord-v2"
+}
+
+def load_model(model_name):
+    if model_name == MODEL_OPTIONS["CLOVA-Donut-CORDv2"]:
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForVision2Seq.from_pretrained(model_name)
+        return pipeline("image-to-text", model=model, tokenizer=tokenizer)
+    else:
+        tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
+        model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True).to("cpu")
+        return pipeline("text-generation", model=model, tokenizer=tokenizer, device=-1)
+
+def clean_text(text): 
+    return re.sub(r'\s+', ' ', text).strip()
+
+def summarize_text(text):
+    text = clean_text(text)
+    length = len(text)
+    sentence_count = 1 if length < 300 else 2 if length < 800 else 3 if length < 1500 else 4
+    try:
+        parser = PlaintextParser.from_string(text, Tokenizer("korean"))
+        if not parser.document.sentences: raise ValueError
+    except:
+        try:
+            parser = PlaintextParser.from_string(text, Tokenizer("english"))
+            if not parser.document.sentences: raise ValueError
+        except:
+            return re.split(r'(?<=[.!?])\s+', text)[:sentence_count]
+    summarizer = TextRankSummarizer()
+    return [str(s) for s in summarizer(parser.document, sentence_count)]
+
+def rewrite_with_llm(sentences, model_choice):
+    if model_choice == "CLOVA-Donut-CORDv2":
+        return "\n".join(sentences)
+    llm_pipeline = load_model(MODEL_OPTIONS[model_choice])
+    joined_text = "\n".join(sentences)
+    prompt = f"다음 문장을 의미는 유지하되, 원문에 없는 내용은 절대 추가하지 말고 자연스럽게 바꿔주세요.\n\n문장:\n{joined_text}"
+    result = llm_pipeline(prompt, max_new_tokens=150, do_sample=False, temperature=0)
+    return result[0]["generated_text"].replace(prompt, "").strip()
+
+def process_url(url, model_choice):
+    try:
+        r = requests.get(url, headers={"User-Agent": "Mozilla/5.0"}, timeout=10)
+        r.raise_for_status()
+        html_content = trafilatura.extract(r.text, output_format="html", include_tables=False, favor_recall=True)
+        markdown_text = md(html_content or r.text, heading_style="ATX")
+        summary_sentences = summarize_text(markdown_text) or ["요약 없음"]
+        paraphrased_text = rewrite_with_llm(summary_sentences, model_choice)
+        return markdown_text, "\n".join(summary_sentences), paraphrased_text
+    except Exception as e:
+        return f"에러 발생: {e}", "요약 없음", "재작성 없음"
+
+iface = gr.Interface(
+    fn=process_url,
+    inputs=[
+        gr.Textbox(label="URL 입력", placeholder="https://example.com"),
+        gr.Dropdown(choices=list(MODEL_OPTIONS.keys()), value="Qwen2.5-1.5B-Instruct", label="재작성 모델 선택")
+    ],
+    outputs=[
+        gr.Markdown(label="추출된 본문"),
+        gr.Textbox(label="자동 요약", lines=5),
+        gr.Textbox(label="자동 재작성 (LLM)", lines=5)
+    ],
+    title="한국어 본문 추출 + 자동 요약 + LLM 재작성",
+    description="URL에서 본문을 추출하고, 자동 요약 후 선택한 모델로 재작성합니다."
+)
+
+if __name__ == "__main__":
+    iface.launch()