Spaces:
Build error
Build error
Deploy chatbot update
Browse files
app.py
CHANGED
|
@@ -1,7 +1,7 @@
|
|
| 1 |
|
| 2 |
import os
|
| 3 |
import pdfplumber
|
| 4 |
-
from huggingface_hub import hf_hub_download
|
| 5 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 6 |
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFaceEndpoint
|
| 7 |
from langchain_community.vectorstores import FAISS
|
|
@@ -10,10 +10,9 @@ import gradio as gr
|
|
| 10 |
|
| 11 |
# 1. ํ๊ฒฝ ๋ณ์ ํ์ธ
|
| 12 |
if "HUGGINGFACEHUB_API_TOKEN" not in os.environ:
|
| 13 |
-
raise ValueError("โ HUGGINGFACEHUB_API_TOKEN ํ๊ฒฝ ๋ณ์๊ฐ
|
| 14 |
-
"HF Space โ Settings โ Secrets์์ ์ถ๊ฐํ์ธ์.")
|
| 15 |
|
| 16 |
-
# 2.
|
| 17 |
repo_id = "meta-llama/Llama-3.2-3B-Instruct"
|
| 18 |
llm = HuggingFaceEndpoint(
|
| 19 |
repo_id=repo_id,
|
|
@@ -22,12 +21,49 @@ llm = HuggingFaceEndpoint(
|
|
| 22 |
task="text-generation"
|
| 23 |
)
|
| 24 |
|
| 25 |
-
# 3. ๋ฐ์ดํฐ์
|
| 26 |
dataset_repo = "dgmos/ericsson-manuals"
|
| 27 |
-
|
| 28 |
-
|
| 29 |
-
|
| 30 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 31 |
|
| 32 |
# 4. PDF โ ํ
์คํธ ์ถ์ถ
|
| 33 |
docs = []
|
|
@@ -44,26 +80,24 @@ for filename in pdf_files:
|
|
| 44 |
".join(texts).strip()
|
| 45 |
if text:
|
| 46 |
docs.append({"page_content": text, "metadata": {"source": filename}})
|
| 47 |
-
print(f"โ
{filename} ์ฒ๋ฆฌ ์๋ฃ (๊ธธ์ด
|
| 48 |
else:
|
| 49 |
print(f"โ ๏ธ ํ
์คํธ ์์: {filename}")
|
| 50 |
except Exception as e:
|
| 51 |
-
print(f"๐จ
|
| 52 |
|
| 53 |
if not docs:
|
| 54 |
-
raise
|
| 55 |
|
| 56 |
# 5. ํ
์คํธ ๋ถํ
|
| 57 |
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
|
| 58 |
texts = splitter.split_documents(docs)
|
| 59 |
|
| 60 |
-
# 6. ์๋ฒ ๋ฉ
|
| 61 |
-
embeddings = HuggingFaceEmbeddings(
|
| 62 |
-
model_name="sentence-transformers/multi-qa-mpnet-base-dot-v1"
|
| 63 |
-
)
|
| 64 |
vectorstore = FAISS.from_documents(texts, embeddings)
|
| 65 |
|
| 66 |
-
# 7.
|
| 67 |
qa_chain = RetrievalQA.from_chain_type(
|
| 68 |
llm=llm,
|
| 69 |
chain_type="stuff",
|
|
@@ -71,7 +105,7 @@ qa_chain = RetrievalQA.from_chain_type(
|
|
| 71 |
)
|
| 72 |
|
| 73 |
# 8. ์ฑ๋ด ํจ์
|
| 74 |
-
def chatbot(query
|
| 75 |
try:
|
| 76 |
response = qa_chain.run(query)
|
| 77 |
return response
|
|
@@ -80,16 +114,11 @@ def chatbot(query: str):
|
|
| 80 |
|
| 81 |
# 9. Gradio UI
|
| 82 |
with gr.Blocks(title="Ericsson ์ฅ๋น ๋ถ์ ์ฑ๋ด") as demo:
|
| 83 |
-
gr.Markdown("# ๐ก Ericsson
|
| 84 |
-
gr.Markdown("
|
| 85 |
-
|
| 86 |
-
query = gr.Textbox(
|
| 87 |
-
label="์ง๋ฌธ ์
๋ ฅ (ํ๊ตญ์ด/์์ด)",
|
| 88 |
-
placeholder="์: 5G ์ค๊ณ๊ธฐ ๋ถ์ํ ๋ฐ์ ์์ธ์?",
|
| 89 |
-
)
|
| 90 |
output = gr.Textbox(label="์๋ต", lines=10)
|
| 91 |
-
btn = gr.Button("๋ถ์
|
| 92 |
-
|
| 93 |
btn.click(chatbot, inputs=query, outputs=output)
|
| 94 |
|
| 95 |
if __name__ == "__main__":
|
|
|
|
| 1 |
|
| 2 |
import os
|
| 3 |
import pdfplumber
|
| 4 |
+
from huggingface_hub import hf_hub_download
|
| 5 |
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 6 |
from langchain_huggingface import HuggingFaceEmbeddings, HuggingFaceEndpoint
|
| 7 |
from langchain_community.vectorstores import FAISS
|
|
|
|
| 10 |
|
| 11 |
# 1. ํ๊ฒฝ ๋ณ์ ํ์ธ
|
| 12 |
if "HUGGINGFACEHUB_API_TOKEN" not in os.environ:
|
| 13 |
+
raise ValueError("โ HUGGINGFACEHUB_API_TOKEN ํ๊ฒฝ ๋ณ์๊ฐ ์ค์ ๋์ง ์์์ต๋๋ค. (Spaces > Settings > Repository secrets)")
|
|
|
|
| 14 |
|
| 15 |
+
# 2. ๋ชจ๋ธ ์ค์
|
| 16 |
repo_id = "meta-llama/Llama-3.2-3B-Instruct"
|
| 17 |
llm = HuggingFaceEndpoint(
|
| 18 |
repo_id=repo_id,
|
|
|
|
| 21 |
task="text-generation"
|
| 22 |
)
|
| 23 |
|
| 24 |
+
# 3. ๋ฐ์ดํฐ์
(PDF ํ์ผ ๋ฆฌ์คํธ)
|
| 25 |
dataset_repo = "dgmos/ericsson-manuals"
|
| 26 |
+
pdf_files = [
|
| 27 |
+
"(20220324) L2 Switch ์ด์ฉ ๋งค๋ด์ผ_Innovation TF_Ver3.1_OCR.pdf",
|
| 28 |
+
"(20230504) 23๋
๊ธฐ์ ๊ต์ก ๊ต์ฌ 1 (LTE)_๊ฐ์นํ์ ํ_OCR.pdf",
|
| 29 |
+
"(20230531) 23๋
๊ธฐ์ ๊ต์ก ๊ต์ฌ 2 (5G)_๊ฐ์นํ์ ํ_OCR.pdf",
|
| 30 |
+
"(20240924) ๋๊ตฌ์ด์ฉ๋ถ TEST BED๋ฅผ ํ์ฉ์ ํตํ ์ค๊ณ๊ธฐ ์ญ๋ํฅ์_OCR.pdf",
|
| 31 |
+
"(7์3์ฃผ)WiFi๊ต์ก(๋ณธ๋ถ ๊ณต์ )_OCR.pdf",
|
| 32 |
+
"1. 5G ๊ธฐ์ง๊ตญ ๊ธฐ์ ํ์ต๊ต์ฌ_Ver4.2_OCR.pdf",
|
| 33 |
+
"1. kt MOS ๋จ๋ถ ๊ธฐ์ (๊ธฐ์ง๊ตญ) ํ์ต๊ต์ฌ_Ver 5.2_OCR.pdf",
|
| 34 |
+
"10G PON ๋ฐ SFP๋ชจ๋ ๊ตฌ๋ถ_OCR.pdf",
|
| 35 |
+
"2. 5G ์ค๊ณ๊ธฐ ๊ธฐ์ ํ์ต๊ต์ฌ_Ver3.6_OCR.pdf",
|
| 36 |
+
"2. kt MOS ๋จ๋ถ ๊ธฐ์ (์ค๊ณ๊ธฐ) ํ์ต๊ต์ฌ_Ver 4.5_OCR.pdf",
|
| 37 |
+
"23๋
๋ฌด์ ์ก์ธ์ค ๊ต์ก ๊ต์(์ค๊ณ๊ธฐ์์ ์ ๋ณต_์ฅ๋น์ด์ฉ๊ธฐ์ค)_OCR.pdf",
|
| 38 |
+
"3. kt MOS ๋จ๋ถ ๋คํธ์ํฌ ํ์ต๊ต์ฌ_Ver 4.7_OCR.pdf",
|
| 39 |
+
"3. ์ฌ๋์์ ํต์ ๋ง ๊ธฐ์ ํ์ต๊ต์ฌ_Ver3.1_OCR.pdf",
|
| 40 |
+
"5G๊ด์ค๊ณ๊ธฐ ์ด์ฉ ๊ถ๊ณ ์V1.0(20200615)_OCR.pdf",
|
| 41 |
+
"Barcode System WEB ์ฌ์ฉ์ ๋งค๋ด์ผ(์๋ณธ)_์์ _OCR.pdf",
|
| 42 |
+
"ELMO(์ ๋ฆฌํ
ํฌ)_OCR.pdf",
|
| 43 |
+
"ELORHA ๊ต์ก์๋ฃ_OCR.pdf",
|
| 44 |
+
"JD745B ๊ฐํธ ์ฌ์ฉ์ ๋งค๋ด์ผ-Viavi (Rev1.0)_OCR.pdf",
|
| 45 |
+
"KELIS(์ ๋ฆฌํ
ํฌ)_OCR.pdf",
|
| 46 |
+
"MMF ์ฌ์
์ฅ ๊ด๋ ๋ฒจ ์ ๋น ์๋ฃ)_OCR.pdf",
|
| 47 |
+
"MS2090A ์ฌ์ฉ์ ๋งค๋ด์ผ_9.4.1(20200612)Trigger_OCR.pdf",
|
| 48 |
+
"Test bed.(์ค๊ณ๊ธฐ ๊ต์)pptx_OCR.pdf",
|
| 49 |
+
"V2824_IMN_KO_080718_OCR.pdf",
|
| 50 |
+
"V2824_UMN_1.03_KO_081217_OCR.pdf",
|
| 51 |
+
"[IP-A]ONE-MUX๊ธฐ์ ๋ฐฉ์ ๋ฐ ํ์ฅ ์ ๋ก ๊ณ ์ฅ์ ๊ฒ Guide_OCR.pdf",
|
| 52 |
+
"[ktMOS๋จ๋ถ]_24๋
(์ ์ ๋ถ์ผ) 1 IP ๋คํธ์ํฌ KT ๋ง ๊ตฌ์กฐ ์ดํด_OCR.pdf",
|
| 53 |
+
"[ktMOS๋จ๋ถ]_24๋
(์ ์ ๋ถ์ผ)_2 IP Access L2์ด์ฉ ๋ฐ ๊ธฐ์ ๋ฐฉ์_OCR.pdf",
|
| 54 |
+
"[ktMOS๋จ๋ถ]_24๋
(์ ์ ๋ถ์ผ)_3 OLT ์ด์ฉ ๋ฐ ๊ธฐ์ ๋ฐฉ์_OCR.pdf",
|
| 55 |
+
"[๊ตฌํ ์๋กํ] ์ ํ์๊ฐ์_OCR.pdf",
|
| 56 |
+
"[์ ํ ์๋กํ] ์ ํ์๊ฐ์_OCR.pdf",
|
| 57 |
+
"[ํ์ฅ2] IP ์ก์ธ์ค ์ฝ์์ ์ ๋ฐ G.EasyOne(์ฅ๋น์ด๊ธฐํ)_KT MOS_๋จ๋ถ_OCR.pdf",
|
| 58 |
+
"[ํ์ฅ3] IP ์ก์ธ์ค A-SDN (์ฅ๋น๊ต์ฒด)_KT MOS_๋จ๋ถ_OCR.pdf",
|
| 59 |
+
"[ํ์ฅ5] IP ์ก์ธ์ค ๊ธด๊ธ๋ณต๊ตฌ(๊ณ ์ฅ์ฌ๋ก)_KT MOS_๋จ๋ถ_OCR.pdf",
|
| 60 |
+
"korea_mobile_frequencies.pdf",
|
| 61 |
+
"๋ค์ฐ ์ค์ฉ๋ OLT V5832XG ๊ต์ก์๋ฃ_OCR.pdf",
|
| 62 |
+
"๋๊ตฌ๋ณธ๋ถ_์ง๊ตฌ๋ ฅ๊ฐํ๊ต์ก_3์ ๊ณ์ธก๊ธฐํ์ฉ๋ฐฉ๋ฒ_OCR.pdf",
|
| 63 |
+
"๋๊ณ_์ญ๋๊ฐํ_๊ต์ก_ํ์ต๊ต์ฌ(์ค๋ฌด๊ณผ์ _์ค๊ณ๊ธฐ_๊ณ์ธก๊ธฐ)_Ver3.0_OCR.pdf",
|
| 64 |
+
"์ค๋ฌด์ ๊ต์ก(10์3์ฃผ)_์ค์ฉ๋ OLT(๊น๋ช
์ค)_OCR.pdf",
|
| 65 |
+
"์ฐจ๋จ๊ธฐ ์ข
๋ฅ ๋ฐ ์ฉ๋_OCR.pdf"
|
| 66 |
+
]
|
| 67 |
|
| 68 |
# 4. PDF โ ํ
์คํธ ์ถ์ถ
|
| 69 |
docs = []
|
|
|
|
| 80 |
".join(texts).strip()
|
| 81 |
if text:
|
| 82 |
docs.append({"page_content": text, "metadata": {"source": filename}})
|
| 83 |
+
print(f"โ
{filename} ์ฒ๋ฆฌ ์๋ฃ (๊ธธ์ด {len(text)}์)")
|
| 84 |
else:
|
| 85 |
print(f"โ ๏ธ ํ
์คํธ ์์: {filename}")
|
| 86 |
except Exception as e:
|
| 87 |
+
print(f"๐จ ์ค๋ฅ ๋ฐ์: {filename} - {e}")
|
| 88 |
|
| 89 |
if not docs:
|
| 90 |
+
raise ValueError("โ PDF์์ ์ถ์ถ๋ ํ
์คํธ๊ฐ ์์ต๋๋ค. (docs ๋ฆฌ์คํธ ๋น์ด์์)")
|
| 91 |
|
| 92 |
# 5. ํ
์คํธ ๋ถํ
|
| 93 |
splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=50)
|
| 94 |
texts = splitter.split_documents(docs)
|
| 95 |
|
| 96 |
+
# 6. ์๋ฒ ๋ฉ ๋ฐ ๋ฒกํฐ DB
|
| 97 |
+
embeddings = HuggingFaceEmbeddings(model_name="sentence-transformers/multi-qa-mpnet-base-dot-v1")
|
|
|
|
|
|
|
| 98 |
vectorstore = FAISS.from_documents(texts, embeddings)
|
| 99 |
|
| 100 |
+
# 7. RAG ์ฒด์ธ
|
| 101 |
qa_chain = RetrievalQA.from_chain_type(
|
| 102 |
llm=llm,
|
| 103 |
chain_type="stuff",
|
|
|
|
| 105 |
)
|
| 106 |
|
| 107 |
# 8. ์ฑ๋ด ํจ์
|
| 108 |
+
def chatbot(query):
|
| 109 |
try:
|
| 110 |
response = qa_chain.run(query)
|
| 111 |
return response
|
|
|
|
| 114 |
|
| 115 |
# 9. Gradio UI
|
| 116 |
with gr.Blocks(title="Ericsson ์ฅ๋น ๋ถ์ ์ฑ๋ด") as demo:
|
| 117 |
+
gr.Markdown("# ๐ก Ericsson ์ฅ๋น ๋งค๋ด์ผ ๊ธฐ๋ฐ ๋ถ๋/๋ถ์ํ ๋ถ์ ์ฑ๋ด")
|
| 118 |
+
gr.Markdown("์
๋ก๋๋ PDF ๋งค๋ด์ผ ๊ธฐ๋ฐ์ผ๋ก ์ง๋ฌธ์ ์
๋ ฅํ์ธ์.")
|
| 119 |
+
query = gr.Textbox(label="์ง๋ฌธ ์
๋ ฅ", placeholder="์: Spurious Emission ์์ธ์?", lines=2)
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
output = gr.Textbox(label="์๋ต", lines=10)
|
| 121 |
+
btn = gr.Button("๋ถ์ ์คํ")
|
|
|
|
| 122 |
btn.click(chatbot, inputs=query, outputs=output)
|
| 123 |
|
| 124 |
if __name__ == "__main__":
|