Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -1,68 +1,58 @@
|
|
| 1 |
import os
|
| 2 |
import glob
|
| 3 |
-
import
|
| 4 |
-
|
| 5 |
from sentence_transformers import SentenceTransformer
|
| 6 |
from sklearn.metrics.pairwise import cosine_similarity
|
|
|
|
| 7 |
|
| 8 |
-
|
|
|
|
|
|
|
| 9 |
model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')
|
| 10 |
|
| 11 |
-
#
|
| 12 |
-
LORE_DIR =
|
| 13 |
|
| 14 |
# Параметры нарезки текста
|
| 15 |
-
CHUNK_SIZE =
|
| 16 |
-
CHUNK_OVERLAP = 100 # перекрытие для
|
| 17 |
|
| 18 |
-
# Загружаем и
|
| 19 |
def load_lore_chunks():
|
| 20 |
chunks = []
|
| 21 |
-
|
| 22 |
-
|
| 23 |
-
|
| 24 |
-
|
| 25 |
-
|
| 26 |
-
|
| 27 |
-
text = ''.join(c if 0x20 <= ord(c) <= 0xFFFF else ' ' for c in text)
|
|
|
|
| 28 |
for i in range(0, len(text), CHUNK_SIZE - CHUNK_OVERLAP):
|
| 29 |
-
chunk = text[i:i+CHUNK_SIZE].strip()
|
| 30 |
if chunk:
|
| 31 |
chunks.append(chunk)
|
| 32 |
return chunks
|
| 33 |
|
| 34 |
-
# Загружаем
|
| 35 |
-
print("
|
| 36 |
lore_chunks = load_lore_chunks()
|
| 37 |
-
if not lore_chunks:
|
| 38 |
-
print("⚠️ Внимание: нет данных для поиска.")
|
| 39 |
lore_embeddings = model.encode(lore_chunks)
|
| 40 |
-
print(f"
|
| 41 |
|
| 42 |
-
#
|
| 43 |
def find_best_answer(question):
|
| 44 |
question_embedding = model.encode([question])[0]
|
| 45 |
similarities = cosine_similarity([question_embedding], lore_embeddings)[0]
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
response = "\n\n".join(best_chunks)
|
| 49 |
-
return response
|
| 50 |
-
|
| 51 |
-
# Gradio интерфейс
|
| 52 |
-
with gr.Blocks() as demo:
|
| 53 |
-
gr.Markdown("## 🧛♂️ ЛОР-БОТ: задавай вопросы о мире!")
|
| 54 |
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
["Какие кланы есть у вампиров?"],
|
| 59 |
-
["Чем оборотни отличаются от ликантропов?"],
|
| 60 |
-
["Где находится замок теней?"]
|
| 61 |
-
],
|
| 62 |
-
title="Лор-бот",
|
| 63 |
-
theme="soft"
|
| 64 |
-
)
|
| 65 |
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
|
|
|
|
|
| 1 |
import os
|
| 2 |
import glob
|
| 3 |
+
import uvicorn
|
| 4 |
+
from fastapi import FastAPI
|
| 5 |
from sentence_transformers import SentenceTransformer
|
| 6 |
from sklearn.metrics.pairwise import cosine_similarity
|
| 7 |
+
import numpy as np
|
| 8 |
|
| 9 |
+
app = FastAPI()
|
| 10 |
+
|
| 11 |
+
# Загружаем модель для создания эмбеддингов
|
| 12 |
model = SentenceTransformer('sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2')
|
| 13 |
|
| 14 |
+
# Папка с файлами лора
|
| 15 |
+
LORE_DIR = './lore'
|
| 16 |
|
| 17 |
# Параметры нарезки текста
|
| 18 |
+
CHUNK_SIZE = 1000 # символов
|
| 19 |
+
CHUNK_OVERLAP = 100 # перекрытие кусков для связности текста
|
| 20 |
|
| 21 |
+
# Загружаем и обрабатываем лор
|
| 22 |
def load_lore_chunks():
|
| 23 |
chunks = []
|
| 24 |
+
file_paths = glob.glob(os.path.join(LORE_DIR, '*.txt'))
|
| 25 |
+
|
| 26 |
+
for path in file_paths:
|
| 27 |
+
with open(path, 'r', encoding='utf-8') as f:
|
| 28 |
+
text = f.read()
|
| 29 |
+
# чистим мусорные символы
|
| 30 |
+
text = ''.join(c if 0x20 <= ord(c) <= 0xFFFF else ' ' for c in text)
|
| 31 |
+
# разбиваем на кусочки
|
| 32 |
for i in range(0, len(text), CHUNK_SIZE - CHUNK_OVERLAP):
|
| 33 |
+
chunk = text[i:i + CHUNK_SIZE].strip()
|
| 34 |
if chunk:
|
| 35 |
chunks.append(chunk)
|
| 36 |
return chunks
|
| 37 |
|
| 38 |
+
# Загружаем чанки и строим эмбеддинги
|
| 39 |
+
print("Идёт загрузка файлов...")
|
| 40 |
lore_chunks = load_lore_chunks()
|
|
|
|
|
|
|
| 41 |
lore_embeddings = model.encode(lore_chunks)
|
| 42 |
+
print(f"Загружено {len(lore_chunks)} частей текста.")
|
| 43 |
|
| 44 |
+
# Функция для поиска лучшего ответа
|
| 45 |
def find_best_answer(question):
|
| 46 |
question_embedding = model.encode([question])[0]
|
| 47 |
similarities = cosine_similarity([question_embedding], lore_embeddings)[0]
|
| 48 |
+
best_idx = np.argmax(similarities)
|
| 49 |
+
return lore_chunks[best_idx]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 50 |
|
| 51 |
+
@app.get("/")
|
| 52 |
+
def read_root():
|
| 53 |
+
return {"message": "Добро пожаловать в Лор-Бота!"}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
+
@app.get("/ask/")
|
| 56 |
+
def ask_question(q: str):
|
| 57 |
+
answer = find_best_answer(q)
|
| 58 |
+
return {"question": q, "answer": answer}
|