Spaces:
Sleeping
Sleeping
Update app.py
Browse files
app.py
CHANGED
|
@@ -10,18 +10,18 @@ from langchain_google_genai import ChatGoogleGenerativeAI
|
|
| 10 |
from huggingface_hub import snapshot_download
|
| 11 |
|
| 12 |
# ==================================================
|
| 13 |
-
# 設定
|
| 14 |
# ==================================================
|
| 15 |
|
| 16 |
PAPER_DIR = "papers"
|
| 17 |
-
DB_DIR = "/tmp/chroma_db" #
|
| 18 |
NOTE_FILE = "research_notes.txt"
|
| 19 |
|
| 20 |
os.makedirs(PAPER_DIR, exist_ok=True)
|
| 21 |
|
| 22 |
|
| 23 |
# ==================================================
|
| 24 |
-
# Gemini /
|
| 25 |
# ==================================================
|
| 26 |
|
| 27 |
API_KEY = os.getenv("GEMINI_API_KEY")
|
|
@@ -31,17 +31,19 @@ if API_KEY is None:
|
|
| 31 |
"GEMINI_API_KEY が設定されていません。環境変数を確認してください。"
|
| 32 |
)
|
| 33 |
|
|
|
|
| 34 |
embedding_model = HuggingFaceEmbeddings(
|
| 35 |
model="sentence-transformers/all-MiniLM-L6-v2"
|
| 36 |
)
|
| 37 |
|
|
|
|
| 38 |
llm = ChatGoogleGenerativeAI(
|
| 39 |
model="gemini-2.5-flash",
|
| 40 |
temperature=0.3
|
| 41 |
)
|
| 42 |
|
| 43 |
# ==================================================
|
| 44 |
-
# PDF → DB 登録
|
| 45 |
# ==================================================
|
| 46 |
|
| 47 |
def rebuild_database():
|
|
@@ -65,8 +67,9 @@ def rebuild_database():
|
|
| 65 |
print(f"PDF読み込みエラー ({pdf}): {e}")
|
| 66 |
|
| 67 |
if not all_docs:
|
| 68 |
-
return "読み込み可能なPDFテキストがありませんでした
|
| 69 |
|
|
|
|
| 70 |
splitter = RecursiveCharacterTextSplitter(
|
| 71 |
chunk_size=1000,
|
| 72 |
chunk_overlap=200
|
|
@@ -74,14 +77,14 @@ def rebuild_database():
|
|
| 74 |
|
| 75 |
chunks = splitter.split_documents(all_docs)
|
| 76 |
|
| 77 |
-
# 既存のDBディレクトリ
|
| 78 |
if os.path.exists(DB_DIR):
|
| 79 |
try:
|
| 80 |
shutil.rmtree(DB_DIR)
|
| 81 |
except Exception as e:
|
| 82 |
-
print(f"DB削除
|
| 83 |
|
| 84 |
-
#
|
| 85 |
Chroma.from_documents(
|
| 86 |
documents=chunks,
|
| 87 |
embedding=embedding_model,
|
|
@@ -98,6 +101,7 @@ def rebuild_database():
|
|
| 98 |
def get_db():
|
| 99 |
if not os.path.exists(DB_DIR):
|
| 100 |
return None
|
|
|
|
| 101 |
return Chroma(
|
| 102 |
persist_directory=DB_DIR,
|
| 103 |
embedding_function=embedding_model
|
|
@@ -105,7 +109,7 @@ def get_db():
|
|
| 105 |
|
| 106 |
|
| 107 |
# ==================================================
|
| 108 |
-
# PDF追加
|
| 109 |
# ==================================================
|
| 110 |
|
| 111 |
def upload_pdfs(files):
|
|
@@ -118,10 +122,15 @@ def upload_pdfs(files):
|
|
| 118 |
PAPER_DIR,
|
| 119 |
os.path.basename(file.name)
|
| 120 |
)
|
| 121 |
-
|
| 122 |
-
|
|
|
|
|
|
|
|
|
|
| 123 |
|
|
|
|
| 124 |
msg = rebuild_database()
|
|
|
|
| 125 |
return f"{count}件アップロード\n{msg}"
|
| 126 |
|
| 127 |
|
|
@@ -131,9 +140,11 @@ def upload_pdfs(files):
|
|
| 131 |
|
| 132 |
def search_method(question):
|
| 133 |
db = get_db()
|
|
|
|
| 134 |
if db is None:
|
| 135 |
return "PDFを登録してください"
|
| 136 |
|
|
|
|
| 137 |
docs = db.similarity_search(question, k=5)
|
| 138 |
context = "\n\n".join([d.page_content for d in docs])
|
| 139 |
|
|
@@ -152,15 +163,16 @@ def search_method(question):
|
|
| 152 |
|
| 153 |
【回答】
|
| 154 |
"""
|
|
|
|
| 155 |
try:
|
| 156 |
response = llm.invoke(prompt)
|
| 157 |
return response.content
|
| 158 |
except Exception as e:
|
| 159 |
-
return f"Gemini
|
| 160 |
|
| 161 |
|
| 162 |
# ==================================================
|
| 163 |
-
# データ
|
| 164 |
# ==================================================
|
| 165 |
|
| 166 |
try:
|
|
@@ -169,9 +181,9 @@ try:
|
|
| 169 |
repo_type="dataset",
|
| 170 |
local_dir=PAPER_DIR
|
| 171 |
)
|
| 172 |
-
print("データセットのダウンロードに成功しました
|
| 173 |
except Exception as e:
|
| 174 |
-
print(f"データセットのダウンロードに失敗しました (手動
|
| 175 |
|
| 176 |
if not os.path.exists(DB_DIR):
|
| 177 |
print(rebuild_database())
|
|
@@ -183,6 +195,7 @@ if not os.path.exists(DB_DIR):
|
|
| 183 |
|
| 184 |
def search_experiment(question):
|
| 185 |
db = get_db()
|
|
|
|
| 186 |
if db is None:
|
| 187 |
return "PDFを登録してください"
|
| 188 |
|
|
@@ -198,7 +211,7 @@ def search_experiment(question):
|
|
| 198 |
・データセット
|
| 199 |
・学習率
|
| 200 |
・Epoch数
|
| 201 |
-
|
| 202 |
・比較手法
|
| 203 |
・実験結果
|
| 204 |
|
|
@@ -210,15 +223,16 @@ def search_experiment(question):
|
|
| 210 |
|
| 211 |
【回答】
|
| 212 |
"""
|
|
|
|
| 213 |
try:
|
| 214 |
response = llm.invoke(prompt)
|
| 215 |
return response.content
|
| 216 |
except Exception as e:
|
| 217 |
-
return f"Gemini
|
| 218 |
|
| 219 |
|
| 220 |
# ==================================================
|
| 221 |
-
# 保存
|
| 222 |
# ==================================================
|
| 223 |
|
| 224 |
def save_note(question, answer):
|
|
@@ -229,20 +243,24 @@ def save_note(question, answer):
|
|
| 229 |
|
| 230 |
try:
|
| 231 |
with open(NOTE_FILE, "a", encoding="utf-8") as f:
|
| 232 |
-
f.write("\n"
|
|
|
|
|
|
|
| 233 |
f.write(f"日時: {now}\n\n")
|
| 234 |
f.write(f"質問:\n{question}\n\n")
|
| 235 |
-
f.write(f"回答:\n{answer}\n
|
|
|
|
| 236 |
return "保存完了"
|
| 237 |
except Exception as e:
|
| 238 |
return f"ファイル保存エラー: {e}"
|
| 239 |
|
| 240 |
|
| 241 |
# ==================================================
|
| 242 |
-
# UI (Gradio)
|
| 243 |
# ==================================================
|
| 244 |
|
| 245 |
with gr.Blocks() as demo:
|
|
|
|
| 246 |
gr.Markdown("# 研究支援RAGアプリ")
|
| 247 |
|
| 248 |
# タブ1: 論文追加
|
|
@@ -252,7 +270,9 @@ with gr.Blocks() as demo:
|
|
| 252 |
file_types=[".pdf"],
|
| 253 |
label="論文PDF"
|
| 254 |
)
|
|
|
|
| 255 |
upload_btn = gr.Button("登録")
|
|
|
|
| 256 |
upload_status = gr.Textbox(label="状態")
|
| 257 |
|
| 258 |
upload_btn.click(
|
|
@@ -264,9 +284,16 @@ with gr.Blocks() as demo:
|
|
| 264 |
# タブ2: 手法・専門用語検索
|
| 265 |
with gr.Tab("手法・専門用語検索"):
|
| 266 |
method_question = gr.Textbox(label="質問")
|
| 267 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 268 |
method_btn = gr.Button("検索")
|
| 269 |
-
|
|
|
|
|
|
|
| 270 |
method_status = gr.Textbox(label="保存状態")
|
| 271 |
|
| 272 |
method_btn.click(
|
|
@@ -274,6 +301,7 @@ with gr.Blocks() as demo:
|
|
| 274 |
inputs=method_question,
|
| 275 |
outputs=method_answer
|
| 276 |
)
|
|
|
|
| 277 |
method_save.click(
|
| 278 |
fn=save_note,
|
| 279 |
inputs=[method_question, method_answer],
|
|
@@ -283,20 +311,19 @@ with gr.Blocks() as demo:
|
|
| 283 |
# タブ3: 実験内容検索
|
| 284 |
with gr.Tab("実験内容検索"):
|
| 285 |
exp_question = gr.Textbox(label="質問")
|
| 286 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 287 |
exp_btn = gr.Button("検索")
|
| 288 |
-
|
|
|
|
|
|
|
| 289 |
exp_status = gr.Textbox(label="保存状態")
|
| 290 |
|
| 291 |
exp_btn.click(
|
| 292 |
fn=search_experiment,
|
| 293 |
inputs=exp_question,
|
| 294 |
-
outputs=exp_answer
|
| 295 |
-
)
|
| 296 |
-
exp_save.click(
|
| 297 |
-
fn=save_note,
|
| 298 |
-
inputs=[exp_question, exp_answer],
|
| 299 |
-
outputs=exp_status
|
| 300 |
-
)
|
| 301 |
-
|
| 302 |
-
demo.launch()
|
|
|
|
| 10 |
from huggingface_hub import snapshot_download
|
| 11 |
|
| 12 |
# ==================================================
|
| 13 |
+
# 設定
|
| 14 |
# ==================================================
|
| 15 |
|
| 16 |
PAPER_DIR = "papers"
|
| 17 |
+
DB_DIR = "/tmp/chroma_db" # Read-onlyエラー回避のため、一時ディレクトリを指定
|
| 18 |
NOTE_FILE = "research_notes.txt"
|
| 19 |
|
| 20 |
os.makedirs(PAPER_DIR, exist_ok=True)
|
| 21 |
|
| 22 |
|
| 23 |
# ==================================================
|
| 24 |
+
# Gemini / Embedding モデル設定
|
| 25 |
# ==================================================
|
| 26 |
|
| 27 |
API_KEY = os.getenv("GEMINI_API_KEY")
|
|
|
|
| 31 |
"GEMINI_API_KEY が設定されていません。環境変数を確認してください。"
|
| 32 |
)
|
| 33 |
|
| 34 |
+
# 埋め込みモデルの初期化
|
| 35 |
embedding_model = HuggingFaceEmbeddings(
|
| 36 |
model="sentence-transformers/all-MiniLM-L6-v2"
|
| 37 |
)
|
| 38 |
|
| 39 |
+
# Geminiモデルの初期化 (安定性の高い2.5-flashを採用)
|
| 40 |
llm = ChatGoogleGenerativeAI(
|
| 41 |
model="gemini-2.5-flash",
|
| 42 |
temperature=0.3
|
| 43 |
)
|
| 44 |
|
| 45 |
# ==================================================
|
| 46 |
+
# PDF → DB 登録・再構築
|
| 47 |
# ==================================================
|
| 48 |
|
| 49 |
def rebuild_database():
|
|
|
|
| 67 |
print(f"PDF読み込みエラー ({pdf}): {e}")
|
| 68 |
|
| 69 |
if not all_docs:
|
| 70 |
+
return "読み込み可能なPDFテキストがありませんでした"
|
| 71 |
|
| 72 |
+
# テキストの最適化分割
|
| 73 |
splitter = RecursiveCharacterTextSplitter(
|
| 74 |
chunk_size=1000,
|
| 75 |
chunk_overlap=200
|
|
|
|
| 77 |
|
| 78 |
chunks = splitter.split_documents(all_docs)
|
| 79 |
|
| 80 |
+
# 既存の古いDBディレクトリを安全に削除して再初期化
|
| 81 |
if os.path.exists(DB_DIR):
|
| 82 |
try:
|
| 83 |
shutil.rmtree(DB_DIR)
|
| 84 |
except Exception as e:
|
| 85 |
+
print(f"既存DBの削除に失敗しました: {e}")
|
| 86 |
|
| 87 |
+
# ベクトルDBの構築
|
| 88 |
Chroma.from_documents(
|
| 89 |
documents=chunks,
|
| 90 |
embedding=embedding_model,
|
|
|
|
| 101 |
def get_db():
|
| 102 |
if not os.path.exists(DB_DIR):
|
| 103 |
return None
|
| 104 |
+
|
| 105 |
return Chroma(
|
| 106 |
persist_directory=DB_DIR,
|
| 107 |
embedding_function=embedding_model
|
|
|
|
| 109 |
|
| 110 |
|
| 111 |
# ==================================================
|
| 112 |
+
# PDF追加
|
| 113 |
# ==================================================
|
| 114 |
|
| 115 |
def upload_pdfs(files):
|
|
|
|
| 122 |
PAPER_DIR,
|
| 123 |
os.path.basename(file.name)
|
| 124 |
)
|
| 125 |
+
try:
|
| 126 |
+
shutil.copy(file.name, dst)
|
| 127 |
+
count += 1
|
| 128 |
+
except Exception as e:
|
| 129 |
+
print(f"ファイルコピーエラー: {e}")
|
| 130 |
|
| 131 |
+
# 追加後にデータベースを再構築
|
| 132 |
msg = rebuild_database()
|
| 133 |
+
|
| 134 |
return f"{count}件アップロード\n{msg}"
|
| 135 |
|
| 136 |
|
|
|
|
| 140 |
|
| 141 |
def search_method(question):
|
| 142 |
db = get_db()
|
| 143 |
+
|
| 144 |
if db is None:
|
| 145 |
return "PDFを登録してください"
|
| 146 |
|
| 147 |
+
# 関連ドキュメントの検索
|
| 148 |
docs = db.similarity_search(question, k=5)
|
| 149 |
context = "\n\n".join([d.page_content for d in docs])
|
| 150 |
|
|
|
|
| 163 |
|
| 164 |
【回答】
|
| 165 |
"""
|
| 166 |
+
|
| 167 |
try:
|
| 168 |
response = llm.invoke(prompt)
|
| 169 |
return response.content
|
| 170 |
except Exception as e:
|
| 171 |
+
return f"Geminiによる生成エラーが発生しました: {e}"
|
| 172 |
|
| 173 |
|
| 174 |
# ==================================================
|
| 175 |
+
# 初回データセット取得 & データベース初期化
|
| 176 |
# ==================================================
|
| 177 |
|
| 178 |
try:
|
|
|
|
| 181 |
repo_type="dataset",
|
| 182 |
local_dir=PAPER_DIR
|
| 183 |
)
|
| 184 |
+
print("データセットのダウンロードに成功しました")
|
| 185 |
except Exception as e:
|
| 186 |
+
print(f"データセットのダウンロードに失敗しました (手動での登録が必要です): {e}")
|
| 187 |
|
| 188 |
if not os.path.exists(DB_DIR):
|
| 189 |
print(rebuild_database())
|
|
|
|
| 195 |
|
| 196 |
def search_experiment(question):
|
| 197 |
db = get_db()
|
| 198 |
+
|
| 199 |
if db is None:
|
| 200 |
return "PDFを登録してください"
|
| 201 |
|
|
|
|
| 211 |
・データセット
|
| 212 |
・学習率
|
| 213 |
・Epoch数
|
| 214 |
+
・評価指標
|
| 215 |
・比較手法
|
| 216 |
・実験結果
|
| 217 |
|
|
|
|
| 223 |
|
| 224 |
【回答】
|
| 225 |
"""
|
| 226 |
+
|
| 227 |
try:
|
| 228 |
response = llm.invoke(prompt)
|
| 229 |
return response.content
|
| 230 |
except Exception as e:
|
| 231 |
+
return f"Geminiによる生成エラーが発生しました: {e}"
|
| 232 |
|
| 233 |
|
| 234 |
# ==================================================
|
| 235 |
+
# 保存
|
| 236 |
# ==================================================
|
| 237 |
|
| 238 |
def save_note(question, answer):
|
|
|
|
| 243 |
|
| 244 |
try:
|
| 245 |
with open(NOTE_FILE, "a", encoding="utf-8") as f:
|
| 246 |
+
f.write("\n")
|
| 247 |
+
f.write("=" * 60)
|
| 248 |
+
f.write("\n")
|
| 249 |
f.write(f"日時: {now}\n\n")
|
| 250 |
f.write(f"質問:\n{question}\n\n")
|
| 251 |
+
f.write(f"回答:\n{answer}\n")
|
| 252 |
+
f.write("\n")
|
| 253 |
return "保存完了"
|
| 254 |
except Exception as e:
|
| 255 |
return f"ファイル保存エラー: {e}"
|
| 256 |
|
| 257 |
|
| 258 |
# ==================================================
|
| 259 |
+
# UI (Gradio) ※インターフェース構成は完全再現
|
| 260 |
# ==================================================
|
| 261 |
|
| 262 |
with gr.Blocks() as demo:
|
| 263 |
+
|
| 264 |
gr.Markdown("# 研究支援RAGアプリ")
|
| 265 |
|
| 266 |
# タブ1: 論文追加
|
|
|
|
| 270 |
file_types=[".pdf"],
|
| 271 |
label="論文PDF"
|
| 272 |
)
|
| 273 |
+
|
| 274 |
upload_btn = gr.Button("登録")
|
| 275 |
+
|
| 276 |
upload_status = gr.Textbox(label="状態")
|
| 277 |
|
| 278 |
upload_btn.click(
|
|
|
|
| 284 |
# タブ2: 手法・専門用語検索
|
| 285 |
with gr.Tab("手法・専門用語検索"):
|
| 286 |
method_question = gr.Textbox(label="質問")
|
| 287 |
+
|
| 288 |
+
method_answer = gr.Textbox(
|
| 289 |
+
label="回答",
|
| 290 |
+
lines=15
|
| 291 |
+
)
|
| 292 |
+
|
| 293 |
method_btn = gr.Button("検索")
|
| 294 |
+
|
| 295 |
+
method_save = gr.Button("保存")
|
| 296 |
+
|
| 297 |
method_status = gr.Textbox(label="保存状態")
|
| 298 |
|
| 299 |
method_btn.click(
|
|
|
|
| 301 |
inputs=method_question,
|
| 302 |
outputs=method_answer
|
| 303 |
)
|
| 304 |
+
|
| 305 |
method_save.click(
|
| 306 |
fn=save_note,
|
| 307 |
inputs=[method_question, method_answer],
|
|
|
|
| 311 |
# タブ3: 実験内容検索
|
| 312 |
with gr.Tab("実験内容検索"):
|
| 313 |
exp_question = gr.Textbox(label="質問")
|
| 314 |
+
|
| 315 |
+
exp_answer = gr.Textbox(
|
| 316 |
+
label="回答",
|
| 317 |
+
lines=15
|
| 318 |
+
)
|
| 319 |
+
|
| 320 |
exp_btn = gr.Button("検索")
|
| 321 |
+
|
| 322 |
+
exp_save = gr.Button("保存") # 整合性を保つため gr.Button に修正
|
| 323 |
+
|
| 324 |
exp_status = gr.Textbox(label="保存状態")
|
| 325 |
|
| 326 |
exp_btn.click(
|
| 327 |
fn=search_experiment,
|
| 328 |
inputs=exp_question,
|
| 329 |
+
outputs=exp_answer
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|