Corin1998 commited on
Commit
474b190
·
verified ·
1 Parent(s): 3255054

Upload 8 files

Browse files
Files changed (3) hide show
  1. FROM python:3.dockerfile +21 -0
  2. app.py +29 -73
  3. requirements.txt +0 -5
FROM python:3.dockerfile ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ FROM python:3.10-slim
2
+
3
+ ENV PYTHONDONTWRITEBYTECODE=1 \
4
+ PYTHONUNBUFFERED=1 \
5
+ PIP_NO_CACHE_DIR=1 \
6
+ PORT=7860 \
7
+ UVICORN_WORKERS=1
8
+
9
+ WORKDIR /app
10
+
11
+ RUN apt-get update && apt-get install -y --no-install-recommends \
12
+ build-essential \
13
+ && rm -rf /var/lib/apt/lists/*
14
+
15
+ COPY requirements.txt /app/requirements.txt
16
+ RUN pip install --upgrade pip && pip install -r requirements.txt
17
+
18
+ COPY . /app
19
+ RUN mkdir -p /app/data/pdf /app/data/index
20
+
21
+ CMD ["sh", "-c", "uvicorn app:app --host 0.0.0.0 --port ${PORT}"]
app.py CHANGED
@@ -1,4 +1,3 @@
1
- # app.py — FastAPI + Gradio (mounted at "/") for Hugging Face Spaces (Docker SDK)
2
  from __future__ import annotations
3
  import os, json, yaml, subprocess, sys, pathlib, traceback
4
  from typing import List, Dict, Tuple
@@ -6,12 +5,9 @@ from typing import List, Dict, Tuple
6
  from fastapi import FastAPI, Body
7
  from fastapi.responses import JSONResponse
8
  from fastapi.middleware.cors import CORSMiddleware
9
-
10
  import gradio as gr
11
 
12
- # =========================
13
- # Config (failsafe loading)
14
- # =========================
15
  DEFAULT_CFG = {
16
  "app_name": "IR/ESG RAG Bot (OpenAI, 8 languages)",
17
  "embedding_model": "text-embedding-3-large",
@@ -60,39 +56,26 @@ except Exception as e:
60
  CFG = DEFAULT_CFG
61
  CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
62
 
63
- # =========================
64
- # Paths / Lazy imports
65
- # =========================
66
  INDEX_PATH = pathlib.Path("data/index/faiss.index")
67
  META_PATH = pathlib.Path("data/index/meta.jsonl")
68
 
69
  def _lazy_imports():
70
- """遅延インポート(初期化安定化)"""
71
  global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
72
- import faiss
73
- import numpy as np
74
  from openai_client import embed_texts, chat
75
  from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
76
  return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
77
 
78
- def _index_exists() -> bool:
79
- return INDEX_PATH.exists() and META_PATH.exists()
80
-
81
- def _check_api_key() -> bool:
82
- return bool(os.getenv("OPENAI_API_KEY"))
83
 
84
- # =========================
85
- # Retrieval helpers
86
- # =========================
87
  _INDEX = None
88
  _METAS = None
89
 
90
  def _ensure_index_loaded():
91
  global _INDEX, _METAS
92
- if _INDEX is not None and _METAS is not None:
93
- return
94
- if not _index_exists():
95
- raise RuntimeError("index_not_ready")
96
  faiss, *_ = _lazy_imports()
97
  _INDEX = faiss.read_index(str(INDEX_PATH))
98
  _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
@@ -113,16 +96,11 @@ def _search(q: str):
113
  sims, idxs = sims[0], idxs[0]
114
  picked, seen = [], set()
115
  for score, idx in zip(sims, idxs):
116
- if score < SCORETH:
117
- continue
118
- c = _METAS[idx]
119
- key = (c["source"], c["page"])
120
- if key in seen:
121
- continue
122
- seen.add(key)
123
- picked.append({**c, "score": float(score)})
124
- if len(picked) >= TOP_K:
125
- break
126
  return picked
127
 
128
  def _format_context(chunks: List[Dict]) -> str:
@@ -143,15 +121,14 @@ _LANG_INSTRUCTIONS = {
143
  "it": "Rispondi in italiano.",
144
  }
145
 
146
- def generate_answer(q: str, lang: str = "ja") -> Tuple[str, Dict]:
147
  q = (q or "").strip()
148
- if not q:
149
- return "質問を入力してください。", {}
150
  try:
151
  _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
152
  if detect_out_of_scope(q):
153
  return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
154
- chunks = _search(q)
155
  context = _format_context(chunks)
156
  lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
157
  user_prompt = (
@@ -161,14 +138,11 @@ def generate_answer(q: str, lang: str = "ja") -> Tuple[str, Dict]:
161
  )
162
  messages = [
163
  {"role": "system", "content": CFG["llm"]["system_prompt"]},
164
- {"role": "user", "content": user_prompt},
165
  ]
166
- text = chat(
167
- messages,
168
- model=CFG["llm"]["model"],
169
- max_output_tokens=CFG["llm"]["max_output_tokens"],
170
- temperature=CFG["llm"]["temperature"],
171
- )
172
  text = sanitize(text) + "\n\n" + compliance_block()
173
  meta = {"citations": [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]}
174
  return text, meta
@@ -184,34 +158,24 @@ def generate_answer(q: str, lang: str = "ja") -> Tuple[str, Dict]:
184
  def rebuild_index() -> str:
185
  if not _check_api_key():
186
  return "OPENAI_API_KEY が未設定です。コンソール / Secrets に登���してください。"
187
- pdf_dir = pathlib.Path("data/pdf")
188
- pdf_dir.mkdir(parents=True, exist_ok=True)
189
- if not list(pdf_dir.glob("*.pdf")):
190
- return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
191
  try:
192
  out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
193
- # キャッシュ破棄
194
- global _INDEX, _METAS
195
- _INDEX = None
196
- _METAS = None
197
  return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
198
  except subprocess.CalledProcessError as e:
199
  return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
200
  except Exception as e:
201
  return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
202
 
203
- # =========================
204
- # FastAPI (ASGI)
205
- # =========================
206
  app = FastAPI(title=CFG.get("app_name", "RAG Bot"))
207
  app.add_middleware(
208
- CORSMiddleware,
209
- allow_origins=["*"], allow_credentials=False, allow_methods=["*"], allow_headers=["*"],
210
  )
211
-
212
  @app.get("/health")
213
- def health():
214
- return {"status": "ok"}
215
 
216
  @app.post("/api/answer")
217
  def api_answer(payload: Dict = Body(...)):
@@ -220,19 +184,13 @@ def api_answer(payload: Dict = Body(...)):
220
 
221
  @app.post("/api/rebuild")
222
  def api_rebuild():
223
- msg = rebuild_index()
224
- return JSONResponse({"message": msg})
225
-
226
- # =========================
227
- # Gradio UI (mounted at "/")
228
- # =========================
229
- LANGS = CFG["languages"]["preferred"]
230
- LABELS = CFG["languages"].get("labels", {l: l for l in LANGS})
231
 
 
 
232
  with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
233
  gr.Markdown("# IR・ESG開示RAG(OpenAI API)— 8言語対応")
234
- if CFG_ERR:
235
- gr.Markdown(f"**構成警告**: {CFG_ERR}")
236
  with gr.Row():
237
  q = gr.Textbox(label="質問 / Question", lines=3, placeholder="例: 2024年度のGHG排出量(スコープ1-3)は?")
238
  with gr.Row():
@@ -240,9 +198,7 @@ with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
240
  with gr.Row():
241
  ask = gr.Button("回答する / Answer", variant="primary")
242
  rebuild = gr.Button("インデックス再構築(ingest.py 実行)")
243
- ans = gr.Markdown()
244
- cites = gr.JSON(label="根拠メタデータ / Citations")
245
- log = gr.Markdown()
246
  ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
247
  rebuild.click(fn=rebuild_index, outputs=[log])
248
 
 
 
1
  from __future__ import annotations
2
  import os, json, yaml, subprocess, sys, pathlib, traceback
3
  from typing import List, Dict, Tuple
 
5
  from fastapi import FastAPI, Body
6
  from fastapi.responses import JSONResponse
7
  from fastapi.middleware.cors import CORSMiddleware
 
8
  import gradio as gr
9
 
10
+ # ===== config =====
 
 
11
  DEFAULT_CFG = {
12
  "app_name": "IR/ESG RAG Bot (OpenAI, 8 languages)",
13
  "embedding_model": "text-embedding-3-large",
 
56
  CFG = DEFAULT_CFG
57
  CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
58
 
 
 
 
59
  INDEX_PATH = pathlib.Path("data/index/faiss.index")
60
  META_PATH = pathlib.Path("data/index/meta.jsonl")
61
 
62
  def _lazy_imports():
 
63
  global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
64
+ import faiss, numpy as np
 
65
  from openai_client import embed_texts, chat
66
  from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
67
  return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
68
 
69
+ def _index_exists(): return INDEX_PATH.exists() and META_PATH.exists()
70
+ def _check_api_key(): return bool(os.getenv("OPENAI_API_KEY"))
 
 
 
71
 
 
 
 
72
  _INDEX = None
73
  _METAS = None
74
 
75
  def _ensure_index_loaded():
76
  global _INDEX, _METAS
77
+ if _INDEX is not None and _METAS is not None: return
78
+ if not _index_exists(): raise RuntimeError("index_not_ready")
 
 
79
  faiss, *_ = _lazy_imports()
80
  _INDEX = faiss.read_index(str(INDEX_PATH))
81
  _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
 
96
  sims, idxs = sims[0], idxs[0]
97
  picked, seen = [], set()
98
  for score, idx in zip(sims, idxs):
99
+ if score < SCORETH: continue
100
+ c = _METAS[idx]; key = (c["source"], c["page"])
101
+ if key in seen: continue
102
+ seen.add(key); picked.append({**c, "score": float(score)})
103
+ if len(picked) >= TOP_K: break
 
 
 
 
 
104
  return picked
105
 
106
  def _format_context(chunks: List[Dict]) -> str:
 
121
  "it": "Rispondi in italiano.",
122
  }
123
 
124
+ def generate_answer(q: str, lang: str = "ja"):
125
  q = (q or "").strip()
126
+ if not q: return "質問を入力してください。", {}
 
127
  try:
128
  _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
129
  if detect_out_of_scope(q):
130
  return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
131
+ chunks = _search(q)
132
  context = _format_context(chunks)
133
  lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
134
  user_prompt = (
 
138
  )
139
  messages = [
140
  {"role": "system", "content": CFG["llm"]["system_prompt"]},
141
+ {"role": "user", "content": user_prompt},
142
  ]
143
+ text = chat(messages, model=CFG["llm"]["model"],
144
+ max_output_tokens=CFG["llm"]["max_output_tokens"],
145
+ temperature=CFG["llm"]["temperature"])
 
 
 
146
  text = sanitize(text) + "\n\n" + compliance_block()
147
  meta = {"citations": [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]}
148
  return text, meta
 
158
  def rebuild_index() -> str:
159
  if not _check_api_key():
160
  return "OPENAI_API_KEY が未設定です。コンソール / Secrets に登���してください。"
161
+ pdf_dir = pathlib.Path("data/pdf"); pdf_dir.mkdir(parents=True, exist_ok=True)
162
+ if not list(pdf_dir.glob("*.pdf")): return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
 
 
163
  try:
164
  out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
165
+ global _INDEX, _METAS; _INDEX = None; _METAS = None
 
 
 
166
  return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
167
  except subprocess.CalledProcessError as e:
168
  return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
169
  except Exception as e:
170
  return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
171
 
172
+ # ===== FastAPI =====
 
 
173
  app = FastAPI(title=CFG.get("app_name", "RAG Bot"))
174
  app.add_middleware(
175
+ CORSMiddleware, allow_origins=["*"], allow_credentials=False, allow_methods=["*"], allow_headers=["*"]
 
176
  )
 
177
  @app.get("/health")
178
+ def health(): return {"status": "ok"}
 
179
 
180
  @app.post("/api/answer")
181
  def api_answer(payload: Dict = Body(...)):
 
184
 
185
  @app.post("/api/rebuild")
186
  def api_rebuild():
187
+ return JSONResponse({"message": rebuild_index()})
 
 
 
 
 
 
 
188
 
189
+ # ===== Gradio UI mounted at "/" =====
190
+ LANGS = CFG["languages"]["preferred"]
191
  with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
192
  gr.Markdown("# IR・ESG開示RAG(OpenAI API)— 8言語対応")
193
+ if CFG_ERR: gr.Markdown(f"**構成警告**: {CFG_ERR}")
 
194
  with gr.Row():
195
  q = gr.Textbox(label="質問 / Question", lines=3, placeholder="例: 2024年度のGHG排出量(スコープ1-3)は?")
196
  with gr.Row():
 
198
  with gr.Row():
199
  ask = gr.Button("回答する / Answer", variant="primary")
200
  rebuild = gr.Button("インデックス再構築(ingest.py 実行)")
201
+ ans = gr.Markdown(); cites = gr.JSON(label="根拠メタデータ / Citations"); log = gr.Markdown()
 
 
202
  ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
203
  rebuild.click(fn=rebuild_index, outputs=[log])
204
 
requirements.txt CHANGED
@@ -1,8 +1,3 @@
1
-
2
- ---
3
-
4
- ## 3) `requirements.txt`
5
- ```txt
6
  fastapi==0.112.0
7
  uvicorn[standard]==0.30.5
8
  gradio==4.44.1
 
 
 
 
 
 
1
  fastapi==0.112.0
2
  uvicorn[standard]==0.30.5
3
  gradio==4.44.1