Upload 8 files

app.py

@@ -1,17 +1,18 @@
-# app.py — Gradio SDK / Lazy index loading / Rebuild index button
+# app.py — Gradio SDK / 8 languages / Lazy index loading / Rebuild index button
 from __future__ import annotations
-import os, json, yaml, subprocess, sys, time, pathlib
+import os, json, yaml, subprocess, sys, pathlib
 from typing import List, Dict, Tuple
 
 import gradio as gr
 
+# ---- Load config ----
 CFG = yaml.safe_load(open("config.yaml", encoding="utf-8"))
 
 INDEX_PATH = pathlib.Path("data/index/faiss.index")
 META_PATH = pathlib.Path("data/index/meta.jsonl")
 
-# ---- Lazy imports (index系は必要時にのみ読み込む) ----
-def lazy_imports():
+# ---- Lazy imports ----
+def _lazy_imports():
     global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
     import faiss
     import numpy as np
@@ -19,13 +20,13 @@ def lazy_imports():
     from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
     return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
 
-def index_exists() -> bool:
+def _index_exists() -> bool:
     return INDEX_PATH.exists() and META_PATH.exists()
 
-def check_api_key() -> bool:
+def _check_api_key() -> bool:
     return bool(os.getenv("OPENAI_API_KEY"))
 
-# ---- Search helpers (load on demand) ----
+# ---- Globals (Lazy) ----
 _INDEX = None
 _METAS = None
 
@@ -33,20 +34,20 @@ def _ensure_index_loaded():
     global _INDEX, _METAS
     if _INDEX is not None and _METAS is not None:
         return
-    if not index_exists():
+    if not _index_exists():
         raise RuntimeError("index_not_ready")
-    faiss, np, *_ = lazy_imports()
+    faiss, *_ = _lazy_imports()
     _INDEX = faiss.read_index(str(INDEX_PATH))
     _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
 
 def _embed_query(q: str):
-    _, np, embed_texts, *_ = lazy_imports()
+    _, np, embed_texts, *_ = _lazy_imports()
     v = np.array(embed_texts([q], CFG["embedding_model"])[0], dtype="float32")
     v = v / (np.linalg.norm(v) + 1e-12)
     return v[None, :]
 
 def _search(q: str):
-    faiss, np, *_ = lazy_imports()
+    faiss, np, *_ = _lazy_imports()
     _ensure_index_loaded()
     TOP_K = CFG["retrieval"]["top_k"]
     SCORE_TH = CFG["retrieval"]["score_threshold"]
@@ -72,41 +73,44 @@ def _format_context(chunks: List[Dict]) -> str:
 
 # ---- Handlers ----
 def rebuild_index() -> str:
-    if not check_api_key():
+    if not _check_api_key():
         return "OPENAI_API_KEY が未設定です。Spaces → Settings → Secrets で登録してください。"
     pdf_dir = pathlib.Path("data/pdf")
-    pdfs = list(pdf_dir.glob("*.pdf"))
-    if not pdfs:
+    pdf_dir.mkdir(parents=True, exist_ok=True)
+    if not list(pdf_dir.glob("*.pdf")):
         return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
     try:
-        # ingest.py を実行して index を作る
         out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
-        return f"✅ インデックス生成完了:\n{out.stdout[-800:]}"
+        # キャッシュ破棄
+        global _INDEX, _METAS
+        _INDEX = None
+        _METAS = None
+        return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
     except subprocess.CalledProcessError as e:
-        return f"❌ インデックス生成に失敗:\n{e.stdout}\n{e.stderr}"
+        return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
+
+_LANG_INSTRUCTIONS = {
+    "ja": "回答は日本語で出力してください。",
+    "en": "Answer in English.",
+    "zh": "请用中文回答。",
+    "ko": "한국어로 답변하세요.",
+    "fr": "Répondez en français.",
+    "de": "Bitte auf Deutsch antworten.",
+    "es": "Responde en español.",
+    "it": "Rispondi in italiano.",
+}
 
 def generate_answer(q: str, lang: str):
     q = (q or "").strip()
     if not q:
         return "質問を入力してください。", {}
     try:
-        _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = lazy_imports()
+        _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
         if detect_out_of_scope(q):
             return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
         chunks = _search(q)
         context = _format_context(chunks)
-
-        lang_note = {
-            "ja": "回答は日本語で出力してください。",
-            "en": "Answer in English.",
-            "zh": "请用中文回答。",
-            "ko": "한국어로 답변하세요.",
-            "fr": "Répondez en français.",
-            "de": "Bitte auf Deutsch antworten.",
-            "es": "Responde en español.",
-            "it": "Rispondi in italiano.",
-        }.get(lang, "Answer in the user's language.")
-
+        lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
         user_prompt = (
             "以下のコンテキストのみを根拠に、簡潔かつ正確に回答してください。\n"
             "必ず箇条書きで根拠(文書名とページ)を列挙してください。\n"
@@ -120,20 +124,17 @@ def generate_answer(q: str, lang: str):
         text = sanitize(text) + "\n\n" + compliance_block()
         citations = [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]
         return text, {"citations": citations}
-
     except RuntimeError as e:
         if str(e) == "index_not_ready":
-            msg = (
+            return (
                 "⚠️ インデックスがまだありません。\n"
                 "1) data/pdf/ にPDFを置く\n"
-                "2) 下の『インデックス再構築』ボタンを押す（OpenAI APIキー必須）\n"
-            )
-            return msg, {}
+                "2) 『インデックス再構築』ボタンを押す（OpenAI APIキー必須）\n"
+            ), {}
         raise
 
 # ---- UI ----
 LANGS = CFG["languages"]["preferred"]
-LABELS = CFG["languages"].get("labels", {l: l for l in LANGS})
 
 with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
     gr.Markdown("# IR・ESG開示RAG（OpenAI API）— 8言語対応")
@@ -151,4 +152,5 @@ with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
     ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
     rebuild.click(fn=rebuild_index, outputs=[log])
 
-demo  # ← Gradio SDK はこの変数を自動起動します
+# Gradio SDK はこの変数を自動検出して起動します
+demo

config.yaml

@@ -1,31 +1,36 @@
-app_name "IR/ESG RAG Bot(OpenAI,8 languages)
-embedding_model:"text-embedding-3-large"
+app_name: "IR/ESG RAG Bot (OpenAI, 8 languages)"
+embedding_model: "text-embedding-3-large"
 normalize_embeddings: true
+
 chunk:
- target_chars:1400
- overlap_chars:180
+  target_chars: 1400
+  overlap_chars: 180
+
 retrieval:
-  top_k:6
-  score_threshold:0.15
-  mmr_lambda:0.3
+  top_k: 6
+  score_threshold: 0.15
+  mmr_lambda: 0.3
+
 llm:
- model:"gpt-4"
- max_output_tokens:700
- temperature:0.2
- system_prompt: |
- 　あなたは上場企業のIR/ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
- 　文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
- language:
-   preferred:[ja,en,zh,ko,de,es,it,fr]
-   labels:
-     ja:"日本語"
-     en:"English"
-     zh:"中文"
-     ko:"한국어"
-     de:"Deutsch"
-     es:"Español"
-     it:"Italiano"
-     fr:"Français"
+  model: "gpt-4o-mini"
+  max_output_tokens: 700
+  temperature: 0.2
+  system_prompt: |-
+    あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
+    文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
+
+languages:
+  preferred: [ja, en, zh, ko, fr, de, es, it]
+  labels:
+    ja: "日本語"
+    en: "English"
+    zh: "中文"
+    ko: "한국어"
+    fr: "Français"
+    de: "Deutsch"
+    es: "Español"
+    it: "Italiano"
+
 logging:
   save_qa: true
-  path:"logs/qa_log.jsonl"
+  path: "logs/qa_log.jsonl"

guardrails.py

@@ -7,10 +7,10 @@ ALLOWED_TOPICS = [
 ]
 OUT_OF_SCOPE_PATTERNS = [r"採用の可否", r"未公開情報", r"株価予想", r"インサイダー", r"個人情報"]
 
-# 簡易PIIマスク（郵便・電話っぽい連番・メール）
+# 簡易PIIマスク（郵便・電話・メール）
 PII = re.compile(
-    r"(\d{3}-\d{4})"                # 郵便番号 123-4567
-    r"|(\d{2,4}-\d{2,4}-\d{3,4})"   # 電話番号 03-1234-5678 など
+    r"(\d{3}-\d{4})"                 # 郵便番号
+    r"|(\d{2,4}-\d{2,4}-\d{3,4})"    # 電話番号
     r"|([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)"  # メール
 )
 

ingest.py

@@ -17,7 +17,7 @@ NORMALIZE = CFG.get("normalize_embeddings", True)
 DATA_DIR = pathlib.Path("data")
 PDF_DIR = DATA_DIR / "pdf"
 INDEX_DIR = DATA_DIR / "index"
-META_PATH = INDEX_DIR / "meta.jsonl"      # ← app.py に合わせて .jsonl
+META_PATH = INDEX_DIR / "meta.jsonl"      # app.py と一致
 INDEX_PATH = INDEX_DIR / "faiss.index"
 
 def read_pdf_with_pages(path: str) -> List[Tuple[int, str]]:
@@ -37,7 +37,7 @@ def split_chunks(pages: List[Tuple[int, str]], target_chars: int, overlap_chars:
         start = 0
         while start < len(text):
             end = min(len(text), start + target_chars)
-            chunk = text[start:end]          # ← タイポ修正（strart -> start）
+            chunk = text[start:end]
             if len(chunk.strip()) >= 50:
                 chunks.append({"page": page, "text": chunk})
             start = end - overlap_chars if end - overlap_chars > 0 else end
@@ -57,8 +57,6 @@ def build_index():
     overlap_chars = CFG["chunk"]["overlap_chars"]
 
     texts: List[str] = []
-    metas: List[Dict] = []
-
     for pdf in sorted(PDF_DIR.glob("*.pdf")):
         print(f"Processing {pdf.name}...")
         pages = read_pdf_with_pages(str(pdf))
@@ -67,7 +65,6 @@ def build_index():
             t = c["text"][:1800]
             texts.append(t)
             meta = {"source": pdf.name, "page": c["page"], "text": sanitize(t)}
-            metas.append(meta)
             meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
 
     meta_f.close()

openai_client.py

@@ -16,9 +16,9 @@ def embed_texts(texts: List[str], model: str) -> List[List[float]]:
     resp = client().embeddings.create(model=model, input=texts)
     return [d.embedding for d in resp.data]
 
-# Responses API（app.py 側の呼び出しシグネチャに合わせる）
+# Responses API
 def chat(messages: List[Dict], model: str, max_output_tokens: int = 700, temperature: float = 0.2) -> str:
-    # Responses APIは input=messages 形式
+    # Responses APIは input=messages
     resp = client().responses.create(
         model=model,
         input=messages,

requirements.txt

@@ -6,4 +6,4 @@ gradio==4.44.0
 fastapi==0.112.0
 uvicorn==0.30.5
 httpx==0.27.0
-pydantic==2.8.2
+pydantic==2.8.2