Upload 8 files

app.py

@@ -1,3 +1,214 @@
+# app.py — Failsafe boot for Hugging Face Spaces (Gradio SDK)
+from __future__ import annotations
+import os, json, yaml, subprocess, sys, pathlib, traceback
+from typing import List, Dict
+
 import gradio as gr
-with gr.Blocks() as demo:
-    gr.Markdown("Hello from Gradio Space ✅")
+
+CFG_ERR = None
+
+# ---- load config with fallback ----
+DEFAULT_CFG = {
+    "app_name": "IR/ESG RAG Bot (OpenAI, 8 languages)",
+    "embedding_model": "text-embedding-3-large",
+    "normalize_embeddings": True,
+    "chunk": {"target_chars": 1400, "overlap_chars": 180},
+    "retrieval": {"top_k": 6, "score_threshold": 0.15, "mmr_lambda": 0.3},
+    "llm": {
+        "model": "gpt-4o-mini",
+        "max_output_tokens": 700,
+        "temperature": 0.2,
+        "system_prompt": (
+            "あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。"
+            "回答は常に根拠（文書名・ページ）を箇条書きで示し、文書外の推測や断定は避けます。"
+            "数値は年度と単位を明記し、最新年度を優先してください。"
+        ),
+    },
+    "languages": {
+        "preferred": ["ja", "en", "zh", "ko", "fr", "de", "es", "it"],
+        "labels": {
+            "ja": "日本語", "en": "English", "zh": "中文", "ko": "한국어",
+            "fr": "Français", "de": "Deutsch", "es": "Español", "it": "Italiano",
+        },
+    },
+}
+
+CFG_PATH = "config.yaml"
+try:
+    if os.path.exists(CFG_PATH):
+        with open(CFG_PATH, encoding="utf-8") as f:
+            CFG = yaml.safe_load(f) or {}
+        # merge defaults (shallow)
+        def _merge(dst, src):
+            for k, v in src.items():
+                if k not in dst:
+                    dst[k] = v
+        _merge(CFG, DEFAULT_CFG)
+        for sec in ("chunk", "retrieval", "llm", "languages"):
+            if sec in DEFAULT_CFG:
+                if sec not in CFG or not isinstance(CFG[sec], dict):
+                    CFG[sec] = DEFAULT_CFG[sec]
+                else:
+                    _merge(CFG[sec], DEFAULT_CFG[sec])
+    else:
+        CFG = DEFAULT_CFG
+        CFG_ERR = "config.yaml が見つかりません。デフォルト設定で起動しました。"
+except Exception as e:
+    CFG = DEFAULT_CFG
+    CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
+
+INDEX_PATH = pathlib.Path("data/index/faiss.index")
+META_PATH = pathlib.Path("data/index/meta.jsonl")
+
+# ---- Lazy imports ----
+def _lazy_imports():
+    global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    import faiss                    # pip: faiss-cpu
+    import numpy as np
+    from openai_client import embed_texts, chat
+    from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+
+def _index_exists() -> bool:
+    return INDEX_PATH.exists() and META_PATH.exists()
+
+def _check_api_key() -> bool:
+    return bool(os.getenv("OPENAI_API_KEY"))
+
+# ---- Globals (Lazy) ----
+_INDEX = None
+_METAS = None
+
+def _ensure_index_loaded():
+    global _INDEX, _METAS
+    if _INDEX is not None and _METAS is not None:
+        return
+    if not _index_exists():
+        raise RuntimeError("index_not_ready")
+    faiss, *_ = _lazy_imports()
+    _INDEX = faiss.read_index(str(INDEX_PATH))
+    _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
+
+def _embed_query(q: str):
+    _, np, embed_texts, *_ = _lazy_imports()
+    v = np.array(embed_texts([q], CFG["embedding_model"])[0], dtype="float32")
+    v = v / (np.linalg.norm(v) + 1e-12)
+    return v[None, :]
+
+def _search(q: str):
+    faiss, np, *_ = _lazy_imports()
+    _ensure_index_loaded()
+    TOP_K = CFG["retrieval"]["top_k"]
+    SCORE_TH = CFG["retrieval"]["score_threshold"]
+    qv = _embed_query(q)
+    sims, idxs = _INDEX.search(qv, TOP_K * 4)
+    sims, idxs = sims[0], idxs[0]
+    picked, seen = [], set()
+    for score, idx in zip(sims, idxs):
+        if score < SCORE_TH:
+            continue
+        c = _METAS[idx]
+        key = (c["source"], c["page"])
+        if key in seen:
+            continue
+        seen.add(key)
+        picked.append({**c, "score": float(score)})
+        if len(picked) >= TOP_K:
+            break
+    return picked
+
+def _format_context(chunks: List[Dict]) -> str:
+    return "\n".join([f"- 出典: {c['source']} p.{c['page']} | 抜粋: {c['text'][:180].replace('\n',' ')}…" for c in chunks])
+
+# ---- Handlers ----
+def rebuild_index() -> str:
+    if not _check_api_key():
+        return "OPENAI_API_KEY が未設定です。Spaces → Settings → Secrets で登録してください。"
+    pdf_dir = pathlib.Path("data/pdf")
+    pdf_dir.mkdir(parents=True, exist_ok=True)
+    if not list(pdf_dir.glob("*.pdf")):
+        return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
+    try:
+        out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
+        # キャッシュ破棄
+        global _INDEX, _METAS
+        _INDEX = None
+        _METAS = None
+        return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
+    except subprocess.CalledProcessError as e:
+        return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
+    except Exception as e:
+        return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
+
+_LANG_INSTRUCTIONS = {
+    "ja": "回答は日本語で出力してください。",
+    "en": "Answer in English.",
+    "zh": "请用中文回答。",
+    "ko": "한국어로 답변하세요.",
+    "fr": "Répondez en français.",
+    "de": "Bitte auf Deutsch antworten.",
+    "es": "Responde en español.",
+    "it": "Rispondi in italiano.",
+}
+
+def generate_answer(q: str, lang: str):
+    q = (q or "").strip()
+    if not q:
+        return "質問を入力してください。", {}
+    try:
+        _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
+        if detect_out_of_scope(q):
+            return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
+        chunks = _search(q)
+        context = _format_context(chunks)
+        lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
+        user_prompt = (
+            "以下のコンテキストのみを根拠に、簡潔かつ正確に回答してください。\n"
+            "必ず箇条書きで根拠(文書名とページ)を列挙してください。\n"
+            f"{lang_note}\n\n[コンテキスト]\n{context}\n\n[質問]\n{q}"
+        )
+        messages = [
+            {"role": "system", "content": CFG["llm"]["system_prompt"]},
+            {"role": "user", "content": user_prompt},
+        ]
+        text = chat(messages, model=CFG["llm"]["model"],
+                    max_output_tokens=CFG["llm"]["max_output_tokens"],
+                    temperature=CFG["llm"]["temperature"])
+        text = sanitize(text) + "\n\n" + compliance_block()
+        citations = [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]
+        return text, {"citations": citations}
+    except RuntimeError as e:
+        if str(e) == "index_not_ready":
+            return ("⚠️ インデックスがまだありません。\n"
+                    "1) data/pdf/ にPDFを置く\n"
+                    "2) 『インデックス再構築』ボタンを押す（OpenAI APIキー必須）\n"), {}
+        raise
+    except Exception as e:
+        return "❌ 実行時エラー: " + str(e) + "\n" + traceback.format_exc()[-1200:], {}
+
+# ---- UI ----
+LANGS = CFG["languages"]["preferred"]
+LABELS = CFG["languages"].get("labels", {l: l for l in LANGS})
+
+with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
+    gr.Markdown("# IR・ESG開示RAG（OpenAI API）— 8言語対応")
+    # config.yaml の読み込みエラー・警告を上部に可視化
+    if CFG_ERR:
+        gr.Markdown(f"**構成警告**: {CFG_ERR}")
+
+    with gr.Row():
+        q = gr.Textbox(label="質問 / Question", lines=3, placeholder="例: 2024年度のGHG排出量(スコープ1-3)は？")
+    with gr.Row():
+        lang = gr.Dropdown(choices=LANGS, value=LANGS[0], label="回答言語 / Output language")
+    with gr.Row():
+        ask = gr.Button("回答する / Answer", variant="primary")
+        rebuild = gr.Button("インデックス再構築（ingest.py 実行）")
+    ans = gr.Markdown()
+    cites = gr.JSON(label="根拠メタデータ / Citations")
+    log = gr.Markdown()
+
+    ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
+    rebuild.click(fn=rebuild_index, outputs=[log])
+
+# Gradio SDK はこの変数を自動検出して起動します
+demo