Upload 8 files

README.md

@@ -1,12 +1,45 @@
 ---
-title: IRESG RAG BOT
-emoji: 🏆
+title: IR・ESG RAG Bot (OpenAI, 8 languages)
+emoji: 📊
 colorFrom: yellow
-colorTo: green
+colorTo: blue
 sdk: gradio
-sdk_version: 5.43.1
+sdk_version: "4.44.0"
 app_file: app.py
 pinned: false
+python_version: "3.10"
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+
+## クイックスタート
+1. `data/pdf/` にIR/ESG PDFを配置
+2. `pip install -r requirements.txt`
+3. `python ingest.py` → `data/index/` 生成
+4. `export OPENAI_API_KEY=...`（必要に応じて `OPENAI_BASE_URL`）
+5. `python app.py` → Gradio UI / `/api/answer`
+
+
+## 埋め込みサンプル
+```html
+<script>
+async function askRag(question, lang="ja"){
+const r = await fetch("https://<your-host>/api/answer",{
+method:"POST", headers:{"Content-Type":"application/json"},
+body: JSON.stringify({question, lang})
+});
+const data = await r.json();
+console.log(data.text, data.citations);
+}
+</script>
+```
+
+
+## モデル推奨
+- 生成: `gpt-4`
+- 埋め込み: `text-embedding-3-large`
+
+
+## 運用Tips
+- PDF直リンク + `#page=<n>` を `meta.jsonl` に保持すれば、根拠クリックで該当ページに飛べます。
+- 年度更新はPDF差替え→`python ingest.py`。CI/CDで自動化を推奨。
+- ログには個人情報を含めない。

app.py

@@ -0,0 +1,218 @@
+# app.py — Failsafe boot for Hugging Face Spaces (Gradio SDK)
+from __future__ import annotations
+import os, json, yaml, subprocess, sys, pathlib, traceback
+from typing import List, Dict
+
+import gradio as gr
+
+CFG_ERR = None
+
+# ---- load config with fallback ----
+DEFAULT_CFG = {
+    "app_name": "IR/ESG RAG Bot (OpenAI, 8 languages)",
+    "embedding_model": "text-embedding-3-large",
+    "normalize_embeddings": True,
+    "chunk": {"target_chars": 1400, "overlap_chars": 180},
+    "retrieval": {"top_k": 6, "score_threshold": 0.15, "mmr_lambda": 0.3},
+    "llm": {
+        "model": "gpt-4o-mini",
+        "max_output_tokens": 700,
+        "temperature": 0.2,
+        "system_prompt": (
+            "あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。"
+            "回答は常に根拠（文書名・ページ）を箇条書きで示し、文書外の推測や断定は避けます。"
+            "数値は年度と単位を明記し、最新年度を優先してください。"
+        ),
+    },
+    "languages": {
+        "preferred": ["ja", "en", "zh", "ko", "fr", "de", "es", "it"],
+        "labels": {
+            "ja": "日本語", "en": "English", "zh": "中文", "ko": "한국어",
+            "fr": "Français", "de": "Deutsch", "es": "Español", "it": "Italiano",
+        },
+    },
+}
+
+CFG_PATH = "config.yaml"
+try:
+    if os.path.exists(CFG_PATH):
+        with open(CFG_PATH, encoding="utf-8") as f:
+            CFG = yaml.safe_load(f) or {}
+        # merge defaults (shallow)
+        def _merge(dst, src):
+            for k, v in src.items():
+                if k not in dst:
+                    dst[k] = v
+        _merge(CFG, DEFAULT_CFG)
+        for sec in ("chunk", "retrieval", "llm", "languages"):
+            if sec in DEFAULT_CFG:
+                if sec not in CFG or not isinstance(CFG[sec], dict):
+                    CFG[sec] = DEFAULT_CFG[sec]
+                else:
+                    _merge(CFG[sec], DEFAULT_CFG[sec])
+    else:
+        CFG = DEFAULT_CFG
+        CFG_ERR = "config.yaml が見つかりません。デフォルト設定で起動しました。"
+except Exception as e:
+    CFG = DEFAULT_CFG
+    CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
+
+INDEX_PATH = pathlib.Path("data/index/faiss.index")
+META_PATH = pathlib.Path("data/index/meta.jsonl")
+
+# ---- Lazy imports ----
+def _lazy_imports():
+    global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    import faiss                    # pip: faiss-cpu
+    import numpy as np
+    from openai_client import embed_texts, chat
+    from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+
+def _index_exists() -> bool:
+    return INDEX_PATH.exists() and META_PATH.exists()
+
+def _check_api_key() -> bool:
+    return bool(os.getenv("OPENAI_API_KEY"))
+
+# ---- Globals (Lazy) ----
+_INDEX = None
+_METAS = None
+
+def _ensure_index_loaded():
+    global _INDEX, _METAS
+    if _INDEX is not None and _METAS is not None:
+        return
+    if not _index_exists():
+        raise RuntimeError("index_not_ready")
+    faiss, *_ = _lazy_imports()
+    _INDEX = faiss.read_index(str(INDEX_PATH))
+    _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
+
+def _embed_query(q: str):
+    _, np, embed_texts, *_ = _lazy_imports()
+    v = np.array(embed_texts([q], CFG["embedding_model"])[0], dtype="float32")
+    v = v / (np.linalg.norm(v) + 1e-12)
+    return v[None, :]
+
+def _search(q: str):
+    faiss, np, *_ = _lazy_imports()
+    _ensure_index_loaded()
+    TOP_K = CFG["retrieval"]["top_k"]
+    SCORE_TH = CFG["retrieval"]["score_threshold"]
+    qv = _embed_query(q)
+    sims, idxs = _INDEX.search(qv, TOP_K * 4)
+    sims, idxs = sims[0], idxs[0]
+    picked, seen = [], set()
+    for score, idx in zip(sims, idxs):
+        if score < SCORE_TH:
+            continue
+        c = _METAS[idx]
+        key = (c["source"], c["page"])
+        if key in seen:
+            continue
+        seen.add(key)
+        picked.append({**c, "score": float(score)})
+        if len(picked) >= TOP_K:
+            break
+    return picked
+
+def _format_context(chunks: List[Dict]) -> str:
+    lines = []
+    for c in chunks:
+        snippet = c['text'][:180].replace("\n", " ")
+        lines.append(f"- 出典: {c['source']} p.{c['page']} | 抜粋: {snippet}…")
+    return "\n".join(lines)
+
+# ---- Handlers ----
+def rebuild_index() -> str:
+    if not _check_api_key():
+        return "OPENAI_API_KEY が未設定です。Spaces → Settings → Secrets で登録してください。"
+    pdf_dir = pathlib.Path("data/pdf")
+    pdf_dir.mkdir(parents=True, exist_ok=True)
+    if not list(pdf_dir.glob("*.pdf")):
+        return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
+    try:
+        out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
+        # キャッシュ破棄
+        global _INDEX, _METAS
+        _INDEX = None
+        _METAS = None
+        return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
+    except subprocess.CalledProcessError as e:
+        return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
+    except Exception as e:
+        return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
+
+_LANG_INSTRUCTIONS = {
+    "ja": "回答は日本語で出力してください。",
+    "en": "Answer in English.",
+    "zh": "请用中文回答。",
+    "ko": "한국어로 답변하세요.",
+    "fr": "Répondez en français.",
+    "de": "Bitte auf Deutsch antworten.",
+    "es": "Responde en español.",
+    "it": "Rispondi in italiano.",
+}
+
+def generate_answer(q: str, lang: str):
+    q = (q or "").strip()
+    if not q:
+        return "質問を入力してください。", {}
+    try:
+        _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
+        if detect_out_of_scope(q):
+            return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
+        chunks = _search(q)
+        context = _format_context(chunks)
+        lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
+        user_prompt = (
+            "以下のコンテキストのみを根拠に、簡潔かつ正確に回答してください。\n"
+            "必ず箇条書きで根拠(文書名とページ)を列挙してください。\n"
+            f"{lang_note}\n\n[コンテキスト]\n{context}\n\n[質問]\n{q}"
+        )
+        messages = [
+            {"role": "system", "content": CFG["llm"]["system_prompt"]},
+            {"role": "user", "content": user_prompt},
+        ]
+        text = chat(messages, model=CFG["llm"]["model"],
+                    max_output_tokens=CFG["llm"]["max_output_tokens"],
+                    temperature=CFG["llm"]["temperature"])
+        text = sanitize(text) + "\n\n" + compliance_block()
+        citations = [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]
+        return text, {"citations": citations}
+    except RuntimeError as e:
+        if str(e) == "index_not_ready":
+            return ("⚠️ インデックスがまだありません。\n"
+                    "1) data/pdf/ にPDFを置く\n"
+                    "2) 『インデックス再構築』ボタンを押す（OpenAI APIキー必須）\n"), {}
+        raise
+    except Exception as e:
+        return "❌ 実行時エラー: " + str(e) + "\n" + traceback.format_exc()[-1200:], {}
+
+# ---- UI ----
+LANGS = CFG["languages"]["preferred"]
+LABELS = CFG["languages"].get("labels", {l: l for l in LANGS})
+
+with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
+    gr.Markdown("# IR・ESG開示RAG（OpenAI API）— 8言語対応")
+    # config.yaml の読み込みエラー・警告を上部に可視化
+    if CFG_ERR:
+        gr.Markdown(f"**構成警告**: {CFG_ERR}")
+
+    with gr.Row():
+        q = gr.Textbox(label="質問 / Question", lines=3, placeholder="例: 2024年度のGHG排出量(スコープ1-3)は？")
+    with gr.Row():
+        lang = gr.Dropdown(choices=LANGS, value=LANGS[0], label="回答言語 / Output language")
+    with gr.Row():
+        ask = gr.Button("回答する / Answer", variant="primary")
+        rebuild = gr.Button("インデックス再構築（ingest.py 実行）")
+    ans = gr.Markdown()
+    cites = gr.JSON(label="根拠メタデータ / Citations")
+    log = gr.Markdown()
+
+    ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
+    rebuild.click(fn=rebuild_index, outputs=[log])
+
+# Gradio SDK はこの変数を自動検出して起動します
+demo

config.yaml

@@ -0,0 +1,36 @@
+app_name: "IR/ESG RAG Bot (OpenAI, 8 languages)"
+embedding_model: "text-embedding-3-large"
+normalize_embeddings: true
+
+chunk:
+  target_chars: 1400
+  overlap_chars: 180
+
+retrieval:
+  top_k: 6
+  score_threshold: 0.15
+  mmr_lambda: 0.3
+
+llm:
+  model: "gpt-4o-mini"
+  max_output_tokens: 700
+  temperature: 0.2
+  system_prompt: |-
+    あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
+    文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
+
+languages:
+  preferred: [ja, en, zh, ko, fr, de, es, it]
+  labels:
+    ja: "日本語"
+    en: "English"
+    zh: "中文"
+    ko: "한국어"
+    fr: "Français"
+    de: "Deutsch"
+    es: "Español"
+    it: "Italiano"
+
+logging:
+  save_qa: true
+  path: "logs/qa_log.jsonl"

guardrails.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+import re
+
+ALLOWED_TOPICS = [
+    r"IR", r"投資家", r"決算", r"財務", r"ガバナンス", r"統合報告", r"サステナビリティ",
+    r"人的資本", r"リスク", r"セグメント", r"株主", r"資本政策", r"ESG", r"GHG",
+]
+OUT_OF_SCOPE_PATTERNS = [r"採用の可否", r"未公開情報", r"株価予想", r"インサイダー", r"個人情報"]
+
+# 簡易PIIマスク（郵便・電話・メール）
+PII = re.compile(
+    r"(\d{3}-\d{4})"                 # 郵便番号
+    r"|(\d{2,4}-\d{2,4}-\d{3,4})"    # 電話番号
+    r"|([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)"  # メール
+)
+
+SCOPE_HINT = (
+    "このボットはIR/ESG開示文書（統合報告書、サステナ、決算短信、コーポガバ報告）を根拠とするQ&A専用です。"
+)
+
+def detect_out_of_scope(q: str) -> bool:
+    if any(re.search(p, q) for p in OUT_OF_SCOPE_PATTERNS):
+        return True
+    if not any(re.search(p, q) for p in ALLOWED_TOPICS):
+        return True
+    return False
+
+def sanitize(text: str) -> str:
+    return PII.sub("[REDACTED]", text)
+
+def compliance_block() -> str:
+    return "※免責：本回答は公開済みIR/ESG資料に基づく情報提供であり、投資判断を目的としません。"

ingest.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+import json, pathlib
+from typing import List, Dict, Tuple
+
+import numpy as np
+import faiss
+from pypdf import PdfReader
+import yaml
+
+from openai_client import embed_texts
+from guardrails import sanitize
+
+CFG = yaml.safe_load(open("config.yaml", encoding="utf-8"))
+EMB_MODEL = CFG["embedding_model"]
+NORMALIZE = CFG.get("normalize_embeddings", True)
+
+DATA_DIR = pathlib.Path("data")
+PDF_DIR = DATA_DIR / "pdf"
+INDEX_DIR = DATA_DIR / "index"
+META_PATH = INDEX_DIR / "meta.jsonl"      # app.py と一致
+INDEX_PATH = INDEX_DIR / "faiss.index"
+
+def read_pdf_with_pages(path: str) -> List[Tuple[int, str]]:
+    pages: List[Tuple[int, str]] = []
+    reader = PdfReader(path)
+    for i, p in enumerate(reader.pages):
+        txt = p.extract_text() or ""
+        txt = "\n".join(line.strip() for line in txt.splitlines() if line.strip())
+        pages.append((i + 1, txt))
+    return pages
+
+def split_chunks(pages: List[Tuple[int, str]], target_chars: int, overlap_chars: int) -> List[Dict]:
+    chunks: List[Dict] = []
+    for page, text in pages:
+        if not text:
+            continue
+        start = 0
+        while start < len(text):
+            end = min(len(text), start + target_chars)
+            chunk = text[start:end]
+            if len(chunk.strip()) >= 50:
+                chunks.append({"page": page, "text": chunk})
+            start = end - overlap_chars if end - overlap_chars > 0 else end
+    return chunks
+
+def l2_normalize(m: np.ndarray) -> np.ndarray:
+    if not NORMALIZE:
+        return m
+    norms = np.linalg.norm(m, axis=1, keepdims=True) + 1e-12
+    return m / norms
+
+def build_index():
+    INDEX_DIR.mkdir(parents=True, exist_ok=True)
+    meta_f = open(META_PATH, "w", encoding="utf-8")
+
+    target_chars = CFG["chunk"]["target_chars"]
+    overlap_chars = CFG["chunk"]["overlap_chars"]
+
+    texts: List[str] = []
+    for pdf in sorted(PDF_DIR.glob("*.pdf")):
+        print(f"Processing {pdf.name}...")
+        pages = read_pdf_with_pages(str(pdf))
+        chunks = split_chunks(pages, target_chars, overlap_chars)
+        for c in chunks:
+            t = c["text"][:1800]
+            texts.append(t)
+            meta = {"source": pdf.name, "page": c["page"], "text": sanitize(t)}
+            meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+    meta_f.close()
+
+    if not texts:
+        raise SystemExit("Put PDFs under data/pdf/")
+
+    vecs = embed_texts(texts, EMB_MODEL)
+    mat = np.array(vecs, dtype="float32")
+    mat = l2_normalize(mat)
+
+    # コサイン類似（正規化済みベクトル × 内積）
+    index = faiss.IndexFlatIP(mat.shape[1])
+    index.add(mat)
+    faiss.write_index(index, str(INDEX_PATH))
+    print(f"Index {len(texts)} chunks → {INDEX_PATH}")
+
+if __name__ == "__main__":
+    build_index()

openai_client.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+from typing import List, Dict
+from openai import OpenAI
+
+_client = None
+
+def client() -> OpenAI:
+    global _client
+    if _client is None:
+        # OPENAI_API_KEY / OPENAI_BASE_URL は環境変数から取得
+        _client = OpenAI()
+    return _client
+
+# Embeddings
+def embed_texts(texts: List[str], model: str) -> List[List[float]]:
+    resp = client().embeddings.create(model=model, input=texts)
+    return [d.embedding for d in resp.data]
+
+# Responses API
+def chat(messages: List[Dict], model: str, max_output_tokens: int = 700, temperature: float = 0.2) -> str:
+    # Responses APIは input=messages
+    resp = client().responses.create(
+        model=model,
+        input=messages,
+        max_output_tokens=max_output_tokens,
+        temperature=temperature,
+    )
+    return resp.output_text

repository layout

@@ -0,0 +1,15 @@
+ir-esg-rag-openai-8lang/
+├── app.py # Gradio UI + FastAPI (embed可) — 8言語対応
+├── ingest.py # PDF→チャンク→OpenAI Embeddings→FAISS
+├── guardrails.py # スコープ/PII/免責
+├── openai_client.py # Responses API呼び出し・共通ユーティリティ
+├── config.yaml # モデル/閾値/言語
+├── requirements.txt
+├── README.md
+├── data/
+│ ├── pdf/
+│ └── index/
+│ ├── faiss.index
+│ └── meta.jsonl
+└── logs/
+└── qa_log.jsonl

requirements.txt

@@ -0,0 +1,9 @@
+openai>=1.40.0
+faiss-cpu==1.8.0.post1
+pypdf==4.2.0
+PyYAML==6.0.2
+gradio==4.44.0
+fastapi==0.112.0
+uvicorn==0.30.5
+httpx==0.27.0
+pydantic==2.8.2