Upload 8 files

IR:ESG_RAGBot

@@ -0,0 +1,28 @@
+# Dockerfile — Hugging Face Spaces (SDK: docker)
+FROM python:3.10-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PORT=7860 \
+    UVICORN_WORKERS=1
+
+WORKDIR /app
+
+# OS依存（必要に応じて追加）
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    build-essential \
+ && rm -rf /var/lib/apt/lists/*
+
+# 依存インストール
+COPY requirements.txt /app/requirements.txt
+RUN pip install --upgrade pip && pip install -r requirements.txt
+
+# アプリケーション
+COPY . /app
+
+# データディレクトリ
+RUN mkdir -p /app/data/pdf /app/data/index
+
+# Spaces は $PORT を渡す; それにバインドして起動
+CMD ["sh", "-c", "uvicorn app:app --host 0.0.0.0 --port ${PORT}"]

README.md

@@ -1,10 +1,29 @@
 ---
-title: ESG IR RAGbot
-emoji: 🚀
-colorFrom: pink
-colorTo: yellow
+title: IR・ESG RAG Bot (OpenAI, 8 languages) — Docker
+emoji: 📊
+colorFrom: yellow
+colorTo: blue
 sdk: docker
+app_port: 7860
 pinned: false
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# IR・ESG RAG Bot（Docker / FastAPI + Gradio）
+
+- UI: `/`（Gradio）
+- API: `POST /api/answer`（JSON: `{ "question":"...", "lang":"ja" }`）
+- Rebuild Index: `POST /api/rebuild`（ingest.py 実行）
+- Health: `GET /health`
+
+## 使い方（Spaces）
+1. このリポジトリをアップロード（このREADMEを含む）
+2. **Settings → Secrets** に `OPENAI_API_KEY` を登録
+3. `data/pdf/` にPDFを追加（UIからはアップロードしません）
+4. Space を再ビルド
+5. UIで「インデックス再構築」 or `POST /api/rebuild` を叩く → 質問
+
+## ローカル起動（任意）
+```bash
+docker build -t ir-esg-rag .
+docker run -e OPENAI_API_KEY=sk-... -p 7860:7860 -v $(pwd)/data:/app/data ir-esg-rag
+# ブラウザ: http://localhost:7860

app.py

@@ -0,0 +1,250 @@
+# app.py — FastAPI + Gradio (mounted at "/") for Hugging Face Spaces (Docker SDK)
+from __future__ import annotations
+import os, json, yaml, subprocess, sys, pathlib, traceback
+from typing import List, Dict, Tuple
+
+from fastapi import FastAPI, Body
+from fastapi.responses import JSONResponse
+from fastapi.middleware.cors import CORSMiddleware
+
+import gradio as gr
+
+# =========================
+# Config (failsafe loading)
+# =========================
+DEFAULT_CFG = {
+    "app_name": "IR/ESG RAG Bot (OpenAI, 8 languages)",
+    "embedding_model": "text-embedding-3-large",
+    "normalize_embeddings": True,
+    "chunk": {"target_chars": 1400, "overlap_chars": 180},
+    "retrieval": {"top_k": 6, "score_threshold": 0.15, "mmr_lambda": 0.3},
+    "llm": {
+        "model": "gpt-4o-mini",
+        "max_output_tokens": 700,
+        "temperature": 0.2,
+        "system_prompt": (
+            "あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。"
+            "回答は常に根拠（文書名・ページ）を箇条書きで示し、文書外の推測や断定は避けます。"
+            "数値は年度と単位を明記し、最新年度を優先してください。"
+        ),
+    },
+    "languages": {
+        "preferred": ["ja", "en", "zh", "ko", "fr", "de", "es", "it"],
+        "labels": {
+            "ja": "日本語", "en": "English", "zh": "中文", "ko": "한국어",
+            "fr": "Français", "de": "Deutsch", "es": "Español", "it": "Italiano"
+        },
+    },
+}
+CFG_ERR = None
+CFG_PATH = "config.yaml"
+try:
+    if os.path.exists(CFG_PATH):
+        with open(CFG_PATH, encoding="utf-8") as f:
+            CFG = yaml.safe_load(f) or {}
+        def _merge(dst, src):
+            for k, v in src.items():
+                if k not in dst:
+                    dst[k] = v
+        _merge(CFG, DEFAULT_CFG)
+        for sec in ("chunk", "retrieval", "llm", "languages"):
+            if sec in DEFAULT_CFG:
+                if sec not in CFG or not isinstance(CFG[sec], dict):
+                    CFG[sec] = DEFAULT_CFG[sec]
+                else:
+                    _merge(CFG[sec], DEFAULT_CFG[sec])
+    else:
+        CFG = DEFAULT_CFG
+        CFG_ERR = "config.yaml が見つかりません。デフォルト設定で起動しました。"
+except Exception as e:
+    CFG = DEFAULT_CFG
+    CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
+
+# =========================
+# Paths / Lazy imports
+# =========================
+INDEX_PATH = pathlib.Path("data/index/faiss.index")
+META_PATH  = pathlib.Path("data/index/meta.jsonl")
+
+def _lazy_imports():
+    """遅延インポート（初期化安定化）"""
+    global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    import faiss
+    import numpy as np
+    from openai_client import embed_texts, chat
+    from guardrails import detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+    return faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
+
+def _index_exists() -> bool:
+    return INDEX_PATH.exists() and META_PATH.exists()
+
+def _check_api_key() -> bool:
+    return bool(os.getenv("OPENAI_API_KEY"))
+
+# =========================
+# Retrieval helpers
+# =========================
+_INDEX = None
+_METAS = None
+
+def _ensure_index_loaded():
+    global _INDEX, _METAS
+    if _INDEX is not None and _METAS is not None:
+        return
+    if not _index_exists():
+        raise RuntimeError("index_not_ready")
+    faiss, *_ = _lazy_imports()
+    _INDEX = faiss.read_index(str(INDEX_PATH))
+    _METAS = [json.loads(l) for l in open(META_PATH, encoding="utf-8")]
+
+def _embed_query(q: str):
+    _, np, embed_texts, *_ = _lazy_imports()
+    v = np.array(embed_texts([q], CFG["embedding_model"])[0], dtype="float32")
+    v = v / (np.linalg.norm(v) + 1e-12)
+    return v[None, :]
+
+def _search(q: str):
+    faiss, np, *_ = _lazy_imports()
+    _ensure_index_loaded()
+    TOP_K   = CFG["retrieval"]["top_k"]
+    SCORETH = CFG["retrieval"]["score_threshold"]
+    qv = _embed_query(q)
+    sims, idxs = _INDEX.search(qv, TOP_K * 4)
+    sims, idxs = sims[0], idxs[0]
+    picked, seen = [], set()
+    for score, idx in zip(sims, idxs):
+        if score < SCORETH:
+            continue
+        c = _METAS[idx]
+        key = (c["source"], c["page"])
+        if key in seen:
+            continue
+        seen.add(key)
+        picked.append({**c, "score": float(score)})
+        if len(picked) >= TOP_K:
+            break
+    return picked
+
+def _format_context(chunks: List[Dict]) -> str:
+    lines = []
+    for c in chunks:
+        snippet = c["text"][:180].replace("\n", " ")
+        lines.append(f"- 出典: {c['source']} p.{c['page']} | 抜粋: {snippet}…")
+    return "\n".join(lines)
+
+_LANG_INSTRUCTIONS = {
+    "ja": "回答は日本語で出力してください。",
+    "en": "Answer in English.",
+    "zh": "请用中文回答。",
+    "ko": "한국어로 답변하세요.",
+    "fr": "Répondez en français.",
+    "de": "Bitte auf Deutsch antworten.",
+    "es": "Responde en español.",
+    "it": "Rispondi in italiano.",
+}
+
+def generate_answer(q: str, lang: str = "ja") -> Tuple[str, Dict]:
+    q = (q or "").strip()
+    if not q:
+        return "質問を入力してください。", {}
+    try:
+        _, _, _, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT = _lazy_imports()
+        if detect_out_of_scope(q):
+            return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。", {}
+        chunks  = _search(q)
+        context = _format_context(chunks)
+        lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's language.")
+        user_prompt = (
+            "以下のコンテキストのみを根拠に、簡潔かつ正確に回答してください。\n"
+            "必ず箇条書きで根拠(文書名とページ)を列挙してください。\n"
+            f"{lang_note}\n\n[コンテキスト]\n{context}\n\n[質問]\n{q}"
+        )
+        messages = [
+            {"role": "system", "content": CFG["llm"]["system_prompt"]},
+            {"role": "user",   "content": user_prompt},
+        ]
+        text = chat(
+            messages,
+            model=CFG["llm"]["model"],
+            max_output_tokens=CFG["llm"]["max_output_tokens"],
+            temperature=CFG["llm"]["temperature"],
+        )
+        text = sanitize(text) + "\n\n" + compliance_block()
+        meta = {"citations": [{"source": c["source"], "page": c["page"], "score": round(c["score"], 3)} for c in chunks]}
+        return text, meta
+    except RuntimeError as e:
+        if str(e) == "index_not_ready":
+            return ("⚠️ インデックスがまだありません。\n"
+                    "1) data/pdf/ にPDFを置く\n"
+                    "2) 『インデックス再構築』ボタンまたは /api/rebuild を実行（OpenAI APIキー必須）\n"), {}
+        raise
+    except Exception as e:
+        return "❌ 実行時エラー: " + str(e) + "\n" + traceback.format_exc()[-1200:], {}
+
+def rebuild_index() -> str:
+    if not _check_api_key():
+        return "OPENAI_API_KEY が未設定です。コンソール / Secrets に登録してください。"
+    pdf_dir = pathlib.Path("data/pdf")
+    pdf_dir.mkdir(parents=True, exist_ok=True)
+    if not list(pdf_dir.glob("*.pdf")):
+        return "data/pdf/ にPDFがありません。PDFを置いて再実行してください。"
+    try:
+        out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
+        # キャッシュ破棄
+        global _INDEX, _METAS
+        _INDEX = None
+        _METAS = None
+        return "✅ インデックス生成完了\n```\n" + (out.stdout[-1200:] or "") + "\n```"
+    except subprocess.CalledProcessError as e:
+        return f"❌ インデックス生成に失敗\nstdout:\n{e.stdout}\n\nstderr:\n{e.stderr}"
+    except Exception as e:
+        return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
+
+# =========================
+# FastAPI (ASGI)
+# =========================
+app = FastAPI(title=CFG.get("app_name", "RAG Bot"))
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"], allow_credentials=False, allow_methods=["*"], allow_headers=["*"],
+)
+
+@app.get("/health")
+def health():
+    return {"status": "ok"}
+
+@app.post("/api/answer")
+def api_answer(payload: Dict = Body(...)):
+    text, meta = generate_answer(payload.get("question", ""), payload.get("lang", "ja"))
+    return JSONResponse({"text": text, **meta})
+
+@app.post("/api/rebuild")
+def api_rebuild():
+    msg = rebuild_index()
+    return JSONResponse({"message": msg})
+
+# =========================
+# Gradio UI (mounted at "/")
+# =========================
+LANGS  = CFG["languages"]["preferred"]
+LABELS = CFG["languages"].get("labels", {l: l for l in LANGS})
+
+with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
+    gr.Markdown("# IR・ESG開示RAG（OpenAI API）— 8言語対応")
+    if CFG_ERR:
+        gr.Markdown(f"**構成警告**: {CFG_ERR}")
+    with gr.Row():
+        q = gr.Textbox(label="質問 / Question", lines=3, placeholder="例: 2024年度のGHG排出量(スコープ1-3)は？")
+    with gr.Row():
+        lang = gr.Dropdown(choices=LANGS, value=LANGS[0], label="回答言語 / Output language")
+    with gr.Row():
+        ask = gr.Button("回答する / Answer", variant="primary")
+        rebuild = gr.Button("インデックス再構築（ingest.py 実行）")
+    ans = gr.Markdown()
+    cites = gr.JSON(label="根拠メタデータ / Citations")
+    log = gr.Markdown()
+    ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
+    rebuild.click(fn=rebuild_index, outputs=[log])
+
+from gradio.routes import mount_gradio_app
+mount_gradio_app(app, demo, path="/")

config.yaml

@@ -0,0 +1,36 @@
+app_name: "IR/ESG RAG Bot (OpenAI, 8 languages)"
+embedding_model: "text-embedding-3-large"
+normalize_embeddings: true
+
+chunk:
+  target_chars: 1400
+  overlap_chars: 180
+
+retrieval:
+  top_k: 6
+  score_threshold: 0.15
+  mmr_lambda: 0.3
+
+llm:
+  model: "gpt-4o-mini"
+  max_output_tokens: 700
+  temperature: 0.2
+  system_prompt: |-
+    あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
+    文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
+
+languages:
+  preferred: [ja, en, zh, ko, fr, de, es, it]
+  labels:
+    ja: "日本語"
+    en: "English"
+    zh: "中文"
+    ko: "한국어"
+    fr: "Français"
+    de: "Deutsch"
+    es: "Español"
+    it: "Italiano"
+
+logging:
+  save_qa: true
+  path: "logs/qa_log.jsonl"

guardrails.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+import re
+
+ALLOWED_TOPICS = [
+    r"IR", r"投資家", r"決算", r"財務", r"ガバナンス", r"統合報告", r"サステナビリティ",
+    r"人的資本", r"リスク", r"セグメント", r"株主", r"資本政策", r"ESG", r"GHG",
+]
+OUT_OF_SCOPE_PATTERNS = [r"採用の可否", r"未公開情報", r"株価予想", r"インサイダー", r"個人情報"]
+
+# 簡易PIIマスク（郵便・電話・メール）
+PII = re.compile(
+    r"(\d{3}-\d{4})"                 # 郵便番号
+    r"|(\d{2,4}-\d{2,4}-\d{3,4})"    # 電話番号
+    r"|([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)"  # メール
+)
+
+SCOPE_HINT = (
+    "このボットはIR/ESG開示文書（統合報告書、サステナ、決算短信、コーポガバ報告）を根拠とするQ&A専用です。"
+)
+
+def detect_out_of_scope(q: str) -> bool:
+    if any(re.search(p, q) for p in OUT_OF_SCOPE_PATTERNS):
+        return True
+    if not any(re.search(p, q) for p in ALLOWED_TOPICS):
+        return True
+    return False
+
+def sanitize(text: str) -> str:
+    return PII.sub("[REDACTED]", text)
+
+def compliance_block() -> str:
+    return "※免責：本回答は公開済みIR/ESG資料に基づく情報提供であり、投資判断を目的としません。"

ingest.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+import json, pathlib
+from typing import List, Dict, Tuple
+
+import numpy as np
+import faiss
+from pypdf import PdfReader
+import yaml
+
+from openai_client import embed_texts
+from guardrails import sanitize
+
+CFG = yaml.safe_load(open("config.yaml", encoding="utf-8"))
+EMB_MODEL = CFG["embedding_model"]
+NORMALIZE = CFG.get("normalize_embeddings", True)
+
+DATA_DIR = pathlib.Path("data")
+PDF_DIR = DATA_DIR / "pdf"
+INDEX_DIR = DATA_DIR / "index"
+META_PATH = INDEX_DIR / "meta.jsonl"      # app.py と一致
+INDEX_PATH = INDEX_DIR / "faiss.index"
+
+def read_pdf_with_pages(path: str) -> List[Tuple[int, str]]:
+    pages: List[Tuple[int, str]] = []
+    reader = PdfReader(path)
+    for i, p in enumerate(reader.pages):
+        txt = p.extract_text() or ""
+        txt = "\n".join(line.strip() for line in txt.splitlines() if line.strip())
+        pages.append((i + 1, txt))
+    return pages
+
+def split_chunks(pages: List[Tuple[int, str]], target_chars: int, overlap_chars: int) -> List[Dict]:
+    chunks: List[Dict] = []
+    for page, text in pages:
+        if not text:
+            continue
+        start = 0
+        while start < len(text):
+            end = min(len(text), start + target_chars)
+            chunk = text[start:end]
+            if len(chunk.strip()) >= 50:
+                chunks.append({"page": page, "text": chunk})
+            start = end - overlap_chars if end - overlap_chars > 0 else end
+    return chunks
+
+def l2_normalize(m: np.ndarray) -> np.ndarray:
+    if not NORMALIZE:
+        return m
+    norms = np.linalg.norm(m, axis=1, keepdims=True) + 1e-12
+    return m / norms
+
+def build_index():
+    INDEX_DIR.mkdir(parents=True, exist_ok=True)
+    meta_f = open(META_PATH, "w", encoding="utf-8")
+
+    target_chars = CFG["chunk"]["target_chars"]
+    overlap_chars = CFG["chunk"]["overlap_chars"]
+
+    texts: List[str] = []
+    for pdf in sorted(PDF_DIR.glob("*.pdf")):
+        print(f"Processing {pdf.name}...")
+        pages = read_pdf_with_pages(str(pdf))
+        chunks = split_chunks(pages, target_chars, overlap_chars)
+        for c in chunks:
+            t = c["text"][:1800]
+            texts.append(t)
+            meta = {"source": pdf.name, "page": c["page"], "text": sanitize(t)}
+            meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+    meta_f.close()
+
+    if not texts:
+        raise SystemExit("Put PDFs under data/pdf/")
+
+    vecs = embed_texts(texts, EMB_MODEL)
+    mat = np.array(vecs, dtype="float32")
+    mat = l2_normalize(mat)
+
+    # コサイン類似（正規化済みベクトル × 内積）
+    index = faiss.IndexFlatIP(mat.shape[1])
+    index.add(mat)
+    faiss.write_index(index, str(INDEX_PATH))
+    print(f"Index {len(texts)} chunks → {INDEX_PATH}")
+
+if __name__ == "__main__":
+    build_index()

openai_client.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+from typing import List, Dict
+from openai import OpenAI
+
+_client = None
+
+def client() -> OpenAI:
+    global _client
+    if _client is None:
+        # OPENAI_API_KEY / OPENAI_BASE_URL は環境変数から取得
+        _client = OpenAI()
+    return _client
+
+# Embeddings
+def embed_texts(texts: List[str], model: str) -> List[List[float]]:
+    resp = client().embeddings.create(model=model, input=texts)
+    return [d.embedding for d in resp.data]
+
+# Responses API
+def chat(messages: List[Dict], model: str, max_output_tokens: int = 700, temperature: float = 0.2) -> str:
+    resp = client().responses.create(
+        model=model,
+        input=messages,
+        max_output_tokens=max_output_tokens,
+        temperature=temperature,
+    )
+    return resp.output_text

requirements.txt

@@ -0,0 +1,14 @@
+
+---
+
+## 3) `requirements.txt`
+```txt
+fastapi==0.112.0
+uvicorn[standard]==0.30.5
+gradio==4.44.1
+openai>=1.40.0
+faiss-cpu==1.8.0.post1
+pypdf==4.2.0
+PyYAML==6.0.2
+httpx==0.27.0
+pydantic==2.8.2