Upload 8 files

README.md

@@ -1,12 +1,36 @@
----
-title: IR ESG RAG Bot
-emoji: 🌍
-colorFrom: red
-colorTo: indigo
-sdk: gradio
-sdk_version: 5.43.1
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+```markdown
+# IR・ESG開示RAGボット（OpenAI API版・8言語対応）
+
+
+## クイックスタート
+1. `data/pdf/` にIR/ESG PDFを配置
+2. `pip install -r requirements.txt`
+3. `python ingest.py` → `data/index/` 生成
+4. `export OPENAI_API_KEY=...`（必要に応じて `OPENAI_BASE_URL`）
+5. `python app.py` → Gradio UI / `/api/answer`
+
+
+## 埋め込みサンプル
+```html
+<script>
+async function askRag(question, lang="ja"){
+const r = await fetch("https://<your-host>/api/answer",{
+method:"POST", headers:{"Content-Type":"application/json"},
+body: JSON.stringify({question, lang})
+});
+const data = await r.json();
+console.log(data.text, data.citations);
+}
+</script>
+```
+
+
+## モデル推奨
+- 生成: `gpt-4o-mini`
+- 埋め込み: `text-embedding-3-large`
+
+
+## 運用Tips
+- PDF直リンク + `#page=<n>` を `meta.jsonl` に保持すれば、根拠クリックで該当ページに飛べます。
+- 年度更新はPDF差替え→`python ingest.py`。CI/CDで自動化を推奨。
+- ログには個人情報を含めない。

app.py

@@ -0,0 +1,110 @@
+from__future__import __annotations
+import os, json,yaml
+from typing import List,Dict,Tuple
+
+import gradio as gr
+import faiss ,numpy as np
+
+from openai_client import embed_texts,chat
+from guardrails import detect_out_of_scope,sanitize,copliance_block,SCOPE_HINT
+
+CFG=yaml.safe_load(open("config.yaml",encoding="utf-8"))
+EMB_MODEL=CFG["emb_model"]
+TOP_K=CFG["retrieval"]["top_k"]
+SCORE_TH=CFG["retrieval"]["score_threshold"]
+LLM_MODEL=CFG["llm"]["model"]
+LANGS =CFG["languages"]["preferred"]
+LABELS =CFG["languages"].get("labels",{l: l for l in LANGS})
+
+INDEX = faiss.read_index("data/index/index.faiss")
+METAS = [json.loads(l) for l in open("data/index/meta.json", encoding="utf-8")]
+
+def embed_query(q:str)->np.ndarray:
+    v = np.array(embed_texts([q], EMB_MODEL)[0],dtype=np.float32)
+    v =  v/(np.linalg.norm(v)+1e-12)
+    return v[None,:]
+
+def search(q:str, top_k:int =TOP_K)->List[Dict]:
+    qv = embed_query(q)
+    sims, idxs = INDEX.search(qv, top_k*4)
+    sims, idxs = sims[0], idxs[0]
+    picked = []
+    seen = set()
+    for score, idx in zip(sims, idxs):
+        if score < SCORE_TH :
+            continue
+        c = METAS[idx]
+        key = (c["source"], c["page"])
+        if key in seen:
+            continue
+        seen.add(key)
+        picked.append({**c, "score": float(score)})
+        if len(picked) >= top_k:
+            break
+    return picked
+
+def format_context(chunks:List[Dict])->str:
+    return"\n".join([
+        f"-出典：{c['source']} p.{c['page']} |抜粋： {c['text'][:180].replace('\n', ' ')}... "for c in chunks
+    ])
+
+_LANG_INSTRUCTIONS = {
+    "ja":"回答は日本語で出力してください",
+    "en":"Answer in English.",
+    "zh":"请用中文回答。",
+    "ko":"한국어로 대답하십시오.",
+    "fr":"Répondez en français.",
+    "de":"Antworten Sie auf Deutsch.",
+    "es":"Responde en español.",
+    "it":"Rispondi in italiano.",
+}
+
+def generate_answer(q:str, lang:str)->Tuple[str,str]:
+    q=(q or "").strip()
+    if not q:
+        return "", "質問が入力してください。"{}"
+
+    if detect_out_of_scope(q):
+        return f"{SCOPE_HINT}\nIR/ESG関連の事項についてお尋ねください。","{}"
+    
+    chunks = search(q)
+    context = format_context(chunks)
+
+    lang_note = _LANG_INSTRUCTIONS.get(lang, "Answer in the user's preferred language.")
+    user_prompt=(
+        "以下のコンテキストのみを根拠に、簡潔かつ正確に回答してください。\n"
+        "必ず箇条書きで根拠（文書名とページ）を列挙してください。\n"
+        f"{context}\n\n[コンテキスト]\n\n|質問｜\n{q}"
+    )
+
+    messages= [
+        {"role": "system", "content": CFG["llm"]["system_prompt"]},
+        {"role": "user", "content": user_prompt},
+    ]
+
+    text=chat(messages,model=LLM_MODEL,max_output_tokens=CFG["llm"]["max_output_tokens"],temperature=CFG["llm"]["temperature"])
+    text=sanitize(text)+"\n\n" + complicance_block()
+    return text, json.dumps(meta,ensure_ascii=False)
+
+with gr.Blocks(fill_height=True,title=CFG.get("app_name","RAG Bot")) as demo:
+    gr.Markdown("#IR/ESG開示RAG（OpenAI API)-8言語対応")
+    q= gr.Textbox("質問 / Question / 質问 / 질문 / Question / Frage / Pregunta / Domanda",lines=3,placeholder="例：2024年度のGHG排出量（スコープ1-3）は？"）
+    lang = gr.Dropdown(choices=LANGS,value=LANGS[0],label="回答言語/Output Language")
+    ask = gr.Button("回答する// Answer / 回答 / 답변 / Répondre / Antworten / Responder / Rispondere", variant="primary")
+    ans = gr.Markdown()
+    cites = gr.JSON(label="根拠メタデータ/Citations")
+    ask.click(fn=generate_answer, inputs=[q, lang], outputs=[ans, cites])
+
+import fastapi
+api = fastapi.FastAPI()
+app = gr.mount_gradio_app(api,demo,path="/")
+
+@api.post("/api/answer")
+async def api_answer(payload:dict):
+      text,meta=generate_answer(payload.get("question",""), payload.get("lang", "ja"))
+      data =json.loads(meta)
+      return{"text":text,**data}
+if__name__=="__main__":
+    import uvicorn
+    uvicorn.run(app,host="0.0.0.0",port=int(os.getenv("PORT", 7860)))
+    

config.yaml

@@ -0,0 +1,31 @@
+app_name "IR/ESG RAG Bot(OpenAI,8 languages)
+embedding_model:"text-embedding-3-large"
+normalize_embeddings: true
+chunk:
+ target_chars:1400
+ overlap_chars:180
+retrieval:
+  top_k:6
+  score_threshold:0.15
+  mmr_lambda:0.3
+llm:
+ model:"gpt-4"
+ max_output_tokens:700
+ temperature:0.2
+ system_prompt: |
+ 　あなたは上場企業のIR/ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
+ 　文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
+ language:
+   preferred:[ja,en,zh,ko,de,es,it,fr]
+   labels:
+     ja:"日本語"
+     en:"English"
+     zh:"中文"
+     ko:"한국어"
+     de:"Deutsch"
+     es:"Español"
+     it:"Italiano"
+     fr:"Français"
+logging:
+  save_qa: true
+  path:"logs/qa_log.jsonl"

guardrails.py

@@ -0,0 +1,26 @@
+from__future__import __annotations
+import re
+
+ALLOWED_TOPICS =[
+    r"IR",r"投資家",r"決算",r"財務",r"ガバナンス",r"統合報告",r"サステナビリティ",
+    r"人的資本",r"リスク",r"セグメント",r"株主",r"資本政策",r"ESG",r"GHG"
+]
+OUT_OF_SCOPE_PATTERNS =[r"採用の可否",r"未公開情報",r"株価予想",r"インサイダー",r"個人情報"]
+PII = re.compile(r"(\d{3}-\d{4})|\d{2,4}-\d{4}|[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+)")
+
+SCOPE_HINT=(
+    "このボットはIR/ESG開示文書（統合報告書、サステナ、決算短信、コーポガバ報告）を根拠とするQ&A専用です。"
+)
+
+def detect_out_of_scope(q:str)->bool:
+    if any(re.search(p,q)for p in OUT_OF_SCOPE_PATTERNS):
+        return True
+    if not any(re.search(p,q)for p in ALLOWED_TOPICS):
+        return True
+    return False
+
+def sanitize(text:str)->str:
+    return PII.sub("[REDACTED]", text)
+
+def compliance_block()->str:
+    return"※免責：本回答は公開済みIR/ESG資料に基づく情報提供であり、投資判断を目的としません。"

ingest.py

@@ -0,0 +1,87 @@
+from__future__import __annotations
+import os,json,pathlib
+from typing import List,Dict,Tuple
+
+import numpy as np
+import faiss
+from pypdf import PdfReader
+import yaml
+
+from openai_client import embed_texts
+from guardrails import sanitize
+
+CFG = yaml.safe_load(open("config.yaml",encoding="utf-8"))
+EMB_MODEL = CFG["embedding_model"]
+NORMALIZE = CFG.get("normalize_embeddings", True)
+
+DATA_DIR = pathlib.Path("data")
+PDF_DIR = DATA_DIR / "pdf"
+INDEX_DIR = DATA_DIR / "index"
+META_PATH = INDEX_DIR / "meta.json"
+INDEX_PATH = INDEX_DIR / "faiss.index"
+
+def read_pdf_with_pages(path:str)->List[Tuple[int,str]]:
+    pages = []
+    reader = PdfReader(path)
+    for i, p in enumerate(reader.pages):
+        txt = p.extract_text()or""
+        txt="\n".join([line.strip()for line in txt.splitlines()if line.strip()])
+        pages.append((i+1, txt))
+    return pages
+
+def split_chunks(pages:List[Tuple[int,str]],target_chars:int,overlap_chars:int)->List[Dict]:
+    chunks=[]
+    for page,text in pages:
+        if not text: continue
+        start=0
+        while start<len(text):
+            end = min(len(text),start + target_chars)
+            chunk=text[strart:end]
+            if len(chunk.strip())>=50:
+                chunks.append({"page":page,"text":chunk})
+                start = end - overlap_chars if end - overlap_chars > 0 else end
+     return chunks
+
+def l2_normalize(m:np.ndarray)->np.ndarray:
+    if not NORMALIZE:
+        return m
+    norms=np.linalg.norm(m, axis=1, keepdims=True);1e-12
+    return m / norms
+
+def build_index():
+    INDEX_DIR.mkdir(parents=True, exist_ok=True)
+    meta_f = open(META_PATH, "w", encoding="utf-8")
+
+    target_chars = CFG ["chunk"]["target_chars"]
+    overlap_chars = CFG["chunk"]["overlap_chars"]
+
+    texts=[]
+    metas=[]
+
+    for pdf in sorted(PDF_DIR.glob("*.pdf")):
+        print(f"Processing {pdf.name}...")
+        pages = read_pdf_with_pages(pdf)
+        chunks = split_chunks(pages, target_chars, overlap_chars)
+        for c in chunks:
+            t=c["text"][:1800]
+            texts.append(t)
+            meta={"source":pdf.name, "page":c["page"],"text":sanitize(t)}
+            metas.append(meta)
+            meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
+
+        meta_f.close()
+
+        if not texts:
+            raise SystemExit("Put PDFs under data/pdf/ ")
+        
+        vecs = embed_texts(texts,EMB_MODEL)
+        mat=np.array(vecs, dtype="float32")
+        mat = l2_normalize(mat)
+
+        index = faiss.IndexFlatL2(mat.shape[1])
+        index.add(mat)
+        faiss.write_index(index, str(INDEX_PATH))
+        print(f"Index {len(texts)} chunks → {INDEX_PATH}")
+
+if __name__ == "__main__":
+    build_index()

openai_client.py

@@ -0,0 +1,24 @@
+from__future__import annoutations
+from typing import List, Dict
+from openai import OpenAI
+
+_client = None
+
+def client() -> OpenAI:
+    global _client
+    if _client is None:
+        _client = OpenAI()
+    return _client
+
+def embed_text(text: List[str],model:str) -> List[List[float]]:
+    response = client().embeddings.create(model=model,input=texts)
+    return [d.embedding for d in response.data]
+
+def chat(messages:List[Dict],model:str,max_output_tokens:int =700,temperature:float=0.2) -> str:
+    response = client().responses.create(
+        model=model,
+        messages=messages,
+        max_tokens=max_output_tokens,
+        temperature=temperature,
+    )
+    return response.output_text

repository layout

@@ -0,0 +1,15 @@
+ir-esg-rag-openai-8lang/
+├── app.py # Gradio UI + FastAPI (embed可) — 8言語対応
+├── ingest.py # PDF→チャンク→OpenAI Embeddings→FAISS
+├── guardrails.py # スコープ/PII/免責
+├── openai_client.py # Responses API呼び出し・共通ユーティリティ
+├── config.yaml # モデル/閾値/言語
+├── requirements.txt
+├── README.md
+├── data/
+│ ├── pdf/
+│ └── index/
+│ ├── faiss.index
+│ └── meta.jsonl
+└── logs/
+└── qa_log.jsonl

requirements.txt

@@ -0,0 +1,9 @@
+openai>=1.40.0
+faiss-cpu==1.8.0.post1
+pypdf==4.2.0
+PyYAML==6.0.2
+gradio==4.44.0
+fastapi==0.112.0
+uvicorn==0.30.5
+httpx==0.27.0
+pydantic==2.8.2