Upload 7 files

app.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-import os, json, yaml, subprocess, sys, pathlib, traceback, shutil, re
+import os, json, yaml, subprocess, sys, pathlib, traceback, shutil, re, getpass, stat
 from typing import List, Dict, Tuple, Iterable, Optional
 
 from fastapi import FastAPI, Body
@@ -56,11 +56,18 @@ except Exception as e:
     CFG = DEFAULT_CFG
     CFG_ERR = "config.yaml 読み込みエラー: " + str(e)
 
-# ===== paths & utils =====
-INDEX_PATH = pathlib.Path("data/index/faiss.index")
-META_PATH  = pathlib.Path("data/index/meta.jsonl")
-PDF_DIR    = pathlib.Path("data/pdf")
+# ===== absolute paths =====
+BASE_DIR  = pathlib.Path(__file__).resolve().parent
+DATA_DIR  = BASE_DIR / "data"
+INDEX_DIR = DATA_DIR / "index"
+PDF_DIR   = DATA_DIR / "pdf"
+
+INDEX_PATH = INDEX_DIR / "faiss.index"
+META_PATH  = INDEX_DIR / "meta.jsonl"
+
+# ensure dirs
 PDF_DIR.mkdir(parents=True, exist_ok=True)
+INDEX_DIR.mkdir(parents=True, exist_ok=True)
 
 def _lazy_imports():
     global faiss, np, embed_texts, chat, detect_out_of_scope, sanitize, compliance_block, SCOPE_HINT
@@ -172,14 +179,18 @@ def _safe_name(name: str) -> str:
 
 def save_uploaded_pdfs(file_paths: Optional[Iterable[str]]) -> Tuple[int, List[str], List[str]]:
     """
-    Gradioのgr.Files(type='filepath') から渡される一時ファイルパス群を data/pdf/ にコピー
-    return: (保存数, 保存先パス一覧, スキップorエラー一覧)
+    gr.Files(type='filepath') からの一時ファイル群を data/pdf/ に保存
+    return: (保存数, 保存先, スキップ/エラー)
     """
     saved, skipped = [], []
     if not file_paths:
         return 0, saved, ["アップロードされたPDFがありません。"]
 
-    PDF_DIR.mkdir(parents=True, exist_ok=True)
+    try:
+        PDF_DIR.mkdir(parents=True, exist_ok=True)
+    except Exception as e:
+        return 0, [], [f"data/pdf の作成に失敗: {e}"]
+
     for fp in file_paths:
         if not fp:
             continue
@@ -192,8 +203,11 @@ def save_uploaded_pdfs(file_paths: Optional[Iterable[str]]) -> Tuple[int, List[s
             continue
         dst = PDF_DIR / _safe_name(src.name)
         try:
-            shutil.copy2(src, dst)
+            # 権限問題を避けるため copyfile（メタデータを引き継がない）
+            shutil.copyfile(src, dst)
             saved.append(str(dst))
+        except PermissionError as e:
+            skipped.append(f"{src.name}: Permission denied（{dst}）。Dockerfileの所有権設定を確認してください。")
         except Exception as e:
             skipped.append(f"{src.name}: コピー失敗 ({e})")
     return len(saved), saved, skipped
@@ -202,7 +216,7 @@ def upload_and_rebuild(file_paths: Optional[List[str]]) -> str:
     n, saved, skipped = save_uploaded_pdfs(file_paths)
     msg = []
     if n > 0:
-        msg.append(f"📥 {n} 件のPDFを data/pdf/ に保存しました。")
+        msg.append(f"📥 {n} 件のPDFを {PDF_DIR} に保存しました。")
         msg.extend([f"- {p}" for p in saved[:10]])
     if skipped:
         msg.append("⚠️ スキップ/エラー:")
@@ -216,9 +230,9 @@ def rebuild_index() -> str:
     if not _check_api_key():
         return "OPENAI_API_KEY が未設定です。コンソール / Secrets に登録してください。"
     if not list(PDF_DIR.glob("*.pdf")):
-        return "data/pdf/ にPDFがありません。PDFをアップロードして再実行してください。"
+        return f"{PDF_DIR} にPDFがありません。PDFをアップロードして再実行してください。"
     try:
-        out = subprocess.run([sys.executable, "ingest.py"], capture_output=True, text=True, check=True)
+        out = subprocess.run([sys.executable, str(BASE_DIR / "ingest.py")], capture_output=True, text=True, check=True)
         # キャッシュ破棄
         global _INDEX, _METAS
         _INDEX = None
@@ -229,6 +243,29 @@ def rebuild_index() -> str:
     except Exception as e:
         return "❌ 予期せぬエラー: " + str(e) + "\n" + traceback.format_exc()[-1200:]
 
+# ===== File-system diagnose (optional) =====
+def fs_diagnose() -> str:
+    lines = []
+    lines.append(f"User: {getpass.getuser()}")
+    lines.append(f"CWD : {os.getcwd()}")
+    for p in [BASE_DIR, DATA_DIR, PDF_DIR, INDEX_DIR]:
+        try:
+            st = p.stat()
+            mode = stat.filemode(st.st_mode)
+            lines.append(f"{p}  exists={p.exists()}  owner={st.st_uid}:{st.st_gid}  mode={mode}")
+        except Exception as e:
+            lines.append(f"{p}  stat error: {e}")
+    # 書き込みテスト
+    try:
+        test = PDF_DIR / "_write_test.tmp"
+        with open(test, "wb") as f:
+            f.write(b"ok")
+        test.unlink()
+        lines.append("WRITE TEST: OK (data/pdf に書き込み可能)")
+    except Exception as e:
+        lines.append(f"WRITE TEST: NG -> {e}")
+    return "```\n" + "\n".join(lines) + "\n```"
+
 # ===== FastAPI =====
 app = FastAPI(title=CFG.get("app_name", "RAG Bot"))
 app.add_middleware(
@@ -278,9 +315,11 @@ with gr.Blocks(fill_height=True, title=CFG.get("app_name", "RAG Bot")) as demo:
         uploads = gr.Files(label="PDFをドラッグ＆ドロップ（複数可）", file_types=[".pdf"], type="filepath")
     with gr.Row():
         up_btn = gr.Button("アップロード → インデックス再構築", variant="secondary")
+        diag_btn = gr.Button("📋 ストレージ診断")
     up_log = gr.Markdown()
+    diag_log = gr.Markdown()
     up_btn.click(fn=upload_and_rebuild, inputs=[uploads], outputs=[up_log])
+    diag_btn.click(fn=fs_diagnose, outputs=[diag_log])
 
 from gradio.routes import mount_gradio_app
 mount_gradio_app(app, demo, path="/")
-

config.yaml

@@ -1,16 +1,8 @@
 app_name: "IR/ESG RAG Bot (OpenAI, 8 languages)"
 embedding_model: "text-embedding-3-large"
 normalize_embeddings: true
-
-chunk:
-  target_chars: 1400
-  overlap_chars: 180
-
-retrieval:
-  top_k: 6
-  score_threshold: 0.15
-  mmr_lambda: 0.3
-
+chunk: { target_chars: 1400, overlap_chars: 180 }
+retrieval: { top_k: 6, score_threshold: 0.15, mmr_lambda: 0.3 }
 llm:
   model: "gpt-4o-mini"
   max_output_tokens: 700
@@ -18,19 +10,6 @@ llm:
   system_prompt: |-
     あなたは上場企業のIR・ESG開示に特化したRAGアシスタントです。回答は常に根拠（文書名・ページ）を箇条書きで示し、
     文書外の推測や断定は避けます。数値は年度と単位を明記し、最新年度を優先してください。
-
 languages:
   preferred: [ja, en, zh, ko, fr, de, es, it]
-  labels:
-    ja: "日本語"
-    en: "English"
-    zh: "中文"
-    ko: "한국어"
-    fr: "Français"
-    de: "Deutsch"
-    es: "Español"
-    it: "Italiano"
-
-logging:
-  save_qa: true
-  path: "logs/qa_log.jsonl"
+  labels: { ja: 日本語, en: English, zh: 中文, ko: 한국어, fr: Français, de: Deutsch, es: Español, it: Italiano }

ingest.py

@@ -10,18 +10,22 @@ import yaml
 from openai_client import embed_texts
 from guardrails import sanitize
 
-CFG = yaml.safe_load(open("config.yaml", encoding="utf-8"))
-EMB_MODEL = CFG["embedding_model"]
-NORMALIZE = CFG.get("normalize_embeddings", True)
-
-DATA_DIR = pathlib.Path("data")
-PDF_DIR = DATA_DIR / "pdf"
+# absolute paths
+BASE_DIR  = pathlib.Path(__file__).resolve().parent
+DATA_DIR  = BASE_DIR / "data"
+PDF_DIR   = DATA_DIR / "pdf"
 INDEX_DIR = DATA_DIR / "index"
-META_PATH = INDEX_DIR / "meta.jsonl"      # app.py と一致
+META_PATH = INDEX_DIR / "meta.jsonl"
 INDEX_PATH = INDEX_DIR / "faiss.index"
 
-def read_pdf_with_pages(path: str) -> List[Tuple[int, str]]:
-    pages: List[Tuple[int, str]] = []
+CFG = yaml.safe_load(open(BASE_DIR / "config.yaml", encoding="utf-8"))
+EMB_MODEL = CFG["embedding_model"]
+NORMALIZE = CFG.get("normalize_embeddings", True)
+target_chars = CFG["chunk"]["target_chars"]
+overlap_chars = CFG["chunk"]["overlap_chars"]
+
+def read_pdf_with_pages(path: str):
+    pages = []
     reader = PdfReader(path)
     for i, p in enumerate(reader.pages):
         txt = p.extract_text() or ""
@@ -29,8 +33,8 @@ def read_pdf_with_pages(path: str) -> List[Tuple[int, str]]:
         pages.append((i + 1, txt))
     return pages
 
-def split_chunks(pages: List[Tuple[int, str]], target_chars: int, overlap_chars: int) -> List[Dict]:
-    chunks: List[Dict] = []
+def split_chunks(pages: List[Tuple[int, str]], target_chars: int, overlap_chars: int):
+    chunks = []
     for page, text in pages:
         if not text:
             continue
@@ -51,33 +55,27 @@ def l2_normalize(m: np.ndarray) -> np.ndarray:
 
 def build_index():
     INDEX_DIR.mkdir(parents=True, exist_ok=True)
-    meta_f = open(META_PATH, "w", encoding="utf-8")
-
-    target_chars = CFG["chunk"]["target_chars"]
-    overlap_chars = CFG["chunk"]["overlap_chars"]
-
-    texts: List[str] = []
-    for pdf in sorted(PDF_DIR.glob("*.pdf")):
-        print(f"Processing {pdf.name}...")
-        pages = read_pdf_with_pages(str(pdf))
-        chunks = split_chunks(pages, target_chars, overlap_chars)
-        for c in chunks:
-            t = c["text"][:1800]
-            texts.append(t)
-            meta = {"source": pdf.name, "page": c["page"], "text": sanitize(t)}
-            meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
-
-    meta_f.close()
+    with open(META_PATH, "w", encoding="utf-8") as meta_f:
+        texts: List[str] = []
+        for pdf in sorted(PDF_DIR.glob("*.pdf")):
+            print(f"Processing {pdf.name}...")
+            pages = read_pdf_with_pages(str(pdf))
+            chunks = split_chunks(pages, target_chars, overlap_chars)
+            for c in chunks:
+                t = c["text"][:1800]
+                texts.append(t)
+                meta = {"source": pdf.name, "page": c["page"], "text": sanitize(t)}
+                meta_f.write(json.dumps(meta, ensure_ascii=False) + "\n")
 
-    if not texts:
+    if not META_PATH.exists() or META_PATH.stat().st_size == 0:
         raise SystemExit("Put PDFs under data/pdf/")
 
+    # embed
     vecs = embed_texts(texts, EMB_MODEL)
     mat = np.array(vecs, dtype="float32")
     mat = l2_normalize(mat)
 
-    # コサイン類似（正規化済みベクトル × 内積）
-    index = faiss.IndexFlatIP(mat.shape[1])
+    index = faiss.IndexFlatIP(mat.shape[1])  # cosine via normalized dot
     index.add(mat)
     faiss.write_index(index, str(INDEX_PATH))
     print(f"Index {len(texts)} chunks → {INDEX_PATH}")