Update app.py

app.py

@@ -1,25 +1,50 @@
-import gradio as gr
-from openai import OpenAI
+import os
+import re
+import json
+import glob
+import time
+import random
+import sqlite3
+import threading
+import tempfile
+import inspect
 from datetime import datetime, timedelta
+
+import gradio as gr
 import pandas as pd
-import os, random, json, glob, re, sqlite3, threading, time
+from openai import OpenAI
+from datasets import Dataset
 
-# --- API / 設定 ---
+# =========================
+# Config
+# =========================
 API_KEY = os.getenv("API_KEY")
-BASE_URL = "https://openrouter.ai/api/v1"
-client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
+BASE_URL = os.getenv("BASE_URL", "https://openrouter.ai/api/v1")
+HF_TOKEN = os.getenv("HF_TOKEN")  # pushするなら必要
+DATASET_REPO = os.getenv("DATASET_REPO", "Toya0421/reading_exercise_logging")
+
+# pushを有効にするか（負荷テスト中は 0 推奨）
+ENABLE_HF_PUSH = os.getenv("ENABLE_HF_PUSH", "0") == "1"
+
+# push間隔（秒）
+PUSH_INTERVAL_SEC = int(os.getenv("PUSH_INTERVAL_SEC", "300"))  # 5分
+# push時に出すparquet一時ファイル名
+PARQUET_NAME = "data.parquet"
 
 LOG_DB = "reading_logs.sqlite"
 CACHE_DIR = "rewrite_cache"
 os.makedirs(CACHE_DIR, exist_ok=True)
 
-# --- passage_information.xlsx ---
+client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
+
+# passage_information.xlsx 読み込み
 passage_info_df = pd.read_excel("passage_information.xlsx")
 
-# ======================================================
-#  DB（SQLite）: 速い・同時アクセスに強い（WAL）
-# ======================================================
+# =========================
+# SQLite (WAL) for logs
+# =========================
 _db_lock = threading.Lock()
+_push_lock = threading.Lock()
 
 def init_db():
     with _db_lock:
@@ -42,8 +67,11 @@ def init_db():
 
 init_db()
 
+def now_jst_iso():
+    return (datetime.utcnow() + timedelta(hours=9)).isoformat()
+
 def save_log(entry: dict):
-    # 1イベント=1INSERT（軽い）
+    # 1イベント=1INSERT（軽量）
     with _db_lock:
         conn = sqlite3.connect(LOG_DB, check_same_thread=False)
         conn.execute("PRAGMA journal_mode=WAL;")
@@ -62,16 +90,54 @@ def save_log(entry: dict):
         conn.commit()
         conn.close()
 
-# ======================================================
-#  passages 管理
-# ======================================================
-def load_passage_file(text_id: int):
-    path = f"passages/pg{text_id}.txt"
-    if not os.path.exists(path):
-        return None
-    with open(path, "r", encoding="utf-8") as f:
-        return f.read()
-
+def export_sqlite_to_parquet(parquet_path: str):
+    # DB全体を読み出してparquet化（pushは低頻度でOK）
+    with _db_lock:
+        conn = sqlite3.connect(LOG_DB, check_same_thread=False)
+        df = pd.read_sql_query("SELECT * FROM logs ORDER BY id ASC", conn)
+        conn.close()
+    df.to_parquet(parquet_path, index=False)
+
+def push_to_hub_if_enabled():
+    if not ENABLE_HF_PUSH:
+        return
+    if not HF_TOKEN:
+        print("[WARN] ENABLE_HF_PUSH=1 ですが HF_TOKEN がありません。pushをスキップします。")
+        return
+
+    # pushが重なると壊れるのでロック
+    if not _push_lock.acquire(blocking=False):
+        return
+    try:
+        tmp_dir = tempfile.mkdtemp()
+        parquet_path = os.path.join(tmp_dir, PARQUET_NAME)
+        export_sqlite_to_parquet(parquet_path)
+        dataset = Dataset.from_parquet(parquet_path)
+        dataset.push_to_hub(DATASET_REPO, token=HF_TOKEN)
+        print(f"[INFO] Pushed logs to hub: {DATASET_REPO} ({len(dataset)} rows)")
+    except Exception as e:
+        print(f"[ERROR] push_to_hub failed: {e}")
+    finally:
+        _push_lock.release()
+
+def start_periodic_pusher():
+    # Spacesはプロセスが1つとは限らないので、push頻度は低め推奨
+    if not ENABLE_HF_PUSH:
+        return
+
+    def loop():
+        while True:
+            time.sleep(PUSH_INTERVAL_SEC)
+            push_to_hub_if_enabled()
+
+    th = threading.Thread(target=loop, daemon=True)
+    th.start()
+
+start_periodic_pusher()
+
+# =========================
+# Passages
+# =========================
 def list_passage_ids():
     files = glob.glob("passages/pg*.txt")
     ids = []
@@ -84,38 +150,49 @@ def list_passage_ids():
 
 ALL_PASSAGE_IDS = list_passage_ids()
 
-def get_new_passage_random(used_passages: set):
+def load_passage_file(text_id: int):
+    path = f"passages/pg{text_id}.txt"
+    if not os.path.exists(path):
+        return None
+    with open(path, "r", encoding="utf-8") as f:
+        return f.read()
+
+def get_new_passage_random(used_passages_list):
+    # used_passages_list: list[int]
+    used = set(used_passages_list or [])
     if not ALL_PASSAGE_IDS:
-        return None, None, None, used_passages
+        return None, None, None, []
 
-    available = [pid for pid in ALL_PASSAGE_IDS if pid not in used_passages]
+    available = [pid for pid in ALL_PASSAGE_IDS if pid not in used]
     if not available:
-        used_passages.clear()
+        used.clear()
         available = list(ALL_PASSAGE_IDS)
 
-    text_id = random.choice(available)
-    used_passages.add(text_id)
+    pid = random.choice(available)
+    used.add(pid)
 
-    text = load_passage_file(text_id)
+    text = load_passage_file(pid)
     if text is None:
-        return None, None, None, used_passages
+        return None, None, None, list(used)
 
-    row = passage_info_df[passage_info_df["Text#"] == text_id]
+    row = passage_info_df[passage_info_df["Text#"] == pid]
     orig_level = None if len(row) == 0 else row.iloc[0]["flesch_score"]
-    return text_id, text, orig_level, used_passages
+    return pid, text, orig_level, list(used)
 
-# ======================================================
-#  Rewrite（キャッシュ付き）
-# ======================================================
+# =========================
+# Rewrite (cached)
+# =========================
 def rewrite_cache_path(passage_id: int, level: int):
     return os.path.join(CACHE_DIR, f"pg{passage_id}_lv{level}.json")
 
 def rewrite_level(text: str, target_level: int, passage_id: int):
-    # ディスクキャッシュ
     cpath = rewrite_cache_path(passage_id, target_level)
     if os.path.exists(cpath):
-        with open(cpath, "r", encoding="utf-8") as f:
-            return json.load(f)["rewritten"]
+        try:
+            with open(cpath, "r", encoding="utf-8") as f:
+                return json.load(f)["rewritten"]
+        except Exception:
+            pass
 
     level_to_flesch = {1: 90, 2: 70, 3: 55, 4: 40, 5: 25}
     target_flesch = level_to_flesch[int(target_level)]
@@ -139,16 +216,17 @@ excluding the title, author name, source information, chapter number, annotation
     )
     rewritten = resp.choices[0].message.content.strip()
 
-    # キャッシュ保存
-    with open(cpath, "w", encoding="utf-8") as f:
-        json.dump({"rewritten": rewritten}, f, ensure_ascii=False)
+    try:
+        with open(cpath, "w", encoding="utf-8") as f:
+            json.dump({"rewritten": rewritten}, f, ensure_ascii=False)
+    except Exception:
+        pass
 
     return rewritten
 
 def split_pages(text, max_words=300):
     sentences = re.split(r'(?<=[.!?])\s+', text.strip())
     pages, current, wc = [], [], 0
-
     for s in sentences:
         w = s.split()
         if wc + len(w) > max_words and current:
@@ -157,32 +235,21 @@ def split_pages(text, max_words=300):
         else:
             current.append(s)
             wc += len(w)
-
     if current:
         pages.append(" ".join(current))
-
     return pages or [text]
 
-def now_jst_iso():
-    return (datetime.utcnow() + timedelta(hours=9)).isoformat()
-
-# ======================================================
-#  ハンドラ（セッション状態: gr.State）
-# ======================================================
-# state = {
-#   "user_id": str,
-#   "level": int,
-#   "used_passages": set(int),
-# }
-
+# =========================
+# Gradio handlers (stateful per user)
+# state = {"user_id": str|None, "level": int|None, "used_passages": list[int]}
+# =========================
 def start_test(student_id, level_input, state):
     if state is None:
-        state = {"user_id": None, "level": None, "used_passages": set()}
+        state = {"user_id": None, "level": None, "used_passages": []}
 
-    # 必須チェック
     if not student_id or str(student_id).strip() == "":
         return (
-            "", "", json.dumps([]), 0, 0, "", "", "",
+            "", "", "[]", 0, 0, "", "", "",
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
@@ -191,9 +258,8 @@ def start_test(student_id, level_input, state):
 
     state["user_id"] = str(student_id).strip()
     state["level"] = int(level_input)
-    state["used_passages"] = set()
+    state["used_passages"] = []
 
-    # startログ（軽い）
     save_log({
         "user_id": state["user_id"],
         "assigned_level": state["level"],
@@ -204,10 +270,12 @@ def start_test(student_id, level_input, state):
         "page_text": None
     })
 
-    pid, text, orig_lev, state["used_passages"] = get_new_passage_random(state["used_passages"])
+    pid, text, orig_lev, used_list = get_new_passage_random(state["used_passages"])
+    state["used_passages"] = used_list
+
     if text is None:
         return (
-            "教材が見つかりません", "", json.dumps([]), 0, 0, "", "", "",
+            "教材が見つかりません", "", "[]", 0, 0, "", "", "",
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
@@ -220,7 +288,7 @@ def start_test(student_id, level_input, state):
 
     prev_upd = gr.update(interactive=False, visible=False)
     next_upd = gr.update(interactive=(total > 1), visible=(total > 1))
-    finish_upd = gr.update(interactive=(total == 1), visible=True if total == 1 else False)
+    finish_upd = gr.update(interactive=(total == 1), visible=(total == 1))
 
     save_log({
         "user_id": state["user_id"],
@@ -239,7 +307,7 @@ def start_test(student_id, level_input, state):
         0,
         total,
         str(pid),
-        str(orig_lev) if orig_lev is not None else "",
+        "" if orig_lev is None else str(orig_lev),
         str(state["level"]),
         prev_upd, next_upd, finish_upd,
         state
@@ -248,15 +316,15 @@ def start_test(student_id, level_input, state):
 def next_page(pages_json, current_page, total_pages, pid, orig_lev, state):
     pages = json.loads(pages_json) if pages_json else []
     if not pages:
-        return ("", "", json.dumps([]), 0,
+        return ("", "", "[]", 0,
                 gr.update(interactive=False, visible=False),
                 gr.update(interactive=False, visible=False),
                 gr.update(interactive=False, visible=False),
                 state)
 
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": int(pid),
         "original_level": float(orig_lev) if orig_lev not in ("", None) else None,
         "action_time": now_jst_iso(),
@@ -264,11 +332,13 @@ def next_page(pages_json, current_page, total_pages, pid, orig_lev, state):
         "page_text": None
     })
 
-    new_page = min(int(current_page) + 1, int(total_pages) - 1)
+    total_pages = int(total_pages)
+    current_page = int(current_page)
+    new_page = min(current_page + 1, total_pages - 1)
 
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": int(pid),
         "original_level": float(orig_lev) if orig_lev not in ("", None) else None,
         "action_time": now_jst_iso(),
@@ -277,7 +347,7 @@ def next_page(pages_json, current_page, total_pages, pid, orig_lev, state):
     })
 
     prev_upd = gr.update(interactive=(new_page > 0), visible=(new_page > 0))
-    next_visible = (new_page < int(total_pages) - 1)
+    next_visible = (new_page < total_pages - 1)
     next_upd = gr.update(interactive=next_visible, visible=next_visible)
     finish_upd = gr.update(interactive=(not next_visible), visible=(not next_visible))
 
@@ -293,15 +363,15 @@ def next_page(pages_json, current_page, total_pages, pid, orig_lev, state):
 def prev_page(pages_json, current_page, total_pages, pid, orig_lev, state):
     pages = json.loads(pages_json) if pages_json else []
     if not pages:
-        return ("", "", json.dumps([]), 0,
+        return ("", "", "[]", 0,
                 gr.update(interactive=False, visible=False),
                 gr.update(interactive=False, visible=False),
                 gr.update(interactive=False, visible=False),
                 state)
 
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": int(pid),
         "original_level": float(orig_lev) if orig_lev not in ("", None) else None,
         "action_time": now_jst_iso(),
@@ -309,11 +379,13 @@ def prev_page(pages_json, current_page, total_pages, pid, orig_lev, state):
         "page_text": None
     })
 
-    new_page = max(int(current_page) - 1, 0)
+    total_pages = int(total_pages)
+    current_page = int(current_page)
+    new_page = max(current_page - 1, 0)
 
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": int(pid),
         "original_level": float(orig_lev) if orig_lev not in ("", None) else None,
         "action_time": now_jst_iso(),
@@ -322,7 +394,7 @@ def prev_page(pages_json, current_page, total_pages, pid, orig_lev, state):
     })
 
     prev_upd = gr.update(interactive=(new_page > 0), visible=(new_page > 0))
-    next_visible = (new_page < int(total_pages) - 1)
+    next_visible = (new_page < total_pages - 1)
     next_upd = gr.update(interactive=next_visible, visible=next_visible)
     finish_upd = gr.update(interactive=(not next_visible), visible=(not next_visible))
 
@@ -337,8 +409,8 @@ def prev_page(pages_json, current_page, total_pages, pid, orig_lev, state):
 
 def finish_or_retire(pages_json, current_page, pid, orig_lev, action, state):
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": int(pid),
         "original_level": float(orig_lev) if orig_lev not in ("", None) else None,
         "action_time": now_jst_iso(),
@@ -346,10 +418,12 @@ def finish_or_retire(pages_json, current_page, pid, orig_lev, action, state):
         "page_text": None
     })
 
-    new_pid, new_text, new_orig_lev, state["used_passages"] = get_new_passage_random(state["used_passages"])
+    new_pid, new_text, new_orig_lev, used_list = get_new_passage_random(state.get("used_passages", []))
+    state["used_passages"] = used_list
+
     if new_text is None:
         return (
-            "教材がありません", "", json.dumps([]), 0, 0, "", "", "",
+            "教材がありません", "", "[]", 0, 0, "", "", "",
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
             gr.update(interactive=False, visible=False),
@@ -362,11 +436,11 @@ def finish_or_retire(pages_json, current_page, pid, orig_lev, action, state):
 
     prev_upd = gr.update(interactive=False, visible=False)
     next_upd = gr.update(interactive=(total > 1), visible=(total > 1))
-    finish_upd = gr.update(interactive=(total == 1), visible=True if total == 1 else False)
+    finish_upd = gr.update(interactive=(total == 1), visible=(total == 1))
 
     save_log({
-        "user_id": state["user_id"],
-        "assigned_level": state["level"],
+        "user_id": state.get("user_id"),
+        "assigned_level": state.get("level"),
         "passage_id": new_pid,
         "original_level": new_orig_lev,
         "action_time": now_jst_iso(),
@@ -381,16 +455,57 @@ def finish_or_retire(pages_json, current_page, pid, orig_lev, action, state):
         0,
         total,
         str(new_pid),
-        str(new_orig_lev) if new_orig_lev is not None else "",
+        "" if new_orig_lev is None else str(new_orig_lev),
         str(state["level"]),
         prev_upd, next_upd, finish_upd,
         state
     )
 
-# ======================================================
-#  UI
-# ======================================================
-custom_css = """/* ここはあなたのCSSをそのまま貼ってOK */"""
+# =========================
+# Gradio queue/launch (version-safe)
+# =========================
+def safe_queue(blocks: gr.Blocks, concurrency: int = 8, max_size: int = 64):
+    sig = inspect.signature(blocks.queue)
+    kwargs = {}
+    # Gradioのバージョン差を吸収
+    if "default_concurrency_limit" in sig.parameters:
+        kwargs["default_concurrency_limit"] = concurrency
+    elif "concurrency_limit" in sig.parameters:
+        kwargs["concurrency_limit"] = concurrency
+    elif "concurrency_count" in sig.parameters:
+        kwargs["concurrency_count"] = concurrency
+    if "max_size" in sig.parameters:
+        kwargs["max_size"] = max_size
+    return blocks.queue(**kwargs)
+
+def safe_launch(blocks: gr.Blocks):
+    sig = inspect.signature(blocks.launch)
+    kwargs = {}
+    # Spacesだと server_name/server_port は不要なことが多いが、あっても問題ない
+    if "server_name" in sig.parameters:
+        kwargs["server_name"] = "0.0.0.0"
+    if "server_port" in sig.parameters and os.getenv("PORT"):
+        kwargs["server_port"] = int(os.getenv("PORT"))
+    # max_threadsがある版だけ付ける
+    if "max_threads" in sig.parameters:
+        kwargs["max_threads"] = 16
+    return blocks.launch(**kwargs)
+
+# =========================
+# UI
+# =========================
+custom_css = """
+.big-text {
+    font-size: 22px !important;
+    line-height: 1.8 !important;
+    font-family: "Noto Sans", sans-serif !important;
+}
+.reading-area {
+    padding: 20px !important;
+    border-radius: 12px !important;
+    border: 1px solid #ccc !important;
+}
+"""
 
 with gr.Blocks(css=custom_css) as demo:
     gr.Markdown("# 📚 Reading Exercise")
@@ -399,7 +514,7 @@ with gr.Blocks(css=custom_css) as demo:
     level_input = gr.Dropdown(choices=[1,2,3,4,5], label="Reading Level", value=3)
     start_btn = gr.Button("スタート")
 
-    text_display = gr.Textbox(label="教材", lines=18, interactive=False)
+    text_display = gr.Textbox(label="教材", lines=18, interactive=False, elem_classes=["big-text", "reading-area"])
     page_display = gr.Textbox(label="進行状況", lines=1, interactive=False)
 
     hidden_pages = gr.Textbox(visible=False)
@@ -416,7 +531,7 @@ with gr.Blocks(css=custom_css) as demo:
 
     retire_btn = gr.Button("リタイア")
 
-    state = gr.State({"user_id": None, "level": None, "used_passages": set()})
+    state = gr.State({"user_id": None, "level": None, "used_passages": []})
 
     start_btn.click(
         fn=start_test,
@@ -469,7 +584,6 @@ with gr.Blocks(css=custom_css) as demo:
         ],
     )
 
-# ★同時実行を制御（重要）
-demo.queue(concurrency_count=8, max_size=64)
-
-demo.launch()
+# queueはバージョン差があるので安全ラッパ
+safe_queue(demo, concurrency=8, max_size=64)
+safe_launch(demo)