Update app.py

app.py

@@ -4,6 +4,7 @@ import glob
 import csv
 import threading
 import shutil
+import random
 from datetime import datetime, timedelta
 from typing import Optional
 
@@ -11,6 +12,9 @@ import gradio as gr
 import textstat
 from openai import OpenAI
 
+# ★追加：xlsx読込
+import pandas as pd
+
 # =========================
 # 設定（元コード踏襲）
 # =========================
@@ -33,6 +37,9 @@ os.makedirs(REWRITE_DIR, exist_ok=True)
 
 PASSAGES_DIR = os.getenv("PASSAGES_DIR", "passages")
 
+# ★追加：Excel
+PASSAGE_INFO_XLSX = os.getenv("PASSAGE_INFO_XLSX", "passage_information.xlsx")
+
 if not API_KEY:
     raise RuntimeError("API_KEY is not set (env: API_KEY)")
 
@@ -44,147 +51,77 @@ _rewrite_sem = threading.Semaphore(REWRITE_CONCURRENCY)
 _stop_flag_lock = threading.Lock()
 _stop_flag = False
 
+
 # =========================
-# passages の列挙
+# ユーティリティ
 # =========================
-def list_passage_files_sorted(passages_dir: str) -> list[tuple[int, str]]:
-    pattern = os.path.join(passages_dir, "pg*.txt")
-    files = glob.glob(pattern)
-
-    items: list[tuple[int, str]] = []
-    for fp in files:
-        name = os.path.basename(fp)
-        m = re.match(r"pg(\d+)\.txt$", name)
-        if m:
-            items.append((int(m.group(1)), fp))
-    items.sort(key=lambda x: x[0])
-    return items
-
-
 def load_text(path: str) -> str:
     with open(path, "r", encoding="utf-8") as f:
         return f.read()
 
 
+def target_flesch_from_level(level: int) -> int:
+    level_to_flesch = {1: 90, 2: 75, 3: 65, 4: 55, 5: 40}
+    return level_to_flesch[int(level)]
+
+
+def passage_path_from_text_id(text_id: int, passages_dir: str) -> str:
+    # 元コードの pg{ID}.txt に合わせる
+    return os.path.join(passages_dir, f"pg{text_id}.txt")
+
+
 # =========================
-# ★追加：プロンプト選択
+# Excel読み込み（Text# と flesch_score）
 # =========================
-PROMPT_MODES = {
-    "既存プロンプト": "legacy",
-    "AIプロンプトセット（FRE制御）": "final_v1",
-}
+def load_passage_info(xlsx_path: str) -> pd.DataFrame:
+    if not os.path.exists(xlsx_path):
+        raise FileNotFoundError(f"Not found: {xlsx_path}")
 
+    df = pd.read_excel(xlsx_path)
 
-def prompt_tag_from_mode(prompt_mode: str) -> str:
-    """
-    ダウンロードファイル名で判別できるように、ファイル名安全な短いタグにする
-    """
-    return "legacy" if prompt_mode == "legacy" else "finalv1"
+    # 必須列チェック（スペースや大小の揺れを許容）
+    cols = {c.strip(): c for c in df.columns}
+    if "Text#" not in cols or "flesch_score" not in cols:
+        raise ValueError(
+            "passage_information.xlsx must contain columns: 'Text#' and 'flesch_score'"
+        )
 
+    df = df.rename(columns={cols["Text#"]: "Text#", cols["flesch_score"]: "flesch_score"})
 
-def build_prompt(text: str, target_level: int, prompt_mode: str) -> str:
-    """
-    prompt_mode:
-      - "legacy": 元の1本プロンプト（現状踏襲）
-      - "final_v1": 最終プロンプトセット（Constraints重視）
-    """
-    level_to_flesch = {1: 90, 2: 75, 3: 65, 4: 55, 5: 40}
-    target_flesch = level_to_flesch[int(target_level)]
+    # 型整形
+    df["Text#"] = pd.to_numeric(df["Text#"], errors="coerce").astype("Int64")
+    df["flesch_score"] = pd.to_numeric(df["flesch_score"], errors="coerce")
 
-    if prompt_mode == "legacy":
-        return f"""
-Rewrite the following text to achieve a Flesch Readability Score of {target_flesch}.
+    df = df.dropna(subset=["Text#", "flesch_score"]).copy()
+    df["Text#"] = df["Text#"].astype(int)
+    df["flesch_score"] = df["flesch_score"].astype(float)
+    return df
 
+
+# =========================
+# プロンプト（★1本固定：元の1つ目だけ）
+# =========================
+def build_prompt(text: str, target_level: int) -> str:
+    target_flesch = target_flesch_from_level(target_level)
+
+    return f"""
+Rewrite the following text to achieve a Flesch Readability Score of {target_flesch}.
 Requirements:
 - Extract only the main text, excluding titles, author names, source information, chapter numbers, annotations, footers, etc.
 - Clearly separate sections such as chapters by inserting blank lines between them.
 - Maintain the original meaning faithfully; do not add or remove important information.
-- Rewrite into clear, modern, and natural English suitable for learners at the target level.
-  Avoid archaic expressions, inverted syntax, and literary expressions typical of old picture books.
-  Use simple, modern sentence structures and word order.
+- Rewrite the text in clear, modern English suitable for learners, consistently eliminating archaic, outdated, or literary expressions and unnatural syntax common in older texts, regardless of the target level.
+- Key points to keep in mind when lowering reading comprehension difficulty include keeping sentences short, using familiar vocabulary, prioritizing simple sentence structures, avoiding jargon or providing clear explanations when used, and minimizing figurative language, idioms, and other expressions where meaning cannot be directly inferred from the words themselves.
 - Output only the rewritten text. Do not include explanatory notes or other additional text.
 {text}
-"""
-
-    # ---- final_v1: あなたの「最終プロンプトセット」をコードに落とし込み ----
-    level_name = {
-        1: "Level 90",
-        2: "Level 75",
-        3: "Level 65",
-        4: "Level 55",
-        5: "Level 40",
-    }[int(target_level)]
-
-    level_constraints = {
-        5: [
-            "- Average sentence length: 18–25 words",
-            "- Use abstract nouns (e.g., system, process, structure, responsibility)",
-            "- Allow multi-syllable words",
-            "- Use complex sentences with relative clauses",
-            "- Prefer formal, academic tone",
-            "- Avoid short sentences",
-        ],
-        4: [
-            "- Average sentence length: 14–18 words",
-            "- Mix abstract and concrete vocabulary",
-            "- Avoid very short sentences",
-            "- Limit technical jargon",
-            "- Use clear logical flow",
-        ],
-        3: [
-            "- Average sentence length: 10–14 words",
-            "- Use common vocabulary",
-            "- One main idea per sentence",
-            "- Prefer active voice",
-            "- Avoid abstract nominalizations",
-            "- Split long sentences into shorter independent sentences",
-        ],
-        2: [
-            "- Average sentence length: 7–10 words",
-            "- Use everyday vocabulary only",
-            "- Avoid complex connectors (however, therefore, although)",
-            "- Use short, simple sentences",
-            "- Repeat key ideas using different simple wording",
-        ],
-        1: [
-            "- Average sentence length: 4–6 words",
-            "- Use only basic vocabulary",
-            "- One action or fact per sentence",
-            "- Avoid abstract words completely",
-            "- Avoid explanations and reasoning",
-            "- Prefer many short declarative sentences",
-        ],
-    }[int(target_level)]
-
-    constraints_block = "\n".join(level_constraints)
-
-    return f"""Rewrite the following text for learners at {level_name}.
-
-Requirements:
-- Target Flesch Reading Ease: around {target_flesch} (±3)
-- Keep the original meaning.
-- Do not add new information.
-- Output only the rewritten text.
-
-Constraints:
-{constraints_block}
-
-Additional handling (same as current system):
-- Extract only the portions of the text that should be read as the main body, excluding title/author/source/chapter numbers/annotations/footers.
-- When outputting, make sure sections divided by chapters are clearly distinguishable by leaving a blank line between them.
-
-Text:
-<<<
-{text}
->>>
-"""
+""".strip()
 
 
 # =========================
-# 書き換え（プロンプト切替）
+# 書き換え
 # =========================
-def rewrite_level(text: str, target_level: int, prompt_mode: str) -> str:
-    prompt = build_prompt(text=text, target_level=target_level, prompt_mode=prompt_mode)
+def rewrite_level(text: str, target_level: int) -> str:
+    prompt = build_prompt(text=text, target_level=target_level)
 
     max_tokens_candidates = [3000, 2000, 1500, 1000]
     last_error = None
@@ -223,7 +160,7 @@ def compute_metrics(text: str) -> tuple[float, int]:
 
 
 # =========================
-# CSV追記
+# CSV追記（元コード踏襲：列は増やさない）
 # =========================
 _csv_lock = threading.Lock()
 
@@ -240,7 +177,7 @@ def append_csv_row(row: dict):
 
 
 # =========================
-# txt追記（★ファイル名に prompt_tag を付ける）
+# txt追記（★ファイル名を Text# と Target level が分かる形に）
 # =========================
 _txt_lock = threading.Lock()
 
@@ -248,19 +185,21 @@ _txt_lock = threading.Lock()
 def append_rewrite_txt(
     text_id: int,
     target_level: int,
-    prompt_mode: str,  # ★追加
-    fre: float,
+    original_fre: float,   # ★追加：元教材スコア
+    rewritten_fre: float,
     word_count: int,
     rewritten_text: str,
 ):
     ts = (datetime.utcnow() + timedelta(hours=9)).strftime("%Y-%m-%d %H:%M:%S")
-    tag = prompt_tag_from_mode(prompt_mode)
-    path = os.path.join(REWRITE_DIR, f"Text_{text_id}_{target_level}_{tag}.txt")
+
+    # ★要件：#Text と Target level がわかるファイル名
+    path = os.path.join(REWRITE_DIR, f"Text_{text_id}_Level_{target_level}.txt")
 
     block = (
         f"# Text {text_id}\n"
         f"Target Level: {target_level}\n"
-        f"Flesch Reading Ease: {fre:.2f}\n"
+        f"Original Flesch (from passage_information.xlsx): {original_fre:.2f}\n"
+        f"Rewritten Flesch Reading Ease: {rewritten_fre:.2f}\n"
         f"Word Count: {word_count}\n"
         f"Timestamp (JST): {ts}\n"
         f"Model: {MODEL}\n\n"
@@ -292,44 +231,85 @@ def get_stop() -> bool:
 # UIロジック
 # =========================
 def init_state():
-    files = list_passage_files_sorted(PASSAGES_DIR)
-    return {"files": files, "idx": 0}
+    # ★Excelから Text#/flesch_score 読み込み
+    df = load_passage_info(PASSAGE_INFO_XLSX)
+
+    # stateに保持（元コードの構造を壊さない範囲で追加）
+    return {
+        "df": df,
+        "eligible_ids": [],          # start()でlevelに応じてセット
+        "orig_score_map": {},        # text_id -> original fre
+        "done": 0,
+        "total": 0,
+    }
 
 
-def start(level: int, prompt_mode: str):
+def start(level: int):
     set_stop(False)
     st = init_state()
-    total = len(st["files"])
-    if total == 0:
-        return st, "passages/pg*.txt が見つかりません", "", "", "", None, None
 
-    msg = f"準備完了: {total}件。次に処理するのは #Text {st['files'][0][0]} です。"
-    return st, msg, "", "", "", None, None
+    target_flesch = target_flesch_from_level(level)
+
+    df = st["df"]
+    # ★要件：Targetよりスコアが低い＝難しいもの
+    eligible = df[df["flesch_score"] < float(target_flesch)].copy()
+
+    # passages/pg{Text#}.txt が存在するものだけに絞る（無いと失敗するため）
+    ids = []
+    score_map = {}
+    for _, r in eligible.iterrows():
+        tid = int(r["Text#"])
+        fp = passage_path_from_text_id(tid, PASSAGES_DIR)
+        if os.path.exists(fp):
+            ids.append(tid)
+            score_map[tid] = float(r["flesch_score"])
+
+    if not ids:
+        msg = (
+            f"条件に合う教材がありません。\n"
+            f"- Target FRE: {target_flesch}\n"
+            f"- 条件: flesch_score < {target_flesch}\n"
+            f"- かつ {PASSAGES_DIR}/pg{{Text#}}.txt が存在"
+        )
+        return st, msg, "", "", "", None, None
+
+    random.shuffle(ids)
+    st["eligible_ids"] = ids
+    st["orig_score_map"] = score_map
+    st["done"] = 0
+    st["total"] = len(ids)
 
+    msg = f"準備完了: 条件に合う {st['total']} 件からランダムに処理します（Target FRE={target_flesch}）。"
+    return st, msg, "", f"0 / {st['total']}", "", None, None
 
-def run_one(level: int, prompt_mode: str, state: dict):
+
+def run_one(level: int, state: dict):
     set_stop(False)
 
-    files = state.get("files", [])
-    idx = int(state.get("idx", 0))
-    total = len(files)
+    ids = state.get("eligible_ids", [])
+    done = int(state.get("done", 0))
+    total = int(state.get("total", 0))
+
+    if not ids:
+        return state, "全て処理済みです。", "", f"{done} / {total}", "", None, None
 
-    if idx >= total:
-        return state, "全て処理済みです。", "", "", "", None, None
+    # ★ランダム（ただし未処理リストから先頭を取る：startでshuffle済み）
+    text_id = ids.pop(0)
 
-    text_id, path = files[idx]
+    path = passage_path_from_text_id(text_id, PASSAGES_DIR)
     original = load_text(path)
 
-    rewritten = rewrite_level(original, target_level=level, prompt_mode=prompt_mode)
-    fre, wc = compute_metrics(rewritten)
+    rewritten = rewrite_level(original, target_level=level)
+    rewritten_fre, wc = compute_metrics(rewritten)
 
+    original_fre = float(state.get("orig_score_map", {}).get(text_id, float("nan")))
     ts = (datetime.utcnow() + timedelta(hours=9)).strftime("%Y-%m-%d %H:%M:%S")
 
     append_csv_row({
         "timestamp_jst": ts,
         "Text#": text_id,
         "target_level": level,
-        "flesch_reading_ease": f"{fre:.2f}",
+        "flesch_reading_ease": f"{rewritten_fre:.2f}",
         "word_count": wc,
         "rewritten_text": rewritten
     })
@@ -337,59 +317,65 @@ def run_one(level: int, prompt_mode: str, state: dict):
     append_rewrite_txt(
         text_id=text_id,
         target_level=level,
-        prompt_mode=prompt_mode,  # ★追加
-        fre=fre,
+        original_fre=original_fre,
+        rewritten_fre=rewritten_fre,
         word_count=wc,
         rewritten_text=rewritten,
     )
 
-    state["idx"] = idx + 1
+    done += 1
+    state["done"] = done
+    state["eligible_ids"] = ids
 
-    tag = prompt_tag_from_mode(prompt_mode)
+    saved_txt = os.path.join(REWRITE_DIR, f"Text_{text_id}_Level_{level}.txt")
     header = (
         f"#Text {text_id}\n"
         f"Target Level: {level}\n"
-        f"Prompt Mode: {prompt_mode}\n"
-        f"Flesch Reading Ease: {fre:.2f}\n"
+        f"Original Flesch (xlsx): {original_fre:.2f}\n"
+        f"Rewritten Flesch Reading Ease: {rewritten_fre:.2f}\n"
         f"Word Count: {wc}\n"
         f"Saved CSV: {CSV_PATH}\n"
-        f"Saved TXT: {os.path.join(REWRITE_DIR, f'Text_{text_id}_{level}_{tag}.txt')}"
+        f"Saved TXT: {saved_txt}"
     )
-    progress = f"{state['idx']} / {total}"
+
+    progress = f"{done} / {total}"
     return state, "1件処理しました。", header, progress, rewritten, None, None
 
 
-def run_all(level: int, prompt_mode: str, state: dict):
+def run_all(level: int, state: dict):
     set_stop(False)
 
-    files = state.get("files", [])
-    idx = int(state.get("idx", 0))
-    total = len(files)
+    ids = state.get("eligible_ids", [])
+    done = int(state.get("done", 0))
+    total = int(state.get("total", 0))
 
-    if idx >= total:
-        return state, "全て処理済みです。", "", f"{idx} / {total}", "", None, None
+    if not ids:
+        return state, "全て処理済みです。", "", f"{done} / {total}", "", None, None
 
     last_header = ""
     last_text = ""
 
-    while idx < total:
+    while ids:
         if get_stop():
-            state["idx"] = idx
-            return state, "停止しました。", last_header, f"{idx} / {total}", last_text, None, None
+            state["eligible_ids"] = ids
+            state["done"] = done
+            return state, "停止しました。", last_header, f"{done} / {total}", last_text, None, None
 
-        text_id, path = files[idx]
+        text_id = ids.pop(0)
+        path = passage_path_from_text_id(text_id, PASSAGES_DIR)
         original = load_text(path)
 
-        rewritten = rewrite_level(original, target_level=level, prompt_mode=prompt_mode)
-        fre, wc = compute_metrics(rewritten)
+        rewritten = rewrite_level(original, target_level=level)
+        rewritten_fre, wc = compute_metrics(rewritten)
 
+        original_fre = float(state.get("orig_score_map", {}).get(text_id, float("nan")))
         ts = (datetime.utcnow() + timedelta(hours=9)).strftime("%Y-%m-%d %H:%M:%S")
 
         append_csv_row({
             "timestamp_jst": ts,
             "Text#": text_id,
             "target_level": level,
-            "flesch_reading_ease": f"{fre:.2f}",
+            "flesch_reading_ease": f"{rewritten_fre:.2f}",
             "word_count": wc,
             "rewritten_text": rewritten
         })
@@ -397,28 +383,29 @@ def run_all(level: int, prompt_mode: str, state: dict):
         append_rewrite_txt(
             text_id=text_id,
             target_level=level,
-            prompt_mode=prompt_mode,  # ★追加
-            fre=fre,
+            original_fre=original_fre,
+            rewritten_fre=rewritten_fre,
             word_count=wc,
             rewritten_text=rewritten,
         )
 
-        tag = prompt_tag_from_mode(prompt_mode)
+        done += 1
+        state["done"] = done
+        state["eligible_ids"] = ids
+
+        saved_txt = os.path.join(REWRITE_DIR, f"Text_{text_id}_Level_{level}.txt")
         last_header = (
             f"#Text {text_id}\n"
             f"Target Level: {level}\n"
-            f"Prompt Mode: {prompt_mode}\n"
-            f"Flesch Reading Ease: {fre:.2f}\n"
+            f"Original Flesch (xlsx): {original_fre:.2f}\n"
+            f"Rewritten Flesch Reading Ease: {rewritten_fre:.2f}\n"
             f"Word Count: {wc}\n"
             f"Saved CSV: {CSV_PATH}\n"
-            f"Saved TXT: {os.path.join(REWRITE_DIR, f'Text_{text_id}_{level}_{tag}.txt')}"
+            f"Saved TXT: {saved_txt}"
         )
         last_text = rewritten
 
-        idx += 1
-        state["idx"] = idx
-
-    return state, "全件処理が完了しました。", last_header, f"{idx} / {total}", last_text, None, None
+    return state, "全件処理が完了しました。", last_header, f"{done} / {total}", last_text, None, None
 
 
 def stop():
@@ -446,18 +433,14 @@ def reset_rewrite_folder():
 
 
 # =========================
-# ★追加：ダウンロード機能
+# ダウンロード機能（ファイル名パターン更新）
 # =========================
 def list_generated_txt_files() -> list[str]:
-    # ★ファイル名が Text_{id}_{level}_{tag}.txt になったのでパターン更新
-    files = sorted(glob.glob(os.path.join(REWRITE_DIR, "Text_*_*_*.txt")))
+    files = sorted(glob.glob(os.path.join(REWRITE_DIR, "Text_*_Level_*.txt")))
     return [os.path.basename(f) for f in files]
 
 
 def build_single_txt_path(selected_name: str) -> str:
-    """
-    選択されたtxtをダウンロード用に返す（gr.Fileに渡す）
-    """
     path = os.path.join(REWRITE_DIR, selected_name)
     if not os.path.exists(path):
         raise FileNotFoundError(f"Not found: {path}")
@@ -465,11 +448,6 @@ def build_single_txt_path(selected_name: str) -> str:
 
 
 def build_zip_of_txts(mode: str, n_last: int) -> str:
-    """
-    mode:
-      - 'all' : 全txt
-      - 'last_n' : 最新N個
-    """
     files = sorted(glob.glob(os.path.join(REWRITE_DIR, "Text_*.txt")), key=os.path.getmtime)
     if not files:
         raise FileNotFoundError("No generated txt files yet.")
@@ -477,12 +455,10 @@ def build_zip_of_txts(mode: str, n_last: int) -> str:
     if mode == "last_n":
         files = files[-max(1, int(n_last)):]
 
-    # ZIPは毎回作り直す
     zip_path = os.path.join(OUT_DIR, "rewrite_passages.zip")
     if os.path.exists(zip_path):
         os.remove(zip_path)
 
-    # 一時ディレクトリに集めてからzip（shutil.make_archiveはディレクトリ単位）
     tmp_dir = os.path.join(OUT_DIR, "_zip_tmp")
     if os.path.exists(tmp_dir):
         shutil.rmtree(tmp_dir)
@@ -492,8 +468,6 @@ def build_zip_of_txts(mode: str, n_last: int) -> str:
         shutil.copy(fp, os.path.join(tmp_dir, os.path.basename(fp)))
 
     shutil.make_archive(os.path.join(OUT_DIR, "rewrite_passages"), "zip", tmp_dir)
-
-    # 後片付け
     shutil.rmtree(tmp_dir, ignore_errors=True)
 
     return zip_path
@@ -518,21 +492,15 @@ with gr.Blocks() as demo:
 
     state = gr.State(init_state())
 
-    # ★追加：プロンプト選択UI
-    prompt_mode_ui = gr.Dropdown(
-        choices=list(PROMPT_MODES.keys()),
-        value="既存（シンプル）",
-        label="Prompt Mode（試すプロンプトを選択）"
-    )
-
+    # ★変更：プロンプト選択UIを削除（1本固定）
     level = gr.Dropdown(choices=[1, 2, 3, 4, 5], value=1, label="Target Level (1..5)")
     status = gr.Textbox(label="Status", interactive=False)
-    header = gr.Textbox(label="Result Header (#Text / FRE / Words)", lines=7, interactive=False)
+    header = gr.Textbox(label="Result Header (#Text / FRE / Words)", lines=8, interactive=False)
     progress = gr.Textbox(label="Progress", interactive=False)
     output_text = gr.Textbox(label="Rewritten Text", lines=18, interactive=False)
 
     with gr.Row():
-        start_btn = gr.Button("開始（ファイル読み込み）")
+        start_btn = gr.Button("開始（Excel読み込み＋候補抽出）")
         one_btn = gr.Button("次へ（1件処理）")
         all_btn = gr.Button("全件実行（残りを処理）")
         stop_btn = gr.Button("停止")
@@ -567,27 +535,24 @@ with gr.Blocks() as demo:
         download_csv_btn = gr.Button("CSV（rewrite_scores.csv）をダウンロード")
         download_csv_file = gr.File(label="Download (csv)")
 
-    # 既存の表示（参考）
     gr.Markdown(f"📄 CSVパス: `{CSV_PATH}`")
     gr.Markdown(f"📝 TXTフォルダ: `{REWRITE_DIR}`")
+    gr.Markdown(f"📘 Excel: `{PASSAGE_INFO_XLSX}` / passages dir: `{PASSAGES_DIR}`")
 
-    # ---- 既存ボタン（prompt_mode を追加して渡すだけ）----
-    def _pm_key(pm_label: str) -> str:
-        return PROMPT_MODES.get(pm_label, "legacy")
-
+    # ---- 既存ボタン（prompt_modeなし）----
     start_btn.click(
-        fn=lambda lvl, pm: start(lvl, _pm_key(pm)),
-        inputs=[level, prompt_mode_ui],
+        fn=start,
+        inputs=[level],
         outputs=[state, status, header, progress, output_text, download_one_file, download_zip_file]
     )
     one_btn.click(
-        fn=lambda lvl, pm, st: run_one(lvl, _pm_key(pm), st),
-        inputs=[level, prompt_mode_ui, state],
+        fn=run_one,
+        inputs=[level, state],
         outputs=[state, status, header, progress, output_text, download_one_file, download_zip_file]
     )
     all_btn.click(
-        fn=lambda lvl, pm, st: run_all(lvl, _pm_key(pm), st),
-        inputs=[level, prompt_mode_ui, state],
+        fn=run_all,
+        inputs=[level, state],
         outputs=[state, status, header, progress, output_text, download_one_file, download_zip_file]
     )