import streamlit as st import fitz # PyMuPDF import tempfile import os import re import json import requests from collections import Counter, defaultdict from dotenv import load_dotenv from paddleocr import PaddleOCR from datetime import datetime load_dotenv() GROQ_API_KEY = os.getenv("GROQ_API_KEY") # ------------------------------ # 設定: モデルごとのパラメータ # ------------------------------ MODEL_CONFIG = { "qwen/qwen3-32b": {"temperature": 0.2, "max_tokens": 4000}, "groq/compound": {"temperature": 0.3, "max_tokens": 4000}, "meta-llama/llama-4-maverick-17b-128e-instruct": {"temperature": 0.2, "max_tokens": 5000}, "moonshotai/kimi-k2-instruct-0905": {"temperature": 0.4, "max_tokens": 6000}, "openai/gpt-oss-120b": {"temperature": 0.1, "max_tokens": 6000} } # ------------------------------ # 固定分野辞書(数学例。必要に応じて拡張) # ------------------------------ FIELD_CATEGORIES = [ "微分積分", "数列", "確率", "ベクトル", "整数", "図形", "複素数", "場合の数", "極限", "行列", "その他" ] # ------------------------------ # 出題スコアの重み(デフォルト) # 頻度を最重視(0.5)、未出年数0.3、LLM予測0.2 # ------------------------------ WEIGHTS = { "freq": 0.5, "years": 0.3, "llm": 0.2 } # ------------------------------ # 難易度算出重み(典型性/誘導/発想) # previous design: typicality 0.35, guidance 0.25, creativity 0.40 # ------------------------------ DIFFICULTY_WEIGHTS = {"typicality": 0.35, "guidance": 0.25, "creativity": 0.40} # ------------------------------ # LLM呼び出し関数(Groq OpenAI互換エンドポイント想定) # ------------------------------ def call_llm(prompt: str, model: str): """ Groq chat completions (OpenAI-like) を呼ぶ簡易 wrapper。 返り値は raw text を返す(JSONを返すようにプロンプトで依頼すること)。 """ if GROQ_API_KEY is None: raise RuntimeError("GROQ_API_KEY is not set in environment variables.") cfg = MODEL_CONFIG.get(model, {}) payload = { "model": model, "messages": [{"role": "user", "content": prompt}], "temperature": cfg.get("temperature", 0.2), "max_tokens": cfg.get("max_tokens", 2000) } resp = requests.post( "https://api.groq.com/openai/v1/chat/completions", headers={ "Authorization": f"Bearer {GROQ_API_KEY}", "Content-Type": "application/json" }, json=payload, timeout=120 ) resp.raise_for_status() j = resp.json() # typical OpenAI-like response shape try: return j["choices"][0]["message"]["content"] except Exception: # フォールバックでraw json return json.dumps(j, ensure_ascii=False) # ------------------------------ # OCR(PaddleOCR)初期化 # ------------------------------ # note: 実行環境によっては model_dir 指定や use_gpu などの調整が必要 ocr = PaddleOCR(use_angle_cls=True, lang="japan") # 日本語OCR def ocr_image_file(img_path: str) -> str: """PaddleOCRで画像ファイルを読み取り、テキストを返す(改行つき)。""" try: result = ocr.ocr(img_path, cls=True) except Exception as e: st.warning(f"OCRエラー: {e}") return "" text_lines = [] # result の形式はページごとのリスト等環境によって差があるが通常は result[0]が行リスト if isinstance(result, list) and len(result) > 0 and isinstance(result[0], list): for line in result[0]: # line: [box, (text, confidence)] text_lines.append(line[1][0]) else: # 互換形 for page in result: for line in page: text_lines.append(line[1][0]) return "\n".join(text_lines) def extract_text_from_pdf_bytes(file_bytes: bytes, max_pages: int = 20) -> str: """ PDFバイト列を受け取り、各ページを画像化してOCRでテキストを抽出する(最大 max_pages)。 """ text_accum = [] with fitz.open(stream=file_bytes, filetype="pdf") as doc: pages = min(len(doc), max_pages) for i in range(pages): page = doc[i] # 画像解像度を上げる - scale = 2.0 等で精度向上 mat = fitz.Matrix(2.0, 2.0) pix = page.get_pixmap(matrix=mat, alpha=False) tmp = tempfile.NamedTemporaryFile(suffix=".png", delete=False) try: pix.save(tmp.name) txt = ocr_image_file(tmp.name) text_accum.append(txt) finally: try: tmp.close() os.unlink(tmp.name) except Exception: pass return "\n\n".join(text_accum) # ------------------------------ # 年度抽出(ファイル名から4桁の年を探す) # ------------------------------ def extract_year_from_filename(filename: str): m = re.search(r"(20\d{2}|19\d{2})", filename) if m: try: return int(m.group(0)) except: return None return None # ------------------------------ # 問題分割: 正規表現で一次分割 + LLMに整形させる # ------------------------------ def initial_split_by_regex(text: str): # よくある区切りにより一次分割 # (第\d+問) や "問\d"、"(1)" などで分ける parts = re.split(r"\n{2,}", text) # 空行で分割の一次案 # さらに "第\d+問" を前処理で残す形にして細分化 out = [] for p in parts: sub = re.split(r"(第\s*\d+\s*問|問\s*\d+|^【問題】)", p) if len(sub) > 1: # join fragments that start with marker buf = "" for s in sub: if re.match(r"(第\s*\d+\s*問|問\s*\d+|^【問題】)", s): if buf: out.append(buf.strip()) buf = s else: buf += s if buf: out.append(buf.strip()) else: if p.strip(): out.append(p.strip()) # filter empty out = [o for o in out if o and len(o) > 10] return out def split_questions_via_llm(raw_text: str, model_choice: str): """ raw_text を LLM に投げ、JSONで問題リストを返すように促す。 戻り: list of { "id": str, "text": str } """ # まず一次分割(軽量) candidates = initial_split_by_regex(raw_text) # Join for context but limit length sample_for_prompt = "\n\n---\n\n".join(candidates[:80]) prompt = f""" あなたは日本語の大学入試問題を「独立した小問」ごとに分割する専門家です。 与えられたテキストから、独立した問題ごとに分割し、JSONで次の形式を返してください。 {{"questions": [{{"id": "Q1", "text": "..."} , ...]}} 注意: - id は Q1, Q2 ... の形式で付与してください。 - 各 text は問題本文のみにしてください(選択肢や解答欄がある場合はそれらも含めてください)。 - 出力は **純粋なJSON**だけにしてください(説明文は入れないでください)。 テキスト: \"\"\"{sample_for_prompt}\"\"\" """ raw = call_llm(prompt, model_choice) # try parse JSON from response try: j = json.loads(raw) if isinstance(j, dict) and "questions" in j: return j["questions"] except Exception: # try to extract JSON substring m = re.search(r"(\{.*\}|\[.*\])", raw, flags=re.S) if m: try: j = json.loads(m.group(0)) if isinstance(j, dict) and "questions" in j: return j["questions"] except Exception: pass # fallback: convert candidates to simple dict list out = [] for i, c in enumerate(candidates, start=1): out.append({"id": f"Q{i}", "text": c}) return out # ------------------------------ # 分野分類(複数可) — LLMにJSONで返させる # ------------------------------ def classify_field_llm(problem_text: str, model_choice: str): prompt = f""" 次の問題文を、以下の選択肢から該当する分野を**複数選択可**で選んでください。必ずJSONで返してください。 選択肢: {FIELD_CATEGORIES} 問題: \"\"\"{problem_text}\"\"\" 出力形式(例): {{"fields": ["微分積分", "ベクトル"]}} """ raw = call_llm(prompt, model_choice) # parse JSON try: j = json.loads(raw) return j.get("fields", []) except Exception: m = re.search(r"\{.*\}", raw, flags=re.S) if m: try: j = json.loads(m.group(0)) return j.get("fields", []) except: pass # fallback: simple keyword matching return heuristic_field_match(problem_text) def heuristic_field_match(text: str): text_low = text matched = set() # simple keyword heuristics KEYWORDS = { "微分": "微分積分", "積分": "微分積分", "極限": "極限", "数列": "数列", "ベクトル": "ベクトル", "確率": "確率", "整数": "整数", "行列": "行列", "複素": "複素数", "図形": "図形", "場合の数": "場合の数" } for k, v in KEYWORDS.items(): if k in text_low: matched.add(v) if not matched: return ["その他"] return list(matched) # ------------------------------ # 指標評価(典型性・誘導度・発想) # LLMに0-5で評価させJSONで受け取る # ------------------------------ def evaluate_three_metrics(problem_text: str, model_choice: str): prompt = f""" 以下の問題を、次の3項目で0から5の整数(0が最も低く、5が最も高い)で評価し、 JSON形式で返してください。 1) 典型度 (0=非常に典型的, 5=強い応用) 2) 誘導度 (0=明確に誘導あり, 5=誘導なし) 3) 発想必要度 (0=発想不要, 5=高度な発想が必要) 出力例: {{"typicality": 2, "guidance": 4, "creativity": 3}} 問題: \"\"\"{problem_text}\"\"\" """ raw = call_llm(prompt, model_choice) try: j = json.loads(raw) # Ensure integers and clamp 0-5 for k in ["typicality", "guidance", "creativity"]: if k in j: try: j[k] = int(round(float(j[k]))) except: j[k] = 0 j[k] = max(0, min(5, j[k])) else: j[k] = 0 return j except Exception: # extract numbers heuristically m = re.search(r"\{.*\}", raw, flags=re.S) if m: try: j = json.loads(m.group(0)) for k in ["typicality", "guidance", "creativity"]: if k in j: try: j[k] = int(round(float(j[k]))) except: j[k] = 0 j[k] = max(0, min(5, j[k])) else: j[k] = 0 return j except: pass # fallback defaults return {"typicality": 2, "guidance": 2, "creativity": 2} # ------------------------------ # 難易度スコア計算 → 易/標準/難 # ------------------------------ def compute_difficulty_level(metrics: dict): typical = metrics.get("typicality", 2) guidance = metrics.get("guidance", 2) creativity = metrics.get("creativity", 2) score = ( typical * DIFFICULTY_WEIGHTS["typicality"] + guidance * DIFFICULTY_WEIGHTS["guidance"] + creativity * DIFFICULTY_WEIGHTS["creativity"] ) # 0..5 scale -> thresholds: if score < 2.0: level = "易" elif score < 3.5: level = "標準" else: level = "難" return {"score": round(score, 3), "level": level} # ------------------------------ # LLMによる分野別「今後の出題可能性」スコア(0-10) # 1回のファイル(ある大学の過去問集合)ごとに一括で聞く # ------------------------------ def ask_llm_field_likelihood(problems_text: str, fields: list, model_choice: str): """ fields: list of field strings to query (unique set) returns dict field->likelihood (0-10) """ field_list_str = ", ".join(fields) prompt = f""" あなたは大学入試の出題予測の専門家です。 以下は過去問(複数問題を連結したテキスト)です。与えられた分野ごとに、次年度に「その大学がその分野を出しそうな可能性」を 0 ~ 10 の整数で評価し、JSONで返してください。 0は「まず出ない」、10は「非常に出やすい」を表します。 対象分野: [{field_list_str}] 過去問全文: \"\"\"{problems_text}\"\"\" 出力例: {{"微分積分": 6, "確率": 3}} """ raw = call_llm(prompt, model_choice) try: j = json.loads(raw) # ensure integer 0-10 for k in list(j.keys()): try: j[k] = int(round(float(j[k]))) except: j[k] = 0 j[k] = max(0, min(10, j[k])) return j except Exception: # fallback: empty scores fallback = {f: 5 for f in fields} return fallback # ------------------------------ # 類題 + 解答 生成(分野 × 難易度を与えて1問ずつ生成) # ------------------------------ def generate_similar_problem_and_solution(field: str, difficulty_level: str, context_examples: str, model_choice: str): """ difficulty_level in ("易","標準","難") returns dict with "problem", "solution", "explanation" """ prompt = f""" あなたは大学入試の問題を作る教師です。 分野: {field} 難易度: {difficulty_level} 文脈(過去問の例/作風): \"\"\"{context_examples}\"\"\" この条件で**大学入試らしい類題を1問**作成し、続けて**模範解答**と**丁寧な解説**を作ってください。 出力は**JSONのみ**で、キーは "problem", "solution", "explanation" としてください。 例: {{"problem": "...", "solution": "...", "explanation": "..."}} """ raw = call_llm(prompt, model_choice) try: j = json.loads(raw) return { "problem": j.get("problem", "").strip(), "solution": j.get("solution", "").strip(), "explanation": j.get("explanation", "").strip() } except Exception: # try to extract parts heuristically # fallback: put entire raw into problem and blank solution (not ideal) return {"problem": raw.strip(), "solution": "", "explanation": ""} # ------------------------------ # 出題スコア算出 # freq_norm: normalized frequency 0..1 # years_since_last: integer (0 if unknown) # llm_likelihood: 0..10 -> normalize to 0..1 # ------------------------------ def compute_field_score(freq_norm: float, years_since_last: int, llm_likelihood: int): # normalize years: assume recent = 0 years -> transform to a 0..1 where larger is increases score # We'll cap years_since_last to 5 for scaling years_factor = min(max(years_since_last, 0), 5) / 5.0 # 0..1 llm_norm = min(max(llm_likelihood, 0), 10) / 10.0 # 0..1 score = ( WEIGHTS["freq"] * freq_norm + WEIGHTS["years"] * years_factor + WEIGHTS["llm"] * llm_norm ) return round(score, 4) # ------------------------------ # ユーティリティ: 年度ごとに各分野最終出現年を推定(ファイル名に年があれば) # uploaded_files_meta: list of dict {"name":..., "year": int or None, "problems": [...]} # problems have 'fields' and maybe 'year' info # ------------------------------ def estimate_last_appearance_years(uploaded_files_meta): """ Returns dict field -> latest_year it appears in uploaded_files_meta; None if unknown. """ last_year = {} for f in uploaded_files_meta: year = f.get("year") problems = f.get("problems", []) for p in problems: for fld in p.get("fields", []): if fld not in last_year: last_year[fld] = None if year: if last_year[fld] is None or year > last_year[fld]: last_year[fld] = year return last_year # ------------------------------ # Streamlit UI & main workflow # ------------------------------ st.set_page_config(page_title="過去問解析・出題予測アプリ", layout="wide") st.title("過去問解析・出題予測(PaddleOCR + LLM)") with st.sidebar: st.header("設定") model_choice = st.selectbox("モデル選択", list(MODEL_CONFIG.keys())) max_pages = st.number_input("PDF 最大ページ数(1ファイルあたり)", min_value=1, max_value=20, value=20) gen_count_per_field = st.number_input("生成する予想問/分野あたり(件)", min_value=1, max_value=5, value=2) show_raw_llm = st.checkbox("LLMの生出力を表示(デバッグ)", value=False) uploaded_files = st.file_uploader("過去問PDF(スキャン)をアップロード(最大20ファイル)", type=["pdf"], accept_multiple_files=True) if uploaded_files: if len(uploaded_files) > 20: st.error("最大20ファイルまでです。") else: if st.button("解析開始"): overall_field_counter = Counter() uploaded_meta = [] # list of {name, year, problems: [{id,text,fields,metrics,difficulty_score,level}]} st.info("OCRと解析を開始します。ファイル数とページ数によって時間がかかります。") # iterate files for file in uploaded_files: st.write(f"処理中: {file.name}") raw_bytes = file.read() year = extract_year_from_filename(file.name) # OCR try: raw_text = extract_text_from_pdf_bytes(raw_bytes, max_pages=max_pages) except Exception as e: st.error(f"OCR失敗: {e}") raw_text = "" if not raw_text.strip(): st.warning(f"{file.name} からテキストが抽出できませんでした。") continue # split into questions try: questions = split_questions_via_llm(raw_text, model_choice) except Exception as e: st.warning(f"問題分割でエラー: {e}") # fallback to simple split parts = initial_split_by_regex(raw_text) questions = [{"id": f"Q{i+1}", "text": parts[i]} for i in range(len(parts))] st.write(f"{file.name} -> 検出問題数: {len(questions)}") # For per-file LLM field-likelihood, prepare sample context sample_context_for_llm = "\n\n".join([q["text"] for q in questions[:60]]) # classify + metrics for each question problems_meta = [] for q in questions: qtext = q.get("text", "") qid = q.get("id", f"Q??") # classify fields (multiple) try: fields = classify_field_llm(qtext, model_choice) if not fields: fields = heuristic_field_match(qtext) except Exception as e: fields = heuristic_field_match(qtext) # evaluate metrics try: metrics = evaluate_three_metrics(qtext, model_choice) except Exception as e: metrics = {"typicality": 2, "guidance": 2, "creativity": 2} # compute difficulty diff = compute_difficulty_level(metrics) problems_meta.append({ "id": qid, "text": qtext, "fields": fields, "metrics": metrics, "difficulty_score": diff["score"], "difficulty_level": diff["level"] }) # update overall field counter (複合は両方1カウント) for f in fields: overall_field_counter[f] += 1 # ask LLM for per-field likelihoods (for this file) unique_fields = sorted(list({f for p in problems_meta for f in p["fields"]})) try: llm_field_likelihoods = ask_llm_field_likelihood(sample_context_for_llm, unique_fields, model_choice) except Exception: llm_field_likelihoods = {f: 5 for f in unique_fields} uploaded_meta.append({ "name": file.name, "year": year, "problems": problems_meta, "llm_field_likelihoods": llm_field_likelihoods }) # ===== 集計フェーズ ===== st.success("個別ファイル解析完了。集計を行います。") total_problems = sum(len(m["problems"]) for m in uploaded_meta) st.write(f"合計検出問題数: {total_problems}") # field -> total count across all files field_counts = dict(overall_field_counter) # compute normalized frequency field_freq_norm = {} for fld, cnt in field_counts.items(): field_freq_norm[fld] = cnt / total_problems if total_problems > 0 else 0.0 # estimate last appearance year per field last_years = estimate_last_appearance_years(uploaded_meta) # compute years_since_last relative to max year seen (if any) years_seen = [m["year"] for m in uploaded_meta if m["year"]] current_year_ref = max(years_seen) if years_seen else datetime.now().year years_since_last = {} for fld in field_freq_norm.keys(): last = last_years.get(fld) if last: diff = max(0, current_year_ref - last) years_since_last[fld] = diff else: years_since_last[fld] = 0 # unknown -> 0 # combine llm field likelihoods across files by averaging (normalize 0..10) llm_scores_accum = defaultdict(list) for m in uploaded_meta: for f, v in m.get("llm_field_likelihoods", {}).items(): llm_scores_accum[f].append(v) llm_avg = {} for f, arr in llm_scores_accum.items(): llm_avg[f] = int(round(sum(arr) / max(1, len(arr)))) # compute final field score final_field_scores = {} for fld in field_freq_norm.keys(): freq_norm = field_freq_norm.get(fld, 0.0) yrs = years_since_last.get(fld, 0) llm_v = llm_avg.get(fld, 5) final_score = compute_field_score(freq_norm, yrs, llm_v) final_field_scores[fld] = { "freq_count": field_counts.get(fld, 0), "freq_norm": round(freq_norm, 4), "years_since_last": yrs, "llm_avg": llm_v, "final_score": final_score } # sort fields by final_score desc sorted_fields = sorted(final_field_scores.items(), key=lambda x: x[1]["final_score"], reverse=True) # Display summary table st.subheader("分野ランキング(出題スコア順)") st.table([{ "分野": k, "出題回数": v["freq_count"], "頻度(norm)": v["freq_norm"], "直近未出年数": v["years_since_last"], "LLM予測(0-10)": v["llm_avg"], "出題スコア": v["final_score"] } for k, v in sorted_fields]) # For each top field, generate similar problems + solutions st.subheader("予想問題(分野ごと)") downloadable_data = {} # field -> text content # Prepare context examples (concatenate some examples per field) field_examples = defaultdict(list) for meta in uploaded_meta: for p in meta["problems"]: for f in p["fields"]: if len(field_examples[f]) < 40: # add up to limited examples field_examples[f].append(p["text"]) # For each field selected (sorted by final score), generate requested count for fld, info in sorted_fields: if info["freq_count"] == 0: continue st.markdown(f"### 分野: {fld} (スコア {info['final_score']})") context_examples_text = "\n\n---\n\n".join(field_examples.get(fld, [])[:30]) gen_list = [] for i in range(int(gen_count_per_field)): # choose difficulty attempt: pick distribution from existing problems: we will pick a difficulty target based on freq of difficulties in that field # compute distribution of difficulties for this field diffs = [] for meta in uploaded_meta: for p in meta["problems"]: if fld in p["fields"]: diffs.append(p["difficulty_level"]) # derive a likely difficulty: if no data, default to 標準 target_difficulty = "標準" if diffs: # simple heuristic: sample most common c = Counter(diffs) target_difficulty = c.most_common(1)[0][0] # generate try: gen = generate_similar_problem_and_solution(fld, target_difficulty, context_examples_text, model_choice) except Exception as e: gen = {"problem": f"生成に失敗しました: {e}", "solution": "", "explanation": ""} gen_list.append(gen) # show one by one st.write(f"**予想問 {i+1}(難易度推定: {target_difficulty})**") st.write(gen.get("problem", "")) st.write("**模範解答**") st.write(gen.get("solution", "")) st.write("**解説**") st.write(gen.get("explanation", "")) if show_raw_llm: st.text_area(f"LLM raw for {fld} #{i+1}", value=json.dumps(gen, ensure_ascii=False, indent=2), height=150) # prepare downloadable text out_lines = [] out_lines.append(f"=== 分野: {fld} ===\n") for idx, g in enumerate(gen_list, start=1): out_lines.append(f"【予想問 {idx}】\n{g.get('problem','')}\n\n【模範解答】\n{g.get('solution','')}\n\n【解説】\n{g.get('explanation','')}\n\n\n") downloadable_data[fld] = "\n".join(out_lines) # Offer download for each field st.subheader("ダウンロード") for fld, content in downloadable_data.items(): filename = f"predicted_{fld}.txt" st.download_button(label=f"{fld} の予想問題と解答をダウンロード", data=content, file_name=filename, mime="text/plain") # Option: download a single aggregate file if downloadable_data: aggregate = [] for fld, cont in downloadable_data.items(): aggregate.append(cont) aggregate_text = "\n\n====\n\n".join(aggregate) st.download_button(label="全分野まとめてダウンロード", data=aggregate_text, file_name="predicted_all_fields.txt", mime="text/plain") st.success("完了しました。")