"""Streamlit UI for the GitHub Repository Analyzer.""" import json import os import sys from pathlib import Path # Load .env from project root before anything else APP_ROOT = Path(__file__).resolve().parent PROJECT_ROOT = APP_ROOT.parent if str(PROJECT_ROOT) not in sys.path: sys.path.insert(0, str(PROJECT_ROOT)) _env = PROJECT_ROOT / ".env" if _env.exists(): for _line in _env.read_text(encoding="utf-8").splitlines(): _line = _line.strip() if _line and not _line.startswith("#") and "=" in _line: _k, _, _v = _line.partition("=") os.environ.setdefault(_k.strip(), _v.strip()) import streamlit as st from src.repo_analyzer import Config, analyze_full from src.repo_analyzer.claims import claims_for_reporting from src.repo_analyzer.pdf_parser import parse_pdf_to_sections from src.repo_analyzer.pdf_parser_marker import ( is_marker_available, parse_pdf_to_sections_marker, ) from src.repo_analyzer.pdf_report import build_pdf_report from src.repo_analyzer.report import build_text_report LEADERBOARD_ROOT = PROJECT_ROOT / "leaderboard" LEADERBOARD_MANIFEST = LEADERBOARD_ROOT / "manifest.json" CHECK_LABELS = { "readme": "README", "license": "License", "commits": "Commits (>5)", "execution_files": "Entry-point files", "requirements": "Requirements file", "tests": "Tests", "data_files": "Data files", "experiment_scripts": "Experiment scripts", } def load_leaderboard_manifest() -> list[dict]: """Return the lightweight saved-report index.""" if not LEADERBOARD_MANIFEST.exists(): return [] data = json.loads(LEADERBOARD_MANIFEST.read_text(encoding="utf-8")) if not isinstance(data, list): raise ValueError("leaderboard/manifest.json must contain a JSON array") return sorted(data, key=lambda entry: (entry.get("language", ""), entry.get("rank", 0))) def filter_leaderboard_manifest(manifest: list[dict], language: str) -> list[dict]: """Return saved reports for one language sorted by rank.""" entries = [entry for entry in manifest if entry.get("language") == language] return sorted(entries, key=lambda entry: entry.get("rank", 0)) def resolve_leaderboard_path(path_text: str) -> Path: """Resolve a manifest path and keep it inside the leaderboard folder.""" path = Path(path_text) candidate = path if path.is_absolute() else PROJECT_ROOT / path candidate = candidate.resolve() leaderboard_root = LEADERBOARD_ROOT.resolve() if not candidate.is_relative_to(leaderboard_root): raise ValueError(f"Refusing to load path outside leaderboard: {path_text}") return candidate def load_saved_report(entry: dict) -> dict: """Load a saved report JSON from the manifest entry selected by the user.""" report_path = resolve_leaderboard_path(entry["json_path"]) return json.loads(report_path.read_text(encoding="utf-8")) def clear_pdf_cache(prefix: str) -> None: for suffix in ("pdf_en", "pdf_ru"): st.session_state.pop(f"{prefix}_{suffix}", None) def render_report( report: dict, *, paper_name: str | None = None, repo_url: str | None = None, pdf_cache_prefix: str, openrouter_key: str | None = None, model: str | None = None, file_stem: str = "report", show_back_button: bool = False, back_button_key: str = "saved_report_back", back_state_prefix: str = "leaderboard", ) -> None: """Render the report UI used by both fresh and saved analyses.""" summary = report["summary"] score = summary["score"] breakdown = summary["score_breakdown"] repo_type = summary["repo_type"] display_repo_url = repo_url or report.get("repo_url", "") # ── Сontrols ─────────────────────────────────────────────────────────────── col_dl0, col_dl1, col_dl2, col_dl3, col_dl4 = st.columns([1, 0.5, 0.5, 1, 2]) with col_dl0: if show_back_button: if st.button(":material/arrow_back: Back to saved reports", key=back_button_key, type="primary"): st.session_state.pop(f"{back_state_prefix}_selected", None) st.session_state.pop(f"{back_state_prefix}_report", None) st.session_state.pop(f"{back_state_prefix}_report_path", None) st.session_state.pop(f"{back_state_prefix}_loading", None) clear_pdf_cache(pdf_cache_prefix) st.rerun() with col_dl1: st.download_button( ":material/download: Download JSON report", data=json.dumps(report, ensure_ascii=False, indent=2), file_name=f"{file_stem}.json", mime="application/json", key=f"{pdf_cache_prefix}_download_json", ) with col_dl2: pdf_en_key = f"{pdf_cache_prefix}_pdf_en" if pdf_en_key not in st.session_state: st.session_state[pdf_en_key] = build_pdf_report(report, lang="en", paper_name=paper_name) st.download_button( ":material/file_save: Download PDF (EN)", data=st.session_state[pdf_en_key], file_name=f"{file_stem}_en.pdf", mime="application/pdf", key=f"{pdf_cache_prefix}_download_pdf_en", ) with col_dl3: pdf_ru_key = f"{pdf_cache_prefix}_pdf_ru" if pdf_ru_key not in st.session_state: st.session_state[pdf_ru_key] = build_pdf_report( report, lang="ru", openrouter_key=openrouter_key, model=model, paper_name=paper_name, ) st.download_button( ":material/file_save: Download PDF (RU)", data=st.session_state[pdf_ru_key], file_name=f"{file_stem}_ru.pdf", mime="application/pdf", key=f"{pdf_cache_prefix}_download_pdf_ru", ) with col_dl4: with st.expander("Text report"): st.code(build_text_report(report), language=None) st.divider() # ── Score ───────────────────────────────────────────────────────────────── col_score, col_meta = st.columns([1, 3]) with col_score: color = "var(--app-success)" if score >= 70 else "var(--app-warning)" if score >= 40 else "var(--app-danger)" st.markdown( f"
" f"
Sanity Score:
" f"{score}" f"/100" f"
", unsafe_allow_html=True, ) with col_meta: if paper_name: st.markdown(f"**Student report:** {paper_name}") if display_repo_url: st.markdown(f"**Repository:** {display_repo_url}") st.markdown(f"**Type:** {repo_type.replace('_', ' ').title()}") st.markdown(f"**Analyzed:** {report.get('analyzed_at', 'unknown')}") st.divider() # ── Repository checks ───────────────────────────────────────────────────── st.subheader("Repository Checks") applicable_checks = [ (key, label) for key, label in CHECK_LABELS.items() if key in breakdown and breakdown[key].get("applicable") is not False ] if applicable_checks: cols = st.columns(min(len(applicable_checks), 4)) for i, (key, label) in enumerate(applicable_checks): bd = breakdown.get(key, {}) passed = bd.get("passed", False) weight = bd.get("weight", 0) earned = bd.get("earned", 0) icon = "✅" if passed else "❌" with cols[i % len(cols)]: st.metric(label=f"{icon} {label}", value=f"{earned}/{weight} pts") else: st.info("No repository checks are available for this report.") st.divider() # ── Code quality ────────────────────────────────────────────────────────── st.subheader("Code Quality (informational)") col1, col2 = st.columns(2) with col1: syntax = summary["syntax"] ok = syntax.get("ok") if ok is True: st.success(f"Syntax: {syntax.get('summary', 'ok')}") elif ok is False: st.error(f"Syntax errors: {syntax.get('summary', '')}") for err in syntax.get("errors", [])[:5]: st.code(err, language=None) else: st.warning(f"Syntax: {syntax.get('summary', 'unavailable')}") with col2: docs = summary["docstrings"] pct = docs.get("coverage_pct") if pct is not None: st.metric("Docstring coverage", f"{pct}%", help=docs.get("summary", "")) else: st.warning(f"Docstrings: {docs.get('summary', 'unavailable')}") # ── Claims analysis ─────────────────────────────────────────────────────── if "claims_analysis" in report: st.divider() ca = report["claims_analysis"] stats = ca["stats"] rate_pct = stats["implementation_rate_pct"] source_total = stats.get("source_total", stats["total"]) scored_total = stats.get("scored_total", stats["total"]) excluded_low = stats.get("excluded_low_verifiability", 0) excluded_invalid = stats.get("excluded_invalid_verifiability", 0) hidden_low_confidence = stats.get("hidden_low_confidence", 0) selection = ca.get("selection", {}) st.subheader("Code-Claims Analysis") caption_parts = [] if selection.get("only_high_medium_verifiability"): caption_parts.append( f"Verified {scored_total} of {source_total} extracted claims; " f"excluded {excluded_low} low-verifiability and {excluded_invalid} missing/invalid claims." ) if "hide_low_confidence" in selection: caption_parts.append( f"Showing {stats['total']} claims; hidden {hidden_low_confidence} low-confidence claims. " "Implementation rate is calculated from shown claims." ) if caption_parts: st.caption(" ".join(caption_parts)) c1, c2, c3 = st.columns(3) c1.metric("Claims shown", f"{stats['total']} / {scored_total}") c2.metric("Implemented", f"{stats['implemented']} / {stats['total']}") rate_color = ( "var(--app-success)" if rate_pct >= 70 else "var(--app-warning)" if rate_pct >= 40 else "var(--app-danger)" ) c3.markdown( f"
" f"
Implementation rate
" f"{rate_pct}%" f"
", unsafe_allow_html=True, ) st.markdown("#### Claims") claims_to_render = claims_for_reporting(ca) for claim in claims_to_render: impl = claim.get("implementation", {}) done = impl.get("implemented", False) conf = impl.get("confidence", "") evidence = impl.get("evidence_file") or "" explanation = impl.get("explanation", "") icon = "✅" if done else "❌" cat = claim.get("category", "") val = claim.get("value") with st.expander(f"{icon} {claim.get('claim', '')}"): cols2 = st.columns([2, 2]) with cols2[0]: st.markdown(f"**Category:** `{cat}`") if val: st.markdown(f"**Value:** `{val}`") st.markdown(f"**Verifiability:** {claim.get('verifiability', '')}") if claim.get("section_name"): st.markdown(f"**Section:** {claim['section_name']}") with cols2[1]: st.markdown(f"**Confidence:** {conf}") if evidence: st.markdown(f"**File:** `{evidence}`") if claim.get("original_text"): st.markdown(f"*\"{claim['original_text']}\"*") if explanation: st.info(explanation) if claim.get("contradiction"): st.warning("⚠️ Contradicts another claim in the paper") def run_analysis( *, repo_url: str, paper_file, ignore_low_verifiability: bool, hide_low_confidence: bool, openrouter_key: str, github_token: str | None, model: str, ) -> dict | None: """Parse optional paper input, run repository analysis, and return a report.""" paper_sections = None pre_extracted_claims = None if paper_file is not None: if paper_file.name.endswith(".pdf"): with st.spinner("Parsing PDF…"): try: pdf_bytes = paper_file.read() if is_marker_available(): try: paper_sections = parse_pdf_to_sections_marker( pdf_bytes, openrouter_key=openrouter_key or None, model=model or None, ) parser_used = "marker" except Exception as marker_err: st.warning(f"marker failed ({marker_err}), falling back to pymupdf.") paper_sections = parse_pdf_to_sections(pdf_bytes) parser_used = "pymupdf" else: paper_sections = parse_pdf_to_sections(pdf_bytes) parser_used = "pymupdf" st.success(f"Parsed {len(paper_sections)} sections from PDF (via {parser_used}).") except Exception as e: st.error(f"PDF parsing failed: {e}") return None else: try: data = json.loads(paper_file.read().decode("utf-8")) except Exception as e: st.error(f"Could not parse JSON: {e}") return None if isinstance(data, dict) and "result" in data and isinstance(data["result"], list): pre_extracted_claims = data["result"] st.success(f"Loaded {len(pre_extracted_claims)} pre-extracted claims — skipping PDF parsing.") elif isinstance(data, list) and data and "claim" in data[0]: pre_extracted_claims = data st.success(f"Loaded {len(pre_extracted_claims)} pre-extracted claims — skipping PDF parsing.") else: paper_sections = data st.success("Loaded sections JSON.") config = Config( repo_url=repo_url, openrouter_key=openrouter_key, github_token=github_token or None, model=model, ) progress_bar = st.progress(0.0) status = st.empty() def on_progress(msg: str, pct: float) -> None: progress_bar.progress(min(pct, 1.0)) status.text(msg) try: report = analyze_full( config, paper_sections=paper_sections, pre_extracted_claims=pre_extracted_claims, ignore_low_verifiability=ignore_low_verifiability, hide_low_confidence=hide_low_confidence, on_progress=on_progress, ) except Exception as e: progress_bar.empty() status.empty() st.error(f"Analysis failed: {e}") return None progress_bar.progress(1.0) status.empty() progress_bar.empty() return report def render_new_analysis_tab( *, analyze_btn: bool, repo_url: str, paper_file, ignore_low_verifiability: bool, hide_low_confidence: bool, openrouter_key: str, github_token: str | None, model: str, ) -> None: if analyze_btn: report = run_analysis( repo_url=repo_url, paper_file=paper_file, ignore_low_verifiability=ignore_low_verifiability, hide_low_confidence=hide_low_confidence, openrouter_key=openrouter_key, github_token=github_token, model=model, ) if report is not None: st.session_state["report"] = report st.session_state["paper_name"] = paper_file.name if paper_file is not None else None clear_pdf_cache("live_report") if "report" not in st.session_state: st.info( "Fill in the repository URL and OpenRouter API key, then click **Analyze**.\n\n" "Optionally upload a PDF article to verify which claims from the paper are implemented in the code." ) return render_report( st.session_state["report"], paper_name=st.session_state.get("paper_name"), repo_url=repo_url, pdf_cache_prefix="live_report", openrouter_key=openrouter_key, model=model, file_stem="report", show_back_button=False, ) def select_saved_report(entry: dict, *, language: str) -> None: state_prefix = f"leaderboard_{language}" st.session_state[f"{state_prefix}_selected"] = entry st.session_state.pop(f"{state_prefix}_report", None) st.session_state.pop(f"{state_prefix}_report_path", None) st.session_state[f"{state_prefix}_loading"] = True clear_pdf_cache(f"saved_report_{language}") st.rerun() def render_saved_report_list(manifest: list[dict], *, language: str) -> None: st.subheader("List of Saved Reports") if not manifest: st.info(f"No saved {language.upper()} reports found in leaderboard/manifest.json.") return headers = st.columns([0.1, 3, 0.5, 0.5], vertical_alignment="center") headers[0].markdown("**#**") headers[1].markdown("**Student Last Name**") headers[2].markdown("**Sanity Score**") headers[3].markdown("**Claim-Code Score**") for entry in manifest: rank = entry.get("rank", "") name = entry.get("name") or entry.get("last_name") or entry.get("label", "") sanity_score = entry.get("sanity_score", entry.get("score", "?")) claim_code_score = entry.get("claim_code_score", "?") cols = st.columns([0.1, 3, 0.5, 0.5], vertical_alignment="center") with cols[0]: st.markdown(f"**{rank}**") with cols[1]: if st.button(name, key=f"saved_report_{language}_open_{rank}", use_container_width=True): select_saved_report(entry, language=language) with cols[2]: st.markdown(f"**{sanity_score}/100**") with cols[3]: st.markdown(f"**{claim_code_score}%**") def render_saved_reports_tab(*, language: str, openrouter_key: str | None, model: str | None) -> None: try: manifest = filter_leaderboard_manifest(load_leaderboard_manifest(), language) except Exception as e: st.error(f"Could not load leaderboard manifest: {e}") return state_prefix = f"leaderboard_{language}" cache_prefix = f"saved_report_{language}" selected = st.session_state.get(f"{state_prefix}_selected") if not selected: render_saved_report_list(manifest, language=language) return selected_path = selected.get("json_path") try: if ( st.session_state.get(f"{state_prefix}_report_path") != selected_path or f"{state_prefix}_report" not in st.session_state ): loading_key = f"{state_prefix}_loading" if st.session_state.pop(loading_key, False): with st.status("Loading saved report...", expanded=False) as status: st.session_state[f"{state_prefix}_report"] = load_saved_report(selected) st.session_state[f"{state_prefix}_report_path"] = selected_path status.update(label="Saved report loaded.", state="complete", expanded=False) else: with st.spinner("Loading saved report..."): st.session_state[f"{state_prefix}_report"] = load_saved_report(selected) st.session_state[f"{state_prefix}_report_path"] = selected_path except Exception as e: st.error(f"Could not load saved report: {e}") return label = selected.get("label", "saved_report") render_report( st.session_state[f"{state_prefix}_report"], paper_name=label, pdf_cache_prefix=cache_prefix, openrouter_key=openrouter_key, model=model, file_stem=label, show_back_button=True, back_button_key=f"{cache_prefix}_back", back_state_prefix=state_prefix, ) st.set_page_config( page_title="🐝 OSA.Edu", page_icon="🔍", layout="wide", ) st.markdown( """

🔍🐝 OSA.Edu

Evaluates thesis project repositories on a 0-100 scale and verifies paper claims.

""", unsafe_allow_html=True, ) # ── Sidebar ─────────────────────────────────────────────────────────────────── with st.sidebar: st.header("Repository") repo_url = st.text_input( "GitHub URL", placeholder="https://github.com/owner/repo", ) st.header("Paper") st.caption("Upload a PDF, pre-parsed sections JSON, or pre-extracted claims JSON to enable claim verification.") paper_file = st.file_uploader("Article", type=["pdf", "json"]) ignore_low_verifiability = st.checkbox( "Verify and show only high/medium-verifiability claims", value=True, help=( "Low, missing, and invalid verifiability values are excluded before code verification " "and do not affect the implementation rate." ), ) hide_low_confidence = st.checkbox( "Hide low-confidence verification results", value=True, help="Hidden low-confidence results are excluded from reports and the implementation rate.", ) st.header("API Keys") openrouter_key = st.text_input( "OpenRouter API Key", type="password", value=os.environ.get("OPENROUTER_KEY", ""), ) github_token = st.text_input( "GitHub Token (optional)", type="password", value=os.environ.get("GITHUB_TOKEN", ""), help="Needed for private repos or to avoid rate limits.", ) st.header("Model") model = st.selectbox( "Model", options=[ "openai/gpt-4o", "openai/gpt-4o-mini", "deepseek/deepseek-v3", "anthropic/claude-3.5-haiku", "anthropic/claude-3.5-sonnet", ], label_visibility="collapsed", accept_new_options=True, ) analyze_btn = st.button( "Analyze", type="primary", disabled=not (repo_url and openrouter_key), use_container_width=True, ) # ── Main tabs ───────────────────────────────────────────────────────────────── app_css = """ """ st.markdown(app_css, unsafe_allow_html=True) tab_saved_reports_en, tab_saved_reports_ru, tab_new_analysis = st.tabs( [ ":material/list: Saved Reports EN", ":material/list: Saved Reports RU", ":material/add: New Analysis", ] ) with tab_saved_reports_en: render_saved_reports_tab(language="en", openrouter_key=openrouter_key or None, model=model or None) with tab_saved_reports_ru: render_saved_reports_tab(language="ru", openrouter_key=openrouter_key or None, model=model or None) with tab_new_analysis: render_new_analysis_tab( analyze_btn=analyze_btn, repo_url=repo_url, paper_file=paper_file, ignore_low_verifiability=ignore_low_verifiability, hide_low_confidence=hide_low_confidence, openrouter_key=openrouter_key, github_token=github_token, model=model, )