OSA.Edu / src /cli.py
tityos's picture
fix: update leaderboard
bab3dab
Raw
History Blame Contribute Delete
12.7 kB
#!/usr/bin/env python3
"""CLI entry point for the repository analyzer."""
import argparse
import json
import os
import re
import sys
from pathlib import Path
# Load .env from project root
_env = Path(__file__).parent / ".env"
if _env.exists():
for _line in _env.read_text(encoding="utf-8").splitlines():
_line = _line.strip()
if _line and not _line.startswith("#") and "=" in _line:
_k, _, _v = _line.partition("=")
os.environ.setdefault(_k.strip(), _v.strip())
from src.repo_analyzer import Config, analyze_full
from src.repo_analyzer.pdf_report import Lang, build_pdf_report
from src.repo_analyzer.report import build_text_report
def _resolve_keys(args) -> tuple[str, str, str]:
openrouter_key = args.openrouter_key or os.environ.get("OPENROUTER_KEY", "")
if not openrouter_key:
print("Error: OpenRouter API key required. Use --openrouter-key or set OPENROUTER_KEY.", file=sys.stderr)
sys.exit(1)
github_token = getattr(args, "token", None) or os.environ.get("GITHUB_TOKEN", "")
model = getattr(args, "model", None) or os.environ.get("OPENROUTER_MODEL", "openai/gpt-4o-mini")
return openrouter_key, github_token, model
def _extract_repo_url(repo_txt: str) -> str | None:
"""Return the first https URL found in repo.txt content."""
urls = re.findall(r"https?://\S+", repo_txt)
if not urls:
return None
return urls[0].rstrip(".,)")
def _load_claims(path: Path) -> list | None:
data = json.loads(path.read_text(encoding="utf-8"))
if isinstance(data, dict) and "result" in data:
return data["result"]
if isinstance(data, list):
return data
return None
def _save_pdfs(
report: dict,
langs: list[Lang],
out_dir: Path,
stem: str,
openrouter_key: str,
model: str,
paper_name: str | None = None,
) -> None:
"""Generate and write PDF report(s) for each requested language."""
for lang in langs:
pdf_bytes = build_pdf_report(
report,
lang=lang,
openrouter_key=openrouter_key if lang == "ru" else None,
model=model if lang == "ru" else None,
paper_name=paper_name,
)
pdf_path = out_dir / f"{stem}_{lang}.pdf"
pdf_path.write_bytes(pdf_bytes)
print(f" pdf ({lang}) -> {pdf_path.name}", file=sys.stderr)
# ── single run ────────────────────────────────────────────────────────────────
def cmd_run(args) -> None:
openrouter_key, github_token, model = _resolve_keys(args)
config = Config(
repo_url=args.repo,
openrouter_key=openrouter_key,
github_token=github_token or None,
model=model,
)
paper_sections = None
pre_extracted_claims = None
paper_name = None
if args.claims_json:
claims_path = Path(args.claims_json)
pre_extracted_claims = _load_claims(claims_path)
if pre_extracted_claims is None:
print("Error: unrecognised claims JSON format.", file=sys.stderr)
sys.exit(1)
# derive paper name from filename, e.g. "sheet_1_14_05_Π“Π°Π»ΠΈΠ½_claims_result" β†’ "sheet_1_14_05_Π“Π°Π»ΠΈΠ½"
paper_name = re.sub(r"_claims_result$", "", claims_path.stem)
elif args.paper_sections:
paper_sections = json.loads(Path(args.paper_sections).read_text(encoding="utf-8"))
try:
report = analyze_full(config, paper_sections=paper_sections, pre_extracted_claims=pre_extracted_claims)
except Exception as e:
print(f"Error: {e}", file=sys.stderr)
sys.exit(1)
out_dir = Path(args.output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
stem = f"{paper_name}_repo_analysis" if paper_name else "report"
json_path = out_dir / f"{stem}.json"
txt_path = out_dir / f"{stem}.txt"
with open(json_path, "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, indent=2)
with open(txt_path, "w", encoding="utf-8") as f:
f.write(build_text_report(report, paper_name=paper_name))
print(f"Saved: {json_path}", file=sys.stderr)
print(f" {txt_path}", file=sys.stderr)
if args.langs:
_save_pdfs(report, args.langs, out_dir, stem, openrouter_key, model, paper_name=paper_name)
if not args.json_only:
print("\n" + build_text_report(report, paper_name=paper_name) + "\n")
print(json.dumps(report, ensure_ascii=False, indent=2))
# ── batch run ─────────────────────────────────────────────────────────────────
def cmd_batch(args) -> None:
openrouter_key, github_token, model = _resolve_keys(args)
data_dir = Path(args.data_dir)
if not data_dir.exists():
print(f"Error: data directory '{data_dir}' does not exist.", file=sys.stderr)
sys.exit(1)
results_root = Path(args.results_dir) if args.results_dir else data_dir / "results"
results_root.mkdir(parents=True, exist_ok=True)
folders = sorted(f for f in data_dir.iterdir() if f.is_dir() and f.name.startswith("sheet_"))
print(f"Found {len(folders)} student folders in {data_dir}", file=sys.stderr)
print(f"Results -> {results_root}\n", file=sys.stderr)
ok = skipped = errors = 0
for folder in folders:
name = folder.name
out_dir = results_root / name
out_json = out_dir / f"{name}_repo_analysis.json"
# ── repo URL ──────────────────────────────────────────────────────────
repo_file = folder / "repo.txt"
if not repo_file.exists():
print(f" SKIP {name}: no repo.txt", file=sys.stderr)
skipped += 1
continue
repo_url = _extract_repo_url(repo_file.read_text(encoding="utf-8"))
if not repo_url:
print(f" SKIP {name}: no valid URL in repo.txt", file=sys.stderr)
skipped += 1
continue
# ── claims JSON ───────────────────────────────────────────────────────
claims_file = folder / f"{name}_claims_result.json"
if not claims_file.exists():
print(f" SKIP {name}: no claims JSON", file=sys.stderr)
skipped += 1
continue
claims = _load_claims(claims_file)
if claims is None:
print(f" SKIP {name}: unrecognised claims format", file=sys.stderr)
skipped += 1
continue
# ── skip already analyzed (unless --force) ────────────────────────────
if out_json.exists() and not args.force:
missing_pdfs = (
[lang for lang in args.langs if not (out_dir / f"{name}_repo_analysis_{lang}.pdf").exists()]
if args.langs
else []
)
if not missing_pdfs:
print(f" SKIP {name}: already analyzed (use --force to rerun)", file=sys.stderr)
skipped += 1
continue
# Analysis done, only PDFs missing β€” load cached report
print(f" PDF {name}: generating missing PDF(s): {missing_pdfs}", file=sys.stderr)
report = json.loads(out_json.read_text(encoding="utf-8"))
_save_pdfs(report, missing_pdfs, out_dir, f"{name}_repo_analysis", openrouter_key, model, paper_name=name)
ok += 1
continue
print(f" RUN {name} ({len(claims)} claims) {repo_url[:70]}", file=sys.stderr)
config = Config(
repo_url=repo_url,
openrouter_key=openrouter_key,
github_token=github_token or None,
model=model,
)
try:
report = analyze_full(config, pre_extracted_claims=claims)
except Exception as e:
print(f" ERROR {name}: {e}", file=sys.stderr)
errors += 1
continue
out_dir.mkdir(parents=True, exist_ok=True)
out_txt = out_dir / f"{name}_repo_analysis.txt"
with open(out_json, "w", encoding="utf-8") as f:
json.dump(report, f, ensure_ascii=False, indent=2)
with open(out_txt, "w", encoding="utf-8") as f:
f.write(build_text_report(report))
score = report.get("summary", {}).get("score", "?")
print(f" score={score} -> {out_dir.name}/", file=sys.stderr)
if args.langs:
_save_pdfs(report, args.langs, out_dir, f"{name}_repo_analysis", openrouter_key, model, paper_name=name)
ok += 1
print(f"\nDone: {ok} analyzed, {skipped} skipped, {errors} errors", file=sys.stderr)
# ── argument parser ───────────────────────────────────────────────────────────
def _add_pdf_args(p: argparse.ArgumentParser) -> None:
p.add_argument(
"--pdf",
dest="langs",
nargs="+",
metavar="LANG",
choices=["en", "ru", "both"],
default=[],
help=("Generate PDF report(s). Pass one or more of: en ru both. " "Example: --pdf en ru or --pdf both"),
)
def main() -> None:
parser = argparse.ArgumentParser(description="Analyze GitHub repositories and verify claims against code.")
# shared flags
shared = argparse.ArgumentParser(add_help=False)
shared.add_argument("--token", default=None, help="GitHub token (or GITHUB_TOKEN env var)")
shared.add_argument(
"--openrouter-key", default=None, dest="openrouter_key", help="OpenRouter API key (or OPENROUTER_KEY env var)"
)
shared.add_argument("--model", default=None, help="OpenRouter model ID")
sub = parser.add_subparsers(dest="command", required=True)
# ── run ───────────────────────────────────────────────────────────────────
p_run = sub.add_parser("run", parents=[shared], help="Analyze a single repository.")
p_run.add_argument("--repo", required=True, help="GitHub repository URL")
p_run.add_argument(
"--paper-sections",
default=None,
dest="paper_sections",
metavar="FILE",
help="Sections JSON from parse_pdf_with_marker.py (enables claim extraction).",
)
p_run.add_argument(
"--claims-json",
default=None,
dest="claims_json",
metavar="FILE",
help="Pre-extracted claims JSON (skips extraction, goes straight to verification).",
)
p_run.add_argument(
"--output-dir", default="./results", dest="output_dir", help="Directory to save results (default: ./results)"
)
p_run.add_argument(
"--json-only", action="store_true", dest="json_only", help="Print JSON only, skip human-readable text"
)
_add_pdf_args(p_run)
p_run.set_defaults(func=cmd_run)
# ── batch ─────────────────────────────────────────────────────────────────
p_batch = sub.add_parser(
"batch",
parents=[shared],
help="Bulk-analyze all student folders in a data directory. "
"Each folder must contain repo.txt and *_claims_result.json.",
)
p_batch.add_argument(
"--data-dir",
default="./data",
dest="data_dir",
help="Root data directory with student folders (default: ./data)",
)
p_batch.add_argument(
"--results-dir", default=None, dest="results_dir", help="Where to save results (default: <data-dir>/results)"
)
p_batch.add_argument(
"--force", action="store_true", help="Re-analyze folders that already have a repo_analysis.json"
)
_add_pdf_args(p_batch)
p_batch.set_defaults(func=cmd_batch)
args = parser.parse_args()
# Expand "both" β†’ ["en", "ru"] and deduplicate
if hasattr(args, "langs") and args.langs:
expanded = []
for lang in args.langs:
expanded += ["en", "ru"] if lang == "both" else [lang]
args.langs = list(dict.fromkeys(expanded)) # deduplicate, preserve order
args.func(args)
if __name__ == "__main__":
main()