f

.gitignore

@@ -1,2 +1,5 @@
 .vscode
 venv*
+.env
+__pycache__/**
+*.pyc

site/backend/.gitignore

@@ -1,2 +1,5 @@
 .vscode
 venv*
+.env
+__pycache__/
+*.pyc

src/demo_cli.py

@@ -2,7 +2,7 @@ from pathlib import Path
 import numpy as np
 import faiss
 from sentence_transformers import SentenceTransformer
-from src.data_io import read_jsonl
+from data_io import read_jsonl
 
 MODEL_PATH = Path("artifacts/models/finetuned_mpnet")
 INDEX_DIR = Path("artifacts/indexes/finetuned")

src/evaluate.py

@@ -1,105 +1,258 @@
-import json
-from pathlib import Path
-import numpy as np
-import faiss
-from sentence_transformers import SentenceTransformer
-from src.data_io import load_pairs, read_jsonl
-
-def load_index(lang: str, alias: str):
-    base = Path("artifacts/indexes") / alias
-    idx_path = base / f"{lang}.faiss"
-    meta_path = base / f"{lang}_meta.jsonl"
-    index = faiss.read_index(str(idx_path))
-    meta = read_jsonl(str(meta_path))
-    pos_to_id = {int(x["pos"]): x["id"] for x in meta}
-    return index, meta, pos_to_id
-
-def metrics_from_ranks(ranks, ks=(1, 3, 5, 10)):
-    out = {}
-    for k in ks:
-        out[f"recall@{k}"] = float(np.mean([1.0 if r is not None and r < k else 0.0 for r in ranks]))
-    rr = []
-    dcg = []
-    for r in ranks:
-        if r is None:
-            rr.append(0.0)
-            dcg.append(0.0)
-        else:
-            rr.append(1.0 / (r + 1.0))
-            dcg.append(1.0 / np.log2(r + 2.0))
-    out["mrr@10"] = float(np.mean(rr))
-    out["ndcg@10"] = float(np.mean(dcg))
-    return out
-
-def eval_model(model_name: str, index_alias: str, test_path: str, top_k=10):
-    model = SentenceTransformer(model_name)
-
-    test = load_pairs(test_path)
-    groups = {"ru": [x for x in test if x["lang"] == "ru"], "kz": [x for x in test if x["lang"] == "kz"]}
-
-    results = {
-        "model": model_name,
-        "index_alias": index_alias,
-        "test_path": test_path,
-        "top_k": top_k,
-        "by_lang": {},
-    }
-
-    all_ranks = []
-
-    for lang, items in groups.items():
-        if not items:
-            results["by_lang"][lang] = {"count": 0}
-            continue
-
-        index, meta, pos_to_id = load_index(lang, index_alias)
-
-        queries = [x["query"] for x in items]
-        q_emb = model.encode(queries, batch_size=64, convert_to_numpy=True, normalize_embeddings=True, show_progress_bar=True).astype(np.float32)
-        scores, idxs = index.search(q_emb, top_k)
-
-        ranks = []
-        for i, x in enumerate(items):
-            target = x["positive_id"]
-            found_rank = None
-            for r in range(top_k):
-                did = pos_to_id.get(int(idxs[i, r]))
-                if did == target:
-                    found_rank = r
-                    break
-            ranks.append(found_rank)
-
-        all_ranks.extend(ranks)
-
-        results["by_lang"][lang] = {
-            "count": len(items),
-            **metrics_from_ranks(ranks, ks=(1, 3, 5, 10)),
-        }
-
-    results["overall"] = {
-        "count": len(all_ranks),
-        **metrics_from_ranks(all_ranks, ks=(1, 3, 5, 10)),
-    }
-    return results
-
-def main():
-    test_path = "data/legal_assistant_test.jsonl"
-
-    models = [
-        ("mpnet_base", "paraphrase-multilingual-mpnet-base-v2"),
-        ("labse", "sentence-transformers/LaBSE"),
-    ]
-
-    finetuned_dir = Path("artifacts/models/finetuned_mpnet")
-    if finetuned_dir.exists():
-        models.append(("finetuned", str(finetuned_dir)))
-
-    out_dir = Path("artifacts/reports")
-    out_dir.mkdir(parents=True, exist_ok=True)
-
-    for alias, model_name in models:
-        r = eval_model(model_name, alias, test_path, top_k=10)
-        (out_dir / f"eval_{alias}.json").write_text(json.dumps(r, ensure_ascii=False, indent=2), encoding="utf-8")
-
-if __name__ == "__main__":
-    main()
+import json
+from pathlib import Path
+
+import faiss
+import numpy as np
+from sentence_transformers import SentenceTransformer
+
+from src.data_io import load_pairs, read_jsonl
+
+
+def load_index(lang: str, alias: str):
+    base = Path("artifacts/indexes") / alias
+    idx_path = base / f"{lang}.faiss"
+    meta_path = base / f"{lang}_meta.jsonl"
+    index = faiss.read_index(str(idx_path))
+    meta = read_jsonl(str(meta_path))
+    pos_to_id = {int(x["pos"]): x["id"] for x in meta}
+    return index, meta, pos_to_id
+
+
+def _stats_from_values(values):
+    if not values:
+        return {
+            "mean": None,
+            "median": None,
+            "p10": None,
+            "p90": None,
+        }
+    arr = np.array(values, dtype=float)
+    return {
+        "mean": float(np.mean(arr)),
+        "median": float(np.median(arr)),
+        "p10": float(np.percentile(arr, 10)),
+        "p90": float(np.percentile(arr, 90)),
+    }
+
+
+def metrics_from_ranks(ranks, ks=(1, 3, 5, 10)):
+    out = {}
+    for k in ks:
+        hits = [1.0 if r is not None and r < k else 0.0 for r in ranks]
+        hit_rate = float(np.mean(hits)) if ranks else 0.0
+        out[f"recall@{k}"] = hit_rate
+        out[f"hit@{k}"] = hit_rate
+        out[f"precision@{k}"] = float(np.mean([h / k for h in hits])) if ranks else 0.0
+
+    rr = []
+    dcg = []
+    for r in ranks:
+        if r is None:
+            rr.append(0.0)
+            dcg.append(0.0)
+        else:
+            rr.append(1.0 / (r + 1.0))
+            dcg.append(1.0 / np.log2(r + 2.0))
+    out["mrr@10"] = float(np.mean(rr)) if rr else 0.0
+    out["ndcg@10"] = float(np.mean(dcg)) if dcg else 0.0
+    out["not_found_rate"] = float(np.mean([1.0 if r is None else 0.0 for r in ranks])) if ranks else 0.0
+    return out
+
+
+def eval_model(model_name: str, index_alias: str, test_path: str, top_k=10):
+    model = SentenceTransformer(model_name)
+
+    test = load_pairs(test_path)
+    groups = {
+        "ru": [x for x in test if x["lang"] == "ru"],
+        "kz": [x for x in test if x["lang"] == "kz"],
+    }
+
+    results = {
+        "model": model_name,
+        "index_alias": index_alias,
+        "test_path": test_path,
+        "top_k": top_k,
+        "by_lang": {},
+    }
+
+    all_ranks = []
+    all_top1_scores = []
+    all_top1_scores_tp = []
+    all_top1_scores_fp = []
+    all_margins = []
+    all_coverage_ids = set()
+    total_corpus_size = 0
+
+    for lang, items in groups.items():
+        if not items:
+            results["by_lang"][lang] = {"count": 0}
+            continue
+
+        index, meta, pos_to_id = load_index(lang, index_alias)
+        total_corpus_size += len(meta)
+
+        queries = [x["query"] for x in items]
+        q_emb = model.encode(
+            queries,
+            batch_size=64,
+            convert_to_numpy=True,
+            normalize_embeddings=True,
+            show_progress_bar=True,
+        ).astype(np.float32)
+        scores, idxs = index.search(q_emb, top_k)
+
+        ranks = []
+        top1_scores = []
+        top1_scores_tp = []
+        top1_scores_fp = []
+        margins = []
+        coverage_ids = set()
+        for i, x in enumerate(items):
+            target = x["positive_id"]
+            found_rank = None
+            top_scores = [float(s) for s in scores[i].tolist()]
+            for r in range(top_k):
+                pos = int(idxs[i, r])
+                did = pos_to_id.get(pos)
+                if did is None:
+                    continue
+                coverage_ids.add(did)
+                if did == target:
+                    found_rank = r
+                    break
+            ranks.append(found_rank)
+
+            if top_scores:
+                top1 = top_scores[0]
+                top1_scores.append(top1)
+                if found_rank == 0:
+                    top1_scores_tp.append(top1)
+                else:
+                    top1_scores_fp.append(top1)
+
+            if len(top_scores) >= 2:
+                margins.append(top_scores[0] - top_scores[1])
+
+        all_ranks.extend(ranks)
+        all_top1_scores.extend(top1_scores)
+        all_top1_scores_tp.extend(top1_scores_tp)
+        all_top1_scores_fp.extend(top1_scores_fp)
+        all_margins.extend(margins)
+        all_coverage_ids.update(coverage_ids)
+
+        found_ranks_1based = [r + 1 for r in ranks if r is not None]
+        rank_stats = _stats_from_values(found_ranks_1based)
+        rank_stats.update(
+            {
+                "found_count": len(found_ranks_1based),
+                "not_found_count": len(ranks) - len(found_ranks_1based),
+                "not_found_rate": float(np.mean([1.0 if r is None else 0.0 for r in ranks])) if ranks else 0.0,
+            }
+        )
+
+        score_stats = _stats_from_values(top1_scores)
+        margin_stats = _stats_from_values(margins)
+        coverage = {
+            "unique_ids": len(coverage_ids),
+            "corpus_size": len(meta),
+            "coverage_ratio": float(len(coverage_ids) / len(meta)) if meta else 0.0,
+        }
+
+        results["by_lang"][lang] = {
+            "count": len(items),
+            **metrics_from_ranks(ranks, ks=(1, 3, 5, 10)),
+            "rank_stats": {
+                "mean_rank": rank_stats["mean"],
+                "median_rank": rank_stats["median"],
+                "p10_rank": rank_stats["p10"],
+                "p90_rank": rank_stats["p90"],
+                "found_count": rank_stats["found_count"],
+                "not_found_count": rank_stats["not_found_count"],
+                "not_found_rate": rank_stats["not_found_rate"],
+            },
+            "score_stats": {
+                "top1_score": score_stats,
+                "margin_top1_top2": margin_stats,
+            },
+            "coverage": coverage,
+            "distributions": {
+                "ranks": [r if r is not None else -1 for r in ranks],
+                "top1_scores": top1_scores,
+                "top1_scores_tp": top1_scores_tp,
+                "top1_scores_fp": top1_scores_fp,
+                "margins": margins,
+            },
+        }
+
+    overall_found_ranks_1based = [r + 1 for r in all_ranks if r is not None]
+    overall_rank_stats = _stats_from_values(overall_found_ranks_1based)
+    overall_rank_stats.update(
+        {
+            "found_count": len(overall_found_ranks_1based),
+            "not_found_count": len(all_ranks) - len(overall_found_ranks_1based),
+            "not_found_rate": float(np.mean([1.0 if r is None else 0.0 for r in all_ranks])) if all_ranks else 0.0,
+        }
+    )
+
+    overall_score_stats = _stats_from_values(all_top1_scores)
+    overall_margin_stats = _stats_from_values(all_margins)
+    overall_coverage = {
+        "unique_ids": len(all_coverage_ids),
+        "corpus_size": total_corpus_size,
+        "coverage_ratio": float(len(all_coverage_ids) / total_corpus_size) if total_corpus_size else 0.0,
+    }
+
+    results["overall"] = {
+        "count": len(all_ranks),
+        **metrics_from_ranks(all_ranks, ks=(1, 3, 5, 10)),
+        "rank_stats": {
+            "mean_rank": overall_rank_stats["mean"],
+            "median_rank": overall_rank_stats["median"],
+            "p10_rank": overall_rank_stats["p10"],
+            "p90_rank": overall_rank_stats["p90"],
+            "found_count": overall_rank_stats["found_count"],
+            "not_found_count": overall_rank_stats["not_found_count"],
+            "not_found_rate": overall_rank_stats["not_found_rate"],
+        },
+        "score_stats": {
+            "top1_score": overall_score_stats,
+            "margin_top1_top2": overall_margin_stats,
+        },
+        "coverage": overall_coverage,
+        "distributions": {
+            "ranks": [r if r is not None else -1 for r in all_ranks],
+            "top1_scores": all_top1_scores,
+            "top1_scores_tp": all_top1_scores_tp,
+            "top1_scores_fp": all_top1_scores_fp,
+            "margins": all_margins,
+        },
+    }
+    return results
+
+
+def main():
+    test_path = "data/legal_assistant_test.jsonl"
+
+    models = [
+        ("mpnet_base", "paraphrase-multilingual-mpnet-base-v2"),
+        ("labse", "sentence-transformers/LaBSE"),
+    ]
+
+    finetuned_dir = Path("artifacts/models/finetuned_mpnet")
+    if finetuned_dir.exists():
+        models.append(("finetuned", str(finetuned_dir)))
+
+    out_dir = Path("artifacts/reports")
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    for alias, model_name in models:
+        r = eval_model(model_name, alias, test_path, top_k=10)
+        (out_dir / f"eval_{alias}.json").write_text(
+            json.dumps(r, ensure_ascii=False, indent=2),
+            encoding="utf-8",
+        )
+
+
+if __name__ == "__main__":
+    main()

src/plot_eval.py

@@ -1,196 +1,522 @@
-import json
-from pathlib import Path
-import matplotlib.pyplot as plt
-import numpy as np
-
-def read_json(path):
-    return json.loads(Path(path).read_text(encoding="utf-8"))
-
-def pick_models(files):
-    items = []
-    for p in files:
-        try:
-            j = read_json(p)
-            items.append((Path(p).stem, j))
-        except Exception:
-            pass
-    return items
-
-def metric_value(obj, scope, lang, metric):
-    if scope == "overall":
-        return obj.get("overall", {}).get(metric, None)
-    if scope == "by_lang":
-        return obj.get("by_lang", {}).get(lang, {}).get(metric, None)
-    return None
-
-def save_recall_plot(models, scope, lang, out_path):
-    ks = [1, 3, 5, 10]
-    x = np.arange(len(ks))
-    width = 0.8 / max(1, len(models))
-
-    plt.figure()
-    for i, (name, obj) in enumerate(models):
-        vals = []
-        for k in ks:
-            v = metric_value(obj, scope, lang, f"recall@{k}")
-            vals.append(0.0 if v is None else float(v))
-        plt.bar(x + (i - (len(models) - 1) / 2) * width, vals, width=width, label=obj.get("model", name))
-
-    plt.xticks(x, [f"@{k}" for k in ks])
-    title = "Recall@k"
-    if scope == "overall":
-        plt.title(f"{title} (overall)")
-    else:
-        plt.title(f"{title} ({lang})")
-    plt.ylabel("score")
-    ymax = max([0.0] + [max([metric_value(o, scope, lang, f"recall@{k}") or 0.0 for k in ks]) for _, o in models])
-    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
-    plt.legend()
-    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=180)
-    plt.close()
-
-def save_rank_metrics_plot(models, scope, lang, out_path):
-    metrics = ["mrr@10", "ndcg@10"]
-    x = np.arange(len(metrics))
-    width = 0.8 / max(1, len(models))
-
-    plt.figure()
-    for i, (name, obj) in enumerate(models):
-        vals = []
-        for m in metrics:
-            v = metric_value(obj, scope, lang, m)
-            vals.append(0.0 if v is None else float(v))
-        plt.bar(x + (i - (len(models) - 1) / 2) * width, vals, width=width, label=obj.get("model", name))
-
-    plt.xticks(x, metrics)
-    title = "Ranking metrics"
-    if scope == "overall":
-        plt.title(f"{title} (overall)")
-    else:
-        plt.title(f"{title} ({lang})")
-    plt.ylabel("score")
-    ymax = max([0.0] + [max([metric_value(o, scope, lang, m) or 0.0 for m in metrics]) for _, o in models])
-    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
-    plt.legend()
-    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=180)
-    plt.close()
-
-def save_recall_curve_plot(models, scope, lang, out_path):
-    ks = [1, 3, 5, 10]
-    xs = np.array(ks, dtype=float)
-
-    plt.figure()
-    for name, obj in models:
-        ys = []
-        for k in ks:
-            v = metric_value(obj, scope, lang, f"recall@{k}")
-            ys.append(0.0 if v is None else float(v))
-        plt.plot(xs, ys, marker="o", label=obj.get("model", name))
-
-    plt.xticks(xs, [f"@{k}" for k in ks])
-    title = "Recall@k vs k"
-    if scope == "overall":
-        plt.title(f"{title} (overall)")
-    else:
-        plt.title(f"{title} ({lang})")
-    plt.xlabel("k")
-    plt.ylabel("recall")
-    ymax = max([0.0] + [max([metric_value(o, scope, lang, f"recall@{k}") or 0.0 for k in ks]) for _, o in models])
-    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
-    plt.legend()
-    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=180)
-    plt.close()
-
-def model_label_key(obj, name):
-    s = str(obj.get("model", name)).lower()
-    if "labse" in s:
-        return "labse"
-    if "finetuned" in s or "artifacts" in s:
-        return "finetuned"
-    if "paraphrase-multilingual-mpnet-base-v2" in s:
-        return "base"
-    if "mpnet" in s:
-        return "base"
-    return name.lower()
-
-def select_model(models, key):
-    for name, obj in models:
-        if model_label_key(obj, name) == key:
-            return (name, obj)
-    return None
-
-def save_relative_improvement_plot(models, scope, lang, out_path):
-    fin = select_model(models, "finetuned")
-    base = select_model(models, "base")
-    if fin is None or base is None:
-        return
-
-    metrics = ["recall@1", "recall@3", "recall@5", "recall@10", "mrr@10", "ndcg@10"]
-    labels = ["R@1", "R@3", "R@5", "R@10", "MRR@10", "nDCG@10"]
-
-    fin_obj = fin[1]
-    base_obj = base[1]
-
-    vals = []
-    for m in metrics:
-        fv = metric_value(fin_obj, scope, lang, m)
-        bv = metric_value(base_obj, scope, lang, m)
-        fv = 0.0 if fv is None else float(fv)
-        bv = 0.0 if bv is None else float(bv)
-        if bv <= 0:
-            vals.append(np.nan)
-        else:
-            vals.append((fv - bv) / bv * 100.0)
-
-    x = np.arange(len(metrics))
-    plt.figure()
-    plt.bar(x, vals)
-    plt.xticks(x, labels)
-    title = "Relative improvement vs base (%)"
-    if scope == "overall":
-        plt.title(f"{title} (overall)")
-    else:
-        plt.title(f"{title} ({lang})")
-    plt.ylabel("%")
-    plt.axhline(0.0)
-    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
-    plt.tight_layout()
-    plt.savefig(out_path, dpi=180)
-    plt.close()
-
-def main():
-    reports_dir = Path("artifacts/reports")
-    files = sorted([str(p) for p in reports_dir.glob("eval_*.json")])
-    models = pick_models(files)
-
-    if not models:
-        raise SystemExit("No eval_*.json found in artifacts/reports")
-
-    fig_dir = reports_dir / "figures"
-    fig_dir.mkdir(parents=True, exist_ok=True)
-
-    save_recall_plot(models, "overall", None, fig_dir / "recall_overall.png")
-    save_rank_metrics_plot(models, "overall", None, fig_dir / "rank_metrics_overall.png")
-    save_recall_curve_plot(models, "overall", None, fig_dir / "recall_curve_overall.png")
-    save_relative_improvement_plot(models, "overall", None, fig_dir / "relative_improvement_overall.png")
-
-    for lang in ["ru", "kz"]:
-        save_recall_plot(models, "by_lang", lang, fig_dir / f"recall_{lang}.png")
-        save_rank_metrics_plot(models, "by_lang", lang, fig_dir / f"rank_metrics_{lang}.png")
-        save_recall_curve_plot(models, "by_lang", lang, fig_dir / f"recall_curve_{lang}.png")
-        save_relative_improvement_plot(models, "by_lang", lang, fig_dir / f"relative_improvement_{lang}.png")
-
-    summary = {
-        "loaded_reports": [Path(f).name for f in files],
-        "figures": [p.name for p in sorted(fig_dir.glob("*.png"))],
-    }
-    (reports_dir / "figures_summary.json").write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8")
-
-if __name__ == "__main__":
-    main()
+import json
+from pathlib import Path
+
+import matplotlib.pyplot as plt
+import numpy as np
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def pick_models(files):
+    items = []
+    for p in files:
+        try:
+            j = read_json(p)
+            items.append((Path(p).stem, j))
+        except Exception:
+            pass
+    return items
+
+
+def metric_value(obj, scope, lang, metric):
+    if scope == "overall":
+        return obj.get("overall", {}).get(metric, None)
+    if scope == "by_lang":
+        return obj.get("by_lang", {}).get(lang, {}).get(metric, None)
+    return None
+
+
+def section(obj, scope, lang):
+    if scope == "overall":
+        return obj.get("overall", {})
+    if scope == "by_lang":
+        return obj.get("by_lang", {}).get(lang, {})
+    return {}
+
+
+def rank_stat_value(obj, scope, lang, key):
+    return section(obj, scope, lang).get("rank_stats", {}).get(key, None)
+
+
+def score_stat_value(obj, scope, lang, group, key):
+    return section(obj, scope, lang).get("score_stats", {}).get(group, {}).get(key, None)
+
+
+def coverage_value(obj, scope, lang, key):
+    return section(obj, scope, lang).get("coverage", {}).get(key, None)
+
+
+def distribution_value(obj, scope, lang, key):
+    return section(obj, scope, lang).get("distributions", {}).get(key, [])
+
+
+def save_recall_plot(models, scope, lang, out_path):
+    ks = [1, 3, 5, 10]
+    x = np.arange(len(ks))
+    width = 0.8 / max(1, len(models))
+
+    plt.figure()
+    for i, (name, obj) in enumerate(models):
+        vals = []
+        for k in ks:
+            v = metric_value(obj, scope, lang, f"recall@{k}")
+            vals.append(0.0 if v is None else float(v))
+        plt.bar(
+            x + (i - (len(models) - 1) / 2) * width,
+            vals,
+            width=width,
+            label=obj.get("model", name),
+        )
+
+    plt.xticks(x, [f"@{k}" for k in ks])
+    title = "Recall@k"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("score")
+    ymax = max(
+        [0.0]
+        + [
+            max(
+                [
+                    metric_value(o, scope, lang, f"recall@{k}") or 0.0
+                    for k in ks
+                ]
+            )
+            for _, o in models
+        ]
+    )
+    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_rank_metrics_plot(models, scope, lang, out_path):
+    metrics = ["mrr@10", "ndcg@10"]
+    x = np.arange(len(metrics))
+    width = 0.8 / max(1, len(models))
+
+    plt.figure()
+    for i, (name, obj) in enumerate(models):
+        vals = []
+        for m in metrics:
+            v = metric_value(obj, scope, lang, m)
+            vals.append(0.0 if v is None else float(v))
+        plt.bar(
+            x + (i - (len(models) - 1) / 2) * width,
+            vals,
+            width=width,
+            label=obj.get("model", name),
+        )
+
+    plt.xticks(x, metrics)
+    title = "Ranking metrics"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("score")
+    ymax = max(
+        [0.0]
+        + [
+            max([metric_value(o, scope, lang, m) or 0.0 for m in metrics])
+            for _, o in models
+        ]
+    )
+    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_precision_plot(models, scope, lang, out_path):
+    ks = [1, 3, 5, 10]
+    x = np.arange(len(ks))
+    width = 0.8 / max(1, len(models))
+
+    plt.figure()
+    any_data = False
+    for i, (name, obj) in enumerate(models):
+        vals = []
+        for k in ks:
+            v = metric_value(obj, scope, lang, f"precision@{k}")
+            if v is not None:
+                any_data = True
+            vals.append(0.0 if v is None else float(v))
+        plt.bar(
+            x + (i - (len(models) - 1) / 2) * width,
+            vals,
+            width=width,
+            label=obj.get("model", name),
+        )
+
+    if not any_data:
+        plt.close()
+        return
+
+    plt.xticks(x, [f"@{k}" for k in ks])
+    title = "Precision@k (single-positive)"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("score")
+    ymax = max(
+        [0.0]
+        + [
+            max(
+                [
+                    metric_value(o, scope, lang, f"precision@{k}") or 0.0
+                    for k in ks
+                ]
+            )
+            for _, o in models
+        ]
+    )
+    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_recall_curve_plot(models, scope, lang, out_path):
+    ks = [1, 3, 5, 10]
+    xs = np.array(ks, dtype=float)
+
+    plt.figure()
+    for name, obj in models:
+        ys = []
+        for k in ks:
+            v = metric_value(obj, scope, lang, f"recall@{k}")
+            ys.append(0.0 if v is None else float(v))
+        plt.plot(xs, ys, marker="o", label=obj.get("model", name))
+
+    plt.xticks(xs, [f"@{k}" for k in ks])
+    title = "Recall@k vs k"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.xlabel("k")
+    plt.ylabel("recall")
+    ymax = max(
+        [0.0]
+        + [
+            max(
+                [
+                    metric_value(o, scope, lang, f"recall@{k}") or 0.0
+                    for k in ks
+                ]
+            )
+            for _, o in models
+        ]
+    )
+    plt.ylim(0, min(1.0, max(0.05, ymax * 1.2)))
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_rank_stats_plot(models, scope, lang, out_path):
+    metrics = [("mean_rank", "Mean"), ("median_rank", "Median"), ("p90_rank", "P90")]
+    x = np.arange(len(metrics))
+    width = 0.8 / max(1, len(models))
+
+    plt.figure()
+    any_data = False
+    for i, (name, obj) in enumerate(models):
+        vals = []
+        for key, _ in metrics:
+            v = rank_stat_value(obj, scope, lang, key)
+            if v is not None:
+                any_data = True
+            vals.append(np.nan if v is None else float(v))
+        plt.bar(
+            x + (i - (len(models) - 1) / 2) * width,
+            vals,
+            width=width,
+            label=obj.get("model", name),
+        )
+
+    if not any_data:
+        plt.close()
+        return
+
+    plt.xticks(x, [m[1] for m in metrics])
+    title = "Rank stats (1-based)"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("rank")
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_rank_distribution_plot(models, scope, lang, out_path):
+    top_k = None
+    for _, obj in models:
+        if "top_k" in obj:
+            top_k = int(obj["top_k"])
+            break
+    if top_k is None:
+        return
+
+    x = np.arange(top_k + 1)
+    width = 0.8 / max(1, len(models))
+
+    plt.figure()
+    any_data = False
+    for i, (name, obj) in enumerate(models):
+        ranks = distribution_value(obj, scope, lang, "ranks")
+        if not ranks:
+            continue
+        any_data = True
+        buckets = [0] * (top_k + 1)
+        for r in ranks:
+            if r is None or r < 0 or r >= top_k:
+                buckets[-1] += 1
+            else:
+                buckets[int(r)] += 1
+        total = max(1, len(ranks))
+        vals = [b / total for b in buckets]
+        plt.bar(
+            x + (i - (len(models) - 1) / 2) * width,
+            vals,
+            width=width,
+            label=obj.get("model", name),
+        )
+
+    if not any_data:
+        plt.close()
+        return
+
+    labels = [str(i + 1) for i in range(top_k)] + ["NF"]
+    plt.xticks(x, labels)
+    title = "Rank distribution"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("share of queries")
+    plt.legend()
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_margin_boxplot(models, scope, lang, out_path):
+    data = []
+    labels = []
+    for name, obj in models:
+        margins = distribution_value(obj, scope, lang, "margins")
+        if margins:
+            data.append(margins)
+            labels.append(obj.get("model", name))
+
+    if not data:
+        return
+
+    plt.figure()
+    plt.boxplot(data, labels=labels, showfliers=False)
+    title = "Score margin (top1 - top2)"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("margin")
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_coverage_plot(models, scope, lang, out_path):
+    vals = []
+    labels = []
+    for name, obj in models:
+        v = coverage_value(obj, scope, lang, "coverage_ratio")
+        if v is not None:
+            vals.append(float(v))
+            labels.append(obj.get("model", name))
+
+    if not vals:
+        return
+
+    x = np.arange(len(vals))
+    plt.figure()
+    plt.bar(x, vals)
+    plt.xticks(x, labels, rotation=15, ha="right")
+    title = "Coverage ratio (unique docs / corpus)"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("ratio")
+    plt.ylim(0, 1.0)
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def save_top1_score_hist(models, scope, lang, out_dir):
+    for name, obj in models:
+        tp = distribution_value(obj, scope, lang, "top1_scores_tp")
+        fp = distribution_value(obj, scope, lang, "top1_scores_fp")
+        if not tp and not fp:
+            continue
+        plt.figure()
+        if tp:
+            plt.hist(tp, bins=20, alpha=0.6, label="top-1 is positive")
+        if fp:
+            plt.hist(fp, bins=20, alpha=0.6, label="top-1 is not positive")
+        title = "Top-1 score distribution"
+        label = obj.get("model", name)
+        if scope == "overall":
+            plt.title(f"{title} ({label}, overall)")
+        else:
+            plt.title(f"{title} ({label}, {lang})")
+        plt.xlabel("similarity score")
+        plt.ylabel("count")
+        plt.legend()
+        Path(out_dir).mkdir(parents=True, exist_ok=True)
+        out_path = (
+            Path(out_dir)
+            / f"top1_score_tp_fp_{model_label_key(obj, name)}_{scope if scope else 'overall'}{'' if lang is None else '_' + lang}.png"
+        )
+        plt.tight_layout()
+        plt.savefig(out_path, dpi=180)
+        plt.close()
+
+
+def model_label_key(obj, name):
+    s = str(obj.get("model", name)).lower()
+    if "labse" in s:
+        return "labse"
+    if "finetuned" in s or "artifacts" in s:
+        return "finetuned"
+    if "paraphrase-multilingual-mpnet-base-v2" in s:
+        return "base"
+    if "mpnet" in s:
+        return "base"
+    return name.lower()
+
+
+def select_model(models, key):
+    for name, obj in models:
+        if model_label_key(obj, name) == key:
+            return (name, obj)
+    return None
+
+
+def save_relative_improvement_plot(models, scope, lang, out_path):
+    fin = select_model(models, "finetuned")
+    base = select_model(models, "base")
+    if fin is None or base is None:
+        return
+
+    metrics = ["recall@1", "recall@3", "recall@5", "recall@10", "mrr@10", "ndcg@10"]
+    labels = ["R@1", "R@3", "R@5", "R@10", "MRR@10", "nDCG@10"]
+
+    fin_obj = fin[1]
+    base_obj = base[1]
+
+    vals = []
+    for m in metrics:
+        fv = metric_value(fin_obj, scope, lang, m)
+        bv = metric_value(base_obj, scope, lang, m)
+        fv = 0.0 if fv is None else float(fv)
+        bv = 0.0 if bv is None else float(bv)
+        if bv <= 0:
+            vals.append(np.nan)
+        else:
+            vals.append((fv - bv) / bv * 100.0)
+
+    x = np.arange(len(metrics))
+    plt.figure()
+    plt.bar(x, vals)
+    plt.xticks(x, labels)
+    title = "Relative improvement vs base (%)"
+    if scope == "overall":
+        plt.title(f"{title} (overall)")
+    else:
+        plt.title(f"{title} ({lang})")
+    plt.ylabel("%")
+    plt.axhline(0.0)
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    plt.tight_layout()
+    plt.savefig(out_path, dpi=180)
+    plt.close()
+
+
+def main():
+    reports_dir = Path("artifacts/reports")
+    files = sorted([str(p) for p in reports_dir.glob("eval_*.json")])
+    models = pick_models(files)
+
+    if not models:
+        raise SystemExit("No eval_*.json found in artifacts/reports")
+
+    fig_dir = reports_dir / "figures"
+    fig_dir.mkdir(parents=True, exist_ok=True)
+
+    save_recall_plot(models, "overall", None, fig_dir / "recall_overall.png")
+    save_rank_metrics_plot(models, "overall", None, fig_dir / "rank_metrics_overall.png")
+    save_recall_curve_plot(models, "overall", None, fig_dir / "recall_curve_overall.png")
+    save_relative_improvement_plot(models, "overall", None, fig_dir / "relative_improvement_overall.png")
+    save_precision_plot(models, "overall", None, fig_dir / "precision_overall.png")
+    save_rank_stats_plot(models, "overall", None, fig_dir / "rank_stats_overall.png")
+    save_rank_distribution_plot(
+        models, "overall", None, fig_dir / "rank_distribution_overall.png"
+    )
+    save_margin_boxplot(models, "overall", None, fig_dir / "score_margin_overall.png")
+    save_coverage_plot(models, "overall", None, fig_dir / "coverage_overall.png")
+    save_top1_score_hist(models, "overall", None, fig_dir)
+
+    for lang in ["ru", "kz"]:
+        save_recall_plot(models, "by_lang", lang, fig_dir / f"recall_{lang}.png")
+        save_rank_metrics_plot(
+            models, "by_lang", lang, fig_dir / f"rank_metrics_{lang}.png"
+        )
+        save_recall_curve_plot(
+            models, "by_lang", lang, fig_dir / f"recall_curve_{lang}.png"
+        )
+        save_relative_improvement_plot(
+            models, "by_lang", lang, fig_dir / f"relative_improvement_{lang}.png"
+        )
+        save_precision_plot(models, "by_lang", lang, fig_dir / f"precision_{lang}.png")
+        save_rank_stats_plot(models, "by_lang", lang, fig_dir / f"rank_stats_{lang}.png")
+        save_rank_distribution_plot(
+            models, "by_lang", lang, fig_dir / f"rank_distribution_{lang}.png"
+        )
+        save_coverage_plot(models, "by_lang", lang, fig_dir / f"coverage_{lang}.png")
+
+    summary = {
+        "loaded_reports": [Path(f).name for f in files],
+        "figures": [p.name for p in sorted(fig_dir.glob("*.png"))],
+    }
+    (reports_dir / "figures_summary.json").write_text(
+        json.dumps(summary, ensure_ascii=False, indent=2),
+        encoding="utf-8",
+    )
+
+
+if __name__ == "__main__":
+    main()