tools/generate_hf_model_card.py

from __future__ import annotations

import argparse
import json
from collections import defaultdict
from pathlib import Path
from typing import Any


ROOT = Path(__file__).resolve().parents[1]
RESULTS = ROOT / "results"
README = ROOT / "README.md"
REPO_ID = "dineth18/CD-Models"

DATASET_LABELS = {
    "dsifn_cd": "DSIFN-CD",
    "levir_cd_test_as_val": "LEVIR-CD+ comparison",
    "sysu_cd": "SYSU-CD",
    "whu_cd": "WHU-CD",
}
DATASET_ORDER = tuple(DATASET_LABELS)
DATASET_ALIASES = {
    "levir_val_as_test": "levir_cd_test_as_val",
}
MODEL_LABELS = {
    "bifa": "BiFA",
    "bit_cd": "BIT-CD",
    "cgnet": "CGNet",
    "changeformer": "ChangeFormer",
    "changemamba": "ChangeMamba",
    "dsamnet": "DSAMNet",
    "fc_ef": "FC-EF",
    "fc_siam_conc": "FC-Siam-conc",
    "fc_siam_diff": "FC-Siam-diff",
    "hanet": "HANet",
    "ifnet": "IFNet",
    "schanger": "SChanger",
    "siam_nestedunet": "Siam-NestedUNet",
    "stanet": "STANet",
}


def load_rows() -> dict[str, list[dict[str, Any]]]:
    grouped: dict[str, list[dict[str, Any]]] = defaultdict(list)
    for path in sorted(RESULTS.glob("*/*/metrics_test.json")):
        with path.open(encoding="utf-8") as handle:
            row = json.load(handle)
        source_dataset = str(row.get("dataset") or path.parent.name)
        dataset = DATASET_ALIASES.get(source_dataset, source_dataset)
        if dataset not in DATASET_LABELS:
            continue
        if row.get("split") != "test" or row.get("status") != "complete":
            continue
        if not isinstance(row.get("f1"), (int, float)):
            continue
        row["_path"] = path
        row["_model"] = str(row.get("model") or path.parents[1].name)
        row["_source_dataset"] = source_dataset
        grouped[dataset].append(row)
    for rows in grouped.values():
        rows.sort(key=lambda row: (-float(row["f1"]), row["_model"]))
    return grouped


def metric(row: dict[str, Any], *keys: str) -> Any:
    for key in keys:
        value = row.get(key)
        if value is not None and value != "":
            return value
    return None


def fmt(value: Any, digits: int = 4) -> str:
    if value is None:
        return "—"
    return f"{float(value):.{digits}f}"


def fmt_profile(value: Any) -> str:
    if value is None:
        return "—"
    number = float(value)
    return f"{number:.2f}"


def checkpoint_path(row: dict[str, Any]) -> Path:
    canonical = (row["_path"].parent / "checkpoints" / "best_model.pth").resolve()
    if canonical.exists():
        return canonical
    raw = metric(row, "checkpoint", "checkpoint_path")
    if raw:
        path = Path(str(raw))
        if path.exists() and path.resolve().is_relative_to(ROOT):
            return path.resolve()
    return canonical


def repo_relative(path: Path) -> Path:
    return path.resolve().relative_to(ROOT)


def download_url(path: Path) -> str:
    relative = repo_relative(path).as_posix()
    return f"https://huggingface.co/{REPO_ID}/resolve/main/{relative}?download=true"


def markdown_table(headers: list[str], rows: list[list[str]]) -> list[str]:
    return [
        "| " + " | ".join(headers) + " |",
        "| " + " | ".join("---" for _ in headers) + " |",
        *("| " + " | ".join(row) + " |" for row in rows),
    ]


def build_results() -> tuple[str, list[Path]]:
    grouped = load_rows()
    checkpoints: list[Path] = []
    lines = [
        "## Results and released checkpoints",
        "",
        "Only completed `metrics_test.json` evaluations are eligible for rankings. Rankings are computed per dataset by test-set F1 (descending), and the canonical `best_model.pth` checkpoint for every ranked model is released.",
        "",
        "All accuracy values are fractions. GPU memory is the PyTorch peak reserved memory for inference, with peak allocated memory used only when reserved memory is absent. FPS is model-only throughput. A dash means the evaluator did not record that metric; values are never estimated.",
        "",
        "The current evaluator records BF1 but not boundary mean IoU (BmIoU), so BmIoU remains explicitly unavailable rather than being inferred from BF1. Boundary IoU and boundary F1 are distinct measures.",
        "",
        "The LEVIR-CD+ comparison combines the project-compatible `levir_cd_test_as_val` and `levir_val_as_test` result records. The ChangeMamba row uses the completed `levir_val_as_test` evaluation over 5,568 samples.",
        "",
        "### Dataset summary",
        "",
    ]
    summary_rows: list[list[str]] = []
    for dataset in DATASET_ORDER:
        rows = grouped.get(dataset, [])
        if not rows:
            continue
        winner = rows[0]
        summary_rows.append(
            [
                DATASET_LABELS[dataset],
                str(len(rows)),
                MODEL_LABELS.get(winner["_model"], winner["_model"]),
                fmt(winner["f1"]),
            ]
        )
    lines.extend(markdown_table(["Dataset", "Tested models", "F1 leader", "Best test F1"], summary_rows))

    headers = [
        "Rank",
        "Model",
        "F1",
        "mIoU",
        "Overall accuracy",
        "Recall",
        "Precision",
        "BF1",
        "BmIoU",
        "GFLOPs",
        "GPU (GB)",
        "FPS",
        "Parameters (M)",
        "Checkpoint",
    ]
    for dataset in DATASET_ORDER:
        rows = grouped.get(dataset, [])
        lines.extend(["", f"### {DATASET_LABELS[dataset]}", ""])
        table_rows: list[list[str]] = []
        for rank, row in enumerate(rows, start=1):
            checkpoint = checkpoint_path(row)
            if not checkpoint.is_file():
                raise FileNotFoundError(
                    f"Tested checkpoint is missing for {row['_model']}/{dataset}: {checkpoint}"
                )
            checkpoints.append(checkpoint)
            link = f"[Download]({download_url(checkpoint)})"
            table_rows.append(
                [
                    str(rank),
                    MODEL_LABELS.get(row["_model"], row["_model"]),
                    fmt(metric(row, "f1", "F1")),
                    fmt(metric(row, "miou", "mIoU")),
                    fmt(metric(row, "oa", "OA")),
                    fmt(metric(row, "recall", "Recall")),
                    fmt(metric(row, "precision", "Precision")),
                    fmt(metric(row, "bf1", "BF1", "boundary_f1")),
                    fmt(metric(row, "bmiou", "BmIoU", "boundary_miou")),
                    fmt_profile(metric(row, "flops_g", "FLOPsG")),
                    fmt_profile(metric(row, "gpu_mem_reserved_peak_gb", "gpu_mem_allocated_peak_gb")),
                    fmt_profile(metric(row, "fps_model_only", "fps", "FPS")),
                    fmt_profile(metric(row, "params_m", "ParamsM")),
                    link,
                ]
            )
        if table_rows:
            lines.extend(markdown_table(headers, table_rows))
        else:
            lines.append("No completed test-set result is available, so no checkpoint is published.")

    lines.extend(
        [
            "",
            "### Datasets without a released checkpoint",
            "",
            "WildFire-S2, KATE-CD-256, the standard LEVIR-CD+ configuration, and Custom-CD currently have no eligible completed test result. Their validation-only or incomplete checkpoints are intentionally not uploaded.",
            "",
        ]
    )
    return "\n".join(lines), checkpoints


def update_readme(results_markdown: str) -> None:
    text = README.read_text(encoding="utf-8")
    current_heading = "## Results and released checkpoints\n"
    legacy_heading = "## Results\n"
    if current_heading in text:
        start = text.index(current_heading)
    else:
        start = text.index(legacy_heading)
    end = text.index("## Installation\n", start)
    text = text[:start] + results_markdown + "\n\n" + text[end:]
    old = (
        "### Trained checkpoints\n\n"
        "The released rank-1 test checkpoints and direct download links are listed in "
        "[Results and released checkpoints](#results-and-released-checkpoints). Local training still "
        "writes checkpoints under `results/{model}/{dataset}/checkpoints/`."
    )
    new = (
        "### Trained checkpoints\n\n"
        "All completed test-run checkpoints and direct download links are listed in "
        "[Results and released checkpoints](#results-and-released-checkpoints). Local training still "
        "writes checkpoints under `results/{model}/{dataset}/checkpoints/`."
    )
    if old in text:
        text = text.replace(old, new)
    elif new not in text:
        raise RuntimeError("Expected trained-checkpoint README text was not found")
    README.write_text(text, encoding="utf-8")


def main() -> None:
    parser = argparse.ArgumentParser()
    parser.add_argument("--write", action="store_true", help="Replace the README result sections")
    args = parser.parse_args()
    markdown, checkpoints = build_results()
    if args.write:
        update_readme(markdown)
    else:
        print(markdown)
    print("Released tested checkpoints:")
    for checkpoint in checkpoints:
        print(repo_relative(checkpoint))


if __name__ == "__main__":
    main()