epoch_1/Mobile-O/analyze_train_log.py

"""
Quick rolling-average loss viewer for train_log.jsonl (written by
step2_finetune_refined.py / step3_finetune_hallucination.py every optimizer
step). Per-step loss is noisy on short-answer VQA data -- this smooths it out
so you can see the actual trend without relying on wandb's UI.

Run:
    python analyze_train_log.py checkpoints/vlm_kvasir_full/train_log.jsonl
    python analyze_train_log.py checkpoints/vlm_kvasir_full/train_log.jsonl --window 100
"""

import argparse
import json
from collections import deque


def main():
    p = argparse.ArgumentParser()
    p.add_argument("log_path")
    p.add_argument("--window", type=int, default=50, help="Rolling average window (steps)")
    p.add_argument("--print_every", type=int, default=200, help="Print a rolling-avg line every N steps")
    args = p.parse_args()

    window = deque(maxlen=args.window)
    epoch_sums = {}
    epoch_counts = {}

    epoch_val_loss = {}

    with open(args.log_path) as f:
        for line in f:
            entry = json.loads(line)

            if "epoch_summary" in entry:
                epoch_val_loss[entry["epoch_summary"]] = entry.get("val_loss")
                continue

            step = entry["step"]
            loss = entry["loss"]
            epoch = entry["epoch"]

            window.append(loss)
            epoch_sums[epoch] = epoch_sums.get(epoch, 0.0) + loss
            epoch_counts[epoch] = epoch_counts.get(epoch, 0) + 1

            if step % args.print_every == 0:
                avg = sum(window) / len(window)
                print(f"step {step:>6} | epoch {epoch} | rolling_avg(last {len(window)}) = {avg:.4f} | last_lr = {entry['lr']:.2e}")

    print("\nPer-epoch average loss:")
    for epoch in sorted(epoch_sums):
        avg = epoch_sums[epoch] / epoch_counts[epoch]
        val = epoch_val_loss.get(epoch)
        val_str = f" | val_loss = {val:.4f}" if val is not None else ""
        print(f"  epoch {epoch}: train_avg = {avg:.4f}  ({epoch_counts[epoch]} steps){val_str}")


if __name__ == "__main__":
    main()