hugging/td_fuse/run_overnight.py

"""
Overnight Pipeline: Merge Klear-Reasoner + 3-Phase Reasoning Heal + Benchmark.

Run this before going to sleep:
    nohup python3 -m td_fuse.run_overnight > overnight.log 2>&1 &

What it does:
    1. Merges Klear-Reasoner-8B into the existing 5-model merge (~10-15 min)
    2. Runs 3-phase reasoning heal (~2-3 hours)
    3. Runs benchmark to see how good the final model is

Total estimated time: 3-4 hours.
"""

import sys
import time


def main():
    total_start = time.time()
    print("=" * 70)
    print("OVERNIGHT PIPELINE: Merge + Reasoning Heal + Benchmark")
    print(f"Started at: {time.strftime('%Y-%m-%d %H:%M:%S')}")
    print("=" * 70)
    sys.stdout.flush()

    # --- Step 1: Merge Klear-Reasoner-8B ---
    print("\n\n>>> STEP 1/3: MERGING KLEAR-REASONER-8B <<<\n")
    sys.stdout.flush()
    from td_fuse.merge_klear import run_merge
    merge_path = run_merge()
    print(f"\n[overnight] Merge complete. Output: {merge_path}")
    sys.stdout.flush()

    # --- Step 2: 3-Phase Reasoning Heal ---
    print("\n\n>>> STEP 2/3: 3-PHASE REASONING HEAL <<<\n")
    sys.stdout.flush()
    from td_fuse.reasoning_heal import run_reasoning_heal
    heal_path = run_reasoning_heal()
    print(f"\n[overnight] Reasoning heal complete. Output: {heal_path}")
    sys.stdout.flush()

    # --- Step 3: Benchmark ---
    print("\n\n>>> STEP 3/3: BENCHMARK <<<\n")
    sys.stdout.flush()
    try:
        from td_fuse.grpo_reason import pre_benchmark
        pre_benchmark(heal_path)
    except Exception as e:
        print(f"[overnight] Benchmark failed: {e}")
        print("[overnight] Model is saved — you can benchmark manually later.")
    sys.stdout.flush()

    # --- Done ---
    total_hours = (time.time() - total_start) / 3600
    print(f"\n\n{'=' * 70}")
    print(f"OVERNIGHT PIPELINE COMPLETE: {total_hours:.1f} hours total")
    print(f"Final model at: {heal_path}")
    print(f"Finished at: {time.strftime('%Y-%m-%d %H:%M:%S')}")
    print(f"{'=' * 70}")
    sys.stdout.flush()


if __name__ == "__main__":
    main()