File size: 917 Bytes
7d4b5f8 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
#!/usr/bin/env python3
"""
Training wrapper with checkpoint support and better logging
"""
import sys
import os
import shutil
from pathlib import Path
# Clear cache before training
print("🧹 Clearing HuggingFace cache...")
cache_dir = Path("/workspace/.hf_home/hub/models--mistralai--Mistral-7B-v0.1")
if cache_dir.exists():
try:
shutil.rmtree(cache_dir)
print("✓ Cache cleared")
except Exception as e:
print(f"⚠️ Cache clear warning: {e}")
# Force unbuffered output for real-time logs
sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', buffering=1)
sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', buffering=1)
print("=" * 70)
print("🚀 Starting Training with Real-time Logging")
print("=" * 70)
sys.stdout.flush()
# Import and run training
sys.path.insert(0, "/workspace/ftt/ft/models/msp")
from ft.finetune_mistral7b import main
if __name__ == "__main__":
main()
|