Update server.py

server.py

@@ -35,10 +35,13 @@ if not HF_TOKEN:
 # Configuration
 HF_DATASET_REPO = "factorstudios/movs"
 READY_VIDEOS_FOLDER = "ready_videos"
+COMPRESSED_FOLDER = "compressed"
 SIZE_THRESHOLD_MB = 100  # Compress videos above 100MB
 CACHE_DIR = "/tmp/video_compress_cache"
 TEMP_DIR = Path("/tmp/video_compression_server")
 TEMP_DIR.mkdir(exist_ok=True)
+PROGRESS_FILE = TEMP_DIR / "compression_progress.json"
+PROGRESS_FILE_REPO = f"{COMPRESSED_FOLDER}/compression_progress.json"
 
 # Global state
 compression_state = {
@@ -91,7 +94,7 @@ def get_video_bitrate(video_path: str) -> float:
 
 def compress_video(input_path: str, output_path: str) -> Dict:
     """
-    Compress video using H.265 with maximum quality preservation.
+    Compress video using H.265 targeting ~50MB with quality preservation.
     Returns dict with compression stats.
     """
     print(f"\n{'='*80}")
@@ -103,50 +106,117 @@ def compress_video(input_path: str, output_path: str) -> Dict:
     original_size = os.path.getsize(input_path) / (1024**2)
     
     print(f"Original: {original_size:.1f} MB | Duration: {duration:.1f}s")
+    print(f"Target: ~50MB with high quality preservation")
     
-    # H.265 encoding with maximum quality
+    # Calculate target bitrate for ~50MB file
+    target_mb = 50
+    target_bitrate_kbps = int((target_mb * 8 * 1024) / max(duration, 1))
+    video_bitrate = max(400, min(target_bitrate_kbps, 1200))
+    
+    print(f"Calculated bitrate: {video_bitrate}kbps")
+    
+    # H.265 encoding with aggressive compression targeting ~50MB
     ffmpeg_cmd = [
         "ffmpeg", "-y",
         "-i", input_path,
         "-c:v", "libx265",
-        "-preset", "veryslow",
-        "-crf", "20",
-        "-x265-params", "aq-mode=3",
-        "-pix_fmt", "yuv420p10le",
+        "-preset", "slow",
+        "-crf", "28",
+        "-b:v", f"{video_bitrate}k",
+        "-maxrate", f"{int(video_bitrate * 1.2)}k",
+        "-bufsize", f"{int(video_bitrate * 2)}k",
+        "-x265-params", "aq-mode=3:log-level=error",
+        "-pix_fmt", "yuv420p",
         "-c:a", "aac",
-        "-b:a", "192k",
+        "-b:a", "128k",
+        "-progress", "pipe:1",
         output_path
     ]
     
-    result = subprocess.run(ffmpeg_cmd, capture_output=True, text=True)
-    
-    if result.returncode != 0:
-        print(f"✗ Encoding failed: {result.stderr[:500]}")
+    try:
+        result = subprocess.run(ffmpeg_cmd, capture_output=True, text=True, timeout=3600)
+        
+        if result.returncode != 0:
+            print(f"✗ Encoding failed with code {result.returncode}")
+            if result.stderr:
+                print(f"Error output: {result.stderr[:500]}")
+            return None
+        
+        if not os.path.exists(output_path):
+            print(f"✗ Output file not created")
+            return None
+        
+        compressed_size = os.path.getsize(output_path) / (1024**2)
+        compression_ratio = original_size / compressed_size
+        saved_mb = original_size - compressed_size
+        
+        print(f"✓ Compressed: {original_size:.1f}MB → {compressed_size:.1f}MB")
+        print(f"  Saved: {saved_mb:.1f}MB ({(saved_mb/original_size*100):.1f}%)")
+        print(f"  Ratio: {compression_ratio:.2f}x")
+        
+        return {
+            "original_size_mb": round(original_size, 2),
+            "compressed_size_mb": round(compressed_size, 2),
+            "saved_mb": round(saved_mb, 2),
+            "compression_ratio": round(compression_ratio, 2),
+            "duration_seconds": duration
+        }
+    except subprocess.TimeoutExpired:
+        print(f"✗ Compression timed out (>1 hour)")
         return None
-    
-    if not os.path.exists(output_path):
-        print(f"✗ Output file not created")
+    except Exception as e:
+        print(f"✗ Compression error: {e}")
         return None
-    
-    compressed_size = os.path.getsize(output_path) / (1024**2)
-    compression_ratio = original_size / compressed_size
-    saved_mb = original_size - compressed_size
-    
-    print(f"✓ Compressed: {original_size:.1f}MB → {compressed_size:.1f}MB")
-    print(f"  Saved: {saved_mb:.1f}MB ({(saved_mb/original_size*100):.1f}%)")
-    print(f"  Ratio: {compression_ratio:.2f}x")
-    
-    return {
-        "original_size_mb": round(original_size, 2),
-        "compressed_size_mb": round(compressed_size, 2),
-        "saved_mb": round(saved_mb, 2),
-        "compression_ratio": round(compression_ratio, 2),
-        "duration_seconds": duration
-    }
+
+
+async def load_progress_file() -> Dict:
+    """Load compression progress from JSON file in HF dataset."""
+    try:
+        # Try to download existing progress file
+        progress_path = hf_hub_download(
+            repo_id=HF_DATASET_REPO,
+            filename=PROGRESS_FILE_REPO,
+            repo_type="dataset",
+            token=HF_TOKEN,
+            cache_dir=CACHE_DIR
+        )
+        
+        if os.path.islink(progress_path):
+            progress_path = os.path.realpath(progress_path)
+        
+        with open(progress_path, 'r') as f:
+            progress = json.load(f)
+        print(f"✓ Loaded progress file: {len(progress.get('compressed', []))} files already processed")
+        return progress
+    except Exception as e:
+        print(f"No existing progress file found, starting fresh: {e}")
+        return {"compressed": [], "failed": [], "last_updated": datetime.now().isoformat()}
+
+
+async def save_progress_file(progress: Dict):
+    """Save and upload compression progress to HF dataset."""
+    try:
+        progress["last_updated"] = datetime.now().isoformat()
+        
+        with open(PROGRESS_FILE, 'w') as f:
+            json.dump(progress, f, indent=2)
+        
+        print(f"Uploading progress file...")
+        upload_file(
+            path_or_fileobj=str(PROGRESS_FILE),
+            path_in_repo=PROGRESS_FILE_REPO,
+            repo_id=HF_DATASET_REPO,
+            repo_type="dataset",
+            token=HF_TOKEN,
+            commit_message="Update compression progress"
+        )
+        print(f"✓ Progress file uploaded")
+    except Exception as e:
+        print(f"Warning: Could not save progress file: {e}")
 
 
 async def scan_and_compress_videos():
-    """Scan ready_videos folder and compress large videos."""
+    """Scan ready_videos folder and compress large videos using persistent progress tracking."""
     if compression_state["is_polling"]:
         print("Already polling, skipping...")
         return
@@ -154,6 +224,15 @@ async def scan_and_compress_videos():
     compression_state["is_polling"] = True
     
     try:
+        print("\n" + "="*80)
+        print("LOADING COMPRESSION PROGRESS")
+        print("="*80)
+        
+        # Load progress file
+        progress = await load_progress_file()
+        compressed_files = {item["path"] for item in progress.get("compressed", [])}
+        failed_files = {item["path"] for item in progress.get("failed", [])}
+        
         print("\n" + "="*80)
         print("SCANNING FOR LARGE VIDEOS")
         print("="*80)
@@ -165,11 +244,18 @@ async def scan_and_compress_videos():
             token=HF_TOKEN
         )
         
-        # Find video files > 100MB in ready_videos folder
+        # Find unprocessed video files
         video_files = []
         for f in files:
             if f.startswith(f"{READY_VIDEOS_FOLDER}/") and f.endswith(".mp4"):
-                # Get file info from HF Hub
+                # Skip if already handled
+                if f in compressed_files:
+                    print(f"  ⊘ {f.split('/')[-1]} (already compressed)")
+                    continue
+                if f in failed_files:
+                    print(f"  ✗ {f.split('/')[-1]} (previously failed)")
+                    continue
+                
                 try:
                     # Parse file path: ready_videos/moviename/segment-XX.mp4
                     parts = f.split("/")
@@ -185,33 +271,16 @@ async def scan_and_compress_videos():
                     print(f"Error parsing file {f}: {e}")
                     continue
         
-        print(f"Found {len(video_files)} video files")
-        
-        # Check sizes and filter large ones
-        large_videos = []
-        for video_info in video_files:
-            # Skip if already processed
-            if any(pv["path"] == video_info["path"] for pv in compression_state["processed_videos"]):
-                print(f"  ⊘ {video_info['file_name']} (already processed)")
-                continue
-            
-            # Skip if already failed
-            if any(fv["path"] == video_info["path"] for fv in compression_state["failed_videos"]):
-                print(f"  ✗ {video_info['file_name']} (previously failed)")
-                continue
-            
-            large_videos.append(video_info)
-        
-        compression_state["total_found"] = len(large_videos)
-        print(f"Unprocessed videos: {len(large_videos)}")
+        compression_state["total_found"] = len(video_files)
+        print(f"Found {len(video_files)} unprocessed video files")
         
-        if not large_videos:
+        if not video_files:
             print("✓ All videos already processed!")
             compression_state["is_polling"] = False
             return
         
         # Process each video
-        for video_info in large_videos:
+        for video_info in video_files:
             compression_state["current_video"] = video_info["file_name"]
             
             try:
@@ -231,18 +300,21 @@ async def scan_and_compress_videos():
                 file_size_mb = os.path.getsize(video_path) / (1024**2)
                 print(f"✓ Downloaded: {file_size_mb:.1f} MB")
                 
-                # Skip if smaller than threshold
+                # Check if meets compression threshold
                 if file_size_mb < SIZE_THRESHOLD_MB:
-                    print(f"⊘ File size {file_size_mb:.1f}MB < {SIZE_THRESHOLD_MB}MB threshold, skipping")
-                    compression_state["processed_videos"].append({
+                    print(f"⊘ Below threshold ({file_size_mb:.1f}MB < {SIZE_THRESHOLD_MB}MB), skipping")
+                    progress["compressed"].append({
                         "path": video_info["path"],
                         "file_name": video_info["file_name"],
                         "status": "skipped",
-                        "reason": f"File size {file_size_mb:.1f}MB below threshold",
+                        "reason": f"Below {SIZE_THRESHOLD_MB}MB threshold",
                         "timestamp": datetime.now().isoformat()
                     })
+                    await save_progress_file(progress)
                     continue
                 
+                print(f"✓ Starting H.265 compression...")
+                
                 # Compress
                 output_path = TEMP_DIR / f"{Path(video_info['file_name']).stem}_compressed.mp4"
                 compression_stats = compress_video(video_path, str(output_path))
@@ -252,9 +324,9 @@ async def scan_and_compress_videos():
                 
                 compression_state["total_compressed"] += 1
                 
-                # Upload compressed video
+                # Upload compressed video to COMPRESSED folder with same structure
                 upload_filename = f"{Path(video_info['file_name']).stem}_compressed.mp4"
-                upload_path = f"{READY_VIDEOS_FOLDER}/{video_info['movie_name']}/{upload_filename}"
+                upload_path = f"{COMPRESSED_FOLDER}/{video_info['movie_name']}/{upload_filename}"
                 
                 print(f"Uploading: {upload_path}")
                 upload_file(
@@ -269,8 +341,8 @@ async def scan_and_compress_videos():
                 compression_state["total_uploaded"] += 1
                 compression_state["total_space_saved_mb"] += compression_stats["saved_mb"]
                 
-                # Track processed video
-                compression_state["processed_videos"].append({
+                # Track in progress file
+                progress["compressed"].append({
                     "path": video_info["path"],
                     "file_name": video_info["file_name"],
                     "upload_path": upload_path,
@@ -279,7 +351,8 @@ async def scan_and_compress_videos():
                     "timestamp": datetime.now().isoformat()
                 })
                 
-                compression_state["compression_stats"][video_info["file_name"]] = compression_stats
+                # Save progress
+                await save_progress_file(progress)
                 
                 # Cleanup
                 try:
@@ -290,50 +363,21 @@ async def scan_and_compress_videos():
             except Exception as e:
                 print(f"✗ Error: {e}")
                 compression_state["last_error"] = str(e)
-                compression_state["failed_videos"].append({
+                progress["failed"].append({
                     "path": video_info["path"],
                     "file_name": video_info["file_name"],
                     "error": str(e),
                     "timestamp": datetime.now().isoformat()
                 })
+                await save_progress_file(progress)
         
-        # Save JSON tracking file
-        print(f"\n{'='*80}")
-        print("SAVING PROCESSING REPORT")
-        print(f"{'='*80}")
-        
-        report = {
-            "timestamp": datetime.now().isoformat(),
-            "total_videos_found": compression_state["total_found"],
-            "total_compressed": compression_state["total_compressed"],
-            "total_uploaded": compression_state["total_uploaded"],
-            "total_space_saved_mb": compression_state["total_space_saved_mb"],
-            "processed_videos": compression_state["processed_videos"],
-            "failed_videos": compression_state["failed_videos"],
-            "compression_stats": compression_state["compression_stats"]
-        }
-        
-        report_path = TEMP_DIR / "compression_report.json"
-        with open(report_path, "w") as f:
-            json.dump(report, f, indent=2)
-        
-        # Upload JSON report
-        print(f"Uploading report...")
-        upload_file(
-            path_or_fileobj=str(report_path),
-            path_in_repo=f"{READY_VIDEOS_FOLDER}/compression_report.json",
-            repo_id=HF_DATASET_REPO,
-            repo_type="dataset",
-            token=HF_TOKEN,
-            commit_message="Update compression processing report"
-        )
-        
-        print(f"✓ Report uploaded")
         print(f"\n{'='*80}")
         print("COMPRESSION COMPLETE")
         print(f"  Compressed: {compression_state['total_compressed']}")
         print(f"  Uploaded: {compression_state['total_uploaded']}")
         print(f"  Space saved: {compression_state['total_space_saved_mb']:.1f} MB")
+        print(f"  Total compressed: {len(progress['compressed'])}")
+        print(f"  Total failed: {len(progress['failed'])}")
         print(f"{'='*80}\n")
         
     except Exception as e:
@@ -432,6 +476,6 @@ async def reset_state():
 
 
 if __name__ == "__main__":
-    print("Starting Video Compression Server on port 7861...")
+    print("Starting Video Compression Server on port 7860...")
     print("Scanning for videos on startup...")
     uvicorn.run(app, host="0.0.0.0", port=7860)