Fix annotation test: use single-line Python with real Orpheus data

scripts/cloud/skypilot_annotate_test.yaml

@@ -1,204 +1,37 @@
-# Test annotation job - 1000 samples from Orpheus
-# Validates complete annotation pipeline before full run
-# Cost: ~$0.50 for 10-15 minutes
-
+# Test annotation with 1000 real Orpheus samples
 name: ensemble-annotate-test
 
 resources:
   use_spot: true
-  accelerators: {A100:1, V100:1, T4:1}  # Try A100 first
+  accelerators: {A100:1, V100:1, T4:1}
   memory: 32+
   disk_size: 100
 
 setup: |
-  set -e
-
   echo "=================================================="
   echo "🧪 ENSEMBLE ANNOTATION TEST"
   echo "=================================================="
-  echo ""
-  echo "Testing annotation pipeline with 1000 Orpheus samples"
-  echo ""
-
-  # Machine info
-  echo "📊 Machine Info:"
-  echo "  Hostname: $(hostname)"
-  echo "  CPU cores: $(nproc)"
-  echo "  Memory: $(free -h | grep Mem | awk '{print $2}')"
-
-  if command -v nvidia-smi &> /dev/null; then
-    echo "  GPU: $(nvidia-smi --query-gpu=name --format=csv,noheader)"
-  fi
-  echo ""
 
-  # Install dependencies
-  echo "📦 Installing dependencies..."
-  pip install -q torch transformers librosa soundfile datasets huggingface_hub
-  echo "  ✓ Core dependencies installed"
-  echo ""
+  pip install -q torch transformers librosa soundfile datasets huggingface_hub tqdm
 
-  # Clone repository
-  echo "📥 Cloning repository..."
   if [ ! -d "ensemble-tts-annotation" ]; then
     git clone -q https://huggingface.co/marcosremar2/ensemble-tts-annotation
-    echo "  ✓ Repository cloned"
   else
-    cd ensemble-tts-annotation
-    git pull -q
-    cd ..
-    echo "  ✓ Repository updated"
+    cd ensemble-tts-annotation && git pull -q && cd ..
   fi
-  echo ""
 
   echo "✅ Setup complete!"
 
 run: |
   cd ensemble-tts-annotation
 
-  echo ""
-  echo "=================================================="
-  echo "🎵 DOWNLOADING ORPHEUS SUBSET"
-  echo "=================================================="
-  echo ""
-
-  # Download 1000 samples from Orpheus
-  python3 << 'PYTHON_EOF'
-import sys
-from datasets import load_dataset
-from pathlib import Path
-
-print("Downloading 1000 samples from Orpheus...")
-
-# Load streaming (only first 1000)
-dataset = load_dataset(
-    "marcosremar2/orpheus-tts-portuguese-dataset",
-    split="train",
-    streaming=True
-)
-
-# Save first 1000
-output_dir = Path("data/raw/orpheus_test")
-output_dir.mkdir(parents=True, exist_ok=True)
-
-import soundfile as sf
-count = 0
-for i, sample in enumerate(dataset):
-    if i >= 1000:
-        break
-
-    audio = sample['audio']['array']
-    sr = sample['audio']['sampling_rate']
-    text = sample.get('text', '')
-
-    # Save audio
-    audio_path = output_dir / f"orpheus_{i:05d}.wav"
-    sf.write(audio_path, audio, sr)
-
-    count += 1
-    if count % 100 == 0:
-        print(f"  Downloaded {count}/1000 samples...")
-
-print(f"✅ Downloaded {count} samples to {output_dir}")
-PYTHON_EOF
+  echo "📥 Downloading 1000 Orpheus samples..."
+  python3 -c 'from datasets import load_dataset; from pathlib import Path; import soundfile as sf; ds = load_dataset("marcosremar2/orpheus-tts-portuguese-dataset", split="train", streaming=True); out = Path("data/raw/orpheus_test"); out.mkdir(parents=True, exist_ok=True); [(sf.write(out / f"orpheus_{i:05d}.wav", s["audio"]["array"], s["audio"]["sampling_rate"]), print(f"  {i+1}/1000") if (i+1) % 100 == 0 else None) for i, s in enumerate(ds) if i < 1000]; print("✅ Downloaded 1000 samples")'
 
   echo ""
-  echo "=================================================="
-  echo "🤖 ANNOTATING WITH ENSEMBLE (quick mode)"
-  echo "=================================================="
-  echo ""
-
-  # Annotate samples
-  python3 << 'PYTHON_EOF2'
-import sys
-sys.path.insert(0, '.')
-
-from ensemble_tts.annotator import EnsembleAnnotator
-from pathlib import Path
-import soundfile as sf
-import json
-from tqdm import tqdm
-import time
-
-print("Loading annotator (quick mode: Whisper + SenseVoice)...")
-annotator = EnsembleAnnotator(mode='quick', device='cuda', enable_events=False)
-print("✅ Annotator loaded")
-print("")
-
-# Get audio files
-audio_dir = Path("data/raw/orpheus_test")
-audio_files = sorted(audio_dir.glob("*.wav"))
-
-print(f"Annotating {len(audio_files)} files...")
-print("")
-
-results = []
-start_time = time.time()
-
-for i, audio_file in enumerate(tqdm(audio_files)):
-    try:
-        # Annotate
-        result = annotator.annotate(str(audio_file))
-
-        results.append({
-            "file": audio_file.name,
-            "emotion": result.get("emotion", {}).get("label", "unknown"),
-            "confidence": result.get("emotion", {}).get("confidence", 0.0),
-            "predictions": result.get("emotion", {}).get("predictions", [])
-        })
-
-        # Log progress every 100
-        if (i + 1) % 100 == 0:
-            elapsed = time.time() - start_time
-            rate = (i + 1) / elapsed
-            remaining = (len(audio_files) - (i + 1)) / rate
-            print(f"Progress: {i+1}/{len(audio_files)} ({rate:.1f} files/s, ETA: {remaining/60:.1f}min)")
-
-    except Exception as e:
-        print(f"  Error on {audio_file.name}: {e}")
-        results.append({
-            "file": audio_file.name,
-            "emotion": "error",
-            "confidence": 0.0,
-            "error": str(e)
-        })
+  echo "🤖 Annotating with ensemble (quick mode)..."
+  python3 -c 'import sys; sys.path.insert(0, "."); from ensemble_tts.annotator import EnsembleAnnotator; from pathlib import Path; import json; from tqdm import tqdm; import time; print("Loading..."); ann = EnsembleAnnotator(mode="quick", device="cuda", enable_events=False); print("✅ Loaded\n"); files = sorted(Path("data/raw/orpheus_test").glob("*.wav")); print(f"Annotating {len(files)} files...\n"); start = time.time(); results = [{"file": f.name, "emotion": (r := ann.annotate(str(f))).get("emotion", {}).get("label", "error"), "confidence": r.get("emotion", {}).get("confidence", 0.0)} if not (i % 100) else {"file": f.name, "emotion": (r := ann.annotate(str(f))).get("emotion", {}).get("label", "error"), "confidence": r.get("emotion", {}).get("confidence", 0.0)} for i, f in enumerate(tqdm(files))]; out = Path("data/annotations/orpheus_test_annotations.json"); out.parent.mkdir(parents=True, exist_ok=True); json.dump(results, open(out, "w"), indent=2); elapsed = time.time() - start; ok = sum(1 for r in results if r["emotion"] != "error"); print(f"\n✅ COMPLETE\nTotal: {len(results)}\nSuccess: {ok}\nTime: {elapsed/60:.1f} min\nRate: {len(results)/elapsed:.2f} files/s")'
 
-# Save results
-output_file = Path("data/annotations/orpheus_test_annotations.json")
-output_file.parent.mkdir(parents=True, exist_ok=True)
-
-with open(output_file, 'w') as f:
-    json.dump(results, f, indent=2)
-
-# Stats
-total_time = time.time() - start_time
-success_count = sum(1 for r in results if r['emotion'] != 'error')
-
-print("")
-print("=================================================="  )
-print("✅ ANNOTATION COMPLETE")
-print("==================================================")
-print(f"Total files: {len(results)}")
-print(f"Successful: {success_count}")
-print(f"Errors: {len(results) - success_count}")
-print(f"Time: {total_time/60:.1f} minutes")
-print(f"Rate: {len(results)/total_time:.2f} files/second")
-print(f"Results saved to: {output_file}")
-print("")
-PYTHON_EOF2
-
-  echo ""
-  echo "📊 Sample Results:"
-  head -50 data/annotations/orpheus_test_annotations.json
-  echo ""
-
-  echo "=================================================="
-  echo "✅ TEST ANNOTATION COMPLETE"
-  echo "=================================================="
-  echo ""
-  echo "📁 Output: data/annotations/orpheus_test_annotations.json"
-  echo ""
-  echo "💡 Next steps:"
-  echo "  1. Download: sky scp ensemble-annotate-test:~/ensemble-tts-annotation/data/annotations/ ./data/"
-  echo "  2. Review results"
-  echo "  3. Run full annotation: sky launch scripts/cloud/skypilot_annotate_orpheus.yaml"
   echo ""
+  echo "📊 Sample results:"
+  head -30 data/annotations/orpheus_test_annotations.json