Update pipeline/orchestrator.py

pipeline/orchestrator.py

@@ -40,6 +40,7 @@ from models.body_recognizer import BodyRecognizer
 from models.motion_detector import MotionDetector
 from scoring.hype_scorer import HypeScorer, SegmentScore
 from scoring.domain_presets import get_domain_preset, Domain
+from scoring.viral_hooks import ViralHookDetector, HookSignal
 
 logger = get_logger("pipeline.orchestrator")
 
@@ -56,6 +57,7 @@ class PipelineStage(Enum):
     DETECTING_PERSON = "detecting_person"
     ANALYZING_MOTION = "analyzing_motion"
     SCORING = "scoring"
+    OPTIMIZING_HOOKS = "optimizing_hooks"
     EXTRACTING_CLIPS = "extracting_clips"
     FINALIZING = "finalizing"
     COMPLETE = "complete"
@@ -123,10 +125,11 @@ class PipelineOrchestrator:
         PipelineStage.EXTRACTING_AUDIO: 0.05,
         PipelineStage.ANALYZING_AUDIO: 0.10,
         PipelineStage.SAMPLING_FRAMES: 0.10,
-        PipelineStage.ANALYZING_VISUAL: 0.30,
+        PipelineStage.ANALYZING_VISUAL: 0.25,
         PipelineStage.DETECTING_PERSON: 0.10,
         PipelineStage.ANALYZING_MOTION: 0.05,
         PipelineStage.SCORING: 0.05,
+        PipelineStage.OPTIMIZING_HOOKS: 0.05,
         PipelineStage.EXTRACTING_CLIPS: 0.10,
         PipelineStage.FINALIZING: 0.05,
     }
@@ -161,6 +164,7 @@ class PipelineOrchestrator:
         self._motion_detector: Optional[MotionDetector] = None
         self._clip_extractor: Optional[ClipExtractor] = None
         self._hype_scorer: Optional[HypeScorer] = None
+        self._hook_detector: Optional[ViralHookDetector] = None
 
         logger.info("PipelineOrchestrator initialized")
 
@@ -359,13 +363,25 @@ class PipelineOrchestrator:
             )
             self._update_progress(PipelineStage.SCORING, 1.0, f"Scored {len(segment_scores)} segments")
 
+            # Viral hook optimization - find best starting points
+            self._update_progress(PipelineStage.OPTIMIZING_HOOKS, 0.0, "Finding viral hooks...")
+            candidates = self._scores_to_candidates(segment_scores, clip_duration)
+
+            # Detect hooks and optimize clip start times
+            hooks = self._detect_viral_hooks(
+                frames, audio_features, visual_features, motion_scores
+            )
+            optimized_candidates = self._optimize_clip_starts(
+                candidates, hooks, num_clips
+            )
+            self._update_progress(PipelineStage.OPTIMIZING_HOOKS, 1.0, f"Optimized {len(optimized_candidates)} clip hooks")
+
             # Clip extraction
             self._update_progress(PipelineStage.EXTRACTING_CLIPS, 0.0, "Extracting clips...")
-            candidates = self._scores_to_candidates(segment_scores, clip_duration)
             clips = self._clip_extractor.extract_clips(
                 video_path,
                 self._temp_dir / "clips",
-                candidates,
+                optimized_candidates,
                 num_clips=num_clips,
             )
             self._update_progress(PipelineStage.EXTRACTING_CLIPS, 1.0, f"Extracted {len(clips)} clips")
@@ -464,6 +480,9 @@ class PipelineOrchestrator:
         preset = get_domain_preset(domain, person_filter_enabled=person_filter)
         self._hype_scorer = HypeScorer(preset=preset)
 
+        # Viral hook detector
+        self._hook_detector = ViralHookDetector(domain=domain)
+
         logger.info("Components initialized")
 
     def _compute_segment_scores(
@@ -589,6 +608,134 @@ class PipelineOrchestrator:
 
         return [motion_map.get(f.action_detected, 0.4) for f in visual_features]
 
+    def _detect_viral_hooks(
+        self,
+        frames: List[SampledFrame],
+        audio_features: List[AudioFeatures],
+        visual_features: List[VisualFeatures],
+        motion_scores: List[float],
+    ) -> List[HookSignal]:
+        """
+        Detect viral hook moments from all available signals.
+
+        Args:
+            frames: Sampled frames with timestamps
+            audio_features: Audio analysis results
+            visual_features: Visual analysis results
+            motion_scores: Motion intensity scores
+
+        Returns:
+            List of detected hook signals
+        """
+        if not self._hook_detector:
+            return []
+
+        # Prepare timestamps
+        frame_timestamps = [f.timestamp for f in frames]
+
+        # Prepare audio signals
+        audio_timestamps = [af.timestamp for af in audio_features] if audio_features else []
+        audio_energy = [af.rms_energy for af in audio_features] if audio_features else None
+        audio_flux = [af.spectral_flux for af in audio_features] if audio_features else None
+        audio_centroid = [af.spectral_centroid for af in audio_features] if audio_features else None
+
+        # Prepare visual signals
+        visual_scores = [vf.hype_score for vf in visual_features] if visual_features else None
+        emotions = [vf.emotion for vf in visual_features] if visual_features else None
+        actions = [vf.action_detected for vf in visual_features] if visual_features else None
+
+        # Use audio timestamps if available (finer granularity), else frame timestamps
+        timestamps = audio_timestamps if audio_timestamps else frame_timestamps
+
+        # Interpolate visual/motion to audio timeline if needed
+        if audio_timestamps and visual_scores and len(visual_scores) != len(audio_timestamps):
+            visual_scores = self._interpolate_scores(frame_timestamps, visual_scores, audio_timestamps)
+            motion_scores = self._interpolate_scores(frame_timestamps, motion_scores, audio_timestamps) if motion_scores else None
+            # For emotions/actions, we'll use nearest neighbor (keep original)
+            emotions = None  # Can't interpolate strings
+            actions = None
+
+        # Detect hooks
+        hooks = self._hook_detector.detect_hooks(
+            timestamps=timestamps,
+            audio_energy=audio_energy,
+            audio_flux=audio_flux,
+            audio_centroid=audio_centroid,
+            visual_scores=visual_scores,
+            motion_scores=motion_scores,
+            emotions=emotions,
+            actions=actions,
+        )
+
+        logger.info(f"Detected {len(hooks)} potential viral hook moments")
+        return hooks
+
+    def _optimize_clip_starts(
+        self,
+        candidates: List[ClipCandidate],
+        hooks: List[HookSignal],
+        num_clips: int,
+    ) -> List[ClipCandidate]:
+        """
+        Optimize clip start times to align with viral hooks.
+
+        Args:
+            candidates: Original clip candidates
+            hooks: Detected hook signals
+            num_clips: Number of clips to extract
+
+        Returns:
+            Optimized clip candidates with adjusted start times
+        """
+        if not hooks or not self._hook_detector:
+            logger.info("No hooks detected, using original clip timings")
+            return candidates
+
+        optimized = []
+
+        # Process top candidates
+        for candidate in candidates[:num_clips * 2]:  # Consider more candidates for optimization
+            # Find best hook-aligned start time
+            adjusted_start, best_hook = self._hook_detector.find_best_clip_start(
+                clip_start=candidate.start_time,
+                clip_end=candidate.end_time,
+                hooks=hooks,
+                allow_adjustment=3.0,  # Allow up to 3 seconds earlier
+            )
+
+            # Create optimized candidate
+            clip_duration = candidate.end_time - candidate.start_time
+
+            # Boost score if we found a good hook
+            hook_boost = 1.0
+            if best_hook:
+                hook_score = self._hook_detector.score_clip_hook_potential(
+                    adjusted_start, clip_duration, hooks
+                )
+                hook_boost = 1.0 + (hook_score * 0.3)  # Up to 30% boost
+
+            optimized.append(ClipCandidate(
+                start_time=adjusted_start,
+                end_time=adjusted_start + clip_duration,
+                hype_score=candidate.hype_score * hook_boost,
+                visual_score=candidate.visual_score,
+                audio_score=candidate.audio_score,
+                motion_score=candidate.motion_score,
+                person_score=candidate.person_score,
+            ))
+
+            if best_hook:
+                logger.debug(
+                    f"Clip {candidate.start_time:.1f}s -> {adjusted_start:.1f}s "
+                    f"(hook: {best_hook.hook_type.value}, boost: {hook_boost:.2f}x)"
+                )
+
+        # Re-sort by boosted score
+        optimized.sort(key=lambda c: c.hype_score, reverse=True)
+
+        logger.info(f"Optimized {len(optimized)} candidates with viral hooks")
+        return optimized
+
     def cleanup(self) -> None:
         """Clean up temporary files and unload models."""
         if self._temp_dir: