Update core/analyze.py

core/analyze.py

@@ -17,18 +17,23 @@ client     = Groq(api_key=api_key)
 
 MIN_DURATION    = 60    # seconds
 MAX_DURATION    = 180   # seconds
-TARGET_DURATION = 90    # ideal segment length for extending short ones
+TARGET_DURATION = 90    # ideal segment length
 
-# ── Backtick fence markers (defined as variables to avoid markdown issues) ──
+# ── Backtick fence markers ──
 _FENCE_JSON = "```json"
 _FENCE      = "```"
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# VALIDATION — no mechanical padding, only natural segments pass
+# ─────────────────────────────────────────────────────────────────────────────
+
 def validate_segments(segments, video_duration=None):
     """
-    ✅ FIX: Instead of discarding short segments, try to EXTEND them
-    symmetrically (pad before + after) to reach MIN_DURATION.
-    Only discard if extension is impossible or duration > MAX_DURATION.
+    Accept only segments the AI correctly identified as complete scenes.
+    Short segments are REJECTED — not padded — to avoid random cuts.
+    The AI is instructed to return full scenes, so if it returns something
+    short it means it only found the punchline, not the full story.
     """
     valid = []
     for seg in segments:
@@ -36,47 +41,27 @@ def validate_segments(segments, video_duration=None):
         end   = seg.get("end_time",   0)
         dur   = end - start
 
-        # ── Too long: hard discard ────────────────────────────────────────────
+        # Clamp end_time to video duration if known
+        if video_duration and end > video_duration:
+            end = video_duration
+            seg["end_time"] = round(end, 2)
+            dur = end - start
+
+        # Too long → hard reject
         if dur > MAX_DURATION:
             logger.warning(
-                f"⚠️ Skipped long segment: {dur:.1f}s "
-                f"[{start}s–{end}s] ({seg.get('title', '')})"
+                f"⚠️  Too long ({dur:.1f}s) — skipped: "
+                f"'{seg.get('title', '')}' [{start}s–{end}s]"
             )
             continue
 
-        # ── Too short: try to extend ──────────────────────────────────────────
+        # Too short → reject and log so we can tune the prompt
         if dur < MIN_DURATION:
-            needed   = MIN_DURATION - dur
-            pad_pre  = needed / 2
-            pad_post = needed / 2
-
-            new_start = max(0.0, start - pad_pre)
-            new_end   = end + pad_post
-
-            # Clamp to video duration if known
-            if video_duration:
-                new_end = min(video_duration, new_end)
-                # If still not long enough, steal more from the front
-                actual_dur = new_end - new_start
-                if actual_dur < MIN_DURATION and new_start > 0:
-                    new_start = max(0.0, new_end - MIN_DURATION)
-
-            actual_dur = new_end - new_start
-            if actual_dur < MIN_DURATION:
-                logger.warning(
-                    f"⚠️ Skipped unextendable segment: "
-                    f"{dur:.1f}s → {actual_dur:.1f}s "
-                    f"[{start}s–{end}s] ({seg.get('title', '')})"
-                )
-                continue
-
-            logger.info(
-                f"🔧 Extended short segment {dur:.1f}s → {actual_dur:.1f}s "
-                f"[{start:.1f}s–{end:.1f}s] → [{new_start:.1f}s–{new_end:.1f}s]"
+            logger.warning(
+                f"⚠️  Too short ({dur:.1f}s) — AI returned only punchline, not full scene. "
+                f"Skipped: '{seg.get('title', '')}' [{start}s–{end}s]"
             )
-            seg["start_time"] = round(new_start, 2)
-            seg["end_time"]   = round(new_end,   2)
-            dur               = actual_dur
+            continue
 
         seg["duration"] = round(dur, 2)
         valid.append(seg)
@@ -85,13 +70,17 @@ def validate_segments(segments, video_duration=None):
     return valid
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# FALLBACK — heuristic when AI returns nothing
+# ─────────────────────────────────────────────────────────────────────────────
+
 def _fallback_segments_from_transcript(transcript: str, video_duration: float) -> list:
     """
-    ✅ FALLBACK: If AI returns nothing useful, generate evenly-spaced segments
-    from the transcript based on timestamp markers [start - end].
-    Picks the most text-dense windows as a heuristic for 'interesting'.
+    If AI returns nothing useful, generate segments from the transcript using
+    the densest word windows around timestamp markers.
+    Only used as a last resort.
     """
-    logger.warning("⚠️ Using fallback segment generator from transcript timestamps")
+    logger.warning("⚠️  Using fallback segment generator from transcript timestamps")
 
     lines = []
     for match in re.finditer(r'\[(\d+\.?\d*)\s*-\s*(\d+\.?\d*)\]\s*(.*)', transcript):
@@ -111,8 +100,8 @@ def _fallback_segments_from_transcript(transcript: str, video_duration: float) -
     max_t  = lines[-1]["end"]
 
     while t + MIN_DURATION <= max_t:
-        w_end     = min(t + window, max_t)
-        in_window = [l for l in lines if l["start"] >= t and l["end"] <= w_end]
+        w_end      = min(t + window, max_t)
+        in_window  = [l for l in lines if l["start"] >= t and l["end"] <= w_end]
         word_count = sum(len(l["text"].split()) for l in in_window)
         candidates.append({
             "start_time":  round(t, 2),
@@ -128,12 +117,10 @@ def _fallback_segments_from_transcript(transcript: str, video_duration: float) -
     if not candidates:
         return []
 
-    # Top 3 by word density, then sort back by time
     candidates.sort(key=lambda x: x["word_count"], reverse=True)
     top = candidates[:3]
     top.sort(key=lambda x: x["start_time"])
 
-    # Deduplicate overlapping windows
     deduped = []
     for c in top:
         if deduped and c["start_time"] < deduped[-1]["end_time"] - 20:
@@ -144,60 +131,76 @@ def _fallback_segments_from_transcript(transcript: str, video_duration: float) -
     return deduped
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# CORE ANALYSIS — single chunk
+# ─────────────────────────────────────────────────────────────────────────────
+
 def analyze_transcript(transcript, video_duration=None):
     """
-    Analyze transcript using Groq API.
-    ✅ FIX: Passes video_duration to validate_segments for smarter extension.
-    ✅ FIX: Falls back to _fallback_segments_from_transcript on 0 valid results.
-    ✅ FIX: Backtick fence strings stored in module-level variables to prevent
-            SyntaxError when the source file is copy-pasted through markdown.
+    Send one transcript chunk to Groq and return validated segments.
     """
 
     prompt = f"""
 You are a viral short-form video editor specializing in TikTok, Reels, and YouTube Shorts.
 Your job is to find COMPLETE, PUBLISH-READY segments — not just funny lines or punchlines.
 
-WARNING — CRITICAL DURATION RULE — VIOLATIONS WILL BE REJECTED:
-- end_time - start_time MUST be between {MIN_DURATION} and {MAX_DURATION} seconds
-- Segments shorter than {MIN_DURATION}s will be AUTOMATICALLY DISCARDED
-- If a funny moment is only 15s, you MUST expand it: go back ~45s for setup and forward ~30s for reaction
-- There are NO exceptions to this rule
-
-THINKING PROCESS — follow these steps for every segment:
-1. Spot an interesting or funny moment in the transcript
-2. Go BACKWARDS to find where the setup or context begins (usually 30–90 seconds before the peak)
-3. Go FORWARDS to find where the natural conclusion or audience reaction ends (usually 15–40 seconds after)
-4. The full segment = setup + build-up + peak + conclusion = {MIN_DURATION} to {MAX_DURATION} seconds total
-
-EXAMPLE OF CORRECT THINKING:
-- You notice a funny moment at 150s
-- The story/setup started at 95s — use that as start_time
-- The conclusion/reaction ends at 220s — use that as end_time
-- Duration = 220 - 95 = 125 seconds (within 60-180) CORRECT
-- WRONG: start_time=145, end_time=165 (20 seconds, just the punchline)
-
-A PUBLISH-READY segment must have ALL of these:
-- A hook in the first 5 seconds that makes viewers want to keep watching
-- A clear setup that gives context so viewers understand without prior knowledge
-- A build-up with tension, humor, or emotion
-- A satisfying payoff or conclusion — not an abrupt cut
-- Standalone: makes complete sense without watching anything before or after
-
-OUTPUT — raw JSON only, no markdown, no explanation:
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+⛔ CRITICAL DURATION RULE — VIOLATIONS WILL BE DELETED
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+- end_time − start_time MUST be between {MIN_DURATION} and {MAX_DURATION} seconds
+- Segments shorter than {MIN_DURATION}s will be DELETED, not fixed
+- "I found a funny line at 150s" is NOT a valid segment
+- "I found a complete story from 95s to 220s that includes setup + peak + conclusion" IS valid
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+✅ MANDATORY THINKING PROCESS
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+For every candidate moment, follow these steps:
+
+Step 1 — Find the INTERESTING MOMENT (the peak/punchline/reveal)
+Step 2 — Scroll BACK in the transcript to find where the SETUP begins
+          (usually 30–90 seconds before the peak)
+          Ask: "When does the viewer first need to start watching to understand this?"
+Step 3 — Scroll FORWARD to find where the NATURAL CONCLUSION ends
+          (usually 15–40 seconds after the peak)
+          Ask: "Is the story/joke/argument fully resolved? Is there a reaction?"
+Step 4 — Check: end_time − start_time is between {MIN_DURATION} and {MAX_DURATION} seconds
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+📌 CORRECT vs WRONG EXAMPLE
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+Funny moment at 150s:
+  ✅ CORRECT: start=95s (setup), end=220s (reaction ends) → 125s duration
+  ❌ WRONG:   start=145s (just before punchline), end=165s → 20s duration
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+📋 A COMPLETE SEGMENT MUST HAVE ALL OF THESE
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+1. HOOK        — something in the first 5s that makes viewers keep watching
+2. SETUP       — enough context so a stranger understands without watching anything before
+3. BUILD-UP    — tension, humor, conflict, or emotion growing
+4. PAYOFF      — the peak moment (punchline / reveal / climax)
+5. CONCLUSION  — resolution, reaction, or natural pause — NOT an abrupt cut
+6. STANDALONE  — makes complete sense on its own
+
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
+🔢 OUTPUT FORMAT — raw JSON only, no markdown, no explanation
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 {{
   "segments": [
     {{
-      "start_time": <float, where the SETUP begins — NOT the funny moment itself>,
-      "end_time": <float, where the CONCLUSION ends — NOT just the punchline>,
-      "title": "<punchy YouTube Shorts title, max 60 chars>",
-      "description": "<1-2 sentences describing the full story>",
-      "reason": "<setup to peak to conclusion arc, max 25 words>"
+      "start_time":  <float — where the SETUP begins, NOT the funny moment>,
+      "end_time":    <float — where the CONCLUSION ends, NOT just the punchline>,
+      "title":       "<punchy YouTube Shorts title, max 60 chars>",
+      "description": "<1-2 sentences describing the full arc: setup → peak → conclusion>",
+      "reason":      "<why this is complete and viral-worthy, max 25 words>"
     }}
   ]
 }}
 
-If no complete story qualifies, return: {{"segments": []}}
+If no complete story qualifies: {{"segments": []}}
 
+━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━
 TRANSCRIPT:
 {transcript}
 """
@@ -213,10 +216,12 @@ TRANSCRIPT:
                     {
                         "role":    "system",
                         "content": (
-                            f"You are a JSON-only assistant. "
-                            f"Output raw JSON only — no markdown, no code blocks, no explanation. "
-                            f"EVERY segment MUST be between {MIN_DURATION} and {MAX_DURATION} seconds. "
-                            f"Always include setup + peak + conclusion, never just the punchline."
+                            "You are a JSON-only assistant. "
+                            "Output raw JSON only — no markdown, no code blocks, no explanation. "
+                            f"Every segment MUST be between {MIN_DURATION} and {MAX_DURATION} seconds. "
+                            "Short segments (punchlines without setup) will be DELETED — not fixed. "
+                            "Always include: full setup before the peak + reaction/conclusion after. "
+                            "Use the timestamps in the transcript to pick natural scene boundaries."
                         ),
                     },
                     {"role": "user", "content": prompt},
@@ -227,9 +232,7 @@ TRANSCRIPT:
             content = response.choices[0].message.content.strip()
             logger.info(f"🤖 AI Raw Response (first 300 chars): {content[:300]}...")
 
-            # ── Strip markdown fences if model ignored system prompt ──────────
-            # Using module-level variables instead of inline literals to prevent
-            # SyntaxError when this file is copy-pasted through markdown editors.
+            # Strip markdown fences if model ignored system prompt
             if _FENCE_JSON in content:
                 content = content.split(_FENCE_JSON)[1].split(_FENCE)[0].strip()
             elif _FENCE in content:
@@ -238,12 +241,11 @@ TRANSCRIPT:
             data         = json.loads(content)
             raw_segments = data.get("segments", [])
 
-            # ✅ Pass video_duration so extension logic has a ceiling
             valid_segments = validate_segments(raw_segments, video_duration=video_duration)
 
-            # ✅ FALLBACK: if AI returned nothing valid, use heuristic generator
+            # Fallback if AI returned nothing valid
             if not valid_segments:
-                logger.warning("⚠️ AI returned 0 valid segments — trying fallback generator")
+                logger.warning("⚠️  AI returned 0 valid segments — trying fallback generator")
                 fallback = _fallback_segments_from_transcript(
                     transcript, video_duration or 0
                 )
@@ -260,89 +262,160 @@ TRANSCRIPT:
             logger.error(f"❌ Error in Groq analysis (attempt {attempt + 1}): {e}")
             if attempt < max_retries - 1:
                 wait = base_delay * (2 ** attempt)
-                logger.warning(f"⚠️ Retrying in {wait}s...")
+                logger.warning(f"⚠️  Retrying in {wait}s...")
                 time.sleep(wait)
 
     logger.error("❌ All retry attempts failed.")
     return {"content": '{"segments": []}'}
 
 
-def smart_chunk_transcript(transcript, max_tokens=4000):
+# ─────────────────────────────────────────────────────────────────────────────
+# SCENE-AWARE CHUNKING
+# ─────────────────────────────────────────────────────────────────────────────
+
+def _sentence_chunk(transcript: str, max_tokens: int) -> list[str]:
     """
-    Split transcript into coherent chunks at sentence boundaries.
-    Adds overlap between chunks so stories that span chunk boundaries aren't lost.
+    Fallback chunker when transcript has no timestamps.
+    Splits at sentence boundaries with overlap.
     """
-    sentences     = transcript.replace('\n', ' ').split('. ')
-    chunks        = []
-    current_chunk = []
+    sentences      = transcript.replace('\n', ' ').split('. ')
+    chunks         = []
+    current_chunk  = []
     current_length = 0
-    overlap_sentences = []
 
     for sentence in sentences:
         sentence_length = len(sentence.split())
-
         if current_length + sentence_length > max_tokens and current_chunk:
-            chunk_text = '. '.join(current_chunk) + '.'
-            chunks.append(chunk_text.strip())
-            overlap_sentences = current_chunk[-5:]
-            current_chunk  = overlap_sentences + [sentence]
+            chunks.append('. '.join(current_chunk) + '.')
+            current_chunk  = current_chunk[-5:] + [sentence]
             current_length = sum(len(s.split()) for s in current_chunk)
         else:
             current_chunk.append(sentence)
             current_length += sentence_length
 
     if current_chunk:
-        chunk_text = '. '.join(current_chunk) + '.'
-        chunks.append(chunk_text.strip())
+        chunks.append('. '.join(current_chunk) + '.')
+    return chunks
+
+
+def smart_chunk_transcript(transcript: str, max_tokens: int = 4000) -> list[str]:
+    """
+    Split transcript into coherent chunks at SCENE BOUNDARIES.
+
+    Strategy:
+    - Parse all [start - end] timestamp lines
+    - Cut only when the chunk is full AND there is a natural gap (>3s) between
+      consecutive lines — a sign of a scene or topic change
+    - Keep 10-line overlap between chunks so cross-boundary stories aren't lost
+    - Falls back to sentence-level splitting if no timestamps found
+    """
+    lines = []
+    for match in re.finditer(r'\[(\d+\.?\d*)\s*-\s*(\d+\.?\d*)\]\s*(.*)', transcript):
+        lines.append({
+            "start": float(match.group(1)),
+            "end":   float(match.group(2)),
+            "text":  match.group(3).strip(),
+        })
+
+    if not lines:
+        logger.info("📦 No timestamps found — using sentence chunker")
+        return _sentence_chunk(transcript, max_tokens)
 
+    chunks        = []
+    current_lines = []
+    current_words = 0
+
+    for i, line in enumerate(lines):
+        current_lines.append(line)
+        current_words += len(line["text"].split())
+
+        is_last = (i + 1 == len(lines))
+        is_full = (current_words >= max_tokens)
+
+        # Detect scene break: gap > 3 seconds between this line and the next
+        next_gap = (
+            lines[i + 1]["start"] - line["end"]
+            if not is_last else 999
+        )
+        at_scene_break = next_gap > 3.0
+
+        if is_last or (is_full and at_scene_break):
+            chunk_text = "\n".join(
+                f"[{l['start']} - {l['end']}] {l['text']}"
+                for l in current_lines
+            )
+            chunks.append(chunk_text)
+            logger.info(
+                f"📦 Chunk {len(chunks)}: "
+                f"{current_lines[0]['start']:.0f}s – {current_lines[-1]['end']:.0f}s "
+                f"({current_words} words)"
+            )
+            # Overlap: keep last 10 lines for context continuity
+            current_lines = current_lines[-10:]
+            current_words = sum(len(l["text"].split()) for l in current_lines)
+
+    logger.info(f"📦 Split into {len(chunks)} scene-aware chunk(s)")
     return chunks
 
 
+# ─────────────────────────────────────────────────────────────────────────────
+# MAIN ENTRY POINT
+# ─────────────────────────────────────────────────────────────────────────────
+
 def analyze_transcript_with_chunking(transcript, video_duration=None):
     """
-    Analyze transcript using smart chunking for long content.
+    Analyze transcript using scene-aware chunking for long content.
     Processes each chunk separately and merges + deduplicates results.
     """
-    if len(transcript.split()) > 3000:
-        logger.info("📦 Transcript too long, using smart chunking...")
+    word_count = len(transcript.split())
+
+    if word_count > 3000:
+        logger.info(f"📦 Transcript is {word_count} words — using scene-aware chunking...")
         chunks       = smart_chunk_transcript(transcript, max_tokens=3000)
         all_segments = []
 
         for i, chunk in enumerate(chunks):
-            logger.info(f"🔄 Processing chunk {i+1}/{len(chunks)}...")
+            logger.info(f"🔄 Processing chunk {i + 1}/{len(chunks)}...")
             result = analyze_transcript(chunk, video_duration=video_duration)
 
             try:
                 data = json.loads(result["content"])
                 if "segments" in data:
                     all_segments.extend(data["segments"])
+                    logger.info(
+                        f"✅ Chunk {i + 1}: "
+                        f"{len(data['segments'])} valid segment(s)"
+                    )
             except Exception as e:
-                logger.warning(f"⚠️ Failed to parse chunk {i+1}: {e}")
+                logger.warning(f"⚠️  Failed to parse chunk {i + 1}: {e}")
                 continue
 
         if all_segments:
+            # Deduplicate: two segments are the same if their start times
+            # round to within 10 seconds of each other
             unique_segments = []
             seen_times      = set()
 
-            for seg in all_segments:
-                time_key = (
-                    f"{round(seg.get('start_time', 0) / 10) * 10}-"
-                    f"{round(seg.get('end_time',   0) / 10) * 10}"
-                )
+            for seg in sorted(all_segments, key=lambda s: s.get("start_time", 0)):
+                time_key = round(seg.get("start_time", 0) / 10) * 10
                 if time_key not in seen_times:
                     unique_segments.append(seg)
                     seen_times.add(time_key)
 
-            logger.info(f"📊 Total unique valid segments: {len(unique_segments)}")
+            logger.info(f"📊 Total unique viral segments found: {len(unique_segments)}")
             return {"content": json.dumps({"segments": unique_segments[:10]})}
 
-        logger.warning("⚠️ No valid segments found across all chunks.")
+        logger.warning("⚠️  No valid segments found across all chunks.")
         return {"content": '{"segments": []}'}
 
+    # Short transcript — process directly without chunking
     return analyze_transcript(transcript, video_duration=video_duration)
 
 
-# ── Testing ───────────────────────────────────────────────────────────────────
+# ─────────────────────────────────────────────────────────────────────────────
+# TESTING
+# ─────────────────────────────────────────────────────────────────────────────
+
 if __name__ == "__main__":
     test_transcript = """
     [0.0 - 5.0] Welcome to today's video about productivity hacks that actually work.
@@ -364,12 +437,12 @@ if __name__ == "__main__":
     try:
         data     = json.loads(result["content"])
         segments = data.get("segments", [])
-        logger.info(f"✅ Found {len(segments)} publish-ready segments:\n")
+        logger.info(f"✅ Found {len(segments)} publish-ready segment(s):\n")
 
         for i, seg in enumerate(segments):
             duration = seg["end_time"] - seg["start_time"]
             logger.info(
-                f"#{i+1} [{seg['start_time']:.0f}s – {seg['end_time']:.0f}s] ({duration:.0f}s)\n"
+                f"#{i + 1} [{seg['start_time']:.0f}s – {seg['end_time']:.0f}s] ({duration:.0f}s)\n"
                 f"  📌 Title:       {seg.get('title', 'N/A')}\n"
                 f"  📝 Description: {seg.get('description', 'N/A')}\n"
                 f"  💡 Story Arc:   {seg.get('reason', 'N/A')}\n"