Update core/analyze.py

core/analyze.py

@@ -1,6 +1,5 @@
-# analyze.py — Full fixed version
-
 import os
+import re
 import time
 import json
 import logging
@@ -12,13 +11,17 @@ load_dotenv()
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 logger = logging.getLogger(__name__)
 
-api_key = os.getenv("GROQ_API_KEY")
+api_key    = os.getenv("GROQ_API_KEY")
 MODEL_NAME = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
-client = Groq(api_key=api_key)
+client     = Groq(api_key=api_key)
+
+MIN_DURATION    = 60    # seconds
+MAX_DURATION    = 180   # seconds
+TARGET_DURATION = 90    # ideal segment length for extending short ones
 
-MIN_DURATION = 60    # seconds
-MAX_DURATION = 180   # seconds
-TARGET_DURATION = 90 # ✅ NEW: ideal segment length for extending short ones
+# ── Backtick fence markers (defined as variables to avoid markdown issues) ──
+_FENCE_JSON = "```json"
+_FENCE      = "```"
 
 
 def validate_segments(segments, video_duration=None):
@@ -35,31 +38,35 @@ def validate_segments(segments, video_duration=None):
 
         # ── Too long: hard discard ────────────────────────────────────────────
         if dur > MAX_DURATION:
-            logger.warning(f"⚠️ Skipped long segment: {dur:.1f}s [{start}s–{end}s] ({seg.get('title','')})")
+            logger.warning(
+                f"⚠️ Skipped long segment: {dur:.1f}s "
+                f"[{start}s–{end}s] ({seg.get('title', '')})"
+            )
             continue
 
         # ── Too short: try to extend ──────────────────────────────────────────
         if dur < MIN_DURATION:
-            needed  = MIN_DURATION - dur
-            pad_pre = needed / 2
+            needed   = MIN_DURATION - dur
+            pad_pre  = needed / 2
             pad_post = needed / 2
 
-            new_start = max(0, start - pad_pre)
+            new_start = max(0.0, start - pad_pre)
             new_end   = end + pad_post
 
             # Clamp to video duration if known
             if video_duration:
                 new_end = min(video_duration, new_end)
-                # If we couldn't get enough at the end, steal from the front
+                # If still not long enough, steal more from the front
                 actual_dur = new_end - new_start
                 if actual_dur < MIN_DURATION and new_start > 0:
-                    new_start = max(0, new_end - MIN_DURATION)
+                    new_start = max(0.0, new_end - MIN_DURATION)
 
             actual_dur = new_end - new_start
             if actual_dur < MIN_DURATION:
                 logger.warning(
-                    f"⚠️ Skipped unextendable segment: {dur:.1f}s→{actual_dur:.1f}s "
-                    f"[{start}s–{end}s] ({seg.get('title','')})"
+                    f"⚠️ Skipped unextendable segment: "
+                    f"{dur:.1f}s → {actual_dur:.1f}s "
+                    f"[{start}s–{end}s] ({seg.get('title', '')})"
                 )
                 continue
 
@@ -80,15 +87,12 @@ def validate_segments(segments, video_duration=None):
 
 def _fallback_segments_from_transcript(transcript: str, video_duration: float) -> list:
     """
-    ✅ NEW FALLBACK: If AI returns nothing useful, generate evenly-spaced
-    segments from the transcript based on timestamp markers [start - end].
-    Tries to pick the most text-dense windows as a heuristic for 'interesting'.
+    ✅ FALLBACK: If AI returns nothing useful, generate evenly-spaced segments
+    from the transcript based on timestamp markers [start - end].
+    Picks the most text-dense windows as a heuristic for 'interesting'.
     """
-    import re
-
     logger.warning("⚠️ Using fallback segment generator from transcript timestamps")
 
-    # Extract all timestamped lines
     lines = []
     for match in re.finditer(r'\[(\d+\.?\d*)\s*-\s*(\d+\.?\d*)\]\s*(.*)', transcript):
         lines.append({
@@ -100,42 +104,40 @@ def _fallback_segments_from_transcript(transcript: str, video_duration: float) -
     if not lines:
         return []
 
-    # Build 90-second windows every 60 seconds, score by word count
     candidates = []
-    step = 60
+    step   = 60
     window = TARGET_DURATION
-
-    t = lines[0]["start"]
-    max_t = lines[-1]["end"]
+    t      = lines[0]["start"]
+    max_t  = lines[-1]["end"]
 
     while t + MIN_DURATION <= max_t:
-        w_end   = min(t + window, max_t)
+        w_end     = min(t + window, max_t)
         in_window = [l for l in lines if l["start"] >= t and l["end"] <= w_end]
         word_count = sum(len(l["text"].split()) for l in in_window)
         candidates.append({
             "start_time":  round(t, 2),
             "end_time":    round(w_end, 2),
             "word_count":  word_count,
-            "title":       f"Highlight at {int(t//60)}m{int(t%60):02d}s",
+            "title":       f"Highlight at {int(t // 60)}m{int(t % 60):02d}s",
             "description": "Auto-detected highlight segment",
             "reason":      "Fallback: highest word-density window",
-            "viral_score": word_count,  # proxy score
+            "viral_score": word_count,
         })
         t += step
 
     if not candidates:
         return []
 
-    # Sort by word density, pick top 3, sort back by time
+    # Top 3 by word density, then sort back by time
     candidates.sort(key=lambda x: x["word_count"], reverse=True)
     top = candidates[:3]
     top.sort(key=lambda x: x["start_time"])
 
-    # Deduplicate overlapping windows (keep higher-scored one)
+    # Deduplicate overlapping windows
     deduped = []
     for c in top:
         if deduped and c["start_time"] < deduped[-1]["end_time"] - 20:
-            continue  # overlaps with previous, skip
+            continue
         deduped.append(c)
 
     logger.info(f"🔧 Fallback generated {len(deduped)} segment(s)")
@@ -147,17 +149,19 @@ def analyze_transcript(transcript, video_duration=None):
     Analyze transcript using Groq API.
     ✅ FIX: Passes video_duration to validate_segments for smarter extension.
     ✅ FIX: Falls back to _fallback_segments_from_transcript on 0 valid results.
+    ✅ FIX: Backtick fence strings stored in module-level variables to prevent
+            SyntaxError when the source file is copy-pasted through markdown.
     """
 
     prompt = f"""
 You are a viral short-form video editor specializing in TikTok, Reels, and YouTube Shorts.
 Your job is to find COMPLETE, PUBLISH-READY segments — not just funny lines or punchlines.
 
-⚠️ CRITICAL DURATION RULE — VIOLATIONS WILL BE REJECTED:
+WARNING — CRITICAL DURATION RULE — VIOLATIONS WILL BE REJECTED:
 - end_time - start_time MUST be between {MIN_DURATION} and {MAX_DURATION} seconds
 - Segments shorter than {MIN_DURATION}s will be AUTOMATICALLY DISCARDED
-- If a funny moment is only 15s, you MUST expand it: go back ~45s for context and forward ~30s for reaction
-- There is NO exception to this rule
+- If a funny moment is only 15s, you MUST expand it: go back ~45s for setup and forward ~30s for reaction
+- There are NO exceptions to this rule
 
 THINKING PROCESS — follow these steps for every segment:
 1. Spot an interesting or funny moment in the transcript
@@ -167,10 +171,10 @@ THINKING PROCESS — follow these steps for every segment:
 
 EXAMPLE OF CORRECT THINKING:
 - You notice a funny moment at 150s
-- The story/setup started at 95s → use that as start_time
-- The conclusion/reaction ends at 220s → use that as end_time
-- Duration = 220 - 95 = 125 seconds ✅ (within 60-180)
-- WRONG → start_time: 145, end_time: 165 (20 seconds, just the punchline) ❌
+- The story/setup started at 95s — use that as start_time
+- The conclusion/reaction ends at 220s — use that as end_time
+- Duration = 220 - 95 = 125 seconds (within 60-180) CORRECT
+- WRONG: start_time=145, end_time=165 (20 seconds, just the punchline)
 
 A PUBLISH-READY segment must have ALL of these:
 - A hook in the first 5 seconds that makes viewers want to keep watching
@@ -187,7 +191,7 @@ OUTPUT — raw JSON only, no markdown, no explanation:
       "end_time": <float, where the CONCLUSION ends — NOT just the punchline>,
       "title": "<punchy YouTube Shorts title, max 60 chars>",
       "description": "<1-2 sentences describing the full story>",
-      "reason": "<setup → peak → conclusion arc, max 25 words>"
+      "reason": "<setup to peak to conclusion arc, max 25 words>"
     }}
   ]
 }}
@@ -212,10 +216,10 @@ TRANSCRIPT:
                             f"You are a JSON-only assistant. "
                             f"Output raw JSON only — no markdown, no code blocks, no explanation. "
                             f"EVERY segment MUST be between {MIN_DURATION} and {MAX_DURATION} seconds. "
-                            f"Think carefully: always include setup + peak + conclusion."
-                        )
+                            f"Always include setup + peak + conclusion, never just the punchline."
+                        ),
                     },
-                    {"role": "user", "content": prompt}
+                    {"role": "user", "content": prompt},
                 ],
                 temperature = 0.3,
             )
@@ -223,7 +227,153 @@ TRANSCRIPT:
             content = response.choices[0].message.content.strip()
             logger.info(f"🤖 AI Raw Response (first 300 chars): {content[:300]}...")
 
-            if "```json" in content:
-                content = content.split("```json")[1].split("```")[0].strip()
-            elif "```" in content:
-                content = content.split("`
+            # ── Strip markdown fences if model ignored system prompt ──────────
+            # Using module-level variables instead of inline literals to prevent
+            # SyntaxError when this file is copy-pasted through markdown editors.
+            if _FENCE_JSON in content:
+                content = content.split(_FENCE_JSON)[1].split(_FENCE)[0].strip()
+            elif _FENCE in content:
+                content = content.split(_FENCE)[1].split(_FENCE)[0].strip()
+
+            data         = json.loads(content)
+            raw_segments = data.get("segments", [])
+
+            # ✅ Pass video_duration so extension logic has a ceiling
+            valid_segments = validate_segments(raw_segments, video_duration=video_duration)
+
+            # ✅ FALLBACK: if AI returned nothing valid, use heuristic generator
+            if not valid_segments:
+                logger.warning("⚠️ AI returned 0 valid segments — trying fallback generator")
+                fallback = _fallback_segments_from_transcript(
+                    transcript, video_duration or 0
+                )
+                valid_segments = validate_segments(fallback, video_duration=video_duration)
+
+            data["segments"] = valid_segments
+            content          = json.dumps(data)
+            logger.info(
+                f"🤖 Parsed: {len(raw_segments)} raw → {len(valid_segments)} valid segments"
+            )
+            return {"content": content}
+
+        except Exception as e:
+            logger.error(f"❌ Error in Groq analysis (attempt {attempt + 1}): {e}")
+            if attempt < max_retries - 1:
+                wait = base_delay * (2 ** attempt)
+                logger.warning(f"⚠️ Retrying in {wait}s...")
+                time.sleep(wait)
+
+    logger.error("❌ All retry attempts failed.")
+    return {"content": '{"segments": []}'}
+
+
+def smart_chunk_transcript(transcript, max_tokens=4000):
+    """
+    Split transcript into coherent chunks at sentence boundaries.
+    Adds overlap between chunks so stories that span chunk boundaries aren't lost.
+    """
+    sentences     = transcript.replace('\n', ' ').split('. ')
+    chunks        = []
+    current_chunk = []
+    current_length = 0
+    overlap_sentences = []
+
+    for sentence in sentences:
+        sentence_length = len(sentence.split())
+
+        if current_length + sentence_length > max_tokens and current_chunk:
+            chunk_text = '. '.join(current_chunk) + '.'
+            chunks.append(chunk_text.strip())
+            overlap_sentences = current_chunk[-5:]
+            current_chunk  = overlap_sentences + [sentence]
+            current_length = sum(len(s.split()) for s in current_chunk)
+        else:
+            current_chunk.append(sentence)
+            current_length += sentence_length
+
+    if current_chunk:
+        chunk_text = '. '.join(current_chunk) + '.'
+        chunks.append(chunk_text.strip())
+
+    return chunks
+
+
+def analyze_transcript_with_chunking(transcript, video_duration=None):
+    """
+    Analyze transcript using smart chunking for long content.
+    Processes each chunk separately and merges + deduplicates results.
+    """
+    if len(transcript.split()) > 3000:
+        logger.info("📦 Transcript too long, using smart chunking...")
+        chunks       = smart_chunk_transcript(transcript, max_tokens=3000)
+        all_segments = []
+
+        for i, chunk in enumerate(chunks):
+            logger.info(f"🔄 Processing chunk {i+1}/{len(chunks)}...")
+            result = analyze_transcript(chunk, video_duration=video_duration)
+
+            try:
+                data = json.loads(result["content"])
+                if "segments" in data:
+                    all_segments.extend(data["segments"])
+            except Exception as e:
+                logger.warning(f"⚠️ Failed to parse chunk {i+1}: {e}")
+                continue
+
+        if all_segments:
+            unique_segments = []
+            seen_times      = set()
+
+            for seg in all_segments:
+                time_key = (
+                    f"{round(seg.get('start_time', 0) / 10) * 10}-"
+                    f"{round(seg.get('end_time',   0) / 10) * 10}"
+                )
+                if time_key not in seen_times:
+                    unique_segments.append(seg)
+                    seen_times.add(time_key)
+
+            logger.info(f"📊 Total unique valid segments: {len(unique_segments)}")
+            return {"content": json.dumps({"segments": unique_segments[:10]})}
+
+        logger.warning("⚠️ No valid segments found across all chunks.")
+        return {"content": '{"segments": []}'}
+
+    return analyze_transcript(transcript, video_duration=video_duration)
+
+
+# ── Testing ───────────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    test_transcript = """
+    [0.0 - 5.0] Welcome to today's video about productivity hacks that actually work.
+    [5.0 - 15.0] The first hack is something I call the 2-minute rule. If something takes less than 2 minutes, do it immediately.
+    [15.0 - 30.0] This simple rule has transformed my life. I used to procrastinate on small tasks, but now I handle them right away.
+    [30.0 - 45.0] The second hack is batching similar tasks together. Instead of checking email 20 times a day, I check it twice.
+    [45.0 - 60.0] This has saved me hours every week. I batch my emails, phone calls, and even errands.
+    [60.0 - 90.0] The third hack is the Pomodoro Technique. Work for 25 minutes, then take a 5-minute break.
+    [90.0 - 120.0] This technique helps me stay focused and avoid burnout. I get more done in less time.
+    [120.0 - 150.0] The fourth hack is to eliminate distractions completely. Turn off notifications, close tabs, and focus.
+    [150.0 - 180.0] When you eliminate distractions, your productivity skyrockets. I finish in 2 hours what used to take 6.
+    [180.0 - 210.0] The fifth and final hack is to review your day every evening. Spend 5 minutes planning tomorrow.
+    [210.0 - 240.0] This evening review changed everything for me. I wake up knowing exactly what to do and I never waste morning time figuring out priorities.
+    """
+
+    logger.info("🧪 Testing AI Analysis...")
+    result = analyze_transcript_with_chunking(test_transcript, video_duration=240.0)
+
+    try:
+        data     = json.loads(result["content"])
+        segments = data.get("segments", [])
+        logger.info(f"✅ Found {len(segments)} publish-ready segments:\n")
+
+        for i, seg in enumerate(segments):
+            duration = seg["end_time"] - seg["start_time"]
+            logger.info(
+                f"#{i+1} [{seg['start_time']:.0f}s – {seg['end_time']:.0f}s] ({duration:.0f}s)\n"
+                f"  📌 Title:       {seg.get('title', 'N/A')}\n"
+                f"  📝 Description: {seg.get('description', 'N/A')}\n"
+                f"  💡 Story Arc:   {seg.get('reason', 'N/A')}\n"
+            )
+    except Exception as e:
+        logger.error(f"❌ Error parsing result: {e}")
+        logger.info(f"Raw result: {result}")