Update core/analyze.py

core/analyze.py

@@ -1,3 +1,5 @@
+# analyze.py — Full fixed version
+
 import os
 import time
 import json
@@ -7,68 +9,168 @@ from dotenv import load_dotenv
 
 load_dotenv()
 
-# Setup Logger
 logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
 logger = logging.getLogger(__name__)
 
-# Configure Groq Client
 api_key = os.getenv("GROQ_API_KEY")
 MODEL_NAME = os.getenv("GROQ_MODEL", "llama-3.3-70b-versatile")
-
 client = Groq(api_key=api_key)
 
-MIN_DURATION = 60   # seconds
-MAX_DURATION = 180  # seconds
+MIN_DURATION = 60    # seconds
+MAX_DURATION = 180   # seconds
+TARGET_DURATION = 90 # ✅ NEW: ideal segment length for extending short ones
 
 
-def validate_segments(segments):
+def validate_segments(segments, video_duration=None):
     """
-    Remove segments that don't meet duration requirements (60-180 seconds).
-    Recalculates duration from start/end times — never trusts AI's duration field.
+    ✅ FIX: Instead of discarding short segments, try to EXTEND them
+    symmetrically (pad before + after) to reach MIN_DURATION.
+    Only discard if extension is impossible or duration > MAX_DURATION.
     """
     valid = []
     for seg in segments:
         start = seg.get("start_time", 0)
-        end = seg.get("end_time", 0)
-        duration = end - start
+        end   = seg.get("end_time",   0)
+        dur   = end - start
 
-        if duration < MIN_DURATION:
-            print(f"⚠️ Skipped short segment: {duration:.1f}s → [{start}s–{end}s] ({seg.get('title', '')})")
+        # ── Too long: hard discard ────────────────────────────────────────────
+        if dur > MAX_DURATION:
+            logger.warning(f"⚠️ Skipped long segment: {dur:.1f}s [{start}s–{end}s] ({seg.get('title','')})")
             continue
 
-        if duration > MAX_DURATION:
-            print(f"⚠️ Skipped long segment: {duration:.1f}s → [{start}s–{end}s] ({seg.get('title', '')})")
-            continue
+        # ── Too short: try to extend ──────────────────────────────────────────
+        if dur < MIN_DURATION:
+            needed  = MIN_DURATION - dur
+            pad_pre = needed / 2
+            pad_post = needed / 2
+
+            new_start = max(0, start - pad_pre)
+            new_end   = end + pad_post
+
+            # Clamp to video duration if known
+            if video_duration:
+                new_end = min(video_duration, new_end)
+                # If we couldn't get enough at the end, steal from the front
+                actual_dur = new_end - new_start
+                if actual_dur < MIN_DURATION and new_start > 0:
+                    new_start = max(0, new_end - MIN_DURATION)
+
+            actual_dur = new_end - new_start
+            if actual_dur < MIN_DURATION:
+                logger.warning(
+                    f"⚠️ Skipped unextendable segment: {dur:.1f}s→{actual_dur:.1f}s "
+                    f"[{start}s–{end}s] ({seg.get('title','')})"
+                )
+                continue
 
-        seg["duration"] = round(duration, 2)
+            logger.info(
+                f"🔧 Extended short segment {dur:.1f}s → {actual_dur:.1f}s "
+                f"[{start:.1f}s–{end:.1f}s] → [{new_start:.1f}s–{new_end:.1f}s]"
+            )
+            seg["start_time"] = round(new_start, 2)
+            seg["end_time"]   = round(new_end,   2)
+            dur               = actual_dur
+
+        seg["duration"] = round(dur, 2)
         valid.append(seg)
 
-    print(f"✅ Valid segments after duration filter: {len(valid)}/{len(segments)}")
+    logger.info(f"✅ Valid segments after filter: {len(valid)}/{len(segments)}")
     return valid
 
 
-def analyze_transcript(transcript):
+def _fallback_segments_from_transcript(transcript: str, video_duration: float) -> list:
+    """
+    ✅ NEW FALLBACK: If AI returns nothing useful, generate evenly-spaced
+    segments from the transcript based on timestamp markers [start - end].
+    Tries to pick the most text-dense windows as a heuristic for 'interesting'.
+    """
+    import re
+
+    logger.warning("⚠️ Using fallback segment generator from transcript timestamps")
+
+    # Extract all timestamped lines
+    lines = []
+    for match in re.finditer(r'\[(\d+\.?\d*)\s*-\s*(\d+\.?\d*)\]\s*(.*)', transcript):
+        lines.append({
+            "start": float(match.group(1)),
+            "end":   float(match.group(2)),
+            "text":  match.group(3).strip(),
+        })
+
+    if not lines:
+        return []
+
+    # Build 90-second windows every 60 seconds, score by word count
+    candidates = []
+    step = 60
+    window = TARGET_DURATION
+
+    t = lines[0]["start"]
+    max_t = lines[-1]["end"]
+
+    while t + MIN_DURATION <= max_t:
+        w_end   = min(t + window, max_t)
+        in_window = [l for l in lines if l["start"] >= t and l["end"] <= w_end]
+        word_count = sum(len(l["text"].split()) for l in in_window)
+        candidates.append({
+            "start_time":  round(t, 2),
+            "end_time":    round(w_end, 2),
+            "word_count":  word_count,
+            "title":       f"Highlight at {int(t//60)}m{int(t%60):02d}s",
+            "description": "Auto-detected highlight segment",
+            "reason":      "Fallback: highest word-density window",
+            "viral_score": word_count,  # proxy score
+        })
+        t += step
+
+    if not candidates:
+        return []
+
+    # Sort by word density, pick top 3, sort back by time
+    candidates.sort(key=lambda x: x["word_count"], reverse=True)
+    top = candidates[:3]
+    top.sort(key=lambda x: x["start_time"])
+
+    # Deduplicate overlapping windows (keep higher-scored one)
+    deduped = []
+    for c in top:
+        if deduped and c["start_time"] < deduped[-1]["end_time"] - 20:
+            continue  # overlaps with previous, skip
+        deduped.append(c)
+
+    logger.info(f"🔧 Fallback generated {len(deduped)} segment(s)")
+    return deduped
+
+
+def analyze_transcript(transcript, video_duration=None):
     """
     Analyze transcript using Groq API.
-    Forces AI to find complete publish-ready stories, not just punchlines.
+    ✅ FIX: Passes video_duration to validate_segments for smarter extension.
+    ✅ FIX: Falls back to _fallback_segments_from_transcript on 0 valid results.
     """
 
     prompt = f"""
 You are a viral short-form video editor specializing in TikTok, Reels, and YouTube Shorts.
 Your job is to find COMPLETE, PUBLISH-READY segments — not just funny lines or punchlines.
 
+⚠️ CRITICAL DURATION RULE — VIOLATIONS WILL BE REJECTED:
+- end_time - start_time MUST be between {MIN_DURATION} and {MAX_DURATION} seconds
+- Segments shorter than {MIN_DURATION}s will be AUTOMATICALLY DISCARDED
+- If a funny moment is only 15s, you MUST expand it: go back ~45s for context and forward ~30s for reaction
+- There is NO exception to this rule
+
 THINKING PROCESS — follow these steps for every segment:
 1. Spot an interesting or funny moment in the transcript
 2. Go BACKWARDS to find where the setup or context begins (usually 30–90 seconds before the peak)
 3. Go FORWARDS to find where the natural conclusion or audience reaction ends (usually 15–40 seconds after)
-4. The full segment = setup + build-up + peak + conclusion = 60 to 180 seconds total
+4. The full segment = setup + build-up + peak + conclusion = {MIN_DURATION} to {MAX_DURATION} seconds total
 
 EXAMPLE OF CORRECT THINKING:
 - You notice a funny moment at 150s
-- The story/setup started at 95s
-- The conclusion/reaction ends at 178s
-- Correct segment → start_time: 95, end_time: 178 (83 seconds) ✅
-- WRONG → start_time: 145, end_time: 158 (13 seconds, just the punchline) ❌
+- The story/setup started at 95s → use that as start_time
+- The conclusion/reaction ends at 220s → use that as end_time
+- Duration = 220 - 95 = 125 seconds ✅ (within 60-180)
+- WRONG → start_time: 145, end_time: 165 (20 seconds, just the punchline) ❌
 
 A PUBLISH-READY segment must have ALL of these:
 - A hook in the first 5 seconds that makes viewers want to keep watching
@@ -77,22 +179,15 @@ A PUBLISH-READY segment must have ALL of these:
 - A satisfying payoff or conclusion — not an abrupt cut
 - Standalone: makes complete sense without watching anything before or after
 
-STRICT RULES:
-- end_time - start_time MUST be between 60 and 180 seconds
-- Never return just a punchline or a reaction — always include the full story arc
-- If you cannot find a complete story that is at least 60 seconds, skip it entirely
-- Natural start: beginning of a thought, scene, or story — never mid-sentence
-- Natural end: after the payoff, conclusion, or audience reaction — never mid-sentence
-
 OUTPUT — raw JSON only, no markdown, no explanation:
 {{
   "segments": [
     {{
-      "start_time": <float, where the setup begins>,
-      "end_time": <float, where the conclusion ends>,
-      "title": "<punchy YouTube Shorts title, max 60 chars, no clickbait>",
-      "description": "<1-2 sentences describing the full story, include relevant keywords for YouTube SEO>",
-      "reason": "<explain the full story arc: setup → peak → conclusion, max 25 words>"
+      "start_time": <float, where the SETUP begins — NOT the funny moment itself>,
+      "end_time": <float, where the CONCLUSION ends — NOT just the punchline>,
+      "title": "<punchy YouTube Shorts title, max 60 chars>",
+      "description": "<1-2 sentences describing the full story>",
+      "reason": "<setup → peak → conclusion arc, max 25 words>"
     }}
   ]
 }}
@@ -104,168 +199,31 @@ TRANSCRIPT:
 """
 
     max_retries = 3
-    base_delay = 5
+    base_delay  = 5
 
     for attempt in range(max_retries):
         try:
             response = client.chat.completions.create(
-                model=MODEL_NAME,
-                messages=[
+                model    = MODEL_NAME,
+                messages = [
                     {
-                        "role": "system",
+                        "role":    "system",
                         "content": (
-                            "You are a JSON-only assistant. "
-                            "Output raw JSON only — no markdown, no code blocks, no explanation. "
-                            "Think carefully about complete story arcs before selecting any segment."
+                            f"You are a JSON-only assistant. "
+                            f"Output raw JSON only — no markdown, no code blocks, no explanation. "
+                            f"EVERY segment MUST be between {MIN_DURATION} and {MAX_DURATION} seconds. "
+                            f"Think carefully: always include setup + peak + conclusion."
                         )
                     },
                     {"role": "user", "content": prompt}
                 ],
-                temperature=0.3,
+                temperature = 0.3,
             )
 
             content = response.choices[0].message.content.strip()
-            print(f"🤖 AI Raw Response (First 300 chars): {content[:300]}...")
+            logger.info(f"🤖 AI Raw Response (first 300 chars): {content[:300]}...")
 
-            # Strip markdown if model ignores system prompt
             if "```json" in content:
                 content = content.split("```json")[1].split("```")[0].strip()
             elif "```" in content:
-                content = content.split("```")[1].split("```")[0].strip()
-
-            # Parse and validate
-            data = json.loads(content)
-            raw_segments = data.get("segments", [])
-            valid_segments = validate_segments(raw_segments)
-            data["segments"] = valid_segments
-
-            content = json.dumps(data)
-            print(f"🤖 Parsed: {len(raw_segments)} raw → {len(valid_segments)} valid segments.")
-            return {"content": content}
-
-        except Exception as e:
-            print(f"❌ Error in Groq analysis (attempt {attempt + 1}): {e}")
-            if attempt < max_retries - 1:
-                wait_time = base_delay * (2 ** attempt)
-                print(f"⚠️ Retrying in {wait_time}s...")
-                time.sleep(wait_time)
-            else:
-                break
-
-    print("❌ All retry attempts failed.")
-    return {"content": '{"segments": []}'}
-
-
-def smart_chunk_transcript(transcript, max_tokens=4000):
-    """
-    Split transcript into coherent chunks at sentence boundaries.
-    Adds overlap between chunks so stories that span chunk boundaries aren't lost.
-    """
-    sentences = transcript.replace('\n', ' ').split('. ')
-    chunks = []
-    current_chunk = []
-    current_length = 0
-    overlap_sentences = []  # last N sentences of previous chunk for context
-
-    for sentence in sentences:
-        sentence_length = len(sentence.split())
-
-        if current_length + sentence_length > max_tokens and current_chunk:
-            chunk_text = '. '.join(current_chunk) + '.'
-            chunks.append(chunk_text.strip())
-
-            # Keep last 5 sentences as overlap for next chunk
-            overlap_sentences = current_chunk[-5:]
-            current_chunk = overlap_sentences + [sentence]
-            current_length = sum(len(s.split()) for s in current_chunk)
-        else:
-            current_chunk.append(sentence)
-            current_length += sentence_length
-
-    if current_chunk:
-        chunk_text = '. '.join(current_chunk) + '.'
-        chunks.append(chunk_text.strip())
-
-    return chunks
-
-
-def analyze_transcript_with_chunking(transcript):
-    """
-    Analyze transcript using smart chunking for long content.
-    Processes each chunk separately and merges + deduplicates results.
-    """
-    if len(transcript.split()) > 3000:
-        logger.info("📦 Transcript too long, using smart chunking...")
-        chunks = smart_chunk_transcript(transcript, max_tokens=3000)
-        all_segments = []
-
-        for i, chunk in enumerate(chunks):
-            logger.info(f"🔄 Processing chunk {i+1}/{len(chunks)}...")
-            result = analyze_transcript(chunk)
-
-            try:
-                data = json.loads(result['content'])
-                if 'segments' in data:
-                    all_segments.extend(data['segments'])
-            except Exception as e:
-                logger.warning(f"⚠️ Failed to parse chunk {i+1}: {e}")
-                continue
-
-        if all_segments:
-            # Deduplicate by time (allow 10s tolerance for overlap chunks)
-            unique_segments = []
-            seen_times = set()
-
-            for seg in all_segments:
-                # Round to nearest 10s to catch near-duplicates from overlap
-                time_key = f"{round(seg.get('start_time', 0) / 10) * 10}-{round(seg.get('end_time', 0) / 10) * 10}"
-                if time_key not in seen_times:
-                    unique_segments.append(seg)
-                    seen_times.add(time_key)
-
-            # Keep AI's original order — AI already ranks by importance (best first)
-
-            logger.info(f"📊 Total unique valid segments: {len(unique_segments)}")
-            return {"content": json.dumps({"segments": unique_segments[:10]})}
-
-        logger.warning("⚠️ No valid segments found across all chunks.")
-        return {"content": '{"segments": []}'}
-
-    return analyze_transcript(transcript)
-
-
-# Testing
-if __name__ == "__main__":
-    test_transcript = """
-    [0.0 - 5.0] Welcome to today's video about productivity hacks that actually work.
-    [5.0 - 15.0] The first hack is something I call the 2-minute rule. If something takes less than 2 minutes, do it immediately.
-    [15.0 - 30.0] This simple rule has transformed my life. I used to procrastinate on small tasks, but now I handle them right away.
-    [30.0 - 45.0] The second hack is batching similar tasks together. Instead of checking email 20 times a day, I check it twice.
-    [45.0 - 60.0] This has saved me hours every week. I batch my emails, phone calls, and even errands.
-    [60.0 - 90.0] The third hack is the Pomodoro Technique. Work for 25 minutes, then take a 5-minute break.
-    [90.0 - 120.0] This technique helps me stay focused and avoid burnout. I get more done in less time.
-    [120.0 - 150.0] The fourth hack is to eliminate distractions completely. Turn off notifications, close tabs, and focus.
-    [150.0 - 180.0] When you eliminate distractions, your productivity skyrockets. I finish in 2 hours what used to take 6.
-    [180.0 - 210.0] The fifth and final hack is to review your day every evening. Spend 5 minutes planning tomorrow.
-    [210.0 - 240.0] This evening review changed everything for me. I wake up knowing exactly what to do and I never waste morning time figuring out priorities.
-    """
-
-    logger.info("🧪 Testing AI Analysis...")
-    result = analyze_transcript_with_chunking(test_transcript)
-
-    try:
-        data = json.loads(result['content'])
-        segments = data.get('segments', [])
-        logger.info(f"✅ Found {len(segments)} publish-ready segments:\n")
-
-        for i, seg in enumerate(segments):
-            duration = seg['end_time'] - seg['start_time']
-            logger.info(
-                f"#{i+1} [{seg['start_time']:.0f}s – {seg['end_time']:.0f}s] ({duration:.0f}s)\n"
-                f"  📌 Title:       {seg.get('title', 'N/A')}\n"
-                f"  📝 Description: {seg.get('description', 'N/A')}\n"
-                f"  💡 Story Arc:   {seg.get('reason', 'N/A')}\n"
-            )
-    except Exception as e:
-        logger.error(f"❌ Error parsing result: {e}")
-        logger.info(f"Raw result: {result}")
+                content = content.split("`