from __future__ import annotations import os import time import shutil import uuid import json import asyncio import base64 import re from typing import List, Optional, Dict, Any from fastapi import FastAPI, UploadFile, File, BackgroundTasks, HTTPException, Form from fastapi.middleware.cors import CORSMiddleware from pydantic import BaseModel, ConfigDict import google.generativeai as genai from google.generativeai.types import HarmCategory, HarmBlockThreshold import cv2 import numpy as np # Configuration GEMINI_API_KEY = os.getenv("GOOGLE_API_KEY") genai.configure(api_key=GEMINI_API_KEY) app = FastAPI(title="BJJ AI Coach - Submission-Aware") app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], ) # --- MODELS --- class TimestampedEvent(BaseModel): time: str title: str description: str category: Optional[str] = "GENERAL" frame_image: Optional[str] = None frame_timestamp: Optional[str] = None model_config = ConfigDict(extra="allow") class Drill(BaseModel): name: str focus_area: str reason: str duration: Optional[str] = "15 min/day" frequency: Optional[str] = "5x/week" class DetailedSkillBreakdown(BaseModel): offense: int defense: int guard: int passing: int standup: int class PerformanceGrades(BaseModel): defense_grade: str offense_grade: str control_grade: str class AnalysisResult(BaseModel): overall_score: int performance_label: str performance_grades: PerformanceGrades skill_breakdown: DetailedSkillBreakdown strengths: List[str] weaknesses: List[str] missed_opportunities: List[TimestampedEvent] key_moments: List[TimestampedEvent] coach_notes: str recommended_drills: List[Drill] db_storage = {} # --- UTILITY FUNCTIONS --- def parse_time_to_seconds(time_str: str) -> Optional[int]: if not time_str: return None match = re.search(r"(\d{1,2}):(\d{2})", time_str) if not match: return None mm, ss = match.groups() return int(mm) * 60 + int(ss) def find_closest_frame(target_time_sec: int, frames: list) -> dict: return min(frames, key=lambda f: abs(f["second"] - target_time_sec)) def attach_frames_to_events(events: List[dict], frames: list): for event in events: try: event_time_sec = parse_time_to_seconds(event.get("time")) if event_time_sec is None: continue closest = find_closest_frame(event_time_sec, frames) event["frame_timestamp"] = closest["timestamp"] event["frame_image"] = base64.b64encode(closest["bytes"]).decode("utf-8") except Exception as e: print(f"āš ļø Frame attachment failed: {e}") event["frame_image"] = None def extract_json_from_text(text: str) -> Dict: text = text.strip() try: return json.loads(text) except: pass if "```json" in text or "```" in text: try: if "```json" in text: text = text.split("```json")[1].split("```")[0] else: text = text.split("```")[1].split("```")[0] return json.loads(text.strip()) except: pass try: start_idx = text.find('{') if start_idx == -1: raise ValueError("No opening brace") brace_count = 0 end_idx = -1 for i in range(start_idx, len(text)): if text[i] == '{': brace_count += 1 elif text[i] == '}': brace_count -= 1 if brace_count == 0: end_idx = i break if end_idx == -1: raise ValueError("No closing brace") json_str = text[start_idx:end_idx+1] return json.loads(json_str) except: pass raise ValueError(f"Could not extract JSON from: {text[:300]}") # --- FRAME EXTRACTION --- def extract_smart_frames(video_path: str) -> tuple: try: cap = cv2.VideoCapture(video_path) if not cap.isOpened(): raise Exception("Cannot open video") fps = cap.get(cv2.CAP_PROP_FPS) total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT)) duration = total_frames / fps if fps > 0 else 0 if duration <= 30: frames_to_extract = 14 elif duration <= 60: frames_to_extract = 16 else: frames_to_extract = 18 print(f"šŸ“¹ Extracting {frames_to_extract} frames from {duration:.1f}s video") metadata = { "duration": round(duration, 2), "fps": round(fps, 2), "frames_extracted": frames_to_extract } frames = [] interval = max(1, total_frames // frames_to_extract) frame_idx = 0 extracted = 0 while cap.isOpened() and extracted < frames_to_extract: ret, frame = cap.read() if not ret: break if frame_idx % interval == 0: h, w = frame.shape[:2] target_h = 720 target_w = int(w * (target_h / h)) resized = cv2.resize(frame, (target_w, target_h)) _, buffer = cv2.imencode('.jpg', resized, [cv2.IMWRITE_JPEG_QUALITY, 85]) timestamp_sec = frame_idx / fps timestamp_str = f"{int(timestamp_sec // 60):02d}:{int(timestamp_sec % 60):02d}" frames.append({ "bytes": buffer.tobytes(), "timestamp": timestamp_str, "second": round(timestamp_sec, 2) }) extracted += 1 frame_idx += 1 cap.release() print(f"āœ“ Extracted {len(frames)} frames") return frames, metadata except Exception as e: if 'cap' in locals(): cap.release() raise Exception(f"Frame extraction failed: {str(e)}") # --- ULTRA-ENHANCED SUBMISSION-AWARE PROMPT --- SUBMISSION_AWARE_PROMPT = """You are an expert BJJ black belt coach analyzing training footage. **ATHLETES:** - User (YOU ARE ANALYZING THIS PERSON): {user_desc} - Opponent: {opp_desc} **VIDEO INFO:** - Duration: {duration}s - Frames: {num_frames} snapshots from the match **FRAME TIMELINE:** {frame_list} ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ CRITICAL: SUBMISSION & TAP DETECTION ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ **YOUR #1 JOB: DETECT IF SOMEONE TAPPED OUT** A "tap" looks like: - āœ… Hand slapping/patting the mat rapidly (2+ times) - āœ… Hand slapping/patting opponent's body rapidly (2+ times) - āœ… Verbal submission (yelling "TAP!" or grimacing in pain) - āœ… Opponent's body going limp/giving up resistance - āœ… Match ending with someone in a submission hold **WATCH THE FINAL 10-15 SECONDS VERY CAREFULLY:** 1. Is someone in a submission position? - Back control with choke grip - Leg entanglement with foot/ankle control - Armbar with arm isolated - Triangle with leg around neck - Mounted with hands near face/neck 2. Do you see ANY tapping motion? - Look for hands moving rapidly - Look for repeated contact with mat/body - Look for opponent's facial expression (pain/grimacing) 3. Does the match end abruptly? - If yes + someone is in submission → LIKELY A TAP ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## SUBMISSION-SPECIFIC GUIDANCE ### LEG LOCKS (Heel Hooks, Ankle Locks, Toe Holds, Knee Bars): **Visual Indicators:** - User has opponent's foot/ankle isolated and controlled - User is arching back or falling back (classic finish motion) - Opponent grimacing, tensing, or trying to escape frantically - Opponent's hand moves to tap **Common Positions:** - 50/50 Guard (both legs entangled) - Ashi Garami (one leg trapped) - Outside Ashi / Saddle (heel hook setup) - Straight Ankle Lock (top or bottom) **CRITICAL:** If you see User controlling opponent's leg in final frames + opponent tapping → User finished with a leg lock ### CHOKES (RNC, Triangle, Guillotine, etc.): **Visual Indicators:** - User's arm(s) around opponent's neck - Opponent's face turning red/strained - Opponent pulling at User's hands desperately - Opponent tapping ### JOINT LOCKS (Armbar, Kimura, Americana): **Visual Indicators:** - User controlling opponent's arm in extended/bent position - Opponent's arm under pressure - Opponent unable to defend or escape - Opponent tapping ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## SCORING RULES (ADJUSTED FOR SUBMISSIONS) **IF USER SUBMITTED OPPONENT:** - Offense: 80-95 (Successful submission = elite offense) - Defense: 70-85 (Wasn't in danger) - Overall: 80-90 (Winning by submission = strong performance) - Performance Label: "STRONG PERFORMANCE" or "EXCELLENT PERFORMANCE" **IF OPPONENT SUBMITTED USER:** - Offense: 40-60 (Couldn't finish) - Defense: 25-40 (Got submitted = failed defense) - Overall: 40-60 (Getting submitted = needs improvement) - Performance Label: "DEVELOPING PERFORMANCE" or "NEEDS IMPROVEMENT" **IF NO SUBMISSION:** - Score based on positional dominance - Most recreational = 55-70 range ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## ANALYSIS FRAMEWORK ### STEP 1: TIMELINE RECONSTRUCTION Document the match flow frame-by-frame: 00:00-00:10 | Position | Who has advantage, what's happening 00:11-00:20 | Position | Details ... continue ... ### STEP 2: SUBMISSION SCAN **Check the final 15 seconds:** - What position are they in? - Is someone controlling a limb/neck/leg? - Do you see tapping motion? - Does opponent look distressed? **If you identify a submission:** - WHO tapped? (User or Opponent) - WHAT was the submission? (Ankle lock, RNC, etc.) - WHEN? (Exact timestamp) ### STEP 3: STRENGTHS & WEAKNESSES **Strengths (EXACTLY 3):** - Must include timestamps - If User won by submission → #1 strength MUST be the finish - Example: "At 0:58 - Successfully finished straight ankle lock, showing excellent leg lock mechanics" **Weaknesses (EXACTLY 3):** - If User got submitted → #1 weakness MUST be the defensive failure - If no submission → Focus on position/technique gaps ### STEP 4: KEY MOMENTS **Must include:** - If submission occurred → It MUST be listed as a key moment - Other significant transitions/attempts ### STEP 5: COACH'S NOTES **If User won by submission:** "You demonstrated strong [position] work leading up to the finish. The submission at [time] showed good technical execution. [Specific details about the setup and finish]." **If User got submitted:** "You were caught in a [submission] at [time]. This indicates a defensive gap that needs immediate attention. [Specific details about how it happened]." ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## OUTPUT FORMAT (JSON ONLY) Output ONLY the JSON object. No markdown, no explanatory text. {{ "overall_score": , "performance_label": "EXCELLENT|STRONG|SOLID|DEVELOPING|NEEDS IMPROVEMENT", "performance_grades": {{ "defense_grade": "", "offense_grade": "", "control_grade": "" }}, "skill_breakdown": {{ "offense": , "defense": , "guard": , "passing": , "standup": }}, "strengths": [ "At 0:XX - [If submission, list it here first]", "At 0:XX - Second strength", "At 0:XX - Third strength" ], "weaknesses": [ "At 0:XX - [If got submitted, list defensive failure here first]", "At 0:XX - Second weakness", "At 0:XX - Third weakness" ], "missed_opportunities": [ {{"time": "00:XX", "title": "...", "description": "...", "category": "SUBMISSION|SWEEP|POSITION"}} ], "key_moments": [ {{"time": "00:XX", "title": "[IF SUBMISSION OCCURRED, LIST IT HERE]", "description": "User/Opponent finished with [technique]", "category": "SUBMISSION"}}, {{"time": "00:XX", "title": "...", "description": "...", "category": "TRANSITION|DEFENSE|SWEEP"}} ], "coach_notes": "150-250 words. If submission occurred, start with: 'The match ended with [winner] finishing [loser] via [technique] at [time].' Then analyze the path to that finish...", "recommended_drills": [ {{"name": "...", "focus_area": "...", "reason": "[If got submitted, drill to prevent that specific submission]", "duration": "15 min/day", "frequency": "5x/week"}}, {{"name": "...", "focus_area": "...", "reason": "...", "duration": "10 min/day", "frequency": "4x/week"}}, {{"name": "...", "focus_area": "...", "reason": "...", "duration": "12 min/day", "frequency": "3x/week"}} ] }} ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ ## FINAL CHECKLIST BEFORE RESPONDING - [ ] Did I check the final 15 seconds for a tap? - [ ] If I saw leg entanglement, did I check if it became a submission? - [ ] If I saw a submission, did I identify WHO tapped? - [ ] Did I score offense 80+ if User won by submission? - [ ] Did I score defense ≤40 if User got submitted? - [ ] Did I list the submission as a key moment if it occurred? - [ ] Did I mention the submission in coach's notes if it occurred? - [ ] Are all timestamps in MM:SS format? - [ ] Is the JSON valid? **REMEMBER:** A match can end in 3 ways: 1. Submission (someone taps) → MOST IMPORTANT TO DETECT 2. Points/Advantage (time runs out) 3. Disqualification (rare) If you see sustained control + opponent distress + tapping motion → IT'S A SUBMISSION! """ # --- ANALYSIS PIPELINE --- async def fast_accurate_analysis( frames: List[Dict], metadata: Dict, user_desc: str, opp_desc: str, activity_type: str, analysis_id: str = None ) -> AnalysisResult: """ Fast 2-agent submission-aware analysis Target: 30-45 seconds """ print("\n" + "="*70) print("šŸŽÆ SUBMISSION-AWARE ANALYSIS (Target: 30-45s)") print("="*70) try: # AGENT 1: GEMINI VISION print("\nšŸ¤– AGENT 1: Gemini Vision Analysis") if analysis_id: db_storage[analysis_id]["progress"] = 50 # Build prompt frame_list = "\n".join([ f"Frame {i+1} @ {f['timestamp']} ({f['second']}s)" for i, f in enumerate(frames) ]) prompt = SUBMISSION_AWARE_PROMPT.format( user_desc=user_desc, opp_desc=opp_desc, duration=metadata["duration"], num_frames=len(frames), frame_list=frame_list ) # Prepare content content = [ { "mime_type": "image/jpeg", "data": base64.b64encode(f["bytes"]).decode("utf-8") } for f in frames ] content.append(prompt) # Call Gemini start = time.time() model = genai.GenerativeModel( model_name="gemini-2.5-flash", generation_config={ "temperature": 0.2, # Slightly higher for better reasoning "response_mime_type": "application/json", } ) response = await asyncio.get_event_loop().run_in_executor( None, lambda: model.generate_content( content, safety_settings={ HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE, HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE, HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE, HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE, } ) ) gemini_time = time.time() - start print(f"āœ“ Gemini analysis: {gemini_time:.2f}s") # AGENT 2: PARSE & ENHANCE print("\nšŸ“Š AGENT 2: Parse & Enhance") if analysis_id: db_storage[analysis_id]["progress"] = 90 # Parse JSON result_data = extract_json_from_text(response.text) # Validate result_data = validate_analysis(result_data) # Attach frames print("šŸ–¼ļø Attaching frames to events...") attach_frames_to_events(result_data.get("missed_opportunities", []), frames) attach_frames_to_events(result_data.get("key_moments", []), frames) if analysis_id: db_storage[analysis_id]["progress"] = 100 total_time = time.time() - start print(f"\nāœ… COMPLETE in {total_time:.2f}s") print("="*70 + "\n") return AnalysisResult(**result_data) except Exception as e: print(f"\nāŒ Analysis failed: {str(e)}") fallback = generate_fallback() if analysis_id: db_storage[analysis_id]["used_fallback"] = True return AnalysisResult(**fallback) def validate_analysis(data: Dict) -> Dict: """Validate and fix analysis data""" if "overall_score" not in data: data["overall_score"] = 65 data["overall_score"] = max(0, min(100, data["overall_score"])) if "performance_label" not in data: score = data["overall_score"] if score >= 85: data["performance_label"] = "EXCELLENT PERFORMANCE" elif score >= 75: data["performance_label"] = "STRONG PERFORMANCE" elif score >= 60: data["performance_label"] = "SOLID PERFORMANCE" else: data["performance_label"] = "DEVELOPING PERFORMANCE" if "performance_grades" not in data: data["performance_grades"] = { "defense_grade": "C+", "offense_grade": "C", "control_grade": "C+" } if "skill_breakdown" not in data: base = data["overall_score"] data["skill_breakdown"] = { "offense": max(0, min(100, base - 5)), "defense": max(0, min(100, base + 3)), "guard": max(0, min(100, base - 2)), "passing": max(0, min(100, base - 10)), "standup": max(0, min(100, base - 13)) } for field in ["strengths", "weaknesses"]: if field not in data or len(data[field]) < 3: default = ["Good structure", "Showed awareness", "Consistent"] if field == "strengths" else ["More aggression", "Improve timing", "Work transitions"] data[field] = default data[field] = data[field][:3] for field in ["missed_opportunities", "key_moments"]: if field not in data or not data[field]: data[field] = [{ "time": "00:30", "title": "Key Moment", "description": "Review footage", "category": "POSITION" }] if "coach_notes" not in data or len(data["coach_notes"]) < 50: data["coach_notes"] = "Focus on fundamentals and consistent positioning." if "recommended_drills" not in data or len(data["recommended_drills"]) < 3: data["recommended_drills"] = [ {"name": "Position Control", "focus_area": "General", "reason": "Improve awareness", "duration": "15 min/day", "frequency": "5x/week"}, {"name": "Guard Work", "focus_area": "Defense", "reason": "Strengthen defense", "duration": "10 min/day", "frequency": "4x/week"}, {"name": "Transitions", "focus_area": "Movement", "reason": "Improve flow", "duration": "12 min/day", "frequency": "3x/week"} ] return data def generate_fallback() -> Dict: return { "overall_score": 65, "performance_label": "SOLID PERFORMANCE", "performance_grades": {"defense_grade": "C+", "offense_grade": "C", "control_grade": "C+"}, "skill_breakdown": {"offense": 60, "defense": 68, "guard": 63, "passing": 55, "standup": 52}, "strengths": ["Maintained defensive structure", "Showed positional awareness", "Consistent movement"], "weaknesses": ["Could be more aggressive", "Improve transition recognition", "Work on timing"], "missed_opportunities": [{"time": "00:30", "title": "Position", "description": "Review for openings", "category": "POSITION"}], "key_moments": [{"time": "00:15", "title": "Exchange", "description": "Positional work", "category": "TRANSITION"}], "coach_notes": "Focus on fundamentals: maintain posture, control distance, look for position improvement.", "recommended_drills": [ {"name": "Positional Sparring", "focus_area": "General", "reason": "Develop awareness", "duration": "15 min/day", "frequency": "5x/week"}, {"name": "Guard Work", "focus_area": "Defense", "reason": "Strengthen defense", "duration": "10 min/day", "frequency": "4x/week"}, {"name": "Position Control", "focus_area": "Control", "reason": "Improve control", "duration": "12 min/day", "frequency": "3x/week"} ] } # --- BACKGROUND TASK --- async def analyze_video_task( analysis_id: str, video_path: str, user_desc: str, opp_desc: str, activity_type: str ): try: db_storage[analysis_id]["status"] = "processing" db_storage[analysis_id]["progress"] = 10 frames, metadata = await asyncio.get_event_loop().run_in_executor( None, extract_smart_frames, video_path ) result = await fast_accurate_analysis( frames, metadata, user_desc, opp_desc, activity_type, analysis_id ) db_storage[analysis_id]["status"] = "completed" db_storage[analysis_id]["data"] = result.model_dump() except Exception as e: print(f"āŒ Task error: {str(e)}") fallback = generate_fallback() db_storage[analysis_id]["status"] = "completed" db_storage[analysis_id]["data"] = fallback db_storage[analysis_id]["used_fallback"] = True finally: try: os.remove(video_path) except: pass # --- API ENDPOINTS --- @app.post("/upload") async def upload_video(file: UploadFile = File(...)): file_name = f"{uuid.uuid4()}_{file.filename}" file_path = f"temp_videos/{file_name}" os.makedirs("temp_videos", exist_ok=True) with open(file_path, "wb") as buffer: shutil.copyfileobj(file.file, buffer) return {"file_name": file_path} @app.post("/analyze") async def start_analysis( video_file_name: str, user_description: str, opponent_description: str, activity_type: str = "Brazilian Jiu-Jitsu", background_tasks: BackgroundTasks = None ): analysis_id = str(uuid.uuid4()) db_storage[analysis_id] = {"status": "queued", "progress": 0} background_tasks.add_task( analyze_video_task, analysis_id, video_file_name, user_description.strip(), opponent_description.strip(), activity_type ) return {"analysis_id": analysis_id} @app.get("/status/{analysis_id}") async def get_status(analysis_id: str): if analysis_id not in db_storage: raise HTTPException(status_code=404, detail="Not found") return db_storage[analysis_id] @app.post("/analyze-complete") async def analyze_complete( file: UploadFile = File(...), user_description: str = Form(...), opponent_description: str = Form(...), activity_type: str = Form("Brazilian Jiu-Jitsu") ): start_time = time.time() file_path = None try: file_name = f"{uuid.uuid4()}_{file.filename}" file_path = f"temp_videos/{file_name}" os.makedirs("temp_videos", exist_ok=True) with open(file_path, "wb") as buffer: shutil.copyfileobj(file.file, buffer) analysis_id = str(uuid.uuid4()) db_storage[analysis_id] = {"status": "processing", "progress": 0} frames, metadata = await asyncio.get_event_loop().run_in_executor( None, extract_smart_frames, file_path ) result = await fast_accurate_analysis( frames, metadata, user_description.strip(), opponent_description.strip(), activity_type, analysis_id ) total_time = time.time() - start_time return { "status": "completed", "data": result.model_dump(), "processing_time": f"{total_time:.2f}s", "used_fallback": db_storage[analysis_id].get("used_fallback", False), "method": "submission_aware" } except Exception as e: print(f"āŒ Error: {str(e)}") fallback = generate_fallback() return { "status": "completed_with_fallback", "data": fallback, "error": str(e), "used_fallback": True } finally: if file_path: try: os.remove(file_path) except: pass @app.get("/health") async def health_check(): return { "status": "healthy", "version": "20.0.0-submission-aware" } @app.get("/") async def root(): return { "message": "BJJ AI Coach - Submission-Aware Edition", "version": "20.0.0", "critical_fixes": [ "Enhanced tap detection (visual + behavioral)", "Explicit submission scoring rules", "Final 15 seconds focus for finishes", ], "features": [ "Detects tapping motion in frames", "Recognizes leg locks, chokes, and joint locks", "Adjusts scoring based on submission outcome", "Target time: 30-45 seconds" ] }