import os, re, json, traceback, subprocess from pathlib import Path from flask import Flask, request, jsonify, send_from_directory from flask_cors import CORS BASE_DIR = Path(__file__).parent STATIC_DIR = BASE_DIR / "static" VIDEOS_DIR = STATIC_DIR / "videos" UPLOAD_DIR = BASE_DIR / "uploads" MODELS_DIR = BASE_DIR / "models" for p in [STATIC_DIR, VIDEOS_DIR, UPLOAD_DIR, MODELS_DIR, STATIC_DIR / "models"]: p.mkdir(exist_ok=True, parents=True) app = Flask(__name__, static_folder=str(STATIC_DIR)) CORS(app) whisper_model = None @app.route("/") def index(): return send_from_directory(str(STATIC_DIR), "index.html") @app.route("/static/") def static_files(path): return send_from_directory(str(STATIC_DIR), path) @app.route("/api/videos") def list_videos(): videos = [] for f in sorted(VIDEOS_DIR.glob("*.mp4")): title = f.stem.replace("_", " ").title() videos.append({"filename": f.name, "title": title, "url": f"/static/videos/{f.name}"}) return jsonify({"videos": videos}) @app.route("/api/process-video", methods=["POST"]) def process_video(): try: video_path = None local_video = request.form.get("local_video", "").strip() if local_video: candidate = (VIDEOS_DIR / local_video).resolve() if str(candidate).startswith(str(VIDEOS_DIR.resolve())) and candidate.exists(): video_path = candidate if video_path is None and "file" in request.files and request.files["file"].filename: uploaded = request.files["file"] safe_name = re.sub(r"[^A-Za-z0-9_.-]", "_", uploaded.filename) video_path = UPLOAD_DIR / safe_name uploaded.save(video_path) if video_path is None: url = request.form.get("url", "").strip() if url: video_path = download_video_from_url(url) if not video_path: return jsonify({"error": "No video found. Choose an internal video or upload an MP4."}), 400 stt = run_stt(str(video_path)) transcript = stt["text"] segments = stt["segments"] tokens = clean_tokens(transcript) signs = text_to_asl_glosses(tokens) timeline = build_sign_timeline(segments, signs) summary = make_simple_summary(transcript, signs) return jsonify({ "video_used": str(Path(video_path).name), "transcript": transcript, "segments": segments, "tokens": tokens[:120], "signs": signs, "timeline": timeline, "summary": summary, "metrics": { "stt": "Whisper base", "glosses": len(signs), "avatar": "Real GLB avatar + gloss animation" } }) except Exception as e: traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route("/api/ask", methods=["POST"]) def ask(): data = request.get_json(force=True) or {} q = data.get("question", "").lower().strip() transcript = data.get("transcript", "").strip() if not q: return jsonify({"answer": "Please write a question."}) if not transcript: return jsonify({"answer": "Process a video first."}) q_tokens = set(clean_tokens(q)) sentences = re.split(r"(?<=[.!?])\s+", transcript) ranked = [] for s in sentences: score = len(q_tokens.intersection(clean_tokens(s))) if score: ranked.append((score, s)) ranked.sort(reverse=True) answer = ranked[0][1] if ranked else transcript[:350] + ("..." if len(transcript) > 350 else "") return jsonify({"answer": answer}) def download_video_from_url(url: str): out = UPLOAD_DIR / "downloaded_video.%(ext)s" cmd = ["yt-dlp", "--no-playlist", "--socket-timeout", "20", "--retries", "2", "-f", "best[ext=mp4]/best", "--merge-output-format", "mp4", "-o", str(out), url] try: r = subprocess.run(cmd, capture_output=True, text=True, timeout=180) if r.returncode != 0: print(r.stderr[:800]) return None files = list(UPLOAD_DIR.glob("downloaded_video.*")) return files[0] if files else None except Exception as e: print("yt-dlp error:", e) return None def run_stt(video_path: str) -> dict: global whisper_model try: import whisper if whisper_model is None: print("Loading Whisper base model. This can take time the first time...") whisper_model = whisper.load_model("base") print("Transcribing:", video_path) result = whisper_model.transcribe(video_path, language="en", fp16=False) text = result.get("text", "").strip() segments = [] for seg in result.get("segments", []): segments.append({ "start": float(seg.get("start", 0)), "end": float(seg.get("end", 0)), "text": seg.get("text", "").strip() }) return {"text": text if text else "No speech detected in this video.", "segments": segments} except Exception as e: print("Whisper failed:", e) return {"text": "Automatic transcription failed. Install dependencies with: pip install -r requirements.txt. Also make sure ffmpeg is installed.", "segments": []} def clean_tokens(text: str): text = text.lower() text = re.sub(r"[^a-z0-9\s]", " ", text) tokens = text.split() stop = {"the","a","an","and","or","to","of","in","on","for","with","is","are","was","were","be","been","being","this","that","it","we","you","your","our","i","they","them","as","at","by","from","can","will","would","should","about","into","than","then","so","if","not","do","does","have","has","had","there","their","what","when","where","why","how","also","just","very","really","more","most","one","two","three","first","second","now","today","here","like"} return [t for t in tokens if len(t) > 2 and t not in stop] ASL_MAP = { "linkedin":"LINKEDIN", "profile":"PROFILE", "picture":"PICTURE", "photo":"PHOTO", "professional":"PROFESSIONAL", "identity":"IDENTITY", "headline":"HEADLINE", "summary":"SUMMARY", "resume":"RESUME", "cv":"RESUME", "career":"CAREER", "job":"JOB", "work":"WORK", "experience":"EXPERIENCE", "education":"EDUCATION", "skill":"SKILL", "skills":"SKILL", "network":"NETWORK", "connect":"CONNECT", "connection":"CONNECT", "message":"MESSAGE", "company":"COMPANY", "business":"BUSINESS", "interview":"INTERVIEW", "recruiter":"RECRUITER", "hire":"HIRE", "hiring":"HIRE", "math":"MATH", "mathematics":"MATH", "number":"NUMBER", "numbers":"NUMBER", "equation":"EQUATION", "equations":"EQUATION", "linear":"LINEAR", "system":"SYSTEM", "systems":"SYSTEM", "solve":"SOLVE", "solution":"SOLUTION", "variable":"VARIABLE", "matrix":"MATRIX", "addition":"ADDITION", "add":"ADDITION", "subtraction":"SUBTRACT", "subtract":"SUBTRACT", "multiply":"MULTIPLY", "division":"DIVIDE", "equal":"EQUAL", "graph":"GRAPH", "function":"FUNCTION", "science":"SCIENCE", "cell":"CELL", "energy":"ENERGY", "plant":"PLANT", "photosynthesis":"PHOTOSYNTHESIS", "learn":"LEARN", "learning":"LEARN", "lesson":"LESSON", "student":"STUDENT", "teacher":"TEACHER", "school":"SCHOOL", "course":"COURSE", "hello":"HELLO", "welcome":"WELCOME", "important":"IMPORTANT", "good":"GOOD", "great":"GREAT", "clear":"CLEAR", "example":"EXAMPLE", "result":"RESULT", "calculate":"CALCULATE", "understand":"UNDERSTAND", "explain":"EXPLAIN", "help":"HELP" } def text_to_asl_glosses(tokens): signs, seen = [], set() for t in tokens: gloss = ASL_MAP.get(t) if gloss and gloss not in seen: signs.append({"gloss": gloss, "source_word": t, "confidence": 0.88, "mode": "dictionary"}) seen.add(gloss) if len(signs) < 5: for t in tokens[:12]: gloss = t.upper() if gloss not in seen: signs.append({"gloss": gloss, "source_word": t, "confidence": 0.60, "mode": "finger-spelling"}) seen.add(gloss) if len(signs) >= 12: break return signs[:18] def build_sign_timeline(segments, global_signs): timeline = [] for seg in segments: tokens = clean_tokens(seg.get("text", "")) local_signs = text_to_asl_glosses(tokens) if not local_signs: continue start, end = float(seg["start"]), float(seg["end"]) duration = max(end - start, 0.8) step = duration / len(local_signs) for i, sign in enumerate(local_signs): timeline.append({ "start": start + i * step, "end": start + (i + 1) * step, "gloss": sign["gloss"], "source_word": sign["source_word"], "mode": sign["mode"] }) if timeline: return timeline[:120] # fallback cycle when Whisper segments fail t = 0.0 for sign in global_signs: timeline.append({"start": t, "end": t + 1.4, "gloss": sign["gloss"], "source_word": sign["source_word"], "mode": sign["mode"]}) t += 1.4 return timeline def make_simple_summary(transcript, signs): return { "first_words": transcript[:220] + ("..." if len(transcript) > 220 else ""), "main_glosses": [s["gloss"] for s in signs[:10]], "note": "The 3D avatar uses a real GLB model. It follows glosses generated from Whisper transcript. This is an MVP rendering layer, not a full ASL production engine yet." } if __name__ == "__main__": print("\nšŸš€ iLearning SignAI GLB avatar demo") print("Open: http://0.0.0.0:7860\n") app.run(host="0.0.0.0", port=7860, debug=False)