Initial Hugging Face deployment

.gitattributes

@@ -1,35 +1,4 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
+*.sh text eol=lf
+Procfile text eol=lf
 *.pt filter=lfs diff=lfs merge=lfs -text
 *.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,16 @@
+.venv/
+venv/
+__pycache__/
+*.pyc
+.env
+*.npz
+*.pth
+#*.pt
+*.onnx
+*.mp4
+# Datasets
+assets/text_to_sign/wlasl2000_filtered_keypoints/
+
+# OS junk
+.DS_Store
+Thumbs.db

Dockerfile

@@ -0,0 +1,22 @@
+FROM python:3.10-slim
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y \
+    libgl1 \
+    libglib2.0-0 \
+    ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+
+RUN pip install --upgrade pip
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY . .
+
+RUN chmod +x start.sh
+
+EXPOSE 7860
+
+CMD ["bash", "start.sh"]

app/api/sign_to_text.py

@@ -0,0 +1,25 @@
+from fastapi import APIRouter, UploadFile, File
+import tempfile
+import os
+from app.services.sign_to_text_service import run_pipeline
+
+router = APIRouter()
+
+
+@router.post("/")
+async def sign_to_text(video: UploadFile = File(...)):
+
+    # read file into memory
+    contents = await video.read()
+
+    # create a safe temporary file (auto deleted)
+    with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmp:
+        tmp.write(contents)
+        temp_path = tmp.name
+
+    try:
+        result = run_pipeline(temp_path)
+        return result
+
+    finally:
+        os.remove(temp_path)

app/api/text_to_sign.py

@@ -0,0 +1,21 @@
+from fastapi import APIRouter
+from fastapi.responses import FileResponse
+from pydantic import BaseModel
+from app.services.text_to_sign_service import generate_video
+
+router = APIRouter()
+
+class TextRequest(BaseModel):
+    text: str
+
+
+@router.post("/")
+def text_to_sign(req: TextRequest):
+
+    video_path = generate_video(req.text)
+
+    return FileResponse(
+        path=video_path,
+        media_type="video/mp4",
+        filename="sign.mp4"
+    )

app/core/config.py

@@ -0,0 +1,14 @@
+import os
+
+BASE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../"))
+
+I3D_WEIGHTS = os.path.join(
+    BASE_DIR,
+   # "assets/I3D_300Weights/FINAL_nslt_300_iters2997_top156.14_top579.94_top1086.98.pt"
+   "assets/I3D_300Weights/FINAL_nslt_100_iters=896_top1=65.89_top5=84.11_top10=89.92 (1).pt"
+)
+
+CLASS_LIST = os.path.join(
+    BASE_DIR,
+    "assets/I3D_modelfiles/wlasl_class_list.txt"
+)

app/core/logger.py

@@ -0,0 +1,6 @@
+import logging
+
+logging.basicConfig(level=logging.INFO)
+
+def log(msg):
+    logging.info(msg)

app/main.py

@@ -0,0 +1,12 @@
+from fastapi import FastAPI
+from app.api import sign_to_text, text_to_sign
+from fastapi.staticfiles import StaticFiles
+
+app = FastAPI(title="Signova API")
+
+app.include_router(sign_to_text.router, prefix="/sign-to-text")
+app.include_router(text_to_sign.router, prefix="/text-to-sign")
+
+@app.get("/")
+def home():
+    return {"message": "Signova API running"}

app/models/i3d_model.py

@@ -0,0 +1,15 @@
+import torch
+from assets.I3D_modelfiles.pytorch_i3d import InceptionI3d
+
+class I3DModel:
+    def __init__(self, weights_path, device):
+        self.device = device
+
+        self.model = InceptionI3d(400, in_channels=3)
+        self.model.replace_logits(100)
+        self.model.load_state_dict(torch.load(weights_path, map_location=device))
+        self.model.to(device)
+        self.model.eval()
+
+    def __call__(self, x):
+        return self.model(x)

app/services/sign_to_text_service.py

@@ -0,0 +1,461 @@
+import cv2
+import torch
+import torch.nn.functional as F
+import numpy as np
+import os
+import requests
+
+from dotenv import load_dotenv
+
+from app.models.i3d_model import I3DModel
+from app.core.config import I3D_WEIGHTS, CLASS_LIST
+from app.utils.mediapipe_utils import extract_hand_status
+
+
+# ================= LOAD ENV =================
+
+load_dotenv()
+
+
+# ================= CONFIG =================
+
+clip_length = 64
+
+device = torch.device(
+    "cuda" if torch.cuda.is_available() else "cpu"
+)
+
+MIN_PAUSE_FRAMES = 5
+MIN_SIGN_FRAMES = 10
+
+TARGET_FPS = 25
+
+
+# ================= LOAD MODEL =================
+
+print(f"Loading I3D model on {device}...")
+
+i3d = I3DModel(I3D_WEIGHTS, device)
+
+with open(CLASS_LIST, "r", encoding="utf-8") as f:
+
+    gloss_list = []
+
+    for line in f.readlines():
+
+        line = line.strip()
+
+        if not line:
+            continue
+
+        # FIX:
+        # "205 FEEL" -> FEEL
+        parts = line.split(maxsplit=1)
+
+        if len(parts) == 2:
+            gloss = parts[1]
+        else:
+            gloss = parts[0]
+
+        gloss_list.append(gloss.upper())
+
+
+# ================= LOAD VIDEO =================
+
+def load_video(video_path):
+
+    print("Processing video...")
+
+    cap = cv2.VideoCapture(video_path)
+
+    if not cap.isOpened():
+        raise Exception(f"Cannot open video: {video_path}")
+
+    original_fps = cap.get(cv2.CAP_PROP_FPS)
+
+    if original_fps <= 0:
+        original_fps = 30
+
+    print(f"Original FPS: {original_fps}")
+
+    frame_skip = max(1, round(original_fps / TARGET_FPS))
+
+    print(f"Frame skip: {frame_skip}")
+
+    frames = []
+    hand_flags = []
+
+    frame_idx = 0
+
+    while True:
+
+        ret, frame = cap.read()
+
+        if not ret:
+            break
+
+        # IMPORTANT:
+        # match kaggle decoding behavior
+        if frame_idx % frame_skip != 0:
+            frame_idx += 1
+            continue
+
+        resized = cv2.resize(frame, (224, 224))
+
+        norm = (resized.astype(np.float32) / 255.0) * 2 - 1
+
+        frames.append(norm)
+
+        hand_flags.append(
+            extract_hand_status(frame)
+        )
+
+        frame_idx += 1
+
+        if len(frames) % 100 == 0:
+            print(f"Processed {len(frames)} frames...")
+
+    cap.release()
+
+    frames = np.array(frames)
+
+    print(f"Loaded {len(frames)} frames")
+
+    return frames, hand_flags
+
+
+# ================= SEGMENTATION =================
+
+def segment_frames(frames, hand_flags):
+
+    segments = []
+
+    start = 0
+    pause = 0
+
+    for i in range(len(frames)):
+
+        # TRUE = pause
+        if hand_flags[i]:
+
+            pause += 1
+
+        else:
+
+            if pause >= MIN_PAUSE_FRAMES:
+
+                end = i - pause
+
+                if end - start >= MIN_SIGN_FRAMES:
+                    segments.append((start, end))
+
+                start = i
+
+            pause = 0
+
+    if len(frames) - start >= MIN_SIGN_FRAMES:
+        segments.append((start, len(frames) - 1))
+
+    return segments
+
+
+# ================= MODEL =================
+
+def predict_segment_topk(segment_frames, topk=5):
+
+    if len(segment_frames) == 0:
+        return []
+
+    indices = np.linspace(
+        0,
+        len(segment_frames) - 1,
+        clip_length
+    ).astype(int)
+
+    clip = segment_frames[indices]
+
+    clip = clip.transpose(3, 0, 1, 2)
+
+    clip_tensor = (
+        torch.from_numpy(clip)
+        .unsqueeze(0)
+        .float()
+        .to(device)
+    )
+
+    with torch.no_grad():
+
+        logits = i3d(clip_tensor)
+
+        logits = torch.mean(logits, dim=2)
+
+        probs = F.softmax(logits, dim=1)
+
+        top_probs, top_indices = torch.topk(
+            probs,
+            k=topk,
+            dim=1
+        )
+
+        top_probs = top_probs.squeeze().cpu().numpy()
+        top_indices = top_indices.squeeze().cpu().numpy()
+
+    results = []
+
+    for idx, prob in zip(top_indices, top_probs):
+
+        gloss = gloss_list[int(idx)]
+
+        results.append((gloss, float(prob)))
+
+    return results
+
+
+# ================= SMART SELECTION =================
+
+def select_best_gloss(topk_results, context):
+
+    candidates = [g for g, _ in topk_results]
+
+    top1 = candidates[0]
+
+    if len(context) == 0:
+        return top1
+
+    context_set = set(context)
+
+    def score(word, is_top1=False):
+
+        s = 0
+
+        if word in context_set:
+            s += 2
+
+        if is_top1:
+            s += 1
+
+        return s
+
+    best_word = top1
+    best_score = score(top1, True)
+
+    for i, word in enumerate(candidates):
+
+        s = score(word, i == 0)
+
+        if s > best_score:
+
+            best_score = s
+            best_word = word
+
+    return best_word
+
+
+# ================= GROQ =================
+
+def translate_with_groq(gloss):
+
+    gloss = gloss.strip()
+
+    if not gloss:
+        return ""
+
+    # ==========================
+    # SINGLE WORD -> SKIP LLM
+    # ==========================
+    words = gloss.split()
+
+    if len(words) == 1:
+
+        word = words[0]
+
+        return word.replace("_", " ").title()
+
+    api_key = os.getenv("GROQ_API_KEY")
+
+    if not api_key:
+        return "Missing GROQ_API_KEY"
+
+    url = "https://api.groq.com/openai/v1/chat/completions"
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+
+    data = {
+        "model": "llama-3.1-8b-instant",
+        "messages": [
+            {
+                "role": "system",
+                "content": """
+You are an ASL gloss to English translator.
+
+Rules:
+- Return ONLY the English translation.
+- Never explain the translation.
+- Never say:
+  'The ASL gloss for...'
+  'Translation:'
+  'English:'
+  'This means...'
+- If input is a sentence, produce one natural English sentence.
+- Output only the translated text.
+"""
+            },
+            {
+                "role": "user",
+                "content": gloss
+            }
+        ],
+        "temperature": 0.2,
+        "max_tokens": 100
+    }
+
+    try:
+
+        r = requests.post(
+            url,
+            headers=headers,
+            json=data
+        )
+
+        if r.status_code != 200:
+            return r.text
+
+        result = (
+            r.json()["choices"][0]["message"]["content"]
+            .strip()
+        )
+
+        bad_prefixes = [
+            "The ASL gloss for",
+            "Translation:",
+            "English:",
+            "This means"
+        ]
+
+        for prefix in bad_prefixes:
+
+            if result.startswith(prefix):
+                return gloss
+
+        return result
+
+    except Exception as e:
+        return str(e)
+    api_key = os.getenv("GROQ_API_KEY")
+
+    if not api_key:
+        return "Missing GROQ_API_KEY"
+
+    url = "https://api.groq.com/openai/v1/chat/completions"
+
+    headers = {
+        "Authorization": f"Bearer {api_key}",
+        "Content-Type": "application/json"
+    }
+
+    data = {
+        "model": "llama-3.1-8b-instant",
+        "messages": [
+            {
+                "role": "system",
+                "content": "You convert ASL gloss into natural English."
+            },
+            {
+                "role": "user",
+                "content": f"""
+Convert ASL gloss to English:
+
+{gloss}
+
+Rules:
+- one sentence
+- correct grammar
+"""
+            }
+        ],
+        "temperature": 0.2,
+        "max_tokens": 100
+    }
+
+    try:
+
+        r = requests.post(
+            url,
+            headers=headers,
+            json=data
+        )
+
+        if r.status_code != 200:
+            return r.text
+
+        return r.json()["choices"][0]["message"]["content"].strip()
+
+    except Exception as e:
+        return str(e)
+
+
+# ================= MAIN =================
+
+def run_pipeline(video_path):
+
+    print("\n" + "=" * 60)
+    print("STARTING SIGN TO TEXT")
+    print("=" * 60)
+
+    frames, hand_flags = load_video(video_path)
+
+    print("Hand flags sample:", hand_flags[:20])
+
+    segments = segment_frames(frames, hand_flags)
+
+    print("Segments:", segments)
+
+    final_glosses = []
+
+    for idx, (s, e) in enumerate(segments):
+
+        segment_frames_data = frames[s:e]
+
+        results = predict_segment_topk(
+            segment_frames_data,
+            topk=5
+        )
+
+        print(f"\n--- Segment {idx+1} ({s}-{e}) ---")
+
+        for i, (g, p) in enumerate(results):
+
+            print(f"{i+1}. {g} ({p:.4f})")
+
+        if len(results) == 0:
+            continue
+
+        selected = select_best_gloss(
+            results,
+            final_glosses
+        )
+
+        print(f"Selected: {selected}")
+
+        final_glosses.append(selected)
+
+    gloss_sentence = " ".join(final_glosses)
+
+    print("\nFINAL GLOSS:", gloss_sentence)
+
+    english = ""
+
+    if gloss_sentence.strip():
+        english = translate_with_groq(
+            gloss_sentence
+        )
+
+    print("ENGLISH:", english)
+
+    return {
+        "gloss": gloss_sentence,
+        "english": english
+    }

app/services/text_to_sign_service.py

@@ -0,0 +1,8 @@
+from pipelines.text_to_sign.run import run_pipeline
+
+from app.models.i3d_model import I3DModel  # (optional future use)
+from app.core.config import CLASS_LIST
+
+
+def generate_video(text):
+    return run_pipeline(text)

app/utils/mediapipe_utils.py

@@ -0,0 +1,30 @@
+import cv2
+hands = None
+HAND_LOWER_THRESHOLD = 0.85 
+def get_hands():
+    global hands
+
+    if hands is None:
+        import mediapipe as mp
+
+        print("Loading MediaPipe...")
+        hands = mp.solutions.hands.Hands(
+            static_image_mode=False,
+            max_num_hands=2
+        )
+
+    return hands
+
+def extract_hand_status(frame):
+
+    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+
+    results = get_hands().process(frame_rgb)
+
+    if results.multi_hand_landmarks:
+        for hand in results.multi_hand_landmarks:
+            for lm in hand.landmark:
+                if lm.y < HAND_LOWER_THRESHOLD:
+                    return False
+
+    return True

app/utils/preprocessing.py

@@ -0,0 +1,5 @@
+import numpy as np
+
+def sample_clip(frames, clip_length=64):
+    idx = np.linspace(0, len(frames)-1, clip_length).astype(int)
+    return frames[idx]

app/utils/video_utils.py

@@ -0,0 +1,19 @@
+#unused file
+import cv2
+import numpy as np
+
+def load_video(path, size=(224,224)):
+    cap = cv2.VideoCapture(path)
+    frames = []
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+        frame = cv2.resize(frame, size)
+        frame = (frame.astype(np.float32) / 255.0) * 2 - 1
+        frames.append(frame)
+
+    cap.release()
+    return np.array(frames)

assets/I3D_300Weights/FINAL_nslt_100_iters=896_top1=65.89_top5=84.11_top10=89.92 (1).pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a61d7dda5f875ce5ebd9d407c56874f77d1cd2aeb4bc7cd0d98a6e1ca4669a0c
+size 49677763

assets/I3D_300Weights/FINAL_nslt_300_iters2997_top156.14_top579.94_top1086.98.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:493baa483880d9a0ac88ff30a749e2a89287a737c16bd9d1e9e119723c617662
+size 50498038

assets/I3D_modelfiles/language.py

@@ -0,0 +1,308 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sat Mar 19 09:37:10 2022
+
+@author: 24412
+"""
+
+from itertools import chain
+import math
+import os
+
+
+from dotenv import load_dotenv
+import nltk
+import random
+import attr
+
+from collections import Counter
+
+load_dotenv("posts/nlp/.env", override=True)
+@attr.s(auto_attribs=True)
+class NGrams:
+    """The N-Gram Language Model
+
+    Args:
+     data: the training data
+     n: the size of the n-grams
+     start_token: string to represent the start of a sentence
+     end_token: string to represent the end of a sentence
+    """
+    data: list
+    n: int
+    start_token: str="<s>"
+    end_token: str="<e>"
+    _start_tokens: list=None
+    _end_tokens: list=None
+    _sentences: list=None
+    _n_grams: list=None
+    _counts: dict=None
+
+    @property
+    def start_tokens(self) -> list:
+        """List of 'n' start tokens"""
+        if self._start_tokens is None:
+            self._start_tokens = [self.start_token] * self.n
+        return self._start_tokens
+    
+    @property
+    def end_tokens(self) -> list:
+        """List of 1 end-tokens"""
+        if self._end_tokens is None:
+            self._end_tokens = [self.end_token]
+        return self._end_tokens
+    
+    @property
+    def sentences(self) -> list:
+        """The data augmented with tags and converted to tuples"""
+        if self._sentences is None:
+            self._sentences = [tuple(self.start_tokens + sentence + self.end_tokens)
+                              for sentence in self.data]
+        return self._sentences
+    
+    @property
+    def n_grams(self) -> list:
+        """The n-grams from the data
+
+        Warning:
+        this flattens the n-grams so there isn't any sentence structure
+        """
+        if self._n_grams is None:
+            self._n_grams = chain.from_iterable([
+                [sentence[cut: cut + self.n] for cut in range(0, len(sentence) - (self.n - 1))]
+                for sentence in self.sentences
+            ])
+        return self._n_grams
+   
+    @property
+    def counts(self) -> Counter:
+        """A count of all n-grams in the data
+
+        Returns:
+          A dictionary that maps a tuple of n-words to its frequency
+        """
+        if self._counts is None:        
+            self._counts = Counter(self.n_grams)
+        return self._counts
+
+@attr.s(auto_attribs=True)
+class Tokenizer:
+    """Tokenizes string sentences
+
+    Args:
+     source: string data to tokenize
+     end_of_sentence: what to split sentences on
+
+    """
+    source: str
+    end_of_sentence: str="\n"
+    _sentences: list=None
+    _tokenized: list=None
+    _training_data: list=None
+
+    
+    @property
+    def sentences(self) -> list:
+      """The data split into sentences"""
+      if self._sentences is None:
+          self._sentences = self.source.split(self.end_of_sentence)
+          self._sentences = (sentence.strip() for sentence in self._sentences)
+          self._sentences = [sentence for sentence in self._sentences if sentence]
+      return self._sentences
+
+    @property
+    def tokenized(self) -> list:
+        """List of tokenized sentence"""
+        if self._tokenized is None:
+            self._tokenized = [nltk.word_tokenize(sentence.lower())
+                              for sentence in self.sentences]
+        return self._tokenized
+    
+    
+@attr.s(auto_attribs=True)
+class TrainTestSplit:
+    """splits up the training and testing sets
+
+    Args:
+     data: list of data to split
+     training_fraction: how much to put in the training set
+     seed: something to seed the random call
+    """
+    data: list
+    training_fraction: float=0.8
+    seed: int=87
+    _shuffled: list=None
+    _training: list=None
+    _testing: list=None
+    _split: int=None
+
+    @property
+    def shuffled(self) -> list:
+        """The data shuffled"""
+        if self._shuffled is None:
+            random.seed(self.seed)
+            self._shuffled = random.sample(self.data, k=len(self.data))
+        return self._shuffled
+
+    @property
+    def split(self) -> int:
+        """The slice value for training and testing"""
+        if self._split is None:
+            self._split = int(len(self.data) * self.training_fraction)
+        return self._split
+
+    @property
+    def training(self) -> list:
+        """The Training Portion of the Set"""
+        if self._training is None:
+            self._training = self.shuffled[0:self.split]
+        return self._training
+
+    @property
+    def testing(self) -> list:
+        """The testing data"""
+        if self._testing is None:
+            self._testing = self.shuffled[self.split:]
+        return self._testing
+def estimate_probability(word: str,
+                         previous_n_gram: tuple, 
+                         n_gram_counts: dict,
+                         n_plus1_gram_counts: dict,
+                         vocabulary_size: int,
+                         k: float=1.0) -> float:
+    """
+    Estimate the probabilities of a next word using the n-gram counts with k-smoothing
+
+    Args:
+       word: next word
+       previous_n_gram: A sequence of words of length n
+       n_gram_counts: Dictionary of counts of n-grams
+       n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
+       vocabulary_size: number of words in the vocabulary
+       k: positive constant, smoothing parameter
+
+    Returns:
+       A probability
+    """
+    previous_n_gram = tuple(previous_n_gram)
+    previous_n_gram_count = n_gram_counts.get(previous_n_gram, 0)
+
+    n_plus1_gram = previous_n_gram + (word,)  
+    n_plus1_gram_count = n_plus1_gram_counts.get(n_plus1_gram, 0)       
+    return (n_plus1_gram_count + k)/(previous_n_gram_count + k * vocabulary_size)
+
+
+def estimate_probabilities(previous_n_gram, n_gram_counts, n_plus1_gram_counts, vocabulary, k=1.0):
+    """
+    Estimate the probabilities of next words using the n-gram counts with k-smoothing
+
+    Args:
+       previous_n_gram: A sequence of words of length n
+       n_gram_counts: Dictionary of counts of (n+1)-grams
+       n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
+       vocabulary: List of words
+       k: positive constant, smoothing parameter
+
+    Returns:
+       A dictionary mapping from next words to the probability.
+    """
+
+    # convert list to tuple to use it as a dictionary key
+    previous_n_gram = tuple(previous_n_gram)
+
+    # add <e> <unk> to the vocabulary
+    # <s> is not needed since it should not appear as the next word
+    vocabulary = vocabulary + ["<e>", "<unk>"]
+    vocabulary_size = len(vocabulary)
+
+    probabilities = {}
+    for word in vocabulary:
+        probability = estimate_probability(word, previous_n_gram, 
+                                           n_gram_counts, n_plus1_gram_counts, 
+                                           vocabulary_size, k=k)
+        probabilities[word] = probability
+
+    return probabilities
+
+def suggest_a_word(previous_tokens, n_gram_counts, n_plus1_gram_counts, vocabulary, k=1.0, start_with=None):
+    """
+    Get suggestion for the next word
+
+    Args:
+       previous_tokens: The sentence you input where each token is a word. Must have length > n 
+       n_gram_counts: Dictionary of counts of (n+1)-grams
+       n_plus1_gram_counts: Dictionary of counts of (n+1)-grams
+       vocabulary: List of words
+       k: positive constant, smoothing parameter
+       start_with: If not None, specifies the first few letters of the next word
+
+    Returns:
+       A tuple of 
+         - string of the most likely next word
+         - corresponding probability
+    """
+
+    # length of previous words
+    n = len(list(n_gram_counts.keys())[0]) 
+
+    # From the words that the user already typed
+    # get the most recent 'n' words as the previous n-gram
+    previous_n_gram = previous_tokens[-n:]
+
+    # Estimate the probabilities that each word in the vocabulary
+    # is the next word,
+    # given the previous n-gram, the dictionary of n-gram counts,
+    # the dictionary of n plus 1 gram counts, and the smoothing constant
+    probabilities = estimate_probabilities(previous_n_gram,
+                                           n_gram_counts, n_plus1_gram_counts,
+                                           vocabulary, k=k)
+
+    # Initialize suggested word to None
+    # This will be set to the word with highest probability
+    suggestion = None
+
+    # Initialize the highest word probability to 0
+    # this will be set to the highest probability 
+    # of all words to be suggested
+    max_prob = 0
+
+    ### START CODE HERE (Replace instances of 'None' with your code) ###
+
+    # For each word and its probability in the probabilities dictionary:
+    for word, prob in probabilities.items(): # complete this line
+
+        # If the optional start_with string is set
+        if start_with is not None: # complete this line
+
+            # Check if the beginning of word does not match with the letters in 'start_with'
+            if not word.startswith(start_with): # complete this line
+
+                # if they don't match, skip this word (move onto the next word)
+                continue # complete this line
+
+        # Check if this word's probability
+        # is greater than the current maximum probability
+        if prob > max_prob: # complete this line
+
+            # If so, save this word as the best suggestion (so far)
+            suggestion = word
+
+            # Save the new maximum probability
+            max_prob = prob
+
+    ### END CODE HERE
+
+    return suggestion, max_prob
+
+
+def get_suggestions(previous_tokens, n_gram_counts_list, vocabulary, k=1.0, start_with=None):
+    #for i in range(model_counts-1):
+    n_gram_counts = n_gram_counts_list[0]
+    n_plus1_gram_counts = n_gram_counts_list[1]
+
+    suggestion = suggest_a_word(previous_tokens, n_gram_counts,
+                                n_plus1_gram_counts, vocabulary,
+                                k=k, start_with=start_with)
+    return " " + str(suggestion[0])
+
+

assets/I3D_modelfiles/pytorch_i3d.py

@@ -0,0 +1,354 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.autograd import Variable
+
+import numpy as np
+
+import os
+import sys
+from collections import OrderedDict
+
+
+class MaxPool3dSamePadding(nn.MaxPool3d):
+    
+    def compute_pad(self, dim, s):
+        if s % self.stride[dim] == 0:
+            return max(self.kernel_size[dim] - self.stride[dim], 0)
+        else:
+            return max(self.kernel_size[dim] - (s % self.stride[dim]), 0)
+
+    def forward(self, x):
+        # compute 'same' padding
+        (batch, channel, t, h, w) = x.size()
+        #print t,h,w
+        out_t = np.ceil(float(t) / float(self.stride[0]))
+        out_h = np.ceil(float(h) / float(self.stride[1]))
+        out_w = np.ceil(float(w) / float(self.stride[2]))
+        #print out_t, out_h, out_w
+        pad_t = self.compute_pad(0, t)
+        pad_h = self.compute_pad(1, h)
+        pad_w = self.compute_pad(2, w)
+        #print pad_t, pad_h, pad_w
+
+        pad_t_f = pad_t // 2
+        pad_t_b = pad_t - pad_t_f
+        pad_h_f = pad_h // 2
+        pad_h_b = pad_h - pad_h_f
+        pad_w_f = pad_w // 2
+        pad_w_b = pad_w - pad_w_f
+
+        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
+        #print x.size()
+        #print pad
+        x = F.pad(x, pad)
+        return super(MaxPool3dSamePadding, self).forward(x)
+    
+
+class Unit3D(nn.Module):
+
+    def __init__(self, in_channels,
+                 output_channels,
+                 kernel_shape=(1, 1, 1),
+                 stride=(1, 1, 1),
+                 padding=0,
+                 activation_fn=F.relu,
+                 use_batch_norm=True,
+                 use_bias=False,
+                 name='unit_3d'):
+        
+        """Initializes Unit3D module."""
+        super(Unit3D, self).__init__()
+        
+        self._output_channels = output_channels
+        self._kernel_shape = kernel_shape
+        self._stride = stride
+        self._use_batch_norm = use_batch_norm
+        self._activation_fn = activation_fn
+        self._use_bias = use_bias
+        self.name = name
+        self.padding = padding
+        
+        self.conv3d = nn.Conv3d(in_channels=in_channels,
+                                out_channels=self._output_channels,
+                                kernel_size=self._kernel_shape,
+                                stride=self._stride,
+                                padding=0, # we always want padding to be 0 here. We will dynamically pad based on input size in forward function
+                                bias=self._use_bias)
+        
+        if self._use_batch_norm:
+            self.bn = nn.BatchNorm3d(self._output_channels, eps=0.001, momentum=0.01)
+
+    def compute_pad(self, dim, s):
+        if s % self._stride[dim] == 0:
+            return max(self._kernel_shape[dim] - self._stride[dim], 0)
+        else:
+            return max(self._kernel_shape[dim] - (s % self._stride[dim]), 0)
+
+            
+    def forward(self, x):
+        # compute 'same' padding
+        (batch, channel, t, h, w) = x.size()
+        #print t,h,w
+        out_t = np.ceil(float(t) / float(self._stride[0]))
+        out_h = np.ceil(float(h) / float(self._stride[1]))
+        out_w = np.ceil(float(w) / float(self._stride[2]))
+        #print out_t, out_h, out_w
+        pad_t = self.compute_pad(0, t)
+        pad_h = self.compute_pad(1, h)
+        pad_w = self.compute_pad(2, w)
+        #print pad_t, pad_h, pad_w
+
+        pad_t_f = pad_t // 2
+        pad_t_b = pad_t - pad_t_f
+        pad_h_f = pad_h // 2
+        pad_h_b = pad_h - pad_h_f
+        pad_w_f = pad_w // 2
+        pad_w_b = pad_w - pad_w_f
+
+        pad = (pad_w_f, pad_w_b, pad_h_f, pad_h_b, pad_t_f, pad_t_b)
+        #print x.size()
+        #print pad
+        x = F.pad(x, pad)
+        #print x.size()        
+
+        x = self.conv3d(x)
+        if self._use_batch_norm:
+            x = self.bn(x)
+        if self._activation_fn is not None:
+            x = self._activation_fn(x)
+        return x
+
+
+
+class InceptionModule(nn.Module):
+    def __init__(self, in_channels, out_channels, name):
+        super(InceptionModule, self).__init__()
+
+        self.b0 = Unit3D(in_channels=in_channels, output_channels=out_channels[0], kernel_shape=[1, 1, 1], padding=0,
+                         name=name+'/Branch_0/Conv3d_0a_1x1')
+        self.b1a = Unit3D(in_channels=in_channels, output_channels=out_channels[1], kernel_shape=[1, 1, 1], padding=0,
+                          name=name+'/Branch_1/Conv3d_0a_1x1')
+        self.b1b = Unit3D(in_channels=out_channels[1], output_channels=out_channels[2], kernel_shape=[3, 3, 3],
+                          name=name+'/Branch_1/Conv3d_0b_3x3')
+        self.b2a = Unit3D(in_channels=in_channels, output_channels=out_channels[3], kernel_shape=[1, 1, 1], padding=0,
+                          name=name+'/Branch_2/Conv3d_0a_1x1')
+        self.b2b = Unit3D(in_channels=out_channels[3], output_channels=out_channels[4], kernel_shape=[3, 3, 3],
+                          name=name+'/Branch_2/Conv3d_0b_3x3')
+        self.b3a = MaxPool3dSamePadding(kernel_size=[3, 3, 3],
+                                stride=(1, 1, 1), padding=0)
+        self.b3b = Unit3D(in_channels=in_channels, output_channels=out_channels[5], kernel_shape=[1, 1, 1], padding=0,
+                          name=name+'/Branch_3/Conv3d_0b_1x1')
+        self.name = name
+
+    def forward(self, x):    
+        b0 = self.b0(x)
+        b1 = self.b1b(self.b1a(x))
+        b2 = self.b2b(self.b2a(x))
+        b3 = self.b3b(self.b3a(x))
+        return torch.cat([b0,b1,b2,b3], dim=1)
+
+
+class InceptionI3d(nn.Module):
+    """Inception-v1 I3D architecture.
+    The model is introduced in:
+        Quo Vadis, Action Recognition? A New Model and the Kinetics Dataset
+        Joao Carreira, Andrew Zisserman
+        https://arxiv.org/pdf/1705.07750v1.pdf.
+    See also the Inception architecture, introduced in:
+        Going deeper with convolutions
+        Christian Szegedy, Wei Liu, Yangqing Jia, Pierre Sermanet, Scott Reed,
+        Dragomir Anguelov, Dumitru Erhan, Vincent Vanhoucke, Andrew Rabinovich.
+        http://arxiv.org/pdf/1409.4842v1.pdf.
+    """
+
+    # Endpoints of the model in order. During construction, all the endpoints up
+    # to a designated `final_endpoint` are returned in a dictionary as the
+    # second return value.
+    VALID_ENDPOINTS = (
+        'Conv3d_1a_7x7',
+        'MaxPool3d_2a_3x3',
+        'Conv3d_2b_1x1',
+        'Conv3d_2c_3x3',
+        'MaxPool3d_3a_3x3',
+        'Mixed_3b',
+        'Mixed_3c',
+        'MaxPool3d_4a_3x3',
+        'Mixed_4b',
+        'Mixed_4c',
+        'Mixed_4d',
+        'Mixed_4e',
+        'Mixed_4f',
+        'MaxPool3d_5a_2x2',
+        'Mixed_5b',
+        'Mixed_5c',
+        'Logits',
+        'Predictions',
+    )
+
+    def __init__(self, num_classes=400, spatial_squeeze=True,
+                 final_endpoint='Logits', name='inception_i3d', in_channels=3, dropout_keep_prob=0.5):
+        """Initializes I3D model instance.
+        Args:
+          num_classes: The number of outputs in the logit layer (default 400, which
+              matches the Kinetics dataset).
+          spatial_squeeze: Whether to squeeze the spatial dimensions for the logits
+              before returning (default True).
+          final_endpoint: The model contains many possible endpoints.
+              `final_endpoint` specifies the last endpoint for the model to be built
+              up to. In addition to the output at `final_endpoint`, all the outputs
+              at endpoints up to `final_endpoint` will also be returned, in a
+              dictionary. `final_endpoint` must be one of
+              InceptionI3d.VALID_ENDPOINTS (default 'Logits').
+          name: A string (optional). The name of this module.
+        Raises:
+          ValueError: if `final_endpoint` is not recognized.
+        """
+
+        if final_endpoint not in self.VALID_ENDPOINTS:
+            raise ValueError('Unknown final endpoint %s' % final_endpoint)
+
+        super(InceptionI3d, self).__init__()
+        self._num_classes = num_classes
+        self._spatial_squeeze = spatial_squeeze
+        self._final_endpoint = final_endpoint
+        self.logits = None
+
+        if self._final_endpoint not in self.VALID_ENDPOINTS:
+            raise ValueError('Unknown final endpoint %s' % self._final_endpoint)
+
+        self.end_points = {}
+        end_point = 'Conv3d_1a_7x7'
+        self.end_points[end_point] = Unit3D(in_channels=in_channels, output_channels=64, kernel_shape=[7, 7, 7],
+                                            stride=(2, 2, 2), padding=(3,3,3),  name=name+end_point)
+        if self._final_endpoint == end_point: return
+        
+        end_point = 'MaxPool3d_2a_3x3'
+        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
+                                                             padding=0)
+        if self._final_endpoint == end_point: return
+        
+        end_point = 'Conv3d_2b_1x1'
+        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=64, kernel_shape=[1, 1, 1], padding=0,
+                                       name=name+end_point)
+        if self._final_endpoint == end_point: return
+        
+        end_point = 'Conv3d_2c_3x3'
+        self.end_points[end_point] = Unit3D(in_channels=64, output_channels=192, kernel_shape=[3, 3, 3], padding=1,
+                                       name=name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'MaxPool3d_3a_3x3'
+        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[1, 3, 3], stride=(1, 2, 2),
+                                                             padding=0)
+        if self._final_endpoint == end_point: return
+        
+        end_point = 'Mixed_3b'
+        self.end_points[end_point] = InceptionModule(192, [64,96,128,16,32,32], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_3c'
+        self.end_points[end_point] = InceptionModule(256, [128,128,192,32,96,64], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'MaxPool3d_4a_3x3'
+        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[3, 3, 3], stride=(2, 2, 2),
+                                                             padding=0)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_4b'
+        self.end_points[end_point] = InceptionModule(128+192+96+64, [192,96,208,16,48,64], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_4c'
+        self.end_points[end_point] = InceptionModule(192+208+48+64, [160,112,224,24,64,64], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_4d'
+        self.end_points[end_point] = InceptionModule(160+224+64+64, [128,128,256,24,64,64], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_4e'
+        self.end_points[end_point] = InceptionModule(128+256+64+64, [112,144,288,32,64,64], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_4f'
+        self.end_points[end_point] = InceptionModule(112+288+64+64, [256,160,320,32,128,128], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'MaxPool3d_5a_2x2'
+        self.end_points[end_point] = MaxPool3dSamePadding(kernel_size=[2, 2, 2], stride=(2, 2, 2),
+                                                             padding=0)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_5b'
+        self.end_points[end_point] = InceptionModule(256+320+128+128, [256,160,320,32,128,128], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Mixed_5c'
+        self.end_points[end_point] = InceptionModule(256+320+128+128, [384,192,384,48,128,128], name+end_point)
+        if self._final_endpoint == end_point: return
+
+        end_point = 'Logits'
+        self.avg_pool = nn.AvgPool3d(kernel_size=[2, 7, 7],
+                                     stride=(1, 1, 1))
+        self.dropout = nn.Dropout(dropout_keep_prob)
+        self.logits  = Unit3D(in_channels=384+384+128+128, output_channels=self._num_classes,
+                             kernel_shape=[1, 1, 1],
+                             padding=0,
+                             activation_fn=None,
+                             use_batch_norm=False,
+                             use_bias=True,
+                             name='logits')
+
+        self.build()
+
+
+    def replace_logits(self, num_classes):
+        self._num_classes = num_classes
+        self.logits = Unit3D(in_channels=384+384+128+128, output_channels=self._num_classes,
+                             kernel_shape=[1, 1, 1],
+                             padding=0,
+                             activation_fn=None,
+                             use_batch_norm=False,
+                             use_bias=True,
+                             name='logits')
+
+    def build(self):
+        for k in self.end_points.keys():
+            self.add_module(k, self.end_points[k])
+        
+    def forward(self, x, pretrained=False, n_tune_layers=-1):
+        if pretrained:
+            assert n_tune_layers >= 0
+
+            freeze_endpoints = self.VALID_ENDPOINTS[:-n_tune_layers]
+            tune_endpoints = self.VALID_ENDPOINTS[-n_tune_layers:]
+        else:
+            freeze_endpoints = []
+            tune_endpoints = self.VALID_ENDPOINTS
+
+        # backbone, no gradient part
+        with torch.no_grad():
+            for end_point in freeze_endpoints:
+                if end_point in self.end_points:
+                    x = self._modules[end_point](x) # use _modules to work with dataparallel
+
+        # backbone, gradient part
+        for end_point in tune_endpoints:
+            if end_point in self.end_points:
+                x = self._modules[end_point](x) # use _modules to work with dataparallel
+
+        # head
+        x = self.logits(self.dropout(self.avg_pool(x)))
+        if self._spatial_squeeze:
+            logits = x.squeeze(3).squeeze(3)
+        # logits is batch X time X classes, which is what we want to work with
+        return logits
+        
+
+    def extract_features(self, x):
+        for end_point in self.VALID_ENDPOINTS:
+            if end_point in self.end_points:
+                x = self._modules[end_point](x)
+        return self.avg_pool(x)

assets/I3D_modelfiles/run.py

@@ -0,0 +1,246 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Feb 25 20:10:06 2022
+
+@author: 24412
+"""
+
+import math
+import os
+import argparse
+
+import matplotlib.pyplot as plt
+
+import torch
+import torch.nn as nn
+
+
+import numpy as np
+
+import torch.nn.functional as F
+from pytorch_i3d import InceptionI3d
+
+import cv2
+from keytotext import pipeline
+
+import language
+from dotenv import load_dotenv
+
+from itertools import chain
+
+import pickle
+
+load_dotenv("posts/nlp/.env", override=True)
+
+
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = '0'
+parser = argparse.ArgumentParser()
+parser.add_argument('-mode', type=str, help='rgb or flow')
+parser.add_argument('-save_model', type=str)
+parser.add_argument('-root', type=str)
+
+args = parser.parse_args()
+
+def load_rgb_frames_from_video():
+    
+    vidcap = cv2.VideoCapture(0)
+
+    
+    frames = []
+    
+    offset = 0
+    text = " "
+    batch = 40
+    text_list = []
+    word_list = []
+    sentence = ""
+    text_count = 0
+    
+    """
+    To maintain the continous flow of actions we bring in the the batch size and offest modulo factor.
+    the batch size and the offset can be varied.
+    
+    """
+    
+    while True:
+        ret, frame1 = vidcap.read()
+        offset = offset + 1
+        font = cv2.FONT_HERSHEY_TRIPLEX
+        
+        if ret == True:
+            
+            w, h, c = frame1.shape
+            sc = 224 / w
+            sx = 224 / h
+            frame = cv2.resize(frame1, dsize=(0, 0), fx=sx, fy=sc)
+            frame1 = cv2.resize(frame1, dsize = (1280,720))
+    
+            frame = (frame / 255.) * 2 - 1
+            
+            if offset > batch:
+                frames.pop(0)
+                frames.append(frame)
+                
+                if offset % 20 == 0:
+                    text = run_on_tensor(torch.from_numpy((np.asarray(frames, dtype=np.float32)).transpose([3, 0, 1, 2])))
+                    if text != " ":
+                        text_count = text_count + 1
+                        
+                        if bool(text_list) != False and bool(word_list) != False and text_list[-1] != text and word_list[-1] != text or bool(text_list) == False:
+                            text_list.append(text)
+                            word_list.append(text)
+                            sentence = sentence + " " + text
+                            
+                        word = language.get_suggestions(text_list, n_gram_counts_list, vocabulary, k = 1.0)
+                        if(word != " ."):
+                            sentence += word
+                            text_list.append(word)
+                        
+                        if(text_count > 2):
+                            sentence = nlp(text_list,**params)
+                        cv2.putText(frame1, sentence, (120, 520), font, 0.9, (0, 255, 255), 2, cv2.LINE_4)
+
+                        
+            else:
+                frames.append(frame)
+                if offset == batch:
+                    text = run_on_tensor(torch.from_numpy((np.asarray(frames, dtype=np.float32)).transpose([3, 0, 1, 2])))
+                    if text != " ":
+                        text_count = text_count + 1
+                        if bool(text_list) != False and bool(word_list) != False and text_list[-1] != text and word_list[-1] != text or bool(text_list) == False:
+                            text_list.append(text)
+                            word_list.append(text)
+                            sentence = sentence + " " + text
+
+                        word = language.get_suggestions(text_list, n_gram_counts_list, vocabulary, k = 1.0)
+                        if(word != " ." ):
+                            sentence += word
+                            text_list.append(word)
+                            
+                        if(text_count > 2):
+                            sentence = nlp(text_list,**params)
+                        cv2.putText(frame1, sentence, (120, 520), font, 0.9, (0, 255, 255), 2, cv2.LINE_4)
+                
+        
+            if cv2.waitKey(1) & 0xFF == ord('q'):
+                break
+            
+            cv2.putText(frame1, sentence, (120, 520), font, 0.9, (0, 255, 255), 2, cv2.LINE_4)
+            cv2.imshow('frame', frame1)
+            
+            if len(text_list) > 10:
+                text_list.pop()
+                text_list.pop()
+                text_list.pop()
+            
+        else:
+            break
+            
+    vidcap.release()
+    cv2.destroyAllWindows()
+    
+
+
+def load_model(weights, num_classes):
+    
+    #Loading the Inception 3D Model
+        
+    global i3d 
+    i3d = InceptionI3d(400, in_channels=3)
+
+    i3d.replace_logits(num_classes)
+    i3d.load_state_dict(torch.load(weights))  # nslt_2000_000700.pt nslt_1000_010800 nslt_300_005100.pt(best_results)  nslt_300_005500.pt(results_reported) nslt_2000_011400
+    i3d.cuda()
+    i3d = nn.DataParallel(i3d)
+    i3d.eval()
+    
+    #Loading the KeytoText model
+    
+    global nlp
+    nlp = pipeline("k2t-new") # The pre-trained models available are 'k2t', 'k2t-base', 'mrm8488/t5-base-finetuned-common_gen', 'k2t-new'
+    global params
+    params = {"do_sample":True, "num_beams": 5, "no_repeat_ngram_size":2, "early_stopping":True}
+    
+    #Loading the NGram model
+    
+    with open("NLP/nlp_data_processed", "rb") as fp:   # Unpickling
+           train_data_processed = pickle.load(fp)
+    
+    global n_gram_counts_list
+    with open("NLP/nlp_gram_counts", "rb") as fp:   # Unpickling
+        n_gram_counts_list = pickle.load(fp)
+        
+    global vocabulary
+    vocabulary = list(set(chain.from_iterable(train_data_processed)))
+    
+    
+    load_rgb_frames_from_video()
+    
+
+def run_on_tensor(ip_tensor):
+
+    ip_tensor = ip_tensor[None, :]
+    
+    t = ip_tensor.shape[2] 
+    ip_tensor.cuda()
+    per_frame_logits = i3d(ip_tensor)
+
+    predictions = F.upsample(per_frame_logits, t, mode='linear')
+
+    predictions = predictions.transpose(2, 1)
+    out_labels = np.argsort(predictions.cpu().detach().numpy()[0])
+    arr = predictions.cpu().detach().numpy()[0] 
+
+    print(float(max(F.softmax(torch.from_numpy(arr[0]), dim=0))))
+    print(wlasl_dict[out_labels[0][-1]])
+    
+    """
+    
+    The 0.5 is threshold value, it varies if the batch sizes are reduced.
+    
+    """
+    if max(F.softmax(torch.from_numpy(arr[0]), dim=0)) > 0.5:
+        return wlasl_dict[out_labels[0][-1]]
+    else:
+        return " " 
+        
+    
+def create_WLASL_dictionary():
+    
+    global wlasl_dict 
+    wlasl_dict = {}
+    
+    with open('preprocess/wlasl_class_list.txt') as file:
+        for line in file:
+            split_list = line.split()
+            if len(split_list) != 2:
+                key = int(split_list[0])
+                value = split_list[1] + " " + split_list[2]
+            else:
+                key = int(split_list[0])
+                value = split_list[1]
+            wlasl_dict[key] = value
+            
+
+if __name__ == '__main__':
+   
+    # ================== test i3d on a dataset ==============
+    # need to add argparse
+
+    mode = 'rgb'
+    num_classes = 2000
+    save_model = './checkpoints/'
+
+    root = '../../data/WLASL2000'
+
+    train_split = 'preprocess/nslt_{}.json'.format(num_classes)
+    weights = 'archived/asl2000/FINAL_nslt_2000_iters=5104_top1=32.48_top5=57.31_top10=66.31.pt'
+    
+    
+    create_WLASL_dictionary()
+    
+    load_model(weights, num_classes)
+        
+    
+    

assets/I3D_modelfiles/wlasl_class_list.txt

@@ -0,0 +1,2000 @@
+0	book
+1	drink
+2	computer
+3	before
+4	chair
+5	go
+6	clothes
+7	who
+8	candy
+9	cousin
+10	deaf
+11	fine
+12	help
+13	no
+14	thin
+15	walk
+16	year
+17	yes
+18	all
+19	black
+20	cool
+21	finish
+22	hot
+23	like
+24	many
+25	mother
+26	now
+27	orange
+28	table
+29	thanksgiving
+30	what
+31	woman
+32	bed
+33	blue
+34	bowling
+35	can
+36	dog
+37	family
+38	fish
+39	graduate
+40	hat
+41	hearing
+42	kiss
+43	language
+44	later
+45	man
+46	shirt
+47	study
+48	tall
+49	white
+50	wrong
+51	accident
+52	apple
+53	bird
+54	change
+55	color
+56	corn
+57	cow
+58	dance
+59	dark
+60	doctor
+61	eat
+62	enjoy
+63	forget
+64	give
+65	last
+66	meet
+67	pink
+68	pizza
+69	play
+70	school
+71	secretary
+72	short
+73	time
+74	want
+75	work
+76	africa
+77	basketball
+78	birthday
+79	brown
+80	but
+81	cheat
+82	city
+83	cook
+84	decide
+85	full
+86	how
+87	jacket
+88	letter
+89	medicine
+90	need
+91	paint
+92	paper
+93	pull
+94	purple
+95	right
+96	same
+97	son
+98	tell
+99	thursday
+100	visit
+101	wait
+102	water
+103	wife
+104	yellow
+105	backpack
+106	bar
+107	brother
+108	cat
+109	check
+110	class
+111	cry
+112	different
+113	door
+114	green
+115	hair
+116	have
+117	headache
+118	inform
+119	knife
+120	laugh
+121	learn
+122	movie
+123	rabbit
+124	read
+125	red
+126	room
+127	run
+128	show
+129	sick
+130	snow
+131	take
+132	tea
+133	teacher
+134	week
+135	why
+136	with
+137	write
+138	yesterday
+139	again
+140	bad
+141	ball
+142	bathroom
+143	blanket
+144	buy
+145	call
+146	coffee
+147	cold
+148	college
+149	copy
+150	cute
+151	daughter
+152	example
+153	far
+154	first
+155	friend
+156	good
+157	happy
+158	home
+159	know
+160	late
+161	leave
+162	list
+163	losear
+164	name
+165	old
+166	person
+167	police
+168	problem
+169	remember
+170	share
+171	soon
+172	stay
+173	sunday
+174	test
+175	tired
+176	trade
+177	travel
+178	window
+179	you
+180	about
+181	approve
+182	arrive
+183	balance
+184	banana
+185	beard
+186	because
+187	boy
+188	business
+189	careful
+190	center
+191	chat
+192	children
+193	christmas
+194	clock
+195	close
+196	convince
+197	country
+198	crash
+199	day
+200	discuss
+201	dress
+202	drive
+203	drop
+204	fat
+205	feel
+206	football
+207	future
+208	game
+209	girl
+210	government
+211	hear
+212	here
+213	hope
+214	house
+215	husband
+216	interest
+217	join
+218	light
+219	live
+220	make
+221	mean
+222	more
+223	most
+224	music
+225	new
+226	none
+227	office
+228	order
+229	pants
+230	party
+231	past
+232	pencil
+233	plan
+234	please
+235	practice
+236	president
+237	restaurant
+238	ride
+239	russia
+240	salt
+241	sandwich
+242	sign
+243	since
+244	small
+245	some
+246	south
+247	student
+248	teach
+249	theory
+250	tomato
+251	train
+252	ugly
+253	war
+254	where
+255	your
+256	always
+257	animal
+258	argue
+259	baby
+260	back
+261	bake
+262	bath
+263	behind
+264	bring
+265	catch
+266	cereal
+267	champion
+268	cheese
+269	cough
+270	crazy
+271	delay
+272	delicious
+273	disappear
+274	divorce
+275	draw
+276	east
+277	easy
+278	egg
+279	environment
+280	father
+281	fault
+282	flower
+283	friendly
+284	glasses
+285	halloween
+286	hard
+287	heart
+288	hour
+289	humble
+290	hurry
+291	improve
+292	internet
+293	jump
+294	kill
+295	law
+296	match
+297	meat
+298	milk
+299	money
+300	month
+301	moon
+302	move
+303	near
+304	nephew
+305	nice
+306	niece
+307	noon
+308	north
+309	not
+310	nurse
+311	off
+312	ok
+313	patient
+314	pay
+315	perspective
+316	potato
+317	sad
+318	saturday
+319	save
+320	scissors
+321	secret
+322	shoes
+323	shop
+324	silly
+325	sister
+326	sleep
+327	sorry
+328	straight
+329	sweet
+330	talk
+331	temperature
+332	tent
+333	thank you
+334	think
+335	throw
+336	today
+337	traffic
+338	understand
+339	use
+340	voice
+341	vomit
+342	vote
+343	wednesday
+344	west
+345	wet
+346	when
+347	which
+348	win
+349	word
+350	afternoon
+351	age
+352	alone
+353	appointment
+354	australia
+355	avoid
+356	balloon
+357	basement
+358	bear
+359	believe
+360	better
+361	blind
+362	bored
+363	bowl
+364	box
+365	bracelet
+366	bread
+367	cafeteria
+368	car
+369	chicken
+370	child
+371	choose
+372	church
+373	cookie
+374	cup
+375	cut
+376	decorate
+377	deep
+378	deer
+379	dentist
+380	dirty
+381	dive
+382	down
+383	dry
+384	ear
+385	earn
+386	earring
+387	elephant
+388	english
+389	escape
+390	expensive
+391	explain
+392	fast
+393	fight
+394	find
+395	fishing
+396	floor
+397	fly
+398	follow
+399	from
+400	get
+401	happen
+402	hello
+403	hit
+404	hospital
+405	idea
+406	important
+407	investigate
+408	japan
+409	jealous
+410	king
+411	kitchen
+412	large
+413	last year
+414	lemon
+415	lettuce
+416	marry
+417	meeting
+418	minute
+419	mirror
+420	miss
+421	monkey
+422	morning
+423	motorcycle
+424	necklace
+425	never
+426	newspaper
+427	night
+428	onion
+429	people
+430	pepper
+431	phone
+432	picture
+433	plus
+434	point
+435	poor
+436	possible
+437	quiet
+438	rain
+439	ready
+440	research
+441	scared
+442	science
+443	score
+444	sentence
+445	shape
+446	sit
+447	slow
+448	snake
+449	soap
+450	soda
+451	speech
+452	star
+453	stink
+454	struggle
+455	stubborn
+456	sunset
+457	surgery
+458	there
+459	tiger
+460	toast
+461	toilet
+462	tomorrow
+463	town
+464	transfer
+465	tree
+466	truck
+467	uncle
+468	vacation
+469	weather
+470	weekend
+471	accept
+472	adult
+473	after
+474	ago
+475	allow
+476	america
+477	angel
+478	answer
+479	any
+480	area
+481	art
+482	asl
+483	aunt
+484	awful
+485	awkward
+486	bacon
+487	bark
+488	bedroom
+489	beer
+490	belt
+491	big
+492	bitter
+493	both
+494	cake
+495	california
+496	canada
+497	carrot
+498	cause
+499	challenge
+500	cheap
+501	chocolate
+502	clean
+503	cloud
+504	compare
+505	complex
+506	contact
+507	continue
+508	corner
+509	correct
+510	curse
+511	dead
+512	demand
+513	depend
+514	desert
+515	desk
+516	develop
+517	dictionary
+518	doll
+519	duty
+520	easter
+521	egypt
+522	elevator
+523	email
+524	end
+525	enter
+526	europe
+527	event
+528	exchange
+529	experience
+530	face
+531	fail
+532	feed
+533	few
+534	flag
+535	food
+536	for
+537	form
+538	freeze
+539	friday
+540	front
+541	giraffe
+542	god
+543	grammar
+544	grandfather
+545	great
+546	greece
+547	guess
+548	guitar
+549	hammer
+550	helicopter
+551	high
+552	history
+553	hungry
+554	hurt
+555	introduce
+556	invest
+557	lazy
+558	library
+559	lie
+560	lobster
+561	love
+562	lucky
+563	magazine
+564	measure
+565	microwave
+566	my
+567	neighbor
+568	number
+569	one
+570	outside
+571	peach
+572	pig
+573	polite
+574	postpone
+575	power
+576	present
+577	price
+578	prison
+579	push
+580	raccoon
+581	really
+582	reason
+583	religion
+584	respect
+585	rule
+586	salad
+587	schedule
+588	sell
+589	serious
+590	shower
+591	single
+592	smart
+593	smile
+594	soft
+595	sound
+596	soup
+597	spain
+598	spray
+599	squirrel
+600	stomach
+601	story
+602	strange
+603	street
+604	stress
+605	strict
+606	strong
+607	sugar
+608	summer
+609	suspect
+610	sweetheart
+611	tease
+612	that
+613	themselves
+614	therapy
+615	thirsty
+616	ticket
+617	top
+618	tuesday
+619	turkey
+620	university
+621	until
+622	watch
+623	weak
+624	wedding
+625	will
+626	wind
+627	world
+628	worry
+629	wow
+630	young
+631	add
+632	airplane
+633	already
+634	also
+635	analyze
+636	and
+637	angry
+638	appear
+639	arm
+640	army
+641	arrest
+642	asia
+643	ask
+644	attitude
+645	attract
+646	bald
+647	barely
+648	baseball
+649	beg
+650	benefit
+651	bite
+652	blame
+653	boat
+654	body
+655	borrow
+656	boss
+657	bother
+658	bottle
+659	bottom
+660	brag
+661	build
+662	bus
+663	butter
+664	calm
+665	cancel
+666	candle
+667	cards
+668	choice
+669	comb
+670	comfortable
+671	conflict
+672	cover
+673	deodorant
+674	dessert
+675	destroy
+676	diamond
+677	dollar
+678	drawer
+679	dream
+680	drunk
+681	duck
+682	during
+683	eagle
+684	early
+685	equal
+686	every
+687	excited
+688	exercise
+689	experiment
+690	expert
+691	fact
+692	fear
+693	feedback
+694	fold
+695	forest
+696	france
+697	frog
+698	funny
+699	garage
+700	goal
+701	golf
+702	gone
+703	gossip
+704	grandmother
+705	grapes
+706	group
+707	guilty
+708	hamburger
+709	hate
+710	head
+711	her
+712	horse
+713	ignore
+714	impossible
+715	in
+716	independent
+717	inside
+718	insurance
+719	island
+720	lecture
+721	lesson
+722	limit
+723	lion
+724	listen
+725	march
+726	math
+727	maybe
+728	mechanic
+729	mom
+730	mouse
+731	much
+732	muscle
+733	museum
+734	must
+735	mustache
+736	negative
+737	nervous
+738	neutral
+739	nose
+740	often
+741	operate
+742	opinion
+743	other
+744	pass
+745	pillow
+746	prepare
+747	pretty
+748	priest
+749	program
+750	promise
+751	put
+752	queen
+753	quit
+754	radio
+755	rat
+756	reject
+757	require
+758	rest
+759	retire
+760	rise
+761	river
+762	rough
+763	see
+764	seem
+765	send
+766	sew
+767	sheep
+768	shine
+769	shock
+770	should
+771	silent
+772	skinny
+773	skirt
+774	sky
+775	sour
+776	special
+777	spirit
+778	staff
+779	stand
+780	steal
+781	stop
+782	stupid
+783	summon
+784	sunrise
+785	swallow
+786	television
+787	to
+788	together
+789	translate
+790	triangle
+791	turtle
+792	underwear
+793	up
+794	upset
+795	violin
+796	volunteer
+797	warm
+798	warn
+799	wash
+800	wine
+801	witness
+802	wood
+803	zero
+804	across
+805	actor
+806	agree
+807	alarm
+808	allergy
+809	almost
+810	announce
+811	apartment
+812	attention
+813	audience
+814	august
+815	autumn
+816	away
+817	beautiful
+818	become
+819	below
+820	best
+821	bet
+822	bicycle
+823	biology
+824	boyfriend
+825	break
+826	bridge
+827	brush
+828	building
+829	busy
+830	camera
+831	camp
+832	caption
+833	care
+834	carry
+835	celebrate
+836	certificate
+837	chain
+838	chance
+839	character
+840	chase
+841	classroom
+842	clear
+843	climb
+844	command
+845	community
+846	complain
+847	compromise
+848	contribute
+849	cop
+850	cost
+851	couch
+852	cracker
+853	curious
+854	curriculum
+855	dad
+856	deny
+857	describe
+858	diaper
+859	diarrhea
+860	disagree
+861	dissolve
+862	divide
+863	dolphin
+864	double
+865	doubt
+866	dumb
+867	earth
+868	earthquake
+869	eight
+870	embarrass
+871	emotion
+872	encourage
+873	energy
+874	enough
+875	evaluate
+876	evidence
+877	exact
+878	exaggerate
+879	expect
+880	fancy
+881	favorite
+882	finally
+883	fox
+884	french
+885	fruit
+886	function
+887	gather
+888	germany
+889	gloves
+890	goat
+891	goodbye
+892	grow
+893	gum
+894	half
+895	hawaii
+896	herself
+897	highway
+898	homework
+899	honor
+900	ice cream
+901	if
+902	increase
+903	india
+904	information
+905	interpret
+906	interview
+907	invite
+908	israel
+909	jesus
+910	keep
+911	keyboard
+912	land
+913	lawyer
+914	lead
+915	lesbian
+916	line
+917	lock
+918	loud
+919	lousy
+920	lunch
+921	machine
+922	mad
+923	memorize
+924	minus
+925	monday
+926	mountain
+927	mouth
+928	mushroom
+929	napkin
+930	neck
+931	next
+932	ocean
+933	open
+934	overwhelm
+935	owe
+936	page
+937	pain
+938	parade
+939	parents
+940	perfect
+941	period
+942	philosophy
+943	photographer
+944	place
+945	policy
+946	popular
+947	pray
+948	predict
+949	presentation
+950	print
+951	printer
+952	private
+953	procrastinate
+954	project
+955	protect
+956	prove
+957	provide
+958	psychology
+959	punish
+960	puzzled
+961	question
+962	quick
+963	rainbow
+964	rake
+965	rehearse
+966	remove
+967	revenge
+968	review
+969	rich
+970	roof
+971	rooster
+972	rude
+973	say
+974	scientist
+975	scotland
+976	screwdriver
+977	second
+978	senate
+979	separate
+980	service
+981	shrimp
+982	shy
+983	side
+984	situation
+985	slave
+986	soccer
+987	socks
+988	solid
+989	sometimes
+990	spider
+991	spin
+992	square
+993	stairs
+994	stare
+995	start
+996	store
+997	straw
+998	structure
+999	suggest
+1000	sun
+1001	support
+1002	suppose
+1003	sure
+1004	surprise
+1005	sweater
+1006	swim
+1007	symbol
+1008	taste
+1009	team
+1010	telephone
+1011	tennis
+1012	their
+1013	then
+1014	thermometer
+1015	thing
+1016	third
+1017	thousand
+1018	three
+1019	tie
+1020	topic
+1021	touch
+1022	tournament
+1023	try
+1024	two
+1025	type
+1026	umbrella
+1027	vegetable
+1028	vocabulary
+1029	waste
+1030	we
+1031	wear
+1032	weekly
+1033	welcome
+1034	winter
+1035	without
+1036	wolf
+1037	worm
+1038	wristwatch
+1039	yourself
+1040	above
+1041	accomplish
+1042	adopt
+1043	advantage
+1044	alphabet
+1045	anniversary
+1046	another
+1047	appropriate
+1048	arrogant
+1049	artist
+1050	background
+1051	bee
+1052	behavior
+1053	bell
+1054	birth
+1055	bless
+1056	blood
+1057	boots
+1058	brave
+1059	breathe
+1060	bright
+1061	bull
+1062	butterfly
+1063	card
+1064	category
+1065	catholic
+1066	cent
+1067	chemistry
+1068	cherry
+1069	china
+1070	chop
+1071	christian
+1072	cigarette
+1073	clever
+1074	clown
+1075	coach
+1076	coat
+1077	cochlear implant
+1078	coconut
+1079	common
+1080	commute
+1081	control
+1082	count
+1083	culture
+1084	daily
+1085	defend
+1086	degree
+1087	demonstrate
+1088	department
+1089	die
+1090	dig
+1091	dinner
+1092	dinosaur
+1093	diploma
+1094	director
+1095	disconnect
+1096	discover
+1097	don't want
+1098	drum
+1099	each
+1100	educate
+1101	electrician
+1102	empty
+1103	england
+1104	establish
+1105	excuse
+1106	eyeglasses
+1107	faculty
+1108	fall in love
+1109	famous
+1110	farm
+1111	fence
+1112	fix
+1113	flexible
+1114	flirt
+1115	flood
+1116	fork
+1117	four
+1118	fun
+1119	funeral
+1120	furniture
+1121	gallaudet
+1122	general
+1123	ghost
+1124	give up
+1125	glass
+1126	gorilla
+1127	gray
+1128	guide
+1129	habit
+1130	health
+1131	hearing aid
+1132	heaven
+1133	heavy
+1134	helmet
+1135	hide
+1136	high school
+1137	hockey
+1138	hold
+1139	honest
+1140	hug
+1141	hunt
+1142	image
+1143	influence
+1144	interesting
+1145	interpreter
+1146	iran
+1147	italy
+1148	jewish
+1149	judge
+1150	key
+1151	label
+1152	leaf
+1153	left
+1154	lend
+1155	less
+1156	linguistics
+1157	lonely
+1158	long
+1159	magic
+1160	mainstream
+1161	major
+1162	manager
+1163	maximum
+1164	me
+1165	mexico
+1166	middle
+1167	mind
+1168	misunderstand
+1169	monster
+1170	multiply
+1171	myself
+1172	nothing
+1173	october
+1174	odd
+1175	offer
+1176	on
+1177	only
+1178	opposite
+1179	out
+1180	overlook
+1181	path
+1182	percent
+1183	physics
+1184	piano
+1185	pick
+1186	pie
+1187	plate
+1188	pocket
+1189	popcorn
+1190	positive
+1191	pregnant
+1192	prevent
+1193	proof
+1194	pumpkin
+1195	purpose
+1196	race
+1197	realize
+1198	recognize
+1199	refuse
+1200	relationship
+1201	repeat
+1202	replace
+1203	report
+1204	represent
+1205	request
+1206	responsibility
+1207	result
+1208	reveal
+1209	ring
+1210	rob
+1211	rock
+1212	role
+1213	rope
+1214	rush
+1215	salute
+1216	satisfy
+1217	search
+1218	selfish
+1219	sensitive
+1220	serve
+1221	seven
+1222	shampoo
+1223	she
+1224	shelf
+1225	shopping
+1226	sign language
+1227	similar
+1228	sing
+1229	six
+1230	skeleton
+1231	sketch
+1232	skill
+1233	sleepy
+1234	slip
+1235	smell
+1236	sneeze
+1237	snob
+1238	specific
+1239	stamp
+1240	steel
+1241	strawberry
+1242	subtract
+1243	subway
+1244	suffer
+1245	surface
+1246	sweep
+1247	swing
+1248	switzerland
+1249	tale
+1250	tan
+1251	teeth
+1252	terrible
+1253	thankful
+1254	they
+1255	through
+1256	tiptoe
+1257	title
+1258	tooth
+1259	toothbrush
+1260	total
+1261	tough
+1262	tradition
+1263	trophy
+1264	trouble
+1265	true
+1266	trust
+1267	under
+1268	verb
+1269	wall
+1270	watermelon
+1271	way
+1272	weird
+1273	while
+1274	wide
+1275	willing
+1276	wish
+1277	wonderful
+1278	workshop
+1279	worse
+1280	wrench
+1281	a
+1282	a lot
+1283	abdomen
+1284	able
+1285	accountant
+1286	action
+1287	active
+1288	activity
+1289	address
+1290	affect
+1291	afraid
+1292	against
+1293	agenda
+1294	ahead
+1295	aim
+1296	alcohol
+1297	algebra
+1298	all day
+1299	amazing
+1300	angle
+1301	apart
+1302	apostrophe
+1303	appetite
+1304	appreciate
+1305	april
+1306	archery
+1307	around
+1308	article
+1309	assistant
+1310	attend
+1311	auction
+1312	authority
+1313	average
+1314	awake
+1315	b
+1316	babysitter
+1317	battle
+1318	beginning
+1319	belief
+1320	bible
+1321	bike
+1322	blend
+1323	bone
+1324	bra
+1325	brief
+1326	broke
+1327	bug
+1328	bully
+1329	button
+1330	cabbage
+1331	cabinet
+1332	calculate
+1333	calculator
+1334	calculus
+1335	camping
+1336	canoe
+1337	careless
+1338	ceiling
+1339	cemetery
+1340	chapter
+1341	choir
+1342	circle
+1343	come
+1344	come here
+1345	company
+1346	complete
+1347	concept
+1348	concern
+1349	confront
+1350	confused
+1351	congress
+1352	connect
+1353	conquer
+1354	constitution
+1355	contract
+1356	court
+1357	crab
+1358	cross
+1359	cruel
+1360	crush
+1361	date
+1362	december
+1363	decrease
+1364	deduct
+1365	defeat
+1366	deposit
+1367	depressed
+1368	detach
+1369	determine
+1370	diabetes
+1371	dime
+1372	dirt
+1373	disgust
+1374	dismiss
+1375	dizzy
+1376	document
+1377	dorm
+1378	dormitory
+1379	downstairs
+1380	economy
+1381	education
+1382	effort
+1383	eighteen
+1384	engage
+1385	engagement
+1386	engineer
+1387	envelope
+1388	erase
+1389	eternity
+1390	evening
+1391	everyday
+1392	expand
+1393	eye
+1394	eyes
+1395	fake
+1396	farmer
+1397	february
+1398	federal
+1399	final
+1400	florida
+1401	flute
+1402	forbid
+1403	forever
+1404	fourth
+1405	free
+1406	french fries
+1407	gamble
+1408	gas
+1409	gasoline
+1410	generation
+1411	geography
+1412	german
+1413	girlfriend
+1414	gold
+1415	grandma
+1416	grass
+1417	grow up
+1418	gun
+1419	haircut
+1420	hard of hearing
+1421	hippopotamus
+1422	his
+1423	honey
+1424	hurricane
+1425	identify
+1426	include
+1427	infection
+1428	innocent
+1429	insect
+1430	instead
+1431	institute
+1432	international
+1433	interrupt
+1434	involve
+1435	jail
+1436	january
+1437	joke
+1438	june
+1439	kangaroo
+1440	karate
+1441	kick
+1442	kneel
+1443	knock
+1444	laptop
+1445	lift
+1446	lightning
+1447	lip
+1448	lipstick
+1449	local
+1450	manage
+1451	mature
+1452	member
+1453	message
+1454	metal
+1455	microphone
+1456	mix
+1457	monthly
+1458	motor
+1459	nation
+1460	network
+1461	nickel
+1462	nineteen
+1463	normal
+1464	november
+1465	nut
+1466	octopus
+1467	once
+1468	organize
+1469	over
+1470	owl
+1471	p
+1472	parachute
+1473	paragraph
+1474	parallel
+1475	pay attention
+1476	peace
+1477	peanut butter
+1478	perfume
+1479	personality
+1480	pet
+1481	philadelphia
+1482	physician
+1483	piece
+1484	pity
+1485	plant
+1486	plenty
+1487	pneumonia
+1488	politics
+1489	position
+1490	pound
+1491	pour
+1492	praise
+1493	preach
+1494	preacher
+1495	prefer
+1496	pressure
+1497	principal
+1498	priority
+1499	professional
+1500	professor
+1501	promote
+1502	prostitute
+1503	proud
+1504	quote
+1505	receive
+1506	recent
+1507	reduce
+1508	refer
+1509	referee
+1510	relate
+1511	relax
+1512	remote control
+1513	rent
+1514	reputation
+1515	resign
+1516	responsible
+1517	ridiculous
+1518	road
+1519	robot
+1520	rubber
+1521	safe
+1522	sauce
+1523	sausage
+1524	scold
+1525	senior
+1526	september
+1527	several
+1528	shame
+1529	shave
+1530	shoot
+1531	shoulder
+1532	silver
+1533	simple
+1534	siren
+1535	size
+1536	skate
+1537	skin
+1538	skunk
+1539	slice
+1540	smooth
+1541	snack
+1542	snowman
+1543	society
+1544	someone
+1545	song
+1546	sore throat
+1547	south america
+1548	spanish
+1549	speak
+1550	speed
+1551	spell
+1552	spoon
+1553	spring
+1554	standard
+1555	statistics
+1556	sticky
+1557	still
+1558	stretch
+1559	surgeon
+1560	sweden
+1561	swimsuit
+1562	sympathy
+1563	take turns
+1564	take up
+1565	technology
+1566	temple
+1567	tender
+1568	thailand
+1569	theater
+1570	them
+1571	theme
+1572	thick
+1573	this
+1574	throat
+1575	tissue
+1576	tongue
+1577	tonight
+1578	tornado
+1579	tower
+1580	tranquil
+1581	transform
+1582	tutor
+1583	twin
+1584	united states
+1585	value
+1586	visitor
+1587	waiter
+1588	wake up
+1589	wallet
+1590	wander
+1591	wash face
+1592	weight
+1593	whale
+1594	whatever
+1595	within
+1596	wonder
+1597	worker
+1598	worthless
+1599	wrap
+1600	accent
+1601	act
+1602	adapt
+1603	adjective
+1604	adjust
+1605	admire
+1606	admit
+1607	advanced
+1608	adverb
+1609	agreement
+1610	aid
+1611	alligator
+1612	amputate
+1613	anatomy
+1614	annoy
+1615	anyway
+1616	approach
+1617	arizona
+1618	assist
+1619	assume
+1620	attorney
+1621	audiologist
+1622	audiology
+1623	austria
+1624	author
+1625	available
+1626	award
+1627	aware
+1628	bank
+1629	baptize
+1630	basic
+1631	battery
+1632	berry
+1633	beside
+1634	between
+1635	binoculars
+1636	blow
+1637	boast
+1638	boil
+1639	bookshelf
+1640	bookstore
+1641	boxing
+1642	braid
+1643	brain
+1644	breakdown
+1645	breakfast
+1646	breeze
+1647	bribe
+1648	brochure
+1649	buffalo
+1650	burp
+1651	bush
+1652	bye
+1653	camel
+1654	candidate
+1655	cannot
+1656	captain
+1657	carnival
+1658	caterpillar
+1659	cheerleader
+1660	chemical
+1661	chicago
+1662	choke
+1663	christ
+1664	click
+1665	closet
+1666	clueless
+1667	clumsy
+1668	collect
+1669	comma
+1670	comment
+1671	commit
+1672	committee
+1673	common sense
+1674	compete
+1675	concentrate
+1676	congratulations
+1677	consider
+1678	construct
+1679	consume
+1680	contest
+1681	conversation
+1682	convert
+1683	cooperate
+1684	counsel
+1685	counselor
+1686	crave
+1687	create
+1688	crocodile
+1689	crown
+1690	cuba
+1691	curtain
+1692	customer
+1693	d
+1694	damage
+1695	dancer
+1696	danger
+1697	dangerous
+1698	dawn
+1699	death
+1700	debate
+1701	debt
+1702	deliver
+1703	democrat
+1704	descend
+1705	design
+1706	detective
+1707	devil
+1708	dice
+1709	difficult
+1710	dining room
+1711	discipline
+1712	discount
+1713	disgusted
+1714	disturb
+1715	division
+1716	done
+1717	drag
+1718	dragon
+1719	drama
+1720	drug
+1721	due
+1722	dull
+1723	dusk
+1724	dvd
+1725	dye
+1726	e
+1727	either
+1728	electricity
+1729	elementary
+1730	else
+1731	emergency
+1732	engine
+1733	enormous
+1734	eraser
+1735	every monday
+1736	every tuesday
+1737	everything
+1738	except
+1739	exhibit
+1740	explode
+1741	express
+1742	f
+1743	fairy
+1744	familiar
+1745	festival
+1746	finance
+1747	fingerspell
+1748	fire
+1749	firefighter
+1750	five
+1751	flatter
+1752	fool
+1753	forgive
+1754	former
+1755	freeway
+1756	from now on
+1757	g
+1758	gang
+1759	gay
+1760	geometry
+1761	get up
+1762	gift
+1763	grab
+1764	graduation
+1765	grateful
+1766	grey
+1767	gymnastics
+1768	h
+1769	hang up
+1770	hanukkah
+1771	heap
+1772	heart attack
+1773	hill
+1774	holy
+1775	hop
+1776	host
+1777	hot dog
+1778	human
+1779	i
+1780	ill
+1781	illegal
+1782	impact
+1783	individual
+1784	inspect
+1785	inspire
+1786	intersection
+1787	ireland
+1788	iron
+1789	j
+1790	jewelry
+1791	journey
+1792	joy
+1793	july
+1794	k
+1795	kid
+1796	kindergarten
+1797	lady
+1798	lamp
+1799	last week
+1800	laundry
+1801	leader
+1802	league
+1803	leak
+1804	legal
+1805	let
+1806	liability
+1807	librarian
+1808	license
+1809	little bit
+1810	loan
+1811	look at
+1812	look for
+1813	lord
+1814	m
+1815	meaning
+1816	melody
+1817	melt
+1818	mention
+1819	microscope
+1820	midnight
+1821	military
+1822	mine
+1823	mistake
+1824	mock
+1825	moose
+1826	mosquito
+1827	motivate
+1828	murder
+1829	n
+1830	narrow
+1831	necessary
+1832	negotiate
+1833	new york
+1834	nine
+1835	northwest
+1836	not yet
+1837	notice
+1838	numerous
+1839	nun
+1840	o
+1841	objective
+1842	obsess
+1843	obtain
+1844	occur
+1845	odor
+1846	olympics
+1847	or
+1848	oral
+1849	our
+1850	overcome
+1851	pack
+1852	painter
+1853	part
+1854	pause
+1855	peaceful
+1856	pear
+1857	peel
+1858	penalty
+1859	pennsylvania
+1860	penny
+1861	permit
+1862	phrase
+1863	pickle
+1864	pilot
+1865	player
+1866	polar bear
+1867	policeman
+1868	poop
+1869	post
+1870	potential
+1871	poverty
+1872	precious
+1873	precipitation
+1874	precise
+1875	preschool
+1876	pride
+1877	prince
+1878	princess
+1879	principle
+1880	process
+1881	profit
+1882	progress
+1883	propaganda
+1884	proper
+1885	psychologist
+1886	public
+1887	publish
+1888	purchase
+1889	pure
+1890	pursue
+1891	put off
+1892	q
+1893	quality
+1894	quarrel
+1895	quarter
+1896	r
+1897	rage
+1898	rather
+1899	real
+1900	recliner
+1901	recommend
+1902	recover
+1903	regular
+1904	release
+1905	relief
+1906	rely
+1907	reply
+1908	rescue
+1909	resist
+1910	restroom
+1911	retreat
+1912	roar
+1913	robber
+1914	roommate
+1915	rose
+1916	ruin
+1917	s
+1918	salary
+1919	saw
+1920	scan
+1921	scream
+1922	sculpture
+1923	sea
+1924	seldom
+1925	sequence
+1926	settle
+1927	shout
+1928	shovel
+1929	sin
+1930	singer
+1931	sixteen
+1932	ski
+1933	skip
+1934	smoking
+1935	sofa
+1936	soldier
+1937	solve
+1938	someday
+1939	something
+1940	somewhere
+1941	soul
+1942	spill
+1943	spit
+1944	spread
+1945	sprint
+1946	squeeze
+1947	stadium
+1948	stepfather
+1949	sting
+1950	stir
+1951	stitch
+1952	stuck
+1953	sue
+1954	sunshine
+1955	superman
+1956	surrender
+1957	suspend
+1958	swimming
+1959	t
+1960	talent
+1961	telescope
+1962	tempt
+1963	ten
+1964	tend
+1965	testify
+1966	texas
+1967	text
+1968	than
+1969	therefore
+1970	thrill
+1971	tobacco
+1972	toilet paper
+1973	tolerate
+1974	torture
+1975	towel
+1976	trip
+1977	truth
+1978	turn
+1979	tv
+1980	u
+1981	unique
+1982	upstairs
+1983	v
+1984	vacant
+1985	vague
+1986	valley
+1987	vampire
+1988	very
+1989	vice president
+1990	viewpoint
+1991	visualize
+1992	vlog
+1993	volleyball
+1994	w
+1995	washington
+1996	waterfall
+1997	weigh
+1998	wheelchair
+1999	whistle

download_assets.py

@@ -0,0 +1,32 @@
+import os
+import gdown
+import zipfile
+
+DATASET_PATH = "assets/text_to_sign/wlasl2000_filtered_keypoints"
+
+# Google Drive file ID
+FILE_ID = "1l8pguR-nfbo5kLPua9wirHVPqiG1BE5h"
+
+# Direct download URL
+URL = f"https://drive.google.com/uc?id={FILE_ID}"
+
+ZIP_PATH = "wlasl2000_filtered_keypoints.zip"
+
+# Download only if dataset doesn't exist
+if not os.path.exists(DATASET_PATH):
+    print("Downloading dataset...")
+
+    gdown.download(URL, ZIP_PATH, quiet=False)
+
+    print("Extracting dataset...")
+
+    with zipfile.ZipFile(ZIP_PATH, "r") as zip_ref:
+        zip_ref.extractall("assets/text_to_sign")
+
+    os.remove(ZIP_PATH)
+
+    print("Dataset ready ✅")
+
+else:
+    print("Dataset already exists ✅")
+    

nixpacks.toml

@@ -0,0 +1,2 @@
+[phases.setup]
+nixPkgs = ["mesa", "glib"]

pipelines/sign_to_text/run.py

@@ -0,0 +1,186 @@
+import cv2
+import torch
+import torch.nn.functional as F
+import numpy as np
+import os
+
+from app.core.config import I3D_WEIGHTS, CLASS_LIST
+from app.models.i3d_model import I3DModel
+from app.utils.mediapipe_utils import hand_detector_instance
+
+# ---------------- CONFIG ----------------
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+clip_length = 64
+MIN_PAUSE_FRAMES = 5
+MIN_SIGN_FRAMES = 10
+
+# ---------------- MODEL ----------------
+i3d_model = None
+
+
+def get_i3d_model():
+    global i3d_model
+
+    if i3d_model is None:
+        print("Loading I3D model...")
+        i3d_model = I3DModel(I3D_WEIGHTS, device)
+
+    return i3d_model
+
+with open(CLASS_LIST, "r") as f:
+    gloss_list = [line.strip().upper() for line in f.readlines()]
+
+
+# ---------------- VIDEO LOADING ----------------
+def load_video(video_path):
+    cap = cv2.VideoCapture(video_path)
+
+    if not cap.isOpened():
+        raise ValueError(f"Cannot open video: {video_path}")
+
+    frames = []
+    hand_flags = []
+
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+
+       #print("FRAME SHAPE:", frame.shape)
+
+        resized = cv2.resize(frame, (224, 224))
+        norm = (resized.astype(np.float32) / 255.0) * 2 - 1
+
+        frames.append(norm)
+        hand_flags.append(hand_detector_instance.detect(frame))
+
+    cap.release()
+
+    frames = np.array(frames)
+    print("TOTAL FRAMES READ:", len(frames))
+
+    return frames, hand_flags
+
+
+# ---------------- SEGMENTATION ----------------
+def segment_frames(frames, hand_flags):
+    segments = []
+    start = 0
+    pause = 0
+
+    for i in range(len(frames)):
+        if hand_flags[i]:
+            pause += 1
+        else:
+            if pause >= MIN_PAUSE_FRAMES:
+                end = i - pause
+                if end - start >= MIN_SIGN_FRAMES:
+                    segments.append((start, end))
+                start = i
+            pause = 0
+
+    if len(frames) - start >= MIN_SIGN_FRAMES:
+        segments.append((start, len(frames) - 1))
+
+    return segments
+
+
+# ---------------- INFERENCE ----------------
+def predict_segment(segment_frames, topk=5):
+
+    if len(segment_frames) < clip_length:
+        return []
+
+    results_accum = {}
+
+    # 🔥 FIX 1: proper sliding window (NOT naive chunking)
+    stride = clip_length // 2
+
+    for start in range(0, len(segment_frames) - clip_length, stride):
+
+        clip = segment_frames[start:start + clip_length]
+
+        # safety check
+        if clip.shape[0] != clip_length:
+            continue
+
+        clip = clip.transpose(3, 0, 1, 2)
+        clip_tensor = torch.from_numpy(clip).unsqueeze(0).to(device)
+
+        with torch.no_grad():
+            model = get_i3d_model()
+            logits = model(clip_tensor)
+            logits = torch.mean(logits, dim=2)
+            probs = F.softmax(logits, dim=1)
+
+            top_probs, top_idx = torch.topk(probs, k=topk, dim=1)
+
+        # 🔥 FIX 2: accumulate scores properly
+        for i, p in zip(top_idx[0], top_probs[0]):
+            idx = int(i)
+            if idx < len(gloss_list):
+                word = gloss_list[idx]
+
+                if word not in results_accum:
+                    results_accum[word] = 0
+
+                results_accum[word] += float(p)
+
+    if not results_accum:
+        return []
+
+    # 🔥 FIX 3: sort final results
+    sorted_results = sorted(results_accum.items(), key=lambda x: x[1], reverse=True)
+
+    return sorted_results[:topk]
+
+
+# ---------------- SELECTION ----------------
+def select_best(results, context):
+    if not results:
+        return ""
+
+    return results[0][0]
+
+
+# ---------------- PIPELINE ----------------
+def run_pipeline(video_path):
+
+    print("\nSTARTING PIPELINE")
+
+    frames, hand_flags = load_video(video_path)
+
+    print("HAND FLAGS SAMPLE:", hand_flags[:20])
+    print("TRUE COUNT:", sum(hand_flags))
+
+    segments = segment_frames(frames, hand_flags)
+
+    print("SEGMENTS:", segments)
+
+    final_glosses = []
+
+    for s, e in segments:
+        segment = frames[s:e]
+
+        results = predict_segment(segment)
+
+        if not results:
+            continue
+
+        selected = select_best(results, final_glosses)
+        final_glosses.append(selected)
+
+    gloss_sentence = " ".join(final_glosses)
+
+    print("\nGLOSS:", gloss_sentence)
+
+    # IMPORTANT: no API test yet
+    english = "API NOT RUN (testing logic only)"
+
+    print("ENGLISH:", english)
+
+    return {
+        "gloss": gloss_sentence,
+        "english": english
+    }

pipelines/text_to_sign/assets_loader.py

@@ -0,0 +1,28 @@
+import json
+import numpy as np
+from pathlib import Path
+from .config import JSON_PATH, LANDMARKS_PATH
+
+print("=" * 60)
+print("Current working directory:", Path.cwd())
+print("JSON_PATH:", JSON_PATH)
+print("JSON exists:", Path(JSON_PATH).exists())
+print("LANDMARKS_PATH:", LANDMARKS_PATH)
+print("LANDMARKS exists:", Path(LANDMARKS_PATH).exists())
+print("=" * 60)
+
+
+def load_landmarks():
+    with open(JSON_PATH) as f:
+        data = json.load(f)
+
+    landmarks_data = np.load(LANDMARKS_PATH, allow_pickle=True)
+    landmarks_array = landmarks_data["landmarks"]
+
+    landmarks_dict = {}
+
+    for sample, lm in zip(data, landmarks_array):
+        gloss = sample["gloss"].lower().strip()
+        landmarks_dict[gloss] = lm
+
+    return landmarks_dict

pipelines/text_to_sign/config.py

@@ -0,0 +1,31 @@
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parents[2]
+
+JSON_PATH = BASE_DIR / "assets" / "text_to_sign" / "filtered_one_video_per_gloss.json"
+
+LANDMARKS_PATH = (
+    BASE_DIR
+    / "assets"
+    / "text_to_sign"
+    / "wlasl2000_filtered_keypoints"
+    / "filtered_landmarks_V3"
+    / "filtered_landmarks_V3.npz"
+)
+FPS = 25
+WIDTH = 1280
+HEIGHT = 720
+
+POSE_CONNECTIONS = [
+    (11,13),(13,15),(12,14),(14,16),(11,12),
+    (23,24),(11,23),(12,24),(23,25),(24,26),
+    (25,27),(26,28)
+]
+
+FINGER_COLORS = {
+    "thumb": (0,128,255),
+    "index": (0,255,0),
+    "middle": (255,255,0),
+    "ring": (255,0,0),
+    "pinky": (255,0,255)
+}

pipelines/text_to_sign/generator.py

@@ -0,0 +1,40 @@
+import re
+
+import numpy as np
+from .assets_loader import load_landmarks
+
+landmarks_dict = None
+
+def get_landmarks():
+    global landmarks_dict
+
+    if landmarks_dict is None:
+        print("Loading landmarks...")
+        landmarks_dict = load_landmarks()
+
+    return landmarks_dict
+
+def generate_frames(mapped_sequence):
+
+    landmarks = get_landmarks()
+
+    all_frames = []
+
+    for item in mapped_sequence:
+
+        if item["type"] == "direct":
+            word = item["mapped"].lower()
+
+            if word in landmarks:
+                frames = [np.array(f, dtype=float) for f in landmarks[word]]
+                all_frames.extend(frames)
+
+        else:
+            word = re.sub(r"[^a-z]", "", item["original"].lower())
+
+            for letter in word:
+                if letter in landmarks:
+                    frames = [np.array(f, dtype=float) for f in landmarks[letter]]
+                    all_frames.extend(frames)
+
+    return all_frames

pipelines/text_to_sign/language.py

@@ -0,0 +1,101 @@
+import re
+import nltk
+from nltk.stem import WordNetLemmatizer
+from groq import Groq
+import os
+from dotenv import load_dotenv
+load_dotenv()
+
+
+try:
+    nltk.data.find("corpora/wordnet")
+except LookupError:
+    nltk.download("wordnet")
+    nltk.download("omw-1.4")
+
+lemmatizer = WordNetLemmatizer()
+client = Groq(api_key=os.getenv("GROQ_API_KEY"))
+
+
+# -----------------------------
+# CLEAN LLM OUTPUT
+# -----------------------------
+def clean_llm_output(text):
+    text = text.strip()
+    text = text.split("\n")[0]
+    text = re.sub(r'[^A-Z0-9\s]', '', text.upper())
+    return text.strip()
+
+
+# -----------------------------
+# CLEAN GLOSS (SAFE)
+# -----------------------------
+def clean_gloss(gloss):
+    gloss = gloss.lower()
+    gloss = re.sub(r'[^a-z\s]', '', gloss)
+    gloss = re.sub(r'\s+', ' ', gloss)
+    return gloss.strip()
+
+# -----------------------------
+# MAP WORD TO VOCAB
+# -----------------------------
+def map_word(word, vocab_set):
+    word = word.lower()
+
+    if word in vocab_set:
+        return {"original": word, "mapped": word, "type": "direct"}
+
+    # fallback → fingerspelling
+    return {"original": word, "mapped": None, "type": "fingerspell"}
+
+
+# -----------------------------
+# LEMMATIZE
+# -----------------------------
+def lemmatize_words(words):
+    return [lemmatizer.lemmatize(w.lower()) for w in words]
+
+
+# -----------------------------
+# LLM → GLOSS (FIXED)
+# -----------------------------
+def llm_text_to_gloss(sentence):
+
+    prompt = f"""
+You are an ASL gloss translator.
+
+TASK:
+Convert the sentence into ASL Gloss using ONLY valid vocabulary words.
+
+STRICT RULES:
+1. Output ONLY space-separated words (NO hyphens, NO merging words)
+2. Do NOT create new words from vocabulary
+3. Use ONLY words from the vocabulary list
+4. Use uppercase only
+5. Remove: IS, AM, ARE, TO, THE, A, AN
+6. Keep natural ASL structure (time → topic → action when possible)
+7. Output ONE single line only
+
+FINGERSPELLING RULE:
+- If a word is NOT in the vocabulary list:
+  → spell it letter by letter separated by spaces
+  → example: "john" → J O H N
+
+INPUT:
+{sentence}
+
+
+OUTPUT:
+"""
+
+    chat = client.chat.completions.create(
+        model="llama-3.1-8b-instant",
+        messages=[
+            {"role": "system", "content": "You output ONLY valid ASL gloss using given vocabulary."},
+            {"role": "user", "content": prompt}
+        ],
+        temperature=0.1
+    )
+
+    return clean_llm_output(chat.choices[0].message.content)
+

pipelines/text_to_sign/renderer.py

@@ -0,0 +1,134 @@
+import cv2
+import numpy as np
+from .config import WIDTH, HEIGHT, FPS, POSE_CONNECTIONS
+
+# =========================
+# COLORS
+# =========================
+WHITE = (255, 255, 255)
+
+# #190124 in HEX
+# RGB = (25, 1, 36)
+# OpenCV uses BGR
+BACKGROUND = (36, 1, 25)
+
+# =========================
+# HAND
+# =========================
+def draw_hand_stylish(img, hand_points, thickness=5):
+    h, w = img.shape[:2]
+
+    pts = [(int(x * w), int(y * h)) for x, y, _ in hand_points]
+
+    fingers = {
+        "thumb": [0, 1, 2, 3, 4],
+        "index": [0, 5, 6, 7, 8],
+        "middle": [0, 9, 10, 11, 12],
+        "ring": [0, 13, 14, 15, 16],
+        "pinky": [0, 17, 18, 19, 20]
+    }
+
+    for idxs in fingers.values():
+        for i in range(len(idxs) - 1):
+            x1, y1 = pts[idxs[i]]
+            x2, y2 = pts[idxs[i + 1]]
+
+            cv2.line(
+                img,
+                (x1, y1),
+                (x2, y2),
+                WHITE,
+                thickness,
+                cv2.LINE_AA
+            )
+
+# =========================
+# POSE
+# =========================
+def draw_pose_lines(img, points, connections, thickness=4):
+    h, w = img.shape[:2]
+
+    pts = [(int(x * w), int(y * h)) for x, y, _ in points]
+
+    for i, j in connections:
+        if i < len(pts) and j < len(pts):
+            cv2.line(
+                img,
+                pts[i],
+                pts[j],
+                WHITE,
+                thickness,
+                cv2.LINE_AA
+            )
+
+# =========================
+# FACE LANDMARKS
+# =========================
+def draw_points_and_connections(img, points):
+    h, w = img.shape[:2]
+
+    pts = [(int(x * w), int(y * h)) for x, y, _ in points]
+
+    for (x, y) in pts:
+        if 0 <= x < w and 0 <= y < h:
+            cv2.circle(
+                img,
+                (x, y),
+                2,
+                WHITE,
+                -1
+            )
+
+# =========================
+# MAIN DRAW
+# =========================
+def draw_skeleton(frame, img):
+    # Pose
+    draw_pose_lines(
+        img,
+        frame[42:75],
+        POSE_CONNECTIONS,
+        thickness=4
+    )
+
+    # Left hand
+    draw_hand_stylish(
+        img,
+        frame[0:21],
+        thickness=4
+    )
+
+    # Right hand
+    draw_hand_stylish(
+        img,
+        frame[21:42],
+        thickness=4
+    )
+
+    # Face
+    draw_points_and_connections(
+        img,
+        frame[75:553]
+    )
+
+# =========================
+# RENDER
+# =========================
+def render(frames, output="output.mp4"):
+    writer = cv2.VideoWriter(
+        output,
+        cv2.VideoWriter_fourcc(*"mp4v"),
+        FPS,
+        (WIDTH, HEIGHT)
+    )
+
+    for frame in frames:
+        # Create background using #190124
+        img = np.zeros((HEIGHT, WIDTH, 3), dtype=np.uint8)
+        img[:] = BACKGROUND
+
+        draw_skeleton(frame, img)
+        writer.write(img)
+
+    writer.release()
+    return output

pipelines/text_to_sign/run.py

@@ -0,0 +1,61 @@
+from .language import (
+    clean_gloss,
+    map_word,
+    lemmatize_words,
+    llm_text_to_gloss,
+)
+
+from .generator import generate_frames, get_landmarks
+from .renderer import render
+
+# ==========================
+# Lazy-loaded vocabulary
+# ==========================
+
+vocab_set = None
+
+
+def get_vocab():
+    global vocab_set
+
+    if vocab_set is None:
+        print("Loading text-to-sign assets...")
+        vocab_set = set(get_landmarks().keys())
+
+    return vocab_set
+
+
+# ==========================
+# Main Pipeline
+# ==========================
+
+def run_pipeline(user_sentence):
+    print("INPUT:", user_sentence)
+
+    # STEP 1: Text → Gloss
+    raw_gloss = llm_text_to_gloss(user_sentence)
+    print("RAW GLOSS:", raw_gloss)
+
+    # STEP 2: Clean gloss
+    cleaned = clean_gloss(raw_gloss)
+    print("CLEANED:", cleaned)
+
+    # STEP 3: Tokenize + Lemmatize
+    tokens = cleaned.split()
+    tokens = lemmatize_words(tokens)
+    print("TOKENS:", tokens)
+
+    # STEP 4: Map words to vocabulary
+    vocab = get_vocab()
+    mapped_sequence = [map_word(token, vocab) for token in tokens]
+    print("MAPPED:", mapped_sequence)
+
+    # STEP 5: Generate landmark frames
+    frames = generate_frames(mapped_sequence)
+    print("FRAMES COUNT:", len(frames))
+
+    # STEP 6: Render video
+    video_path = render(frames)
+    print("VIDEO PATH:", video_path)
+
+    return video_path

requirements.txt

@@ -0,0 +1,95 @@
+absl-py==2.4.0
+annotated-doc==0.0.4
+annotated-types==0.7.0
+anyio==4.13.0
+astunparse==1.6.3
+attrs==26.1.0
+beautifulsoup4==4.14.3
+certifi==2026.4.22
+cffi==2.0.0
+charset-normalizer==3.4.7
+click==8.3.3
+colorama==0.4.6
+contourpy==1.3.2
+cryptography==47.0.0
+cycler==0.12.1
+distro==1.9.0
+exceptiongroup==1.3.1
+fastapi==0.136.1
+filelock==3.29.0
+flatbuffers==25.12.19
+fonttools==4.62.1
+fsspec==2026.4.0
+gast==0.7.0
+gdown==6.0.0
+google-auth==2.50.0
+google-auth-oauthlib==1.3.1
+google-pasta==0.2.0
+groq==1.2.0
+grpcio==1.80.0
+h11==0.16.0
+h5py==3.16.0
+httpcore==1.0.9
+httpx==0.28.1
+idna==3.13
+jax==0.4.34
+jaxlib==0.4.34
+Jinja2==3.1.6
+joblib==1.5.3
+keras==2.15.0
+kiwisolver==1.5.0
+libclang==18.1.1
+Markdown==3.10.2
+MarkupSafe==3.0.3
+matplotlib==3.10.9
+mediapipe==0.10.21
+ml-dtypes==0.2.0
+mpmath==1.3.0
+networkx==3.4.2
+nltk==3.9.4
+numpy==1.26.4
+oauthlib==3.3.1
+opencv-python-headless==4.9.0.80
+opt_einsum==3.4.0
+packaging==26.2
+pillow==12.2.0
+protobuf==4.25.9
+pyasn1==0.6.3
+pyasn1_modules==0.4.2
+pycparser==3.0
+pydantic==2.13.3
+pydantic_core==2.46.3
+pyparsing==3.3.2
+PySocks==1.7.1
+python-dateutil==2.9.0.post0
+python-dotenv==1.2.2
+python-multipart==0.0.27
+regex==2026.4.4
+requests==2.32.3
+requests-oauthlib==2.0.0
+scikit-learn==1.5.0
+scipy==1.15.3
+sentencepiece==0.2.1
+six==1.17.0
+sniffio==1.3.1
+sounddevice==0.5.5
+soupsieve==2.8.4
+starlette==1.0.0
+sympy==1.14.0
+tensorboard==2.15.2
+tensorboard-data-server==0.7.2
+tensorflow==2.15.0
+tensorflow-estimator==2.15.0
+tensorflow-io-gcs-filesystem==0.31.0
+termcolor==3.3.0
+threadpoolctl==3.6.0
+--extra-index-url https://download.pytorch.org/whl/cpu
+torch==2.2.0+cpu
+tqdm==4.67.3
+typing-inspection==0.4.2
+typing_extensions==4.15.0
+urllib3==2.6.3
+uvicorn==0.46.0
+Werkzeug==3.1.8
+wrapt==1.14.2
+# Railway rebuild

start.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+echo "Downloading assets..."
+python download_assets.py
+
+echo "Listing assets..."
+find assets -maxdepth 4
+
+echo "Starting FastAPI..."
+uvicorn app.main:app --host 0.0.0.0 --port 7860