Updated the code for my custom Space

Dockerfile

@@ -0,0 +1,18 @@
+# Hugging Face Spaces expects port 7860
+FROM python:3.10-slim
+
+WORKDIR /app
+
+# System deps (for Pillow / torch)
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libgl1 libglib2.0-0 && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Copy everything in the repo
+COPY . .
+
+EXPOSE 7860
+CMD ["uvicorn", "app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,9 +1,10 @@
 ---
-title: Vocal_Eyes
+title: VocalEyes
+sdk: docker
+app_port: 7860
 emoji: 😎🕶👨‍🦯
 colorFrom: blue
 colorTo: purple
-sdk: gradio
 sdk_version: 6.5.1
 app_file: app.py
 license: mit

app.py

@@ -1,67 +1,67 @@
 import torch
-import torch.nn as nn
-import torch.nn.functional as F
 import torchvision.transforms as T
-import os
+import io
 from PIL import Image
+from fastapi import FastAPI, File, UploadFile, HTTPException
+from fastapi.responses import JSONResponse
+import uvicorn
+
 from RelTR_build import Reltr_model
 from T5_build import T5_model
-import gradio as gr
-
-
-CLASSES = [ 'N/A', 'airplane', 'animal', 'arm', 'bag', 'banana', 'basket', 'beach', 'bear', 'bed', 'bench', 'bike',
-                'bird', 'board', 'boat', 'book', 'boot', 'bottle', 'bowl', 'box', 'boy', 'branch', 'building',
-                'bus', 'cabinet', 'cap', 'car', 'cat', 'chair', 'child', 'clock', 'coat', 'counter', 'cow', 'cup',
-                'curtain', 'desk', 'dog', 'door', 'drawer', 'ear', 'elephant', 'engine', 'eye', 'face', 'fence',
-                'finger', 'flag', 'flower', 'food', 'fork', 'fruit', 'giraffe', 'girl', 'glass', 'glove', 'guy',
-                'hair', 'hand', 'handle', 'hat', 'head', 'helmet', 'hill', 'horse', 'house', 'jacket', 'jean',
-                'kid', 'kite', 'lady', 'lamp', 'laptop', 'leaf', 'leg', 'letter', 'light', 'logo', 'man', 'men',
-                'motorcycle', 'mountain', 'mouth', 'neck', 'nose', 'number', 'orange', 'pant', 'paper', 'paw',
-                'people', 'person', 'phone', 'pillow', 'pizza', 'plane', 'plant', 'plate', 'player', 'pole', 'post',
-                'pot', 'racket', 'railing', 'rock', 'roof', 'room', 'screen', 'seat', 'sheep', 'shelf', 'shirt',
-                'shoe', 'short', 'sidewalk', 'sign', 'sink', 'skateboard', 'ski', 'skier', 'sneaker', 'snow',
-                'sock', 'stand', 'street', 'surfboard', 'table', 'tail', 'tie', 'tile', 'tire', 'toilet', 'towel',
-                'tower', 'track', 'train', 'tree', 'truck', 'trunk', 'umbrella', 'vase', 'vegetable', 'vehicle',
-                'wave', 'wheel', 'window', 'windshield', 'wing', 'wire', 'woman', 'zebra']
-
-REL_CLASSES = ['__background__', 'above', 'across', 'against', 'along', 'and', 'at', 'attached to', 'behind',
-                'belonging to', 'between', 'carrying', 'covered in', 'covering', 'eating', 'flying in', 'for',
-                'from', 'growing on', 'hanging from', 'has', 'holding', 'in', 'in front of', 'laying on',
-                'looking at', 'lying on', 'made of', 'mounted on', 'near', 'of', 'on', 'on back of', 'over',
-                'painted on', 'parked on', 'part of', 'playing', 'riding', 'says', 'sitting on', 'standing on',
-                'to', 'under', 'using', 'walking in', 'walking on', 'watching', 'wearing', 'wears', 'with']
-
-model=Reltr_model()
-tokenizer_2,model_text_2=T5_model() 
-
 
+# ── Label sets ────────────────────────────────────────────────────────────────
+CLASSES = [
+    'N/A', 'airplane', 'animal', 'arm', 'bag', 'banana', 'basket', 'beach', 'bear', 'bed', 'bench', 'bike',
+    'bird', 'board', 'boat', 'book', 'boot', 'bottle', 'bowl', 'box', 'boy', 'branch', 'building',
+    'bus', 'cabinet', 'cap', 'car', 'cat', 'chair', 'child', 'clock', 'coat', 'counter', 'cow', 'cup',
+    'curtain', 'desk', 'dog', 'door', 'drawer', 'ear', 'elephant', 'engine', 'eye', 'face', 'fence',
+    'finger', 'flag', 'flower', 'food', 'fork', 'fruit', 'giraffe', 'girl', 'glass', 'glove', 'guy',
+    'hair', 'hand', 'handle', 'hat', 'head', 'helmet', 'hill', 'horse', 'house', 'jacket', 'jean',
+    'kid', 'kite', 'lady', 'lamp', 'laptop', 'leaf', 'leg', 'letter', 'light', 'logo', 'man', 'men',
+    'motorcycle', 'mountain', 'mouth', 'neck', 'nose', 'number', 'orange', 'pant', 'paper', 'paw',
+    'people', 'person', 'phone', 'pillow', 'pizza', 'plane', 'plant', 'plate', 'player', 'pole', 'post',
+    'pot', 'racket', 'railing', 'rock', 'roof', 'room', 'screen', 'seat', 'sheep', 'shelf', 'shirt',
+    'shoe', 'short', 'sidewalk', 'sign', 'sink', 'skateboard', 'ski', 'skier', 'sneaker', 'snow',
+    'sock', 'stand', 'street', 'surfboard', 'table', 'tail', 'tie', 'tile', 'tire', 'toilet', 'towel',
+    'tower', 'track', 'train', 'tree', 'truck', 'trunk', 'umbrella', 'vase', 'vegetable', 'vehicle',
+    'wave', 'wheel', 'window', 'windshield', 'wing', 'wire', 'woman', 'zebra',
+]
+
+REL_CLASSES = [
+    '__background__', 'above', 'across', 'against', 'along', 'and', 'at', 'attached to', 'behind',
+    'belonging to', 'between', 'carrying', 'covered in', 'covering', 'eating', 'flying in', 'for',
+    'from', 'growing on', 'hanging from', 'has', 'holding', 'in', 'in front of', 'laying on',
+    'looking at', 'lying on', 'made of', 'mounted on', 'near', 'of', 'on', 'on back of', 'over',
+    'painted on', 'parked on', 'part of', 'playing', 'riding', 'says', 'sitting on', 'standing on',
+    'to', 'under', 'using', 'walking in', 'walking on', 'watching', 'wearing', 'wears', 'with',
+]
+
+# ── Image preprocessing ───────────────────────────────────────────────���────────
 transform = T.Compose([
     T.Resize(800),
     T.ToTensor(),
-    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
+    T.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]),
 ])
 
 
+# ── Pipeline components ────────────────────────────────────────────────────────
 def normalize_triplets(triplets):
-                  seen = set()
-                  cleaned = []
-                  for s, r, o in triplets:
-                      key = tuple(sorted((s, r, o)))
-                      if key not in seen:
-                          seen.add(key)
-                          cleaned.append((s, r, o))
-                  return cleaned
+    seen = set()
+    cleaned = []
+    for s, r, o in triplets:
+        key = tuple(sorted((s, r, o)))
+        if key not in seen:
+            seen.add(key)
+            cleaned.append((s, r, o))
+    return cleaned
 
 
 class RelTRSceneGraphExtractor:
-    def __init__(self, model, obj_classes, rel_classes, device="cuda"):
+    def __init__(self, model, obj_classes, rel_classes, device="cpu"):
         self.device = device
-        self.model = model.to(self.device)
-
-        self.model.eval()
+        self.model = model.to(self.device).eval()
         self.obj_classes = obj_classes
         self.rel_classes = rel_classes
-        self.device = device
 
     @torch.no_grad()
     def extract_triplets(self, image_tensor, conf_thresh=0.3):
@@ -74,138 +74,130 @@ class RelTRSceneGraphExtractor:
 
         triplets = []
         for i in range(len(probas_rel)):
-            score = probas_rel[i].max().item()
-            if score < conf_thresh:
+            if probas_rel[i].max().item() < conf_thresh:
                 continue
-
             rel = self.rel_classes[probas_rel[i].argmax()]
             sub = self.obj_classes[probas_sub[i].argmax()]
             obj = self.obj_classes[probas_obj[i].argmax()]
-
             triplets.append((sub, rel, obj))
 
+        return normalize_triplets(triplets)
 
-        triplets = normalize_triplets(triplets)
-
-        return triplets
 
 class SceneGraphToText:
-    def __init__(self):
-        pass
-
     def convert(self, triplets):
-        if len(triplets) == 0:
+        if not triplets:
             return "No clear relationships detected in the image."
+        return " ".join(f"A {s} is {r} a {o}." for s, r, o in triplets)
 
-        sentences = []
-        for sub, rel, obj in triplets:
-            sentences.append(f"A {sub} is {rel} a {obj}.")
-        return " ".join(sentences)
 
 class T5TextGenerator:
-    def __init__(self, model, tokenizer, device="cuda"):
+    def __init__(self, model, tokenizer, device="cpu"):
         self.tokenizer = tokenizer
-        self.model = model.to(device)
+        self.model = model.to(device).eval()
         self.device = device
 
     @torch.no_grad()
-    def generate(self, prompt,  **gen_kwargs):
-        # Tokenize and move each tensor to the right device
+    def generate(self, prompt, **gen_kwargs):
         inputs = self.tokenizer(prompt, return_tensors="pt", truncation=True)
         inputs = {k: v.to(self.device) for k, v in inputs.items()}
-
-        # Pass through the model with any generation kwargs
-        outputs = self.model.generate(
-            **inputs,
-
-            **gen_kwargs
-        )
-
+        outputs = self.model.generate(**inputs, **gen_kwargs)
         return self.tokenizer.decode(outputs[0], skip_special_tokens=True)
 
 
 class RelTR_T5_Pipeline:
-    def __init__(self, reltr_model,model_text,tokenizer, obj_classes, rel_classes, device="cuda"):
-        self.scene_graph = RelTRSceneGraphExtractor(
-            reltr_model, obj_classes, rel_classes, device
-        )
+    def __init__(self, reltr_model, model_text, tokenizer, obj_classes, rel_classes, device="cpu"):
+        self.scene_graph = RelTRSceneGraphExtractor(reltr_model, obj_classes, rel_classes, device)
         self.graph_to_text = SceneGraphToText()
-        self.t5 = T5TextGenerator(model_text,tokenizer,device=device)
+        self.t5 = T5TextGenerator(model_text, tokenizer, device=device)
 
     def run(self, image_tensor):
         triplets = self.scene_graph.extract_triplets(image_tensor)
         scene_text = self.graph_to_text.convert(triplets)
 
-
-        prompt = f"""
-                 Convert the following relationship facts into a short, coherent scene description.
-
+        prompt = f"""Convert the following relationship facts into a short, coherent scene description.
 Rules:
 - Use only the provided facts.
 - Combine related facts into natural sentences.
 - Do not invent new objects or actions.
 - Describe the main subject first.
 - Maximum 3 sentences.
-
 Facts:
-
-                 {scene_text}
-                                          """
-
-
+{scene_text}"""
 
         output = self.t5.generate(
-                                    prompt,
-
-
-                                    max_new_tokens=60,
-                                    do_sample=False,      # still deterministic
-                                    num_beams=5,
-                                    length_penalty=1.2,   # encourages longer summaries
-                                    repetition_penalty=1.3,
-                                    early_stopping=True
-                                )
-
-        return {
-            "triplets": triplets,
-            "scene_text": scene_text,
-            "generated_text": output
-        }
-
-
+            prompt,
+            max_new_tokens=60,
+            do_sample=False,
+            num_beams=5,
+            length_penalty=1.2,
+            repetition_penalty=1.3,
+            early_stopping=True,
+        )
+        return {"triplets": triplets, "scene_text": scene_text, "generated_text": output}
 
 
+# ── Load models once at startup ────────────────────────────────────────────────
+print("Loading models...")
+_reltr = Reltr_model()
+_tokenizer, _t5 = T5_model()
 
 pipeline = RelTR_T5_Pipeline(
-    reltr_model=model,
-    model_text=model_text_2,
-    tokenizer=tokenizer_2,
+    reltr_model=_reltr,
+    model_text=_t5,
+    tokenizer=_tokenizer,
     obj_classes=CLASSES,
     rel_classes=REL_CLASSES,
-    device="cpu"
+    device="cpu",
 )
+print("Models loaded.")
 
-def Scene_to_text(image):
-  img = transform(image).unsqueeze(0)
-  result = pipeline.run(img)
-  return result["generated_text"]
- 
-#Build Gradio app
+# ── FastAPI app ────────────────────────────────────────────────────────────────
+app = FastAPI(
+    title="VocalEyes API",
+    description="Converts an uploaded image into a short scene description via RelTR + T5.",
+    version="1.0.0",
+)
 
 
+@app.get("/")
+def root():
+    return {"status": "ok", "message": "VocalEyes API is running. POST an image to /predict"}
 
-title="Vocal_Eyes😎🕶👨‍🦯"
-description="Converts images into scene graph triplets then into a short factual description."
-# Create examples list from "examples/" directory
-example_list = [["examples/" + example] for example in os.listdir("examples")]
 
-demo = gr.Interface(
-    fn=Scene_to_text,
-    inputs=gr.Image(type="pil"),
-    outputs=[gr.Textbox(label="Description", lines=3)],
-    title=title,
-    description=description,
-    examples=example_list
+@app.post("/predict")
+async def predict(file: UploadFile = File(...)):
+    # ── Validate content type ──────────────────────────────────────────────────
+    if file.content_type not in ("image/jpeg", "image/png", "image/webp", "image/bmp"):
+        raise HTTPException(
+            status_code=415,
+            detail=f"Unsupported file type '{file.content_type}'. Send JPEG, PNG, WEBP, or BMP.",
+        )
 
-)
-demo.launch()
+    # ── Read & preprocess ──────────────────────────────────────────────────────
+    try:
+        raw = await file.read()
+        image = Image.open(io.BytesIO(raw)).convert("RGB")
+    except Exception as e:
+        raise HTTPException(status_code=400, detail=f"Could not read image: {e}")
+
+    img_tensor = transform(image).unsqueeze(0)   # (1, 3, H, W)
+
+    # ── Run pipeline ───────────────────────────────────────────────────────────
+    try:
+        result = pipeline.run(img_tensor)
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=f"Pipeline error: {e}")
+
+    return JSONResponse({
+        "description": result["generated_text"],
+        "triplets": [
+            {"subject": s, "relation": r, "object": o}
+            for s, r, o in result["triplets"]
+        ],
+    })
+
+
+# ── Entry point ────────────────────────────────────────────────────────────────
+if __name__ == "__main__":
+    uvicorn.run("app:app", host="0.0.0.0", port=7860)

requirements.txt

@@ -7,4 +7,7 @@ opencv-python>=4.7.0,<5
 pillow>=9.5.0,<13
 numpy>=1.25.0,<2
 scipy>=1.10.0,<1.18
-huggingface_hub<1.18
+huggingface_hub<1.18
+fastapi>=0.111.0
+uvicorn[standard]>=0.29.0
+python-multipart        # required for UploadFile