Initial commit - Multimodal AI project

.gitignore

@@ -0,0 +1,44 @@
+# Python
+__pycache__/
+*.pyc
+*.pyo
+*.pyd
+
+# Environment
+.env
+.venv
+env/
+venv/
+
+# Conda
+conda-meta/
+
+# Mac
+.DS_Store
+
+# Data
+data/
+!data/.gitkeep
+
+# Models
+models/
+*.pt
+*.pth
+*.bin
+
+# Logs
+*.log
+
+# Jupyter
+.ipynb_checkpoints
+
+# HuggingFace cache
+.cache/
+huggingface/
+
+
+gradio_cached_examples/
+
+
+*.zip
+*.tar

README.md

@@ -0,0 +1,152 @@
+---
+title: Scene Graph Generator
+emoji: 🧠
+colorFrom: blue
+colorTo: purple
+sdk: gradio
+app_file: app.py
+pinned: false
+---
+
+# 🧠 Scene Graph Generator (Multimodal AI System)
+
+A multimodal computer vision system that takes an input image, detects objects, predicts relationships between them, constructs a structured scene graph, and generates a natural language description of the scene.
+
+🔗 **Live Demo (Hugging Face Spaces):**  
+https://huggingface.co/spaces/<your-username>/scene-graph-generator
+
+---
+
+# 🚀 Features
+
+- 🖼️ Object Detection using DETR (ResNet-50)
+- 🔗 Relationship Prediction (Custom Trained Model)
+- 📐 Spatial Reasoning (Hybrid AI with Geometry Rules)
+- 🧩 Scene Graph Construction (Directed Graph)
+- 📊 Graph Visualization (NetworkX + Matplotlib)
+- 🧠 Graph-to-Text Generation (FLAN-T5)
+- 🌐 Interactive UI (Gradio)
+- ☁️ Deployed on Hugging Face Spaces (CPU)
+
+---
+
+# 🧠 How It Works (End-to-End Pipeline)
+
+### 1. Input
+- User uploads an image (JPG/PNG) via Gradio UI
+- Image is converted from PIL → OpenCV format
+
+---
+
+### 2. Object Detection
+- Uses `facebook/detr-resnet-50` from Hugging Face
+- Outputs:
+  - Object labels (COCO classes)
+  - Bounding boxes
+  - Confidence scores
+- Applies threshold (≥ 0.7) to filter noise
+
+---
+
+### 3. Pairwise Object Processing
+- Generates object pairs using `itertools.combinations`
+- Extracts bounding boxes for each pair
+- Creates union region for relation inference
+- Filters duplicate object pairs
+
+---
+
+### 4. Relationship Prediction
+- Custom-trained classifier on Visual Genome subset (~10K samples)
+- Predicts semantic relations:
+  - `on`, `holding`, `behind`, etc.
+- Trained using PyTorch (10 epochs)
+
+---
+
+### 5. Spatial Reasoning (Hybrid AI)
+- Uses bounding box geometry to compute:
+  - `left_of`, `right_of`, `above`, `below`, `near`
+- Hybrid logic:
+  - Semantic relations from model (if confident)
+  - Otherwise fallback to spatial rules
+- Reduces bias (e.g., “everything = on”)
+
+---
+
+### 6. Graph Construction
+- Builds a **directed graph (NetworkX DiGraph)**
+  - Nodes → objects
+  - Edges → relationships
+- Removes duplicates and limits edges for clarity
+
+---
+
+### 7. Graph Visualization
+- Uses NetworkX + Matplotlib
+- Displays:
+  - Directed edges with labels
+  - Clean layout for readability
+
+---
+
+### 8. Graph → Text (NLP)
+- Uses `google/flan-t5-small`
+- Converts structured triples into natural language
+
+Example:
+laptop → on → table
+mouse → next_to → laptop
+
+Output:
+"A laptop is placed on a table with a mouse next to it."
+
+---
+
+### 9. UI (Gradio)
+- Upload image
+- View:
+  - Scene graph
+  - Generated description
+- Fully interactive and browser-based
+
+---
+
+# 🏗️ Tech Stack
+
+- **Computer Vision:** DETR (Hugging Face Transformers)
+- **Deep Learning:** PyTorch
+- **Graph Processing:** NetworkX
+- **NLP:** FLAN-T5
+- **Image Processing:** OpenCV
+- **Frontend/UI:** Gradio
+- **Deployment:** Hugging Face Spaces
+
+---
+
+# 📁 Project Structure
+scene-graph-generator/
+│
+├── app.py
+├── requirements.txt
+├── README.md
+│
+├── src/
+│ ├── pipeline.py
+│ ├── detection.py
+│ ├── spatial_rules.py
+│ ├── relationship_infer.py
+│ ├── scene_graph.py
+│ ├── visualization.py
+│ ├── text_generation.py
+
+---
+
+# ⚙️ Installation (Local Setup)
+
+```bash
+git clone https://github.com/<your-username>/scene-graph-generator.git
+cd scene-graph-generator
+
+pip install -r requirements.txt
+python app.py

app.py

@@ -0,0 +1,53 @@
+import gradio as gr
+import cv2
+import numpy as np
+
+from src.pipeline import run_pipeline
+from src.scene_graph import build_graph
+from src.visualization import visualize_graph
+from src.text_generation import graph_to_text   
+
+
+def process_image(image):
+    try:
+        
+        image_cv = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+
+        
+        relations = run_pipeline(image_cv)
+
+        if not relations or len(relations) == 0:
+            return image, None, "No relationships detected."
+
+     
+        G = build_graph(relations)
+
+       
+        fig = visualize_graph(G)
+
+     
+        caption = graph_to_text(relations)
+
+        return image, fig, caption
+
+    except Exception as e:
+        print("Error:", e)
+        return image, None, "Error processing image."
+
+
+demo = gr.Interface(
+    fn=process_image,
+    inputs=gr.Image(type="pil"),
+    outputs=[
+        gr.Image(label="Input Image"),
+        gr.Plot(label="Scene Graph"),
+        gr.Textbox(label="Generated Description")  
+    ],
+    title="Scene Graph Generator",
+    description="Upload an image → Detect objects → Predict relationships → Generate scene graph + description",
+    theme="soft"
+)
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,16 @@
+torch
+torchvision
+transformers
+datasets
+opencv-python
+networkx
+plotly
+gradio
+numpy
+pillow
+tqdm
+scikit-learn
+kaggle
+timm
+transformers 
+sentencepiece

src/build_relationship_dataset.py

@@ -0,0 +1,73 @@
+import json
+import os
+import cv2
+from tqdm import tqdm
+
+SUBSET_PATH = "data/relationship_dataset/subset.json"
+IMAGE_MAP_PATH = "data/relationship_dataset/image_paths.json"
+
+OUTPUT_DIR = "data/relationship_dataset/images"
+LABELS_PATH = "data/relationship_dataset/labels.json"
+
+os.makedirs(OUTPUT_DIR, exist_ok=True)
+
+with open(SUBSET_PATH) as f:
+    subset = json.load(f)
+
+with open(IMAGE_MAP_PATH) as f:
+    image_map = json.load(f)
+
+labels = []
+
+idx = 0
+
+for item in tqdm(subset):
+    image_id = str(item["image_id"])
+
+    if image_id not in image_map:
+        continue
+
+    img_path = image_map[image_id]
+    img = cv2.imread(img_path)
+
+    if img is None:
+        continue
+
+    h_img, w_img, _ = img.shape
+
+    s = item["subject"]
+    o = item["object"]
+
+    x1 = min(s["x"], o["x"])
+    y1 = min(s["y"], o["y"])
+    x2 = max(s["x"] + s["w"], o["x"] + o["w"])
+    y2 = max(s["y"] + s["h"], o["y"] + o["h"])
+
+    x1 = max(0, x1)
+    y1 = max(0, y1)
+    x2 = min(w_img, x2)
+    y2 = min(h_img, y2)
+
+    crop = img[y1:y2, x1:x2]
+
+    if crop.size == 0:
+        continue
+
+    crop = cv2.resize(crop, (128, 128))
+
+    filename = f"{idx}.jpg"
+    save_path = os.path.join(OUTPUT_DIR, filename)
+
+    cv2.imwrite(save_path, crop)
+
+    labels.append({
+        "image": filename,
+        "predicate": item["predicate"]
+    })
+
+    idx += 1
+
+with open(LABELS_PATH, "w") as f:
+    json.dump(labels, f)
+
+print("Total processed:", idx)

src/config.py

@@ -0,0 +1,12 @@
+MAX_OBJECTS = 5
+CONF_THRESHOLD = 0.7
+
+RELATIONS = [
+    "on",
+    "next_to",
+    "holding",
+    "riding",
+    "behind",
+    "in_front_of",
+    "under"
+]

src/dataset.py

@@ -0,0 +1,30 @@
+import json
+import os
+import cv2
+import torch
+from torch.utils.data import Dataset
+
+class RelationshipDataset(Dataset):
+    def __init__(self, image_dir, label_path):
+        self.image_dir = image_dir
+
+        with open(label_path) as f:
+            self.data = json.load(f)
+
+    def __len__(self):
+        return len(self.data)
+
+    def __getitem__(self, idx):
+        item = self.data[idx]
+
+        img_path = os.path.join(self.image_dir, item["image"])
+        image = cv2.imread(img_path)
+        image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
+        image = image / 255.0
+        image = (image - 0.5) / 0.5
+
+        image = torch.tensor(image, dtype=torch.float32).permute(2, 0, 1)
+
+        label = torch.tensor(item["label"], dtype=torch.long)
+
+        return image, label

src/detection.py

@@ -0,0 +1,38 @@
+from transformers import DetrImageProcessor, DetrForObjectDetection
+import torch
+import cv2
+
+device = "mps" if torch.backends.mps.is_available() else "cpu"
+
+processor = DetrImageProcessor.from_pretrained("facebook/detr-resnet-50")
+model = DetrForObjectDetection.from_pretrained("facebook/detr-resnet-50")
+model.to(device)
+model.eval()
+
+id2label = model.config.id2label
+
+
+def detect(image):
+    inputs = processor(images=image, return_tensors="pt").to(device)
+
+    with torch.no_grad():
+        outputs = model(**inputs)
+
+    target_sizes = torch.tensor([image.shape[:2]]).to(device)
+    results = processor.post_process_object_detection(outputs, target_sizes=target_sizes)[0]
+
+    from src.config import MAX_OBJECTS, CONF_THRESHOLD
+
+    detections = []
+
+    for score, label, box in zip(results["scores"], results["labels"], results["boxes"]):
+        if score.item() > CONF_THRESHOLD:
+            detections.append({
+                "label": id2label[label.item()],
+                "score": score.item(),
+                "box": box.tolist()
+            })
+
+    detections = sorted(detections, key=lambda x: x["score"], reverse=True)[:MAX_OBJECTS]
+
+    return detections

src/encode_labels.py

@@ -0,0 +1,28 @@
+import json
+
+LABELS_PATH = "data/relationship_dataset/labels.json"
+OUTPUT_PATH = "data/relationship_dataset/labels_encoded.json"
+MAP_PATH = "data/relationship_dataset/label_map.json"
+
+with open(LABELS_PATH) as f:
+    labels = json.load(f)
+
+predicates = sorted(list(set([item["predicate"] for item in labels])))
+
+label_map = {p: i for i, p in enumerate(predicates)}
+
+encoded = []
+
+for item in labels:
+    encoded.append({
+        "image": item["image"],
+        "label": label_map[item["predicate"]]
+    })
+
+with open(OUTPUT_PATH, "w") as f:
+    json.dump(encoded, f)
+
+with open(MAP_PATH, "w") as f:
+    json.dump(label_map, f)
+
+print("Classes:", len(label_map))

src/model.py

@@ -0,0 +1,27 @@
+import torch.nn as nn
+
+class RelationshipNet(nn.Module):
+    def __init__(self, num_classes):
+        super().__init__()
+
+        self.model = nn.Sequential(
+            nn.Conv2d(3, 32, 3, padding=1),
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+
+            nn.Conv2d(32, 64, 3, padding=1),
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+
+            nn.Conv2d(64, 128, 3, padding=1),
+            nn.ReLU(),
+            nn.MaxPool2d(2),
+
+            nn.Flatten(),
+            nn.Linear(128 * 16 * 16, 256),
+            nn.ReLU(),
+            nn.Linear(256, num_classes)
+        )
+
+    def forward(self, x):
+        return self.model(x)

src/pipeline.py

@@ -0,0 +1,78 @@
+import cv2
+from itertools import combinations
+
+from src.detection import detect
+from src.relationship_infer import predict
+from src.spatial_rules import get_relation
+
+
+def run_pipeline(image):
+
+    detections = detect(image)
+
+    results = []
+    pair_seen = set()
+
+    for obj1, obj2 in combinations(detections, 2):
+
+        # 🔥 avoid duplicate object pairs (by label)
+        pair_key = (obj1["label"], obj2["label"])
+        if pair_key in pair_seen:
+            continue
+        pair_seen.add(pair_key)
+
+        # boxes: [x1, y1, x2, y2]
+        x1, y1, x2, y2 = obj1["box"]
+        x3, y3, x4, y4 = obj2["box"]
+
+        # convert to (x, y, w, h)
+        box1 = (x1, y1, x2 - x1, y2 - y1)
+        box2 = (x3, y3, x4 - x3, y4 - y3)
+
+        # 🔥 spatial relation (primary)
+        spatial_rel = get_relation(box1, box2)
+
+        # crop region for model
+        x_min = int(min(x1, x3))
+        y_min = int(min(y1, y3))
+        x_max = int(max(x2, x4))
+        y_max = int(max(y2, y4))
+
+        crop = image[y_min:y_max, x_min:x_max]
+
+        if crop.size == 0:
+            continue
+
+        # 🔥 model prediction (secondary)
+        try:
+            model_rel = predict(crop)
+        except:
+            model_rel = None
+
+        # 🔥 hybrid logic
+        if model_rel in ["holding", "sitting_on"]:
+            relation = model_rel
+        else:
+            relation = spatial_rel
+
+        results.append({
+            "subject": obj1["label"],
+            "object": obj2["label"],
+            "relation": relation
+        })
+
+    # 🔥 remove duplicate relations
+    unique = set()
+    clean_results = []
+
+    for r in results:
+        key = (r["subject"], r["object"], r["relation"])
+        if key not in unique:
+            unique.add(key)
+            clean_results.append(r)
+
+    # 🔥 limit number of relations (clean graph)
+    MAX_RELATIONS = 8
+    clean_results = clean_results[:MAX_RELATIONS]
+
+    return clean_results

src/prepare_images.py

@@ -0,0 +1,31 @@
+import json
+import os
+
+INPUT_PATH = "data/relationship_dataset/subset.json"
+OUTPUT_PATH = "data/relationship_dataset/image_paths.json"
+
+IMAGE_DIR_1 = "data/visual_genome/images/VG_100K"
+IMAGE_DIR_2 = "data/visual_genome/images2/VG_100K_2"
+
+with open(INPUT_PATH) as f:
+    data = json.load(f)
+
+image_ids = set([item["image_id"] for item in data])
+
+image_map = {}
+
+for img_id in image_ids:
+    filename = f"{img_id}.jpg"
+
+    path1 = os.path.join(IMAGE_DIR_1, filename)
+    path2 = os.path.join(IMAGE_DIR_2, filename)
+
+    if os.path.exists(path1):
+        image_map[img_id] = path1
+    elif os.path.exists(path2):
+        image_map[img_id] = path2
+
+with open(OUTPUT_PATH, "w") as f:
+    json.dump(image_map, f)
+
+print("Total images found:", len(image_map))

src/relationship_dataset.py

@@ -0,0 +1,66 @@
+import json
+import random
+from tqdm import tqdm
+from src.config import RELATIONS
+
+INPUT_PATH = "data/visual_genome/region_graphs.json"
+OUTPUT_PATH = "data/relationship_dataset/subset.json"
+
+subset_size = 10000
+def normalize_predicate(p):
+    if "on" in p:
+        return "on"
+    if "next_to" in p or "next" in p:
+        return "next_to"
+    if "hold" in p:
+        return "holding"
+    if "ride" in p:
+        return "riding"
+    if "behind" in p:
+        return "behind"
+    if "front" in p:
+        return "in_front_of"
+    if "under" in p:
+        return "under"
+    return None
+
+with open(INPUT_PATH) as f:
+    data = json.load(f)
+
+valid_samples = []
+
+for item in tqdm(data):
+    image_id = item["image_id"]
+
+    for region in item.get("regions", []):
+        objects = region.get("objects", [])
+        obj_map = {obj["object_id"]: obj for obj in objects}
+
+        for rel in region.get("relationships", []):
+            predicate = rel.get("predicate", "").lower().replace(" ", "_")
+
+            normalized = normalize_predicate(predicate)
+
+            if normalized is not None:
+                subject_id = rel.get("subject_id")
+                object_id = rel.get("object_id")
+
+                if subject_id in obj_map and object_id in obj_map:
+                    subject = obj_map[subject_id]
+                    obj = obj_map[object_id]
+
+                    valid_samples.append({
+                        "image_id": image_id,
+                        "predicate": normalized,
+                        "subject": subject,
+                        "object": obj
+                    })
+
+random.shuffle(valid_samples)
+
+subset = valid_samples[:subset_size]
+
+with open(OUTPUT_PATH, "w") as f:
+    json.dump(subset, f)
+
+print("Total samples:", len(subset))

src/relationship_infer.py

@@ -0,0 +1,36 @@
+import torch
+import cv2
+import json
+from src.model import RelationshipNet
+
+MODEL_PATH = "models/relationship_model.pth"
+LABEL_MAP_PATH = "data/relationship_dataset/label_map.json"
+
+device = "mps" if torch.backends.mps.is_available() else "cpu"
+
+with open(LABEL_MAP_PATH) as f:
+    label_map = json.load(f)
+
+inv_map = {v: k for k, v in label_map.items()}
+
+num_classes = len(label_map)
+
+model = RelationshipNet(num_classes)
+model.load_state_dict(torch.load(MODEL_PATH, map_location=device))
+model.to(device)
+model.eval()
+
+
+def predict(image):
+    image = cv2.resize(image, (128, 128))
+    image = image / 255.0
+    image = (image - 0.5) / 0.5
+
+    image = torch.tensor(image, dtype=torch.float32).permute(2, 0, 1)
+    image = image.unsqueeze(0).to(device)
+
+    with torch.no_grad():
+        output = model(image)
+        pred = torch.argmax(output, dim=1).item()
+
+    return inv_map[pred]

src/scene_graph.py

@@ -0,0 +1,25 @@
+import networkx as nx
+
+
+def build_graph(relations):
+    G = nx.DiGraph()
+
+    seen = set()
+
+    for rel in relations:
+        subj = rel["subject"]
+        obj = rel["object"]
+        predicate = rel["relation"]
+
+        key = (subj, obj, predicate)
+
+        
+        if key in seen:
+            continue
+        seen.add(key)
+
+        G.add_node(subj)
+        G.add_node(obj)
+        G.add_edge(subj, obj, label=predicate)
+
+    return G

src/spatial_rules.py

@@ -0,0 +1,21 @@
+def get_relation(subj_box, obj_box):
+    sx, sy, sw, sh = subj_box
+    ox, oy, ow, oh = obj_box
+
+    # centers
+    scx, scy = sx + sw / 2, sy + sh / 2
+    ocx, ocy = ox + ow / 2, oy + oh / 2
+
+    # vertical relation
+    if scy < ocy - 20:
+        return "above"
+    elif scy > ocy + 20:
+        return "below"
+
+    # horizontal relation
+    if scx < ocx - 20:
+        return "left_of"
+    elif scx > ocx + 20:
+        return "right_of"
+
+    return "near"

src/text_generation.py

@@ -0,0 +1,24 @@
+from transformers import AutoTokenizer, AutoModelForSeq2SeqLM
+
+tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-small")
+model = AutoModelForSeq2SeqLM.from_pretrained("google/flan-t5-small")
+
+
+def graph_to_text(relations):
+    if not relations:
+        return "No relationships detected."
+
+    # convert relations to prompt
+    triples = [
+        f"{r['subject']} {r['relation']} {r['object']}"
+        for r in relations
+    ]
+
+    prompt = "Convert the following relationships into a natural sentence:\n"
+    prompt += ", ".join(triples)
+
+    inputs = tokenizer(prompt, return_tensors="pt", truncation=True)
+
+    outputs = model.generate(**inputs, max_length=50)
+
+    return tokenizer.decode(outputs[0], skip_special_tokens=True)

src/visualization.py

@@ -0,0 +1,78 @@
+import plotly.graph_objects as go
+import networkx as nx
+
+
+def visualize_graph(G):
+    pos = nx.spring_layout(G, seed=42)
+
+    edge_x = []
+    edge_y = []
+    edge_text = []
+
+    for u, v, data in G.edges(data=True):
+        x0, y0 = pos[u]
+        x1, y1 = pos[v]
+
+        edge_x += [x0, x1, None]
+        edge_y += [y0, y1, None]
+        edge_text.append(data["label"])
+
+    edge_trace = go.Scatter(
+    x=edge_x,
+    y=edge_y,
+    line=dict(width=1),
+    hoverinfo='none',
+    mode='lines'
+)
+
+    node_x = []
+    node_y = []
+    node_text = []
+
+    for node in G.nodes():
+        x, y = pos[node]
+        node_x.append(x)
+        node_y.append(y)
+        node_text.append(node)
+
+    node_trace = go.Scatter(
+        x=node_x,
+        y=node_y,
+        mode='markers+text',
+        text=node_text,
+        textposition="top center",
+        hoverinfo='text',
+        marker=dict(size=20)
+    )
+
+    fig = go.Figure(data=[edge_trace, node_trace])
+
+    fig.update_layout(
+        showlegend=False,
+        hovermode='closest',
+        margin=dict(b=20, l=5, r=5, t=40)
+    )
+    for u, v in G.edges():
+        x0, y0 = pos[u]
+        x1, y1 = pos[v]
+
+        fig.add_annotation(
+    x=x1,
+    y=y1,
+    ax=x0,
+    ay=y0,
+    xref='x',
+    yref='y',
+    axref='x',
+    ayref='y',
+    showarrow=True,
+    
+    arrowhead=4,        
+    arrowsize=2.5,      
+    arrowwidth=2.5,     
+    arrowcolor="black", 
+
+    opacity=0.9
+)
+
+    return fig

training/train_relationship.py

@@ -0,0 +1,42 @@
+import torch
+from torch.utils.data import DataLoader
+from src.dataset import RelationshipDataset
+from src.model import RelationshipNet
+
+device = "mps" if torch.backends.mps.is_available() else "cpu"
+
+dataset = RelationshipDataset(
+    image_dir="data/relationship_dataset/images",
+    label_path="data/relationship_dataset/labels_encoded.json"
+)
+
+loader = DataLoader(dataset, batch_size=16, shuffle=True)
+
+num_classes = len(set([item["label"] for item in dataset.data]))
+
+model = RelationshipNet(num_classes).to(device)
+
+criterion = torch.nn.CrossEntropyLoss()
+optimizer = torch.optim.Adam(model.parameters(), lr=3e-4)
+
+epochs = 10
+
+for epoch in range(epochs):
+    total_loss = 0
+
+    for images, labels in loader:
+        images = images.to(device)
+        labels = labels.to(device)
+
+        outputs = model(images)
+        loss = criterion(outputs, labels)
+
+        optimizer.zero_grad()
+        loss.backward()
+        optimizer.step()
+
+        total_loss += loss.item()
+
+    print(f"Epoch {epoch+1}, Loss: {total_loss:.4f}")
+
+torch.save(model.state_dict(), "models/relationship_model.pth")