Initial Init

.dockerignore

@@ -0,0 +1,9 @@
+__pycache__
+*.pyc
+*.pyo
+*.pyd
+.env
+.git
+.gitignore
+archive/*
+archive/

Dockerfile

@@ -0,0 +1,19 @@
+# Use an official Python runtime as a base image
+FROM python:3.9
+
+WORKDIR /code
+
+COPY backend/requirements.txt /code/backend/requirements.txt
+
+RUN pip install --no-cache-dir --upgrade -r /code/backend/requirements.txt
+
+RUN useradd -m -u 1000 user
+USER user
+ENV HOME=/home/user
+ENV PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+COPY --chown=user . $HOME/app
+
+CMD ["uvicorn", "backend.app:app", "--host", "0.0.0.0", "--port", "7860"]

README.md

@@ -1,10 +1,17 @@
 ---
-title: SokratesAI
-emoji: 📊
-colorFrom: purple
-colorTo: indigo
+title: DeeperGaze
+emoji: 📈
+colorFrom: blue
+colorTo: pink
 sdk: docker
+sdk_version: "1.0"
+app_file: backend/app.py
+app_port: 7860
 pinned: false
+license: apache-2.0
+short_description: A cutting edge attention visualization app.
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# DeeperGaze
+
+This project is a web application using a GPT-2 model powered by FastAPI and Docker.

backend/app.py

@@ -0,0 +1,84 @@
+from fastapi import FastAPI, HTTPException, Body
+from fastapi.middleware.cors import CORSMiddleware
+from fastapi.staticfiles import StaticFiles
+from fastapi.responses import FileResponse
+from transformers import BertTokenizer, BertModel, pipeline
+import torch as t
+import logging
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+
+# Configure CORS: In production, you might restrict allowed origins
+app.add_middleware(
+    CORSMiddleware,
+    allow_origins=["*"],
+    allow_methods=["*"],
+    allow_headers=["*"],
+)
+
+# Mount static files (frontend) so that visiting "/" serves index.html
+# The directory path "../frontend" works because when running in Docker,
+# our working directory is set to /app, and the frontend folder is at /app/frontend.
+app.mount("/static", StaticFiles(directory="frontend", html=True), name="static")
+
+# Load tokenizer and BERT model
+tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
+try:
+    model = BertModel.from_pretrained('bert-base-uncased', output_attentions=True)
+except Exception as e:
+    logger.error(f"Model loading failed: {e}")
+    raise
+
+@app.post("/process")
+async def process_text(text: str = Body(..., embed=True)):
+    """
+    Process the input text:
+      - Tokenizes the text using BERT's tokenizer
+      - Runs the BERT model to obtain attentions (bidirectional)
+      - Returns the tokens and attention values (rounded to 2 decimals)
+    """
+    try:
+        logger.info(f"Received text: {text}")
+        # Tokenize input text (truncating if needed)
+        inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=512)
+
+        # Run the model without gradient computation (inference mode)
+        with t.no_grad():
+            outputs = model(**inputs)
+        attentions = outputs.attentions  # Tuple of attention tensors for each layer
+
+        decimals = 2
+        # Convert attention tensors to lists with rounded decimals
+        attn_series = t.round(
+            t.tensor([layer_attention.tolist() for layer_attention in attentions], dtype=t.double)
+            .squeeze(), decimals=decimals
+        ).detach().cpu().tolist()
+
+        return {
+            "tokens": tokenizer.convert_ids_to_tokens(inputs["input_ids"][0]),
+            "attention": attn_series
+        }
+    except Exception as e:
+        logger.error(f"Error processing text: {e}")
+        raise HTTPException(status_code=500, detail=str(e))
+
+# Initialize the text generation pipeline (unchanged)
+pipe = pipeline("text2text-generation", model="google/flan-t5-small")
+
+@app.get("/generate")
+def generate(text: str):
+    """
+    Using the text2text-generation pipeline from `transformers`, generate text
+    from the given input text. The model used is `google/flan-t5-small`.
+    """
+    # Use the pipeline to generate text from the given input text
+    output = pipe(text)
+    # Return the generated text in a JSON response
+    return {"output": output[0]["generated_text"]}
+
+@app.get("/")
+async def read_index():
+    return FileResponse("frontend/index.html")

backend/requirements.txt

@@ -0,0 +1,5 @@
+uvicorn[standard]
+fastapi==0.115.7
+torch==2.5.1
+transformers==4.48.1
+python-multipart>=0.0.5

frontend/index.html

@@ -0,0 +1,131 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <meta charset="UTF-8" />
+  <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
+  <title>DeepGaze</title>
+  <link rel="stylesheet" href="static/styles.css" />
+  <!-- Load D3 -->
+  <script src="https://d3js.org/d3.v6.min.js"></script>
+  <style>
+    /* Process button styling */
+    .text-form button {
+      background-color: #800000; /* Bordo red */
+      color: #fff;
+      border: none;
+      padding: 8px 12px;
+      font-size: 14px;
+      border-radius: 4px;
+      cursor: pointer;
+    }
+    /* Info container styling for head and layer display */
+    .info-container {
+      display: flex;
+      align-items: center;
+      margin: 5px 0;
+      font-size: 0.9rem;
+    }
+    .info-container .label {
+      margin-right: 5px;
+    }
+    .info-container .number-box {
+      border: 1px solid #800000;
+      border-radius: 4px;
+      padding: 2px 6px;
+      font-weight: bold;
+      color: #800000;
+      min-width: 20px;
+      text-align: center;
+    }
+  </style>
+</head>
+<body>
+  <!-- Header -->
+  <header>
+    <h1>DeepGaze</h1>
+  </header>
+  <!-- Main container -->
+  <div class="container">
+    <main>
+      <!-- About Attention Section -->
+      <section class="about-section">
+        <h2>About Attention</h2>
+        <p>
+          Transformer networks (the architecture behind Chat-GPT) are built from multiple layers, and each layer is divided into several attention heads.
+          Each head computes its own attention matrix by combining "queries" and "keys"—the fundamental elements that help
+          the network decide how much focus to give to different parts of the input.
+        </p>
+        <p>
+          You can think of each query as a question that a token asks, such as "Are there adjectives in front of me?"
+          Meanwhile, each key serves as a potential answer, carrying the token's characteristics. When the model compares
+          queries with keys, it determines the strength of their match and, therefore, how much influence one token should
+          have on another.
+        </p>
+        <p>
+          For example, consider the phrase "fluffy blue monster." One token might generate a query like, "Is the word in front
+          of me an adjective?" In this case, the tokens "fluffy" and "blue"—which are adjectives—provide keys that answer this
+          question strongly, while "monster," being a noun, offers a weaker response. This interplay of questions (queries)
+          and answers (keys) is what creates the attention matrix for each head.
+        </p>
+        <p>
+          Each attention head focuses on different relationships and patterns within the text, allowing the network to capture
+          a rich and nuanced understanding of the language. Despite the critical role that these attention mechanisms play,
+          it's interesting to note that only about one third of all the weights in a large language model are actually in the
+          attention blocks. So while the famous slogan "attention is all you need" highlights the importance of these connections,
+          in terms of sheer weight, it's only one third of what you really need!
+        </p>
+        <p id="credits">Made with <3 by Ferdi & Samu. Credits for model view below to <a href="https://jessevig.com/" target="_blank">BertViz</a>.</p>
+      </section>
+      
+      <!-- Deep Gaze into Attention Heads Section -->
+      <section class="deep-gaze-section">
+        <h2>A Deep Gaze into Attention Heads</h2>
+        <p>Type in a token sequence (below 50 characters) and hit process. After some loading time, you will be able to see the attention patterns of individual so-called "heads" in the LLM.
+          Each head focuses on different aspects of the input text, and by visualizing these patterns, you can gain insights into how the model processes and understands language.
+        </p>
+
+        <p>Here is an example view of a head, with tokens on each side. If you see a connection between two tokens, it means that the head is paying attention to the relationship between those tokens. This way you can see attention heads which "pay attention" to the previous token, the first token, or other patterns.
+          Click on an attention head to select the respective head in a layer. Afterwards you can hover over tokens to see the attention weights of the selected head for that token. </p>
+        <div id="thumbnailContainer"></div>
+        <!-- Text Input & Process Button -->
+        <form id="textForm" class="text-form">
+          <textarea id="inputText" rows="2" cols="50" maxlength="50" placeholder="Enter your text here..." autofocus></textarea>
+          <button type="submit">Process</button>
+        </form>
+      </section>
+      
+      <!-- Model View Section -->
+      <section class="model_view">
+        <p>Click on a head that looks interesting to gaze deeper into it in the next section:</p>
+
+        <div id="model_view_container">
+          <!-- Thumbnails of attention heads will be rendered here -->
+        </div>
+        <!-- Display for selected head and layer -->
+        <div id="display_info">
+          <div id="display_head" class="info-container">
+            <span class="label">Head:</span>
+            <span class="number-box">-</span>
+          </div>
+          <div id="display_layer" class="info-container">
+            <span class="label">Layer:</span>
+            <span class="number-box">-</span>
+          </div>
+        </div>
+      </section>
+      
+      <!-- Hover Visualization Section -->
+      <section class="hover-visualization">
+        <h2>Hover Visualization</h2>
+        <p>
+          By hovering over each token, you can see which other token is important for that token. The larger the token, the more important it is for the token you are hovering over. The token with the maximal attention is colored in red.
+        </p>
+        <div id="tokenContainer"></div>
+      </section>
+    </main>
+  </div>
+  
+  <!-- External JavaScript (all event listeners and functions are defined here) -->
+  <script src="static/script.js?v=1111"></script>
+</body>
+</html>

frontend/script.js

@@ -0,0 +1,213 @@
+// Form submit handler 2.1
+document.getElementById('textForm').addEventListener('submit', async (e) => {
+  e.preventDefault();
+  const inputText = document.getElementById('inputText').value;
+
+  try {
+    const response = await fetch('/process', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ text: inputText })
+    });
+
+    if (!response.ok) {
+      throw new Error('Network response was not ok');
+    }
+    const data = await response.json();
+
+    // Use data.tokens and data.attention from your POST response
+    // displayOutput(data);
+    displayHoverTokens(data, 0, 0);
+    // Changed call here to pass the entire data object
+    renderModelView(data);
+  } catch (error) {
+    console.error('Error:', error);
+    document.getElementById('output').innerText = 'Error processing text.';
+  }
+});
+
+function renderModelView(data) {
+  // Extract tokens and attention from the data object
+  const tokens = data.tokens;
+  const attention = data.attention;
+  
+  const container = document.getElementById("model_view_container");
+  if (!container) return;
+  container.innerHTML = "";
+  
+  const gridContainer = document.createElement("div");
+  gridContainer.style.display = "grid";
+  gridContainer.style.gridTemplateColumns = "repeat(12, 80px)";
+  gridContainer.style.gridGap = "10px";
+  gridContainer.style.padding = "20px";
+  
+  // Loop over all 12 layers and 12 heads, passing the complete data object
+  for (let layerIdx = 0; layerIdx < 12; layerIdx++) {
+    for (let headIdx = 0; headIdx < 12; headIdx++) {
+      const thumbnail = createAttentionThumbnail(data, layerIdx, headIdx);
+      gridContainer.appendChild(thumbnail);
+    }
+  }
+  
+  container.appendChild(gridContainer);
+}
+
+function createAttentionThumbnail(data, layerIdx, headIdx) {
+  // Extract tokens and attention from the data object
+  const tokens = data.tokens;
+  const attention = data.attention;
+  
+  const width = 80;       
+  const tokenHeight = 15; 
+  const padding = 10; 
+  // Compute the thumbnail height dynamically based on the number of tokens.
+  const height = padding * 2 + tokens.length * tokenHeight;
+  const maxLineWidth = 4;
+  const maxOpacity = 0.8;
+  
+  // Compute the right-side x-coordinate numerically.
+  const xRight = width - padding;
+  
+  // Create a thumbnail container using D3.
+  const thumbnail = d3.select(document.createElement("div"))
+    .style("position", "relative")
+    .style("height", height + "px")
+    .style("width", width + "px")
+    .style("border", "1px solid #ddd")
+    .style("border-radius", "4px")
+    .style("padding", "5px")
+    .style("background", "#fff");
+  
+  // Append an SVG container with fixed dimensions.
+  const svg = thumbnail.append("svg")
+    .attr("width", width)
+    .attr("height", height);
+  
+  // Add header text (e.g., "L4 H4") to show the layer and head number.
+  svg.append("text")
+    .attr("x", width / 2)
+    .attr("y", 15)
+    .attr("text-anchor", "middle")
+    .attr("font-size", "10")
+    .text(`L${layerIdx + 1} H${headIdx + 1}`);
+  
+  // Draw attention lines with per-row normalization.
+  attention[layerIdx][headIdx].forEach((sourceWeights, sourceIdx) => {
+    const rowMax = Math.max(...sourceWeights) || 1;
+    sourceWeights.forEach((weight, targetIdx) => {
+      if (weight > 0.01 && sourceIdx !== targetIdx) {
+        const normalizedWeight = weight / rowMax;
+        svg.append("line")
+          .attr("x1", padding)
+          .attr("y1", padding + sourceIdx * tokenHeight - 5)
+          .attr("x2", xRight)
+          .attr("y2", padding + targetIdx * tokenHeight - 5)
+          .attr("stroke", "#800000") // Bordo red
+          .attr("stroke-width", Math.max(0.5, normalizedWeight * maxLineWidth))
+          .attr("opacity", Math.min(maxOpacity, normalizedWeight * 2))
+          .attr("stroke-linecap", "round");
+      }
+    });
+  });
+  
+  // Click handler remains unchanged, using the passed-in data object.
+  thumbnail.on("click", function() {
+    d3.select("#display_head .number-box").text(headIdx + 1);
+    d3.select("#display_layer .number-box").text(layerIdx + 1);
+    displayHoverTokens(data, layerIdx, headIdx);
+  });
+  
+  return thumbnail.node();
+}
+
+// Function to display the tokens and attention values
+// function displayOutput(data) {
+//   const outputDiv = document.getElementById('output');
+//   outputDiv.innerHTML = `
+//     <h2>Tokens</h2>
+//     <pre>${JSON.stringify(data.tokens, null, 2)}</pre>
+//     <h2>Attention</h2>
+//     <pre>${JSON.stringify(data.attention, null, 2)}</pre>
+//   `;
+// }
+
+function renderTokens(tokens, attentionData, layer_idx, head_idx) {
+  const container = document.getElementById('tokenContainer');
+  container.innerHTML = "";
+
+  tokens.forEach((token, index) => {
+    const span = document.createElement('span');
+    span.textContent = token.replace("Ġ", "") + " ";
+    span.style.fontSize = "32px";
+    span.addEventListener('mouseenter', () => {
+      highlightAttention(index, attentionData, layer_idx, head_idx);
+    });
+    span.addEventListener('mouseleave', () => {
+      resetTokenSizes();
+    });
+    container.appendChild(span);
+  });
+}
+
+function displayHoverTokens(data, layer_idx, head_idx) {
+  let tokens, attentionMatrix;
+  if (!data.tokens || !data.attention) {
+    tokens = ['This', 'is', 'a', 'test', '.'];
+    // Create a dummy attention matrix if missing
+    attentionMatrix = Array(12)
+      .fill(null)
+      .map(() => Array(12).fill(null).map(() => Array(tokens.length).fill(0)));
+  } else {
+    tokens = data.tokens;
+    attentionMatrix = data.attention;
+  }
+  renderTokens(tokens, attentionMatrix, layer_idx, head_idx);
+}
+
+function resetTokenSizes() {
+  const container = document.getElementById("tokenContainer");
+  Array.from(container.children).forEach((span) => {
+    span.style.fontSize = "32px";
+    span.style.color = "#555";
+  });
+}
+
+function highlightAttention(index, attentionData, layer_idx, head_idx) {
+  const container = document.getElementById('tokenContainer');
+  const row = attentionData[layer_idx][head_idx][index];
+
+  if (!row) {
+    console.warn(`No attention data for token index ${index}`);
+    return;
+  }
+  const weights = row;
+  if (!weights.length) {
+    return;
+  }
+
+  // Find the maximum weight
+  const maxWeight = Math.max(...weights) || 1;
+  const baseFontSize = 32;
+  const maxIncrease = 20;
+  
+  const maxIndex = weights.indexOf(maxWeight);
+
+  Array.from(container.children).forEach((span, idx) => {
+    const weight = weights[idx];
+
+    if (typeof weight === 'number') {
+      const newFontSize = baseFontSize + (weight / maxWeight) * maxIncrease;
+      span.style.fontSize = newFontSize + "px";
+
+      if (idx === maxIndex) {
+        span.style.color = "#800000"; // Bordo red
+      } else {
+        span.style.color = "#555"; // Reset color
+      }
+    } else {
+      // For tokens without a corresponding weight, reset styles.
+      span.style.fontSize = baseFontSize + "px";
+      span.style.color = "#555";
+    }
+  });
+}

frontend/styles.css

@@ -0,0 +1,198 @@
+@import url('https://fonts.googleapis.com/css2?family=Roboto:wght@400;700&display=swap');
+@import url('https://fonts.googleapis.com/css2?family=Lora:ital,wght@0,400..700;1,400..700&display=swap');
+
+*,
+*::before,
+*::after {
+  box-sizing: border-box;
+}
+
+/* Basic Resets & Body */
+html, body {
+  margin: 0;
+  padding: 0;
+  font-family: 'Lora', 'Roboto', sans-serif;
+  background-color: #fefdf6; /* Off-white background */
+  color: #333;
+}
+input:focus,
+select:focus,
+textarea:focus,
+button:focus {
+    outline: none;
+}
+/* Header spanning full width */
+header {
+  width: 100%;
+  background-color: #fff;
+  padding: 1rem 2rem;
+  border-bottom: 1px solid #ddd;
+}
+
+header h1 {
+  margin: 0;
+  font-size: 1.8rem;
+  font-weight: 100;
+}
+
+/*
+  Container that splits the page:
+  left (main) ~75%, right (aside) ~25%
+*/
+.container {
+  display: flex;
+  flex-direction: row;
+  min-height: calc(100vh - 60px); /* Keep some height below header */
+}
+
+/* Main content on the left */
+.container main {
+  width: 75%;
+  padding: 2rem;
+  box-sizing: border-box;
+}
+
+/* The aside on the right can be used or left empty */
+.container aside {
+  width: 25%;
+  padding: 2rem;
+  box-sizing: border-box;
+  background-color: #fffff8;
+}
+
+/* ABOUT / DEEP GAZE SECTIONS */
+.about-section {
+  margin-bottom: 2rem;
+}
+
+.about-section h2 {
+  font-size: 1.4rem;
+  margin-bottom: 1rem;
+  font-weight: 100;
+  font-style: italic;
+
+}
+
+.about-section p {
+  line-height: 1.6;
+  margin-bottom: 1rem;
+}
+
+.deep-gaze-section h2 {
+  font-size: 1.4rem;
+  margin-bottom: 0.5rem;
+  font-weight: 100;
+  font-style: italic;
+
+}
+
+.deep-gaze-section p {
+  margin-bottom: 1rem;
+  line-height: 1.6;
+}
+
+/* Selection & Text Form Styles */
+.selection-form,
+.text-form {
+  margin-bottom: 1.5rem;
+}
+
+.selection-form label {
+  margin-right: 0.5rem;
+}
+
+.selection-form select {
+  margin-right: 1rem;
+}
+
+.text-form {
+  display: flex;
+  flex-direction: column;
+  align-items: flex-start;
+  border: none;
+}
+
+textarea {
+  width: 100%;
+  max-width: 100%;
+  font-size: 2rem;
+  font-weight: 700;
+  margin-bottom: 2rem;
+  border: none;
+  resize: none;
+  font-family: 'Roboto', sans-serif;
+  background-color: #fffff8;
+  color: #3a3939;
+  border-radius: 8px;
+  padding: 20px;
+  filter: drop-shadow(0 0 0.75rem #ddd);
+}
+
+
+textarea::placeholder {
+  color: #888;
+  font-family: 'Roboto', sans-serif;
+  font-size:2rem;
+  font-weight: 700;
+
+}
+
+button {
+  background-color: #4b80f9;
+  color: white;
+  border: none;
+  border-radius: 4px;
+  padding: 0.6rem 1rem;
+  cursor: pointer;
+  font-size: 1rem;
+}
+button:hover {
+  background-color: #3f6ddb;
+}
+
+/* Output Area */
+#output {
+  background-color: #fff;
+  border: 1px solid #ddd;
+  padding: 1rem;
+  margin-bottom: 1rem;
+  max-height: 300px;
+  overflow-y: auto;
+}
+
+#output h2 {
+  margin-top: 0;
+  font-size: 1.2rem;
+}
+
+/* Token Container */
+#tokenContainer {
+  margin-top: 1rem;
+}
+
+/* Token highlighting styles */
+#tokenContainer span {
+  cursor: default;
+  transition: font-size 0.9s ease;
+  color: #555;
+  display: inline-block;
+  transition: font-size 0.9s ease;
+  margin-right: 4px;
+  padding: 2px 4px;
+}
+
+.text-form button {
+  background-color: #800000; /* Bordo red */
+  color: #fff; /* Optional: set text color to white for contrast */
+  border: none;
+  padding: 10px 20px;
+  font-size: 16px;
+  cursor: pointer;
+}
+
+#credits {
+  font-size: 0.8rem;
+  color: #888;
+  margin-top: 1rem;
+  font-style: italic;
+}