mobileclip-webnn-gpu

Running

App Files Files Community

Xenova HF Staff commited on Apr 17

Commit

27d3085

verified ·

1 Parent(s): 78f6230

Upload 3 files

Browse files

Files changed (3) hide show

index.html +39 -17
main.js +171 -0
style.css +81 -18

index.html CHANGED Viewed

@@ -1,19 +1,41 @@
 <!doctype html>
-<html>
-	<head>
-		<meta charset="utf-8" />
-		<meta name="viewport" content="width=device-width" />
-		<title>My static Space</title>
-		<link rel="stylesheet" href="style.css" />
-	</head>
-	<body>
-		<div class="card">
-			<h1>Welcome to your static Space!</h1>
-			<p>You can modify this app directly by editing <i>index.html</i> in the Files and versions tab.</p>
-			<p>
-				Also don't forget to check the
-				<a href="https://huggingface.co/docs/hub/spaces" target="_blank">Spaces documentation</a>.
-			</p>
-		</div>
-	</body>
 </html>

 <!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>Transformers.js | real-time CLIP</title>
+    <link rel="stylesheet" href="/style.css" />
+  </head>
+  <body>
+    <h1>Real-time zero-shot image classification (WebGPU)</h1>
+    <h3>
+      Runs locally in your browser w/
+      <a
+        href="https://github.com/huggingface/transformers.js"
+        target="_blank"
+        rel="noopener noreferrer"
+        >🤗 Transformers.js</a
+      >
+    </h3>
+    <div id="container">
+      <video id="video" autoplay muted playsinline></video>
+      <div id="overlay"></div>
+    </div>
+    <div id="controls">
+      <div title="Labels used to perform zero-shot image classification">
+        <label>Labels (comma-separated)</label>
+        <br />
+        <input id="labels" type="text" disabled />
+      </div>
+      <div title="Template used to perform zero-shot image classification">
+        <label>Hypothesis template</label>
+        <br />
+        <input id="template" type="text" value="A photo of a {}" disabled />
+      </div>
+    </div>
+    <label id="status"></label>
+    <script type="module" src="/main.js"></script>
+  </body>
 </html>

main.js ADDED Viewed

	@@ -0,0 +1,171 @@

+import {
+  AutoTokenizer,
+  CLIPTextModelWithProjection,
+  AutoProcessor,
+  CLIPVisionModelWithProjection,
+  RawImage,
+  dot,
+  softmax,
+} from "https://cdn.jsdelivr.net/npm/@huggingface/transformers@3.5.0";
+// Reference the elements that we will need
+const status = document.getElementById("status");
+const container = document.getElementById("container");
+const video = document.getElementById("video");
+const labelsInput = document.getElementById("labels");
+const templateInput = document.getElementById("template");
+const overlay = document.getElementById("overlay");
+status.textContent = "Loading model (88MB)...";
+const model_id = "Xenova/mobileclip_s0";
+let tokenizer, text_model, processor, vision_model;
+try {
+  // Load tokenizer and text model
+  tokenizer = await AutoTokenizer.from_pretrained(model_id);
+  text_model = await CLIPTextModelWithProjection.from_pretrained(model_id, {
+    device: "wasm",
+    dtype: "q8",
+  });
+  // Load processor and vision model
+  processor = await AutoProcessor.from_pretrained(model_id);
+  vision_model = await CLIPVisionModelWithProjection.from_pretrained(model_id, {
+    device: "webnn",
+    dtype: "fp32",
+  });
+} catch (err) {
+  console.error(err);
+  status.textContent = err.message;
+  alert(err.message);
+  throw err;
+}
+labelsInput.disabled = false;
+templateInput.disabled = false;
+status.textContent = "Ready";
+// See `model.logit_scale` parameter of original model
+const exp_logit_scale = Math.exp(4.6052);
+const IMAGE_SIZE = 224;
+const canvas = document.createElement("canvas");
+canvas.width = canvas.height = IMAGE_SIZE;
+const context = canvas.getContext("2d", { willReadFrequently: true });
+let isProcessing = false;
+let previousTime;
+let textEmbeddings;
+let prevTextInputs;
+let prevTemplate;
+let labels;
+function onFrameUpdate() {
+  if (!isProcessing) {
+    isProcessing = true;
+    (async function () {
+      // If text inputs have changed, update the embeddings
+      if (
+        prevTextInputs !== labelsInput.value ||
+        prevTemplate !== templateInput.value
+      ) {
+        textEmbeddings = null;
+        prevTextInputs = labelsInput.value;
+        prevTemplate = templateInput.value;
+        labels = prevTextInputs.split(/\s*,\s*/).filter((x) => x);
+        if (labels.length > 0) {
+          const texts = labels.map((x) =>
+            templateInput.value.replaceAll("{}", x),
+          );
+          const text_inputs = tokenizer(texts, {
+            padding: "max_length", // NB: the model requires max_length padding
+            truncation: true,
+          });
+          // Compute embeddings
+          const { text_embeds } = await text_model(text_inputs);
+          textEmbeddings = text_embeds.normalize().tolist();
+        } else {
+          overlay.innerHTML = "";
+        }
+      }
+      if (textEmbeddings) {
+        // Read the current frame from the video
+        context.drawImage(video, 0, 0, IMAGE_SIZE, IMAGE_SIZE);
+        const pixelData = context.getImageData(
+          0,
+          0,
+          IMAGE_SIZE,
+          IMAGE_SIZE,
+        ).data;
+        const image = new RawImage(pixelData, IMAGE_SIZE, IMAGE_SIZE, 4);
+        const image_inputs = await processor(image);
+        // Compute embeddings
+        const { image_embeds } = await vision_model(image_inputs);
+        const imageEmbedding = image_embeds.normalize().tolist()[0];
+        // Compute similarity
+        const similarities = textEmbeddings.map(
+          (x) => dot(x, imageEmbedding) * exp_logit_scale,
+        );
+        const sortedIndices = softmax(similarities)
+          .map((x, i) => [x, i])
+          .sort((a, b) => b[0] - a[0]);
+        // Update UI
+        overlay.innerHTML = "";
+        for (const [score, index] of sortedIndices) {
+          overlay.appendChild(
+            document.createTextNode(`${labels[index]}: ${score.toFixed(2)}`),
+          );
+          overlay.appendChild(document.createElement("br"));
+        }
+      }
+      if (previousTime !== undefined) {
+        const fps = 1000 / (performance.now() - previousTime);
+        status.textContent = `FPS: ${fps.toFixed(2)}`;
+      }
+      previousTime = performance.now();
+      isProcessing = false;
+    })();
+  }
+  window.requestAnimationFrame(onFrameUpdate);
+}
+// Start the video stream
+navigator.mediaDevices
+  .getUserMedia(
+    { video: true }, // Ask for video
+  )
+  .then((stream) => {
+    // Set up the video and canvas elements.
+    video.srcObject = stream;
+    video.play();
+    const videoTrack = stream.getVideoTracks()[0];
+    const { width, height } = videoTrack.getSettings();
+    video.width = width;
+    video.height = height;
+    // Set container width and height depending on the image aspect ratio
+    const ar = width / height;
+    const [cw, ch] = ar > 720 / 405 ? [720, 720 / ar] : [405 * ar, 405];
+    container.style.width = `${cw}px`;
+    container.style.height = `${ch}px`;
+    // Start the animation loop
+    window.requestAnimationFrame(onFrameUpdate);
+  })
+  .catch((error) => {
+    alert(error);
+  });

style.css CHANGED Viewed

@@ -1,28 +1,91 @@
 body {
-	padding: 2rem;
-	font-family: -apple-system, BlinkMacSystemFont, "Arial", sans-serif;
 }
-h1 {
-	font-size: 16px;
-	margin-top: 0;
 }
-p {
-	color: rgb(107, 114, 128);
-	font-size: 15px;
-	margin-bottom: 10px;
-	margin-top: 5px;
 }
-.card {
-	max-width: 620px;
-	margin: 0 auto;
-	padding: 16px;
-	border: 1px solid lightgray;
-	border-radius: 16px;
 }
-.card p:last-child {
-	margin-bottom: 0;
 }

+* {
+  box-sizing: border-box;
+  padding: 0;
+  margin: 0;
+  font-family: sans-serif;
+}
+html,
+body {
+  height: 100%;
+}
 body {
+  padding: 16px 32px;
+}
+body,
+#container {
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+}
+#controls {
+  display: flex;
+  padding: 1rem;
+  gap: 1rem;
+}
+#controls > div {
+  text-align: center;
+}
+h1,
+h3 {
+  text-align: center;
+}
+h3 {
+  margin-top: 0.5rem;
+}
+#container {
+  position: relative;
+  width: 720px;
+  height: 405px;
+  max-width: 100%;
+  max-height: 100%;
+  border: 2px dashed #d1d5db;
+  border-radius: 0.75rem;
+  overflow: hidden;
+  margin-top: 1rem;
+  background-size: 100% 100%;
+  background-position: center;
+  background-repeat: no-repeat;
+}
+#status {
+  min-height: 16px;
+  margin: 8px 0;
+}
+video {
+  width: 100%;
+  height: 100%;
 }
+input[type="text"] {
+  padding: 0.25rem 0.5rem;
+  border: 1px solid #d1d5db;
+  border-radius: 0.25rem;
+  margin-top: 2px;
 }
+input[type="range"] {
+  margin-top: 6px;
 }
+#overlay {
+  position: absolute;
+  top: 0;
+  left: 0;
+  background-color: rgba(255, 255, 255, 0.9);
+  font-size: 1.25rem;
+  border-radius: 2px;
 }
+#overlay:not(:empty) {
+  padding: 0.5rem;
 }