working to add some llama2 step

package-lock.json

@@ -8,6 +8,7 @@
       "name": "video-quest",
       "version": "0.0.0",
       "dependencies": {
+        "@huggingface/inference": "^2.6.1",
         "@radix-ui/react-accordion": "^1.1.2",
         "@radix-ui/react-avatar": "^1.0.3",
         "@radix-ui/react-checkbox": "^1.0.4",
@@ -183,6 +184,14 @@
         "react-dom": ">=16.8.0"
       }
     },
+    "node_modules/@huggingface/inference": {
+      "version": "2.6.1",
+      "resolved": "https://registry.npmjs.org/@huggingface/inference/-/inference-2.6.1.tgz",
+      "integrity": "sha512-qFYchgOCPeEkZJKiSr7Kz62QwukJtgkeQCT7Q0SSKUcvHpTQVNJp6i/JrJMR4dBdzQysJ1SZDC0pLBBnnskTag==",
+      "engines": {
+        "node": ">=18"
+      }
+    },
     "node_modules/@humanwhocodes/config-array": {
       "version": "0.11.10",
       "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.11.10.tgz",

package.json

@@ -9,6 +9,7 @@
     "lint": "next lint"
   },
   "dependencies": {
+    "@huggingface/inference": "^2.6.1",
     "@radix-ui/react-accordion": "^1.1.2",
     "@radix-ui/react-avatar": "^1.0.3",
     "@radix-ui/react-checkbox": "^1.0.4",

src/app/agents/pirates.ts

@@ -34,10 +34,13 @@ const actionnables = [
   // "ship's wheel",
   // "hat",
   // "barrel",
-  // "cannon",
+  "cannon",
   // "rope",
   // "bucket",
+  "skull",
   "parrot",
+  "lock",
+  "ship",
   // "wooden leg"
 ]
 
@@ -54,9 +57,8 @@ export const agent: Agent = {
     const prompt = [
       `screenshot from an adventure videogame`,
       `inside the hold of a pirate ship`,
-      `with a pirate chest in the center`,
-      `a parrot`,
-      `and a wooden leg`,
+      `a pirate chest in the center with a large lock`,
+      `a parrot on top of it`,
       `at sunset`,
       `unreal engine`,
     ].join(", ")

src/app/agents/server.ts

@@ -0,0 +1,10 @@
+"use server"
+
+import { HfInference } from "@huggingface/inference"
+
+const hfi = new HfInference(process.env.HF_API_TOKEN)
+const hf = hfi.endpoint(`${process.env.HF_INFERENCE_ENDPOINT_URL || ""}`)
+
+export async function decideNextSteps(userAction: string) {
+  return ""
+}

src/app/main.tsx

@@ -13,9 +13,9 @@ import {
 } from "@/components/ui/select"
 
 import { render } from "./render"
-import { Agent, AgentType, Scene } from "./agents/types"
+import { AgentType, Scene } from "./agents/types"
 import { agents, defaultAgent, getAgent } from "./agents"
-import { ImageSegment, RenderedScene } from "./types"
+import { RenderedScene } from "./types"
 
 export default function Main() {
   const [isPending, startTransition] = useTransition()
@@ -28,54 +28,59 @@ export default function Main() {
   })
   const ref = useRef<AgentType>(defaultAgent)
    
-  useEffect(() => {
-    
-    const updateView = async () => {
-      // console.log(`update view..`)
-
-      await startTransition(async () => {
-
-        // console.log(`getting agent..`)
-        const type = ref?.current
-        const agent = getAgent(type)
-
-        // console.log(`asking agent to determine things..`)
-        const scene = agent.simulate()
-
-        // console.log(`rendering scene..`)
-        const newRendered = await render(
-          scene.prompt,
-          scene.actionnables.slice(0, 5) // too many can slow us down it seems
-        )
-
-        if (type !== ref?.current) {
-          console.log("agent type changed! reloading scene")
-          setTimeout(() => { updateView() }, 0)
-          return
-        } 
-
-
-        if (newRendered.assetUrl) {
-          setRendered(newRendered)
-          // console.log(`got a new url: ${newUrl}`)
-          setScene(scene)
-          setTimeout(() => { updateView()}, 1000)
-        } else {
-          // console.log(`going to wait a bit more: ${newUrl}`)
-          setTimeout(() => { updateView()}, newRendered.error ? 6000 : 3000)
-        }
-      })
-    }
-
-    updateView()
 
+  const loadNextScene = async () => {
+    // console.log(`update view..`)
+
+    await startTransition(async () => {
+
+      // console.log(`getting agent..`)
+      const type = ref?.current
+      const agent = getAgent(type)
+
+      // console.log(`asking agent to determine things..`)
+      const scene = agent.simulate()
+
+      // console.log(`rendering scene..`)
+      const newRendered = await render(
+        scene.prompt,
+        scene.actionnables.slice(0, 5) // too many can slow us down it seems
+      )
+
+      if (type !== ref?.current) {
+        console.log("agent type changed! reloading scene")
+        setTimeout(() => { loadNextScene() }, 0)
+        return
+      } 
+
+      if (newRendered.assetUrl) {
+        setRendered(newRendered)
+        // console.log(`got a new url: ${newUrl}`)
+        setScene(scene)
+      }
+    })
+  }
+
+  useEffect(() => {
+    loadNextScene()
   }, [])
 
+  const handleUserAction = (action: string) => {
+    console.log("user action:", action)
+
+    // TODO: ask Llama2 what to do about it
+    // we need a frame and some actionnables,
+    // perhaps even some music or sound effects
+
+    console.log("we don't know what to do, so we just load the next frame!")
+    loadNextScene()
+  }
+
   return (
     <div className="flex flex-col w-full pt-4">
       <div className="flex flex-col space-y-3 px-2">
         <div className="flex flex-row items-center space-x-3">
-          <label className="flex">Select a game:</label>
+          <label className="flex">Select a story:</label>
           <Select
             defaultValue={defaultAgent}
             onValueChange={(value) => {
@@ -97,7 +102,7 @@ export default function Main() {
             </SelectContent>
           </Select>
         </div>
-        <p>Note: changing the model might take up to 1 minute</p>
+        <p>Note: it takes about 1 minute to generate a new game panel</p>
           
         {(scene) ? <div>
           <p>Action: {scene.action}</p>
@@ -111,7 +116,7 @@ export default function Main() {
           </div>)}
         </div>
       </div>
-      <ImageRenderer {...rendered} />
+      <ImageRenderer rendered={rendered} onUserAction={handleUserAction} />
     </div>
   )
 }

src/components/business/image-renderer.tsx

@@ -1,73 +1,115 @@
-import { useRef } from "react"
+import { useEffect, useRef, useState } from "react"
 
 import { ImageSegment, RenderedScene } from "@/app/types"
 
 export const ImageRenderer = ({
-  assetUrl = "",
-  maskBase64 = "",
-  segments = []
-}: RenderedScene) => {
+  rendered: {
+    assetUrl = "",
+    maskBase64 = "",
+    segments = []
+  },
+  onUserAction,
+}: {
+  rendered: RenderedScene
+  onUserAction: (action: string) => void
+}) => {
   const imgRef = useRef<HTMLImageElement | null>(null)
-  const canvas = document.createElement('canvas')
-  const context = canvas.getContext('2d')
-  
+  const canvasRef = useRef<HTMLCanvasElement | null>(null)
+  const contextRef = useRef<CanvasRenderingContext2D | null>(null)
+  const [actionnable, setActionnable] = useState<string>("")
+
+  useEffect(() => {
+    if (maskBase64) {
+      console.log("maskBase64:", maskBase64)
+      const img = new Image();
+      img.onload = function () {
+        canvasRef.current = document.createElement('canvas');
+        canvasRef.current.width = img.width;
+        canvasRef.current.height = img.height;
+        contextRef.current = canvasRef.current.getContext('2d');
+        contextRef.current!.drawImage(img, 0, 0, img.width, img.height);
+      }
+      img.src = "data:image/png;base64," + maskBase64;
+    } else {
+      console.log("error, no maskBase64 detected!")
+    }
+  }, [maskBase64]);
+
+
   const getPixelColor = (x: number, y: number) => {
-    console.log("getting pixel color")
-    if (!context) {
+    if (!contextRef.current) {
       throw new Error("Unable to get context from canvas")
     }
 
-    const imgData = context.getImageData(x, y, 1, 1).data
+    const imgData = contextRef.current.getImageData(x, y, 1, 1).data
 
-    return `[${imgData[0]},${imgData[1]},${imgData[2]},${imgData[3]/255}]`
+
+    const clickedColor = Array.from(imgData.slice(0, 3), value => value / 255);
+
+    return clickedColor
   }
 
-  const handleMouseDown = (event: React.MouseEvent) => {
-    const boundingRect = imgRef.current!.getBoundingClientRect()
+
+  const getSegmentAt = (x: number, y: number): ImageSegment => {
+    if (!contextRef.current) throw new Error("Unable to get context from canvas");
+    if (!maskBase64) throw new Error("Mask is undefined");
+
+    let closestSegment: ImageSegment = {
+      id: 0,
+      box: [],
+      color: [],
+      label: "",
+      score: 0,
+    }
+
+    const clickedColor = getPixelColor(x,y) as any
+
+    if (`${clickedColor}` === "1,1,1") {
+      return closestSegment
+    }
   
-    const x = event.clientX - boundingRect.left
-    const y = event.clientY - boundingRect.top
+    let minDistance = Infinity;
+        
+    segments.forEach(segment => {
+      const segmentColor = segment.color.slice(0,3); // get the RGB part only
+
+      const distance = Math.sqrt(
+        Math.pow(clickedColor[0] - segmentColor[0], 2) +
+        Math.pow(clickedColor[1] - segmentColor[1], 2) +
+        Math.pow(clickedColor[2] - segmentColor[2], 2)
+      );
+      
+      if(distance < minDistance) {
+        minDistance = distance;
+        closestSegment = segment;
+      }
+    });
+
+    return closestSegment;
+  }
+
+  const handleMouseEvent = (event: React.MouseEvent, isClickEvent: boolean = false) => {
+    if (!contextRef.current) return; // Return early if mask image has not been loaded yet
   
-    if (maskBase64) {
-      const image = new Image()
-
-      image.onload = function () {
-        if (context) {
-          context.drawImage(image, 0, 0)
-
-          const clickedColor = getPixelColor(x, y) as any
-          
-          let closestSegment: ImageSegment | null = null
-
-          let minDistance = Infinity
-          
-          segments.forEach(segment => {
-            const segmentColor = segment.color.slice(0,3) // get the RGB part only
-            
-            const distance = Math.sqrt(
-              Math.pow(clickedColor[0] - segmentColor[0], 2) +
-              Math.pow(clickedColor[1] - segmentColor[1], 2) +
-              Math.pow(clickedColor[2] - segmentColor[2], 2)
-            )
-            
-            if(distance < minDistance) {
-              minDistance = distance;
-              closestSegment = segment;
-            }
-          })
+    const boundingRect = imgRef.current!.getBoundingClientRect();
+    const x = event.clientX - boundingRect.left;
+    const y = event.clientY - boundingRect.top;
   
-          console.log("closestSegment:", closestSegment)
-          console.log(closestSegment) // Here is the closest matching segment
-        }
-      }
+    const newSegment = getSegmentAt(x, y)
 
-      image.src = maskBase64
-      // image.src = "data:image/png;base64," + maskBase64;
-    } else {
-      console.log("No mask available, aborting..")
+    if (actionnable !== newSegment.label) {
+      setActionnable(newSegment.label)
     }
-  }
 
+    if (!newSegment.label) { return }
+
+    console.log("actionnable: ", actionnable)
+
+    if (isClickEvent) {
+      console.log("User clicked on " + actionnable);
+      // onUserAction(actionnable);
+    }
+  };
 
   if (!assetUrl) {
     return <div className="flex w-full h-screen items-center justify-center text-center">
@@ -75,14 +117,56 @@ export const ImageRenderer = ({
     </div>
   }
 
+  /*
+        <img
+          src={assetUrl}
+          ref={imgRef}
+          width="1024px"
+          height="512px"
+          className={
+            [
+              "absolute top-0 left-0",
+              actionnable ? "cursor-pointer" : ""
+            ].join(" ")
+          }
+          onMouseDown={(event) => handleMouseEvent(event, true)}
+          onMouseMove={handleMouseEvent}
+        />
+
+        <img
+          src={"data:image/png;base64," + maskBase64}
+          ref={imgRef}
+          width="1024px"
+          height="512px"
+          className={
+            [
+              "absolute top-0 left-0 opacity-30",
+              actionnable ? "cursor-pointer" : ""
+            ].join(" ")
+          }
+          onMouseDown={(event) => handleMouseEvent(event, true)}
+          onMouseMove={handleMouseEvent}
+        />
+        */
+
   return (
     <div className="w-full py-8 px-2">
-      <img
-        src={assetUrl}
-        ref={imgRef}
-        className="w-full rounded-md overflow-hidden"
-        onMouseDown={handleMouseDown}
-      />
+      <div className="relative w-full">
+        <img
+          src={"data:image/png;base64," + maskBase64}
+          ref={imgRef}
+          width="1024px"
+          height="512px"
+          className={
+            [
+              "absolute top-0 left-0 opacity-30",
+              actionnable ? "cursor-pointer" : ""
+            ].join(" ")
+          }
+          onMouseDown={(event) => handleMouseEvent(event, true)}
+          onMouseMove={handleMouseEvent}
+        />
+      </div>
     </div>
   )
 }