Create snail-extension.js

scaffolding/snail-extension.js

@@ -0,0 +1,134 @@
+class Posenet2Scratch {
+  constructor(runtime) {
+    this.runtime = runtime;
+    this.poses = [];
+    this.keypoints = [];
+    this._locale = this.setLocale();
+
+    const video = document.createElement("video");
+    video.width = 480;
+    video.height = 360;
+    video.autoplay = true;
+    video.style.display = "none";
+
+    navigator.mediaDevices
+      .getUserMedia({ video: true, audio: false })
+      .then((stream) => {
+        video.srcObject = stream;
+      });
+
+    video.addEventListener("loadeddata", () => {
+      // Load posenet model and process poses
+      posenet.load().then((net) => {
+        const detectPose = () => {
+          net.estimateMultiplePoses(video, {
+            flipHorizontal: false,
+          }).then((poses) => {
+            this.poses = poses;
+            this.keypoints = poses[0]?.keypoints || [];
+            requestAnimationFrame(detectPose);
+          });
+        };
+        detectPose();
+      });
+    });
+
+    this.runtime.ioDevices.video.enableVideo();
+  }
+
+  setLocale() {
+    const supportedLocales = ["en", "ja", "ja-Hira"];
+    const locale = navigator.language || "en";
+    return supportedLocales.includes(locale) ? locale : "en";
+  }
+
+  getX(args) {
+    const personIndex = parseInt(args.PERSON_NUMBER, 10) - 1;
+    const partIndex = parseInt(args.PART, 10);
+    const pose = this.poses[personIndex];
+
+    if (pose && pose.pose.keypoints[partIndex]) {
+      const x = pose.pose.keypoints[partIndex].position.x;
+      return this.runtime.ioDevices.video.mirror ? 240 - x : x - 240;
+    }
+    return "";
+  }
+
+  getY(args) {
+    const personIndex = parseInt(args.PERSON_NUMBER, 10) - 1;
+    const partIndex = parseInt(args.PART, 10);
+    const pose = this.poses[personIndex];
+
+    if (pose && pose.pose.keypoints[partIndex]) {
+      const y = pose.pose.keypoints[partIndex].position.y;
+      return 180 - y;
+    }
+    return "";
+  }
+
+  getPeopleCount() {
+    return this.poses.length;
+  }
+
+  getInfo() {
+    return {
+      id: "posenet2scratch",
+      name: "Posenet2Scratch",
+      blocks: [
+        {
+          opcode: "getX",
+          blockType: "reporter",
+          text: "[PART] x of person no. [PERSON_NUMBER]",
+          arguments: {
+            PERSON_NUMBER: { type: "string", menu: "personNumbers", defaultValue: "1" },
+            PART: { type: "string", menu: "parts", defaultValue: "0" },
+          },
+        },
+        {
+          opcode: "getY",
+          blockType: "reporter",
+          text: "[PART] y of person no. [PERSON_NUMBER]",
+          arguments: {
+            PERSON_NUMBER: { type: "string", menu: "personNumbers", defaultValue: "1" },
+            PART: { type: "string", menu: "parts", defaultValue: "0" },
+          },
+        },
+        {
+          opcode: "getPeopleCount",
+          blockType: "reporter",
+          text: "people count",
+        },
+      ],
+      menus: {
+        personNumbers: {
+          acceptReporters: true,
+          items: Array.from({ length: 10 }, (_, i) => ({ text: `${i + 1}`, value: `${i + 1}` })),
+        },
+        parts: {
+          acceptReporters: true,
+          items: [
+            { text: "nose", value: "0" },
+            { text: "left eye", value: "1" },
+            { text: "right eye", value: "2" },
+            { text: "left ear", value: "3" },
+            { text: "right ear", value: "4" },
+            { text: "left shoulder", value: "5" },
+            { text: "right shoulder", value: "6" },
+            { text: "left elbow", value: "7" },
+            { text: "right elbow", value: "8" },
+            { text: "left wrist", value: "9" },
+            { text: "right wrist", value: "10" },
+            { text: "left hip", value: "11" },
+            { text: "right hip", value: "12" },
+            { text: "left knee", value: "13" },
+            { text: "right knee", value: "14" },
+            { text: "left ankle", value: "15" },
+            { text: "right ankle", value: "16" },
+          ],
+        },
+      },
+    };
+  }
+}
+
+Scratch.extensions.register(new Posenet2Scratch());