Upload 4 files

README.md

@@ -4,26 +4,23 @@ library_name: unity-sentis
 pipeline_tag: object-detection
 ---
 
-# Blaze Palm palm detector in Unity Sentis Format (Version 1.3.0-pre.3*)
-*Sentis files from 1.4.0 are not compatible with 1.3.0 and above and need to be recreated/downloaded
+# Blaze Face face detector in Unity Sentis (Version 1.4.0-pre.3*)
+*Version 1.3.0 sentis files are not compatible with 1.4.0 and will need to be recreated/downloaded
 
-This is the Blaze Palm model, part of the [MediaPipe hand detection](https://developers.google.com/mediapipe/solutions/vision/hand_landmarker) formatted to work in Unity Sentis 2023
+This is the [Blaze Face model](https://developers.google.com/mediapipe/solutions/vision/face_detector) formatted to work in Unity Sentis 2023
 
 ## How to Use
 * Create a new scene in Unity 2023
-* Install package `com.unity.sentis` version `1.3.0-pre.3` from the package manager
-* Put the hand_detection_lite.sentis file in the Assets/StreamingAssets folder
-* Put a video in the Assets/StreamingAssets folder and set `videoName` variable to the video name
-* Create a RawImage and place it in your scene. Link to this image in the `previewUI` field.
-* Attach a sprite for the bounding box image to the `boundingBoxTexture` field
+* Install `com.unity.sentis` version `1.4.0-pre.3` from the package manager
+* Drag the blazeface.sentis file into the model asset field
+* Put a video in the Assets/StreamingAssets folder and set _videoName variable to the video name
+* Create a RawImage and place it in your scene. Link to this image in the _previewUI field.
+* Attach a sprite or texture for the bounding box image to the BondingBoxSprite or BorderTexture field
 
 ## Preview
 When you get it working you should see something like this:
 
-![preview](blaze_palm_preview.png)
-
-## Information
-This model may have some accuracy issues.
+![preview](blaze_preview.png)
 
 ## Unity Sentis
 Sentis is the inference engine for Unity 2023. More information can be found [here](https://unity.com/products/sentis)

RunBlazePalm.cs

@@ -3,6 +3,7 @@ using Unity.Sentis;
 using UnityEngine.Video;
 using UnityEngine.UI;
 using Lays = Unity.Sentis.Layers;
+using FF = Unity.Sentis.Functional;
 
 /*
  *                   Blaze Palm Inference
@@ -22,11 +23,13 @@ using Lays = Unity.Sentis.Layers;
 
 public class RunBlazePalm : MonoBehaviour
 {
+    public ModelAsset asset;
     //Drag a link to a raw image here:
     public RawImage previewUI = null;
 
     // Put your bounding box sprite image here
-    public Sprite boundingBoxTexture;
+    public Texture2D boundingBoxTexture;
+    public Sprite boundingBoxSprite;
 
     // optional images for palm markers
     public Sprite[] markerTextures;
@@ -58,9 +61,6 @@ public class RunBlazePalm : MonoBehaviour
     //Holds image size
     int size;
 
-    Ops ops;
-    ITensorAllocator allocator;
-
     Model model;
 
     //webcam device name:
@@ -68,8 +68,8 @@ public class RunBlazePalm : MonoBehaviour
 
     bool closing = false;
 
-    const string regressorsOutput = "Identity";
-    const string classificatorsOutput = "Identity_1";
+
+    TensorFloat anchors, centersToCorners;
 
     public struct BoundingBox
     {
@@ -81,8 +81,6 @@ public class RunBlazePalm : MonoBehaviour
 
     void Start()
     {
-        allocator = new TensorCachingAllocator();
-
         //(Note: if using a webcam on mobile get permissions here first)
 
         targetTexture = new RenderTexture(resolution.x, resolution.y, 0);
@@ -93,6 +91,13 @@ public class RunBlazePalm : MonoBehaviour
         SetupModel();
 
         SetupEngine();
+
+        if (boundingBoxSprite == null)
+        {
+            boundingBoxSprite = Sprite.Create(boundingBoxTexture,
+                new Rect(0, 0, boundingBoxTexture.width, boundingBoxTexture.height), 
+                new Vector2(boundingBoxTexture.width / 2, boundingBoxTexture.height / 2));
+        }
     }
 
     void SetupInput()
@@ -179,6 +184,15 @@ public class RunBlazePalm : MonoBehaviour
     }
 
 
+
+    float[] GetGridBoxCoords()
+    {
+        var offsets = new float[2016 * 4];
+        int n = 0;
+        AddGrid(offsets, 24, 2, 8, ref n);
+        AddGrid(offsets, 12, 6, 16, ref n);
+        return offsets;
+    }
     void AddGrid(float[] offsets, int rows, int repeats, int cellWidth, ref int n)
     {
         for (int j = 0; j < repeats * rows * rows; j++)
@@ -189,51 +203,52 @@ public class RunBlazePalm : MonoBehaviour
         }
     }
 
-    float[] GetGridBoxCoords()
-    {
-        var offsets = new float[2016 * 4];
-        int n = 0;
-        AddGrid(offsets, 24, 2, 8, ref n);
-        AddGrid(offsets, 12, 6, 16, ref n);
-        return offsets;
-    }
 
     void SetupModel()
     {
         float[] offsets = GetGridBoxCoords();
 
-        model = ModelLoader.Load(Application.streamingAssetsPath + "/palm_detection_lite.sentis");
+        model = ModelLoader.Load(asset);
+        //model = ModelLoader.Load(Application.streamingAssetsPath + "/palm_detection_lite.sentis");
 
         //We need to add extra layers to the model in order to aggregate the box predicions:
         size = model.inputs[0].shape.ToTensorShape()[2]; // Input tensor width (192)
 
-        //set constants
-        model.AddConstant(new Lays.Constant("0", new int[] { 0 }));
-        model.AddConstant(new Lays.Constant("2", new int[] { 2 }));
-        model.AddConstant(new Lays.Constant("4", new int[] { 4 }));
-        model.AddConstant(new Lays.Constant("maxOutputBoxes", new int[] { maxOutputBoxes }));
-        model.AddConstant(new Lays.Constant("iouThreshold", new float[] { iouThreshold }));
-        model.AddConstant(new Lays.Constant("scoreThreshold", new float[] { scoreThreshold }));
-        model.AddConstant(new Lays.Constant("offsets",
-            new TensorFloat(new TensorShape(1, offsets.Length / 4, 4), offsets)
-         ));
-
-        //add layers
-        model.AddLayer(new Lays.Slice("boxes", regressorsOutput, "0", "4", "2"));
-        model.AddLayer(new Lays.Transpose("scores", classificatorsOutput, new int[] { 0, 2, 1 }));
-        model.AddLayer(new Lays.Add("boxCoords", "boxes", "offsets"));
-        model.AddOutput("boxCoords");
-
-        model.AddLayer(new Lays.NonMaxSuppression("NMS", "boxCoords", "scores",
-            "maxOutputBoxes", "iouThreshold", "scoreThreshold",
-            centerPointBox: Lays.CenterPointBox.Center
-        ));
-        model.AddOutput("NMS");
+        anchors = new TensorFloat(new TensorShape(offsets.Length / 4, 4), offsets);
+
+        centersToCorners = new TensorFloat(new TensorShape(4, 4),
+        new float[]
+        {
+            1,      0,      1,      0,
+            0,      1,      0,      1,
+            -0.5f,  0,      0.5f,   0,
+            0,      -0.5f,  0,      0.5f
+        });
+
+        var model2 = Functional.Compile(
+          input =>
+          {
+              var outputs = model.Forward(input);
+              var regressors = outputs[1][0];                                       //shape=(2016,18)
+              var scores = outputs[0][0].Transpose(0, 1) - scoreThreshold;          //shape=(1,2016) 
+              var boxCoords = regressors[.., 0..4] + FunctionalTensor.FromTensor(anchors);  //(2016,4)            
+              var boxCorners = FF.MatMul(boxCoords, FunctionalTensor.FromTensor(centersToCorners));
+              var indices = FF.NMS(boxCoords, scores, iouThreshold);                //shape=(N)
+              var indices2 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 4 });    //shape=(N,4)
+              var output = FF.Gather(boxCoords, 0, indices2);                       //shape=(N,4)
+              var indices3 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 18 });   //shape=(N,18)
+              var markersOutput = FF.Gather(regressors, 0, indices3);               //shape=(N,18)
+              return (output, markersOutput);
+          },
+          InputDef.FromModel(model)[0]
+       );
+
+        worker = WorkerFactory.CreateWorker(backend, model2);
+
     }
     public void SetupEngine()
     {
-        worker = WorkerFactory.CreateWorker(backend, model);
-        ops = WorkerFactory.CreateOps(backend, allocator);
+        
     }
 
     void DrawPalms(TensorFloat index3, TensorFloat regressors, int NMAX, Vector2 scale)
@@ -243,12 +258,12 @@ public class RunBlazePalm : MonoBehaviour
             //Draw bounding box of the palm
             var box = new BoundingBox
             {
-                centerX = index3[0, n, 0] * scale.x,
-                centerY = index3[0, n, 1] * scale.y,
-                width = index3[0, n, 2] * scale.x,
-                height = index3[0, n, 3] * scale.y
+                centerX = index3[n, 0] * scale.x,
+                centerY = index3[n, 1] * scale.y,
+                width = index3[n, 2] * scale.x,
+                height = index3[n, 3] * scale.y
             };
-            DrawBox(box, boundingBoxTexture);
+            DrawBox(box, boundingBoxSprite);
             if (regressors == null) continue;
             
             //Draw markers starts of fingers
@@ -256,12 +271,12 @@ public class RunBlazePalm : MonoBehaviour
             {
                 var marker = new BoundingBox
                 {
-                    centerX = box.centerX + (regressors[0, n, 4 + j * 2] - regressors[0, n, 0]) * scale.x,
-                    centerY = box.centerY + (regressors[0, n, 4 + j * 2 + 1] - regressors[0, n, 1]) * scale.y,
+                    centerX = box.centerX + (regressors[n, 4 + j * 2] - regressors[n, 0]) * scale.x,
+                    centerY = box.centerY + (regressors[n, 4 + j * 2 + 1] - regressors[n, 1]) * scale.y,
                     width = 4f * scale.x,
                     height = 4f * scale.y,
                 };
-                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : boundingBoxTexture);
+                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : boundingBoxSprite);
             }
         }
     }
@@ -271,26 +286,15 @@ public class RunBlazePalm : MonoBehaviour
         var transform = new TextureTransform();
         transform.SetDimensions(size, size, 3);
         transform.SetTensorLayout(0, 3, 1, 2);
-        using var image0 = TextureConverter.ToTensor(source, transform);
-
-        // Pre-process the image to make input in range (-1..1)
-        //using var image = ops.Mad(image0, 2f, -1f);
-
-        worker.Execute(image0);
-
-        using var boxCoords = worker.PeekOutput("boxCoords") as TensorFloat; //palm coords
-        using var regressors = worker.PeekOutput(regressorsOutput) as TensorFloat; //contains markers
-
-        var NMS = worker.PeekOutput("NMS") as TensorInt;
+        using var image = TextureConverter.ToTensor(source, transform);
 
-        using var boxCoords2 = boxCoords.ShallowReshape(boxCoords.shape.Unsqueeze(0)) as TensorFloat;
-        using var output = ops.GatherND(boxCoords2, NMS, 0);
+        worker.Execute(image);
 
-        using var regressors2 = regressors.ShallowReshape(regressors.shape.Unsqueeze(0)) as TensorFloat;
-        using var markersOutput = ops.GatherND(regressors2, NMS, 0);
+        var output = worker.PeekOutput("output_0") as TensorFloat;
+        var markersOutput = worker.PeekOutput("output_1") as TensorFloat;
 
-        output.MakeReadable();
-        markersOutput.MakeReadable();
+        output.CompleteOperationsAndDownload();
+        markersOutput.CompleteOperationsAndDownload();
 
         ClearAnnotations();
 
@@ -327,9 +331,9 @@ public class RunBlazePalm : MonoBehaviour
 
     void CleanUp()
     {
+        anchors?.Dispose();
+        centersToCorners?.Dispose();
         closing = true;
-        ops?.Dispose();
-        allocator?.Dispose();
         if (webcam) Destroy(webcam);
         if (video) Destroy(video);
         RenderTexture.active = null;

info.js

@@ -1,6 +1,6 @@
 {
    "version" : [
-       "1.3.0-pre.3"
+       "1.4.0-pre.3"
    ]
   
 }

palm_detection_lite.sentis

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9d1746c58054f5419c994d8869e7945dd074d34bbedb27b518cb4d106447973b
-size 3953743
+oid sha256:1598c7c4afc1bfccab382c69e43aaf5c7b099fb4a35f77a8ad32014656b0f6e4
+size 3892980