Upload 4 files

README.md

@@ -4,18 +4,18 @@ library_name: unity-sentis
 pipeline_tag: object-detection
 ---
 
-# Blaze Face face detector in Unity Sentis (Version 1.3.0-pre.3*)
-*Version 1.3.0 Sentis files are not compatible with Sentis 1.4.0 and need to be recreated/downloaded
+# Blaze Face face detector in Unity Sentis (Version 1.4.0-pre.3*)
+*Version 1.3.0 sentis files are not compatible with 1.4.0 and will need to be recreated/downloaded
 
 This is the [Blaze Face model](https://developers.google.com/mediapipe/solutions/vision/face_detector) formatted to work in Unity Sentis 2023
 
 ## How to Use
 * Create a new scene in Unity 2023
-* Install `com.unity.sentis` version `1.3.0-pre.3` from the package manager
-* Put the blazeface.sentis file in the Assets/StreamingAssets folder
+* Install `com.unity.sentis` version `1.4.0-pre.3` from the package manager
+* Drag the blazeface.sentis file into the model asset field
 * Put a video in the Assets/StreamingAssets folder and set _videoName variable to the video name
 * Create a RawImage and place it in your scene. Link to this image in the _previewUI field.
-* Attach a sprite for the bounding box image to the faceTexture field
+* Attach a sprite or texture for the bounding box image to the BondingBoxSprite or BorderTexture field
 
 ## Preview
 When you get it working you should see something like this:

RunBlazeFace.cs

@@ -3,6 +3,7 @@ using Unity.Sentis;
 using UnityEngine.Video;
 using UnityEngine.UI;
 using Lays = Unity.Sentis.Layers;
+using FF = Unity.Sentis.Functional;
 
 /*
  *                   Blaze Face Inference
@@ -11,7 +12,7 @@ using Lays = Unity.Sentis.Layers;
  * Basic inference script for blaze face
  * 
  * Put this script on the Main Camera
- * Put blazeface.sentis in the Assets/StreamingAssets folder
+ * Drawg the blazeface.sentis into the modelAsset field
  * Create a RawImage in the scene
  * Put a link to that image in previewUI
  * Put a video in Assets/StreamingAssets folder and put the name of it int videoName
@@ -22,11 +23,15 @@ using Lays = Unity.Sentis.Layers;
 
 public class RunBlazeFace : MonoBehaviour
 {
+    //Draw the sentis model asset here:
+    public ModelAsset modelAsset;
+
     //Drag a link to a raw image here:
     public RawImage previewUI = null;
 
     // Put your bounding box sprite image here
-    public Sprite faceTexture;
+    public Sprite boundingboxSprite;
+    public Texture2D borderTexture;
 
     // 6 optional sprite images (left eye, right eye, nose, mouth, left ear, right ear)
     public Sprite[] markerTextures;
@@ -58,9 +63,6 @@ public class RunBlazeFace : MonoBehaviour
     //Holds input image size
     int size = 128;
 
-    Ops ops;
-    ITensorAllocator allocator;
-
     Model model;
 
     //webcam device name:
@@ -68,6 +70,8 @@ public class RunBlazeFace : MonoBehaviour
 
     bool closing = false;
 
+    TensorFloat anchors, centersToCorners;
+
     public struct BoundingBox
     {
         public float centerX;
@@ -78,7 +82,6 @@ public class RunBlazeFace : MonoBehaviour
 
     void Start()
     {
-        allocator = new TensorCachingAllocator();
 
         //(Note: if using a webcam on mobile get permissions here first)
 
@@ -88,7 +91,7 @@ public class RunBlazeFace : MonoBehaviour
 
         SetupModel();
 
-        SetupEngine();
+        SetupBoundingBoxSprite();
     }
 
     void SetupInput()
@@ -120,6 +123,16 @@ public class RunBlazeFace : MonoBehaviour
         }
     }
 
+    void SetupBoundingBoxSprite()
+    {
+        if (boundingboxSprite == null)
+        {
+            boundingboxSprite = Sprite.Create(borderTexture,
+                new Rect(0, 0, borderTexture.width, borderTexture.height),
+                new Vector2(borderTexture.width / 2, borderTexture.height / 2));
+        }
+    }
+
     void Update()
     {
         if (inputType == InputType.Webcam)
@@ -198,36 +211,43 @@ public class RunBlazeFace : MonoBehaviour
     {
         float[] offsets = GetGridBoxCoords();
 
-        model = ModelLoader.Load(Application.streamingAssetsPath + "/blazeface.sentis");
+        var model = ModelLoader.Load(modelAsset);
 
         //We need to add extra layers to the model in order to aggregate the box predicions:
 
         size = model.inputs[0].shape.ToTensorShape()[1]; // Input tensor width
 
-        model.AddConstant(new Lays.Constant("0", new int[] { 0 }));
-        model.AddConstant(new Lays.Constant("2", new int[] { 2 }));
-        model.AddConstant(new Lays.Constant("4", new int[] { 4 }));
-        model.AddConstant(new Lays.Constant("offsets", new TensorFloat(new TensorShape(1, 896, 4), offsets)));
-
-        model.AddLayer(new Lays.Slice("boxes", "regressors", "0", "4", "2"));
-        model.AddLayer(new Lays.Transpose("scores", "classificators", new int[] { 0, 2, 1 }));
-        model.AddLayer(new Lays.Add("boxCoords", "boxes", "offsets"));
-        model.AddOutput("boxCoords");
-
-        model.AddConstant(new Lays.Constant("maxOutputBoxes", new int[] { maxOutputBoxes }));
-        model.AddConstant(new Lays.Constant("iouThreshold", new float[] { iouThreshold }));
-        model.AddConstant(new Lays.Constant("scoreThreshold", new float[] { scoreThreshold }));
-
-        model.AddLayer(new Lays.NonMaxSuppression("NMS", "boxCoords", "scores",
-            "maxOutputBoxes", "iouThreshold", "scoreThreshold",
-            centerPointBox: Lays.CenterPointBox.Center
-        ));
-        model.AddOutput("NMS");
-    }
-    public void SetupEngine()
-    {
-        worker = WorkerFactory.CreateWorker(backend, model);
-        ops = WorkerFactory.CreateOps(backend, allocator);
+        anchors = new TensorFloat(new TensorShape(offsets.Length / 4, 4), offsets);
+
+        centersToCorners = new TensorFloat(new TensorShape(4, 4),
+        new float[]
+        {
+            1,      0,      1,      0,
+            0,      1,      0,      1,
+            -0.5f,  0,      0.5f,   0,
+            0,      -0.5f,  0,      0.5f
+        });
+
+        var model2 = Functional.Compile(
+          input =>
+          {
+              var outputs = model.Forward(input);
+              var regressors = outputs[0][0];                                       //shape=(896,16)
+              var scores = outputs[1][0].Transpose(0, 1) - scoreThreshold;          //shape=(1,896) 
+              var boxCoords = regressors[.., 0..4] + FunctionalTensor.FromTensor(anchors);//(896,4)            
+              var boxCorners = FF.MatMul(boxCoords, FunctionalTensor.FromTensor(centersToCorners));
+              var indices = FF.NMS(boxCorners, scores, iouThreshold);                //shape=(N)
+              var indices2 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 4 });    //shape=(N,4)
+              var output = FF.Gather(boxCoords, 0, indices2);                       //shape=(N,4)
+              var indices3 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 16 });   //shape=(N,16)
+              var markersOutput = FF.Gather(regressors, 0, indices3);               //shape=(N,16)
+              return (output, markersOutput);
+          },
+          InputDef.FromModel(model)[0]
+        );
+
+        worker = WorkerFactory.CreateWorker(backend, model2);
+
     }
 
     void DrawFaces(TensorFloat index3, TensorFloat regressors, int NMAX, Vector2 scale)
@@ -237,12 +257,12 @@ public class RunBlazeFace : MonoBehaviour
             //Draw bounding box of face
             var box = new BoundingBox
             {
-                centerX = index3[0, n, 0] * scale.x,
-                centerY = index3[0, n, 1] * scale.y,
-                width = index3[0, n, 2] * scale.x,
-                height = index3[0, n, 3] * scale.y
+                centerX = index3[ n, 0] * scale.x,
+                centerY = index3[ n, 1] * scale.y,
+                width = index3[ n, 2] * scale.x,
+                height = index3[ n, 3] * scale.y
             };
-            DrawBox(box, faceTexture);
+            DrawBox(box, boundingboxSprite);
             if (regressors == null) continue;
             
             //Draw markers for eyes, ears, nose, mouth:
@@ -250,12 +270,12 @@ public class RunBlazeFace : MonoBehaviour
             {
                 var marker = new BoundingBox
                 {
-                    centerX = box.centerX + (regressors[0, n, 4 + j * 2] - regressors[0, n, 0]) * scale.x,
-                    centerY = box.centerY + (regressors[0, n, 4 + j * 2 + 1] - regressors[0, n, 1]) * scale.y,
+                    centerX = box.centerX + (regressors[ n, 4 + j * 2] - regressors[ n, 0]) * scale.x,
+                    centerY = box.centerY + (regressors[ n, 4 + j * 2 + 1] - regressors[ n, 1]) * scale.y,
                     width = 8.0f * scale.x,
                     height = 8.0f * scale.y,
                 };
-                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : faceTexture);
+                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : boundingboxSprite);
             }
         }
     }
@@ -265,31 +285,24 @@ public class RunBlazeFace : MonoBehaviour
         var transform = new TextureTransform();
         transform.SetDimensions(size, size, 3);
         transform.SetTensorLayout(0, 3, 1, 2);
-        using var image0 = TextureConverter.ToTensor(source, transform);
-
-        // Pre-process the image to make input in range (-1..1)
-        using var image = ops.Mad(image0, 2f, -1f);
+        using var image = TextureConverter.ToTensor(source, transform);
 
         worker.Execute(image);
 
-        using var boxCoords = worker.PeekOutput("boxCoords") as TensorFloat; //face coords
-        using var regressors = worker.PeekOutput("regressors") as TensorFloat; //contains markers
-
-        var NM1 = worker.PeekOutput("NMS") as TensorInt;
-
-        using var boxCoords2 = ops.Reshape(boxCoords, boxCoords.shape.Unsqueeze(0));
-        using var output = ops.GatherND(boxCoords2, NM1, 0);
+        using var output = worker.PeekOutput("output_0") as TensorFloat; //face coords
+        using var markersOutput = worker.PeekOutput("output_1") as TensorFloat; //contains markers
 
-        using var regressors2 = ops.Reshape(regressors,regressors.shape.Unsqueeze(0));
-        using var markersOutput = ops.GatherND(regressors2, NM1, 0);
+        output.CompleteOperationsAndDownload();
+        markersOutput.CompleteOperationsAndDownload();
 
-        output.MakeReadable();
-        markersOutput.MakeReadable();
+       // Debug.Log(output.shape + " " + markersOutput.shape);
+        //return;
 
         ClearAnnotations();
 
         Vector2 markerScale = previewUI.rectTransform.rect.size / size;
 
+        //Debug.Log(output.shape + " " + markersOutput.shape);
         DrawFaces(output, markersOutput, output.shape[0], markerScale);
 
     }
@@ -329,8 +342,8 @@ public class RunBlazeFace : MonoBehaviour
     void CleanUp()
     {
         closing = true;
-        ops?.Dispose();
-        allocator?.Dispose();
+        anchors?.Dispose();
+        centersToCorners?.Dispose();
         if (webcam) Destroy(webcam);
         if (video) Destroy(video);
         RenderTexture.active = null;

info.json

@@ -1,11 +1,11 @@
 {
     "code": [
-        "RunBlazeFace.cs"
+      "RunBlazeFace.cs"
     ],
     "models": [
         "blazeface.sentis"
     ],
     "version": [
-         "1.3.0-pre.3"
+        "1.4.0"
     ]
 }