Upload 4 files

README.md

@@ -2,8 +2,8 @@
 library_name: unity-sentis
 pipeline_tag: object-detection
 ---
-# YOLOv8n validated for Unity Sentis (Version 1.3.0-pre.3 only*)
-*Version 1.3.0 Sentis files will not work with version 1.4.0 and above. We are still making some updates to the 1.4.0 files
+# YOLOv8n validated for Unity Sentis (Version 1.4.0-pre.3*)
+*Version 1.3.0 sentis files are not compatible with 1.4.0 and will need to be recreated/downloaded
 
 [YOLOv8n](https://docs.ultralytics.com/models/yolov8/) is a real-time multi-object recognition model confirmed to run in Unity 2023.
 
@@ -12,7 +12,7 @@ First get the package `com.unity.sentis` from the package manager.
 You will also need the Unity UI package.
 
 * Create a new scene in Unity 2023.
-* Install `com.unity.sentis` version `1.3.0-pre.3` from the package manager
+* Install `com.unity.sentis` version `1.4.0-pre.3` from the package manager
 * Add the c# script to the Main Camera.
 * Create a Raw Image in the scene and link it as the `displayImage`
 * Put the yolov8n.sentis file in the Assets/StreamingAssets folder

RunYOLO8n.cs

@@ -4,6 +4,8 @@ using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Video;
 using Lays = Unity.Sentis.Layers;
+using System.IO;
+using FF = Unity.Sentis.Functional;
 
 /*
  *  YOLOv8n Inference Script
@@ -11,7 +13,8 @@ using Lays = Unity.Sentis.Layers;
  * 
  * Place this script on the Main Camera.
  * 
- * Place the yolov8n.sentis file and a *.mp4 video file in the Assets/StreamingAssets folder
+ * Place the yolob8n.sentis file in the asset folder and drag onto the asset field
+ * Place a *.mp4 video file in the Assets/StreamingAssets folder
  * Create a RawImage in your scene and set it as the displayImage field
  * Drag the classes.txt into the labelsAsset field
  * Add a reference to a sprite image for the bounding box and a font for the text
@@ -21,6 +24,8 @@ using Lays = Unity.Sentis.Layers;
 
 public class RunYOLO8n : MonoBehaviour
 {
+    // Drag the yolov8n.sentis file here
+    public ModelAsset asset;
     const string modelName = "yolov8n.sentis";
     // Change this to the name of the video you put in StreamingAssets folder:
     const string videoName = "giraffes.mp4";
@@ -28,15 +33,15 @@ public class RunYOLO8n : MonoBehaviour
     public TextAsset labelsAsset;
     // Create a Raw Image in the scene and link it here:
     public RawImage displayImage;
-    // Link to a bounding box texture here:
-    public Sprite boxTexture;
+    // Link to a bounding box sprite or texture here:
+    public Sprite borderSprite;
+    public Texture2D borderTexture;
     // Link to the font for the labels:
     public Font font;
 
     const BackendType backend = BackendType.GPUCompute;
 
     private Transform displayLocation;
-    private Model model;
     private IWorker engine;
     private string[] labels;
     private RenderTexture targetRT;
@@ -51,15 +56,13 @@ public class RunYOLO8n : MonoBehaviour
 
     private VideoPlayer video;
 
-    List<GameObject> boxPool = new List<GameObject>();
+    List<GameObject> boxPool = new();
 
     [SerializeField, Range(0, 1)] float iouThreshold = 0.5f;
     [SerializeField, Range(0, 1)] float scoreThreshold = 0.5f;
     int maxOutputBoxes = 64;
 
-    //For using tensor operators:
-    Ops ops;
-
+    TensorFloat centersToCorners;
     //bounding box data
     public struct BoundingBox
     {
@@ -70,14 +73,12 @@ public class RunYOLO8n : MonoBehaviour
         public string label;
     }
 
-    
+
     void Start()
     {
         Application.targetFrameRate = 60;
         Screen.orientation = ScreenOrientation.LandscapeLeft;
 
-        ops = WorkerFactory.CreateOps(backend, null);
-
         //Parse neural net labels
         labels = labelsAsset.text.Split('\n');
 
@@ -88,49 +89,50 @@ public class RunYOLO8n : MonoBehaviour
         //Create image to display video
         displayLocation = displayImage.transform;
 
-        //Create engine to run model
-        engine = WorkerFactory.CreateWorker(backend, model);
-
         SetupInput();
-    }
 
+        if (borderSprite == null)
+        {
+            borderSprite = Sprite.Create(borderTexture, new Rect(0, 0, borderTexture.width, borderTexture.height), new Vector2(borderTexture.width / 2, borderTexture.height / 2));
+        }
+    }
     void LoadModel()
     {
+
         //Load model
-        model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName);
-
-        //The classes are also stored here in JSON format:
-        Debug.Log($"Class names: \n{model.Metadata["names"]}");
-
-        //We need to add some layers to choose the best boxes with the NMSLayer
-        
-        //Set constants
-        model.AddConstant(new Lays.Constant("0", new int[] { 0 }));
-        model.AddConstant(new Lays.Constant("1", new int[] { 1 }));
-        model.AddConstant(new Lays.Constant("4", new int[] { 4 }));
-
-
-        model.AddConstant(new Lays.Constant("classes_plus_4", new int[] { numClasses + 4 }));
-        model.AddConstant(new Lays.Constant("maxOutputBoxes", new int[] { maxOutputBoxes }));
-        model.AddConstant(new Lays.Constant("iouThreshold", new float[] { iouThreshold }));
-        model.AddConstant(new Lays.Constant("scoreThreshold", new float[] { scoreThreshold }));
-       
-        //Add layers
-        model.AddLayer(new Lays.Slice("boxCoords0", "output0", "0", "4", "1")); 
-        model.AddLayer(new Lays.Transpose("boxCoords", "boxCoords0", new int[] { 0, 2, 1 }));
-        model.AddLayer(new Lays.Slice("scores0", "output0", "4", "classes_plus_4", "1")); 
-        model.AddLayer(new Lays.ReduceMax("scores", new[] { "scores0", "1" }));
-        model.AddLayer(new Lays.ArgMax("classIDs", "scores0", 1));
-
-        model.AddLayer(new Lays.NonMaxSuppression("NMS", "boxCoords", "scores",
-            "maxOutputBoxes", "iouThreshold", "scoreThreshold",
-            centerPointBox: Lays.CenterPointBox.Center
-        ));
-
-        model.outputs.Clear();
-        model.AddOutput("boxCoords");
-        model.AddOutput("classIDs");
-        model.AddOutput("NMS");
+        //var model1 = ModelLoader.Load(Path.Join(Application.streamingAssetsPath, modelName));
+        var model1 = ModelLoader.Load(asset);
+
+        centersToCorners = new TensorFloat(new TensorShape(4, 4),
+        new float[]
+        {
+                    1,      0,      1,      0,
+                    0,      1,      0,      1,
+                    -0.5f,  0,      0.5f,   0,
+                    0,      -0.5f,  0,      0.5f
+        });
+
+        //Here we transform the output of the model1 by feeding it through a Non-Max-Suppression layer.
+        var model2 = Functional.Compile(
+               input =>
+               {
+                   var modelOutput = model1.Forward(input)[0];
+                   var boxCoords = modelOutput[0, 0..4, ..].Transpose(0, 1);        //shape=(8400,4)
+                   var allScores = modelOutput[0, 4.., ..];                         //shape=(80,8400)
+                   var scores = FF.ReduceMax(allScores, 0) - scoreThreshold;        //shape=(8400)
+                   var classIDs = FF.ArgMax(allScores, 0);                          //shape=(8400) 
+                   var boxCorners = FF.MatMul(boxCoords, FunctionalTensor.FromTensor(centersToCorners));
+                   var indices = FF.NMS(boxCorners, scores, iouThreshold);           //shape=(N)
+                   var indices2 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 4 });//shape=(N,4)
+                   var coords = FF.Gather(boxCoords, 0, indices2);                  //shape=(N,4)
+                   var labelIDs = FF.Gather(classIDs, 0, indices);                  //shape=(N)
+                   return (coords, labelIDs);
+               },
+               InputDef.FromModel(model1)[0]
+         );
+
+        //Create engine to run model
+        engine = WorkerFactory.CreateWorker(backend, model2);
     }
 
     void SetupInput()
@@ -138,7 +140,7 @@ public class RunYOLO8n : MonoBehaviour
         video = gameObject.AddComponent<VideoPlayer>();
         video.renderMode = VideoRenderMode.APIOnly;
         video.source = VideoSource.Url;
-        video.url = Application.streamingAssetsPath + "/" + videoName;
+        video.url = Path.Join(Application.streamingAssetsPath, videoName);
         video.isLooping = true;
         video.Play();
     }
@@ -168,17 +170,11 @@ public class RunYOLO8n : MonoBehaviour
         using var input = TextureConverter.ToTensor(targetRT, imageWidth, imageHeight, 3);
         engine.Execute(input);
 
-        var boxCoords = engine.PeekOutput("boxCoords") as TensorFloat;
-        var NMS = engine.PeekOutput("NMS") as TensorInt;
-        var classIDs = engine.PeekOutput("classIDs") as TensorInt;
+        var output = engine.PeekOutput("output_0") as TensorFloat;
+        var labelIDs = engine.PeekOutput("output_1") as TensorInt;
 
-        using var boxIDs = ops.Slice(NMS, new int[] { 2 }, new int[] { 3 }, new int[] { 1 }, new int[] { 1 });
-        using var boxIDsFlat = boxIDs.ShallowReshape(new TensorShape(boxIDs.shape.length)) as TensorInt;
-        using var output = ops.Gather(boxCoords, boxIDsFlat, 1);
-        using var labelIDs = ops.Gather(classIDs, boxIDsFlat, 2);
-        
-        output.MakeReadable();
-        labelIDs.MakeReadable();
+        output.CompleteOperationsAndDownload();
+        labelIDs.CompleteOperationsAndDownload();
 
         float displayWidth = displayImage.rectTransform.rect.width;
         float displayHeight = displayImage.rectTransform.rect.height;
@@ -186,22 +182,23 @@ public class RunYOLO8n : MonoBehaviour
         float scaleX = displayWidth / imageWidth;
         float scaleY = displayHeight / imageHeight;
 
+        int boxesFound = output.shape[0];
         //Draw the bounding boxes
-        for (int n = 0; n < output.shape[1]; n++)
+        for (int n = 0; n < Mathf.Min(boxesFound, 200); n++)
         {
             var box = new BoundingBox
             {
-                centerX = output[0, n, 0] * scaleX - displayWidth / 2,
-                centerY = output[0, n, 1] * scaleY - displayHeight / 2,
-                width = output[0, n, 2] * scaleX,
-                height = output[0, n, 3] * scaleY,
-                label = labels[labelIDs[0, 0,n]],
+                centerX = output[n, 0] * scaleX - displayWidth / 2,
+                centerY = output[n, 1] * scaleY - displayHeight / 2,
+                width = output[n, 2] * scaleX,
+                height = output[n, 3] * scaleY,
+                label = labels[labelIDs[n]],
             };
-            DrawBox(box, n);
+            DrawBox(box, n, displayHeight * 0.05f);
         }
     }
 
-    public void DrawBox(BoundingBox box , int id)
+    public void DrawBox(BoundingBox box, int id, float fontSize)
     {
         //Create the bounding box graphic or get from pool
         GameObject panel;
@@ -220,10 +217,11 @@ public class RunYOLO8n : MonoBehaviour
         //Set box size
         RectTransform rt = panel.GetComponent<RectTransform>();
         rt.sizeDelta = new Vector2(box.width, box.height);
-        
+
         //Set label text
         var label = panel.GetComponentInChildren<Text>();
         label.text = box.label;
+        label.fontSize = (int)fontSize;
     }
 
     public GameObject CreateNewBox(Color color)
@@ -234,7 +232,7 @@ public class RunYOLO8n : MonoBehaviour
         panel.AddComponent<CanvasRenderer>();
         Image img = panel.AddComponent<Image>();
         img.color = color;
-        img.sprite = boxTexture;
+        img.sprite = borderSprite;
         img.type = Image.Type.Sliced;
         panel.transform.SetParent(displayLocation, false);
 
@@ -263,7 +261,7 @@ public class RunYOLO8n : MonoBehaviour
 
     public void ClearAnnotations()
     {
-        foreach(var box in boxPool)
+        foreach (var box in boxPool)
         {
             box.SetActive(false);
         }
@@ -271,7 +269,7 @@ public class RunYOLO8n : MonoBehaviour
 
     private void OnDestroy()
     {
+        centersToCorners?.Dispose();
         engine?.Dispose();
-        ops?.Dispose();
     }
-}
+}

info.js

@@ -0,0 +1,5 @@
+{
+      "version" : [
+         "1.4.0-pre.2"
+    ]
+}

yolov8n.sentis

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:2a788b23743661e23da08e893fb7efb507e912a017bfe98604b36a9609f9ffb3
-size 12897918
+oid sha256:72c2ccde7dedd160cd8b62907ff2fa06ffe594d4a0fe0d2b13eb270297ca455c
+size 12834028