Update script to Sentis 2.1.2

yolov8n.onnx → Models/yolov8n.onnx

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:00113c3ec2b9fd39142b2adb82c0f67d694eb2856413e14b37602d77eddfea08
-size 6434150
+oid sha256:341ad75c98ff88775c63e899e7cbbf497c13161e3393b95b620a6cab65052811
+size 6435893

README.md

@@ -2,24 +2,21 @@
 library_name: unity-sentis
 pipeline_tag: object-detection
 ---
-# YOLOv8n validated for Unity Sentis (Version 1.4.0-pre.3*)
-*Version 1.3.0 sentis files are not compatible with 1.4.0 and will need to be recreated/downloaded
+# YOLOv8n validated for Unity Sentis (Version 2.1.2)
 
-[YOLOv8n](https://docs.ultralytics.com/models/yolov8/) is a real-time multi-object recognition model confirmed to run in Unity 2023.
+[YOLOv8n](https://docs.ultralytics.com/models/yolov8/) is a real-time multi-object recognition model confirmed to run in Unity 6000.
 
 ## How to Use
-First get the package `com.unity.sentis` from the package manager.
-You will also need the Unity UI package.
-
-* Create a new scene in Unity 2023.
-* Install `com.unity.sentis` version `1.4.0-pre.3` from the package manager
-* Add the c# script to the Main Camera.
-* Create a Raw Image in the scene and link it as the `displayImage`
-* Drag the yolov8n.sentis file into the model asset field
-* Drag the classes.txt on to the labelAssets field
-* Put a video file in the Assets/StreamingAssets folder and set the name of videoName to the filename in the script
-* Set the fields for the bounding box texture sprite (you can [create your own one](https://docs.unity3d.com/Manual/9SliceSprites.html) using a transparent texture or use an inbuilt one) and the font
 
+* Create a new scene in Unity 6000;
+* Install `com.unity.sentis` version `2.1.2` from the package manager;
+* Add the `RunYOLO8n.cs` script to the Main Camera;
+* Drag the `Models/yolov8n.onnx` file into the `Model Asset` field;
+* Drag the `classes.txt` file into the `Classes Asset` field;
+* Create a `GameObject > UI > Raw Image` object in the scene, set its width and height to 640, and link it as the `Display Image` field;
+* Drag the `Border Texture.png` file into the `Border Texture` field;
+* Select an appropriate font in the `Font` field;
+* Put a video file in the `Assets/StreamingAssets` folder and set the `Video Filename` field to the filename of the video.
 
 ## Preview
 If working correctly you should see something like this:

RunYOLO8n.cs

@@ -3,7 +3,6 @@ using Unity.Sentis;
 using UnityEngine;
 using UnityEngine.UI;
 using UnityEngine.Video;
-using Lays = Unity.Sentis.Layers;
 using System.IO;
 using FF = Unity.Sentis.Functional;
 
@@ -11,41 +10,37 @@ using FF = Unity.Sentis.Functional;
  *  YOLOv8n Inference Script
  *  ========================
  * 
- * Place this script on the Main Camera.
- * 
- * Place the yolob8n.sentis file in the asset folder and drag onto the asset field
- * Place a *.mp4 video file in the Assets/StreamingAssets folder
- * Create a RawImage in your scene and set it as the displayImage field
- * Drag the classes.txt into the labelsAsset field
- * Add a reference to a sprite image for the bounding box and a font for the text
+ * Place this script on the Main Camera and set the script parameters according to the tooltips.
  * 
  */
 
-
 public class RunYOLO8n : MonoBehaviour
 {
-    // Drag the yolov8n.sentis file here
-    public ModelAsset asset;
-    const string modelName = "yolov8n.sentis";
-    // Change this to the name of the video you put in StreamingAssets folder:
-    const string videoName = "giraffes.mp4";
-    // Link the classes.txt here:
-    public TextAsset labelsAsset;
-    // Create a Raw Image in the scene and link it here:
+    [Tooltip("Drag a YOLO model .onnx file here")]
+    public ModelAsset modelAsset;
+
+    [Tooltip("Drag the classes.txt here")]
+    public TextAsset classesAsset;
+
+    [Tooltip("Create a Raw Image in the scene and link it here")]
     public RawImage displayImage;
-    // Link to a bounding box sprite or texture here:
-    public Sprite borderSprite;
+
+    [Tooltip("Drag a border box texture here")]
     public Texture2D borderTexture;
-    // Link to the font for the labels:
+
+    [Tooltip("Select an appropriate font for the labels")]
     public Font font;
 
+    [Tooltip("Change this to the name of the video you put in the Assets/StreamingAssets folder")]
+    public string videoFilename = "giraffes.mp4";
+
     const BackendType backend = BackendType.GPUCompute;
 
     private Transform displayLocation;
-    private IWorker engine;
+    private Worker worker;
     private string[] labels;
     private RenderTexture targetRT;
-
+    private Sprite borderSprite;
 
     //Image size for the model
     private const int imageWidth = 640;
@@ -60,9 +55,8 @@ public class RunYOLO8n : MonoBehaviour
 
     [SerializeField, Range(0, 1)] float iouThreshold = 0.5f;
     [SerializeField, Range(0, 1)] float scoreThreshold = 0.5f;
-    int maxOutputBoxes = 64;
 
-    TensorFloat centersToCorners;
+    Tensor<float> centersToCorners;
     //bounding box data
     public struct BoundingBox
     {
@@ -80,7 +74,7 @@ public class RunYOLO8n : MonoBehaviour
         Screen.orientation = ScreenOrientation.LandscapeLeft;
 
         //Parse neural net labels
-        labels = labelsAsset.text.Split('\n');
+        labels = classesAsset.text.Split('\n');
 
         LoadModel();
 
@@ -91,19 +85,15 @@ public class RunYOLO8n : MonoBehaviour
 
         SetupInput();
 
-        if (borderSprite == null)
-        {
-            borderSprite = Sprite.Create(borderTexture, new Rect(0, 0, borderTexture.width, borderTexture.height), new Vector2(borderTexture.width / 2, borderTexture.height / 2));
-        }
+        borderSprite = Sprite.Create(borderTexture, new Rect(0, 0, borderTexture.width, borderTexture.height), new Vector2(borderTexture.width / 2, borderTexture.height / 2));
     }
     void LoadModel()
     {
 
         //Load model
-        //var model1 = ModelLoader.Load(Path.Join(Application.streamingAssetsPath, modelName));
-        var model1 = ModelLoader.Load(asset);
+        var model1 = ModelLoader.Load(modelAsset);
 
-        centersToCorners = new TensorFloat(new TensorShape(4, 4),
+        centersToCorners = new Tensor<float>(new TensorShape(4, 4),
         new float[]
         {
                     1,      0,      1,      0,
@@ -113,26 +103,20 @@ public class RunYOLO8n : MonoBehaviour
         });
 
         //Here we transform the output of the model1 by feeding it through a Non-Max-Suppression layer.
-        var model2 = Functional.Compile(
-               input =>
-               {
-                   var modelOutput = model1.Forward(input)[0];
-                   var boxCoords = modelOutput[0, 0..4, ..].Transpose(0, 1);        //shape=(8400,4)
-                   var allScores = modelOutput[0, 4.., ..];                         //shape=(80,8400)
-                   var scores = FF.ReduceMax(allScores, 0) - scoreThreshold;        //shape=(8400)
-                   var classIDs = FF.ArgMax(allScores, 0);                          //shape=(8400) 
-                   var boxCorners = FF.MatMul(boxCoords, FunctionalTensor.FromTensor(centersToCorners));
-                   var indices = FF.NMS(boxCorners, scores, iouThreshold);           //shape=(N)
-                   var indices2 = indices.Unsqueeze(-1).BroadcastTo(new int[] { 4 });//shape=(N,4)
-                   var coords = FF.Gather(boxCoords, 0, indices2);                  //shape=(N,4)
-                   var labelIDs = FF.Gather(classIDs, 0, indices);                  //shape=(N)
-                   return (coords, labelIDs);
-               },
-               InputDef.FromModel(model1)[0]
-         );
-
-        //Create engine to run model
-        engine = WorkerFactory.CreateWorker(backend, model2);
+        var graph = new FunctionalGraph();
+        var inputs = graph.AddInputs(model1);
+        var modelOutput = FF.Forward(model1, inputs)[0];                        //shape=(1,84,8400)
+        var boxCoords = modelOutput[0, 0..4, ..].Transpose(0, 1);               //shape=(8400,4)
+        var allScores = modelOutput[0, 4.., ..];                                //shape=(80,8400)
+        var scores = FF.ReduceMax(allScores, 0);                                //shape=(8400)
+        var classIDs = FF.ArgMax(allScores, 0);                                 //shape=(8400)
+        var boxCorners = FF.MatMul(boxCoords, FF.Constant(centersToCorners));   //shape=(8400,4)
+        var indices = FF.NMS(boxCorners, scores, iouThreshold, scoreThreshold); //shape=(N)
+        var coords = FF.IndexSelect(boxCoords, 0, indices);                     //shape=(N,4)
+        var labelIDs = FF.IndexSelect(classIDs, 0, indices);                    //shape=(N)
+
+        //Create worker to run model
+        worker = new Worker(graph.Compile(coords, labelIDs), backend);
     }
 
     void SetupInput()
@@ -140,7 +124,7 @@ public class RunYOLO8n : MonoBehaviour
         video = gameObject.AddComponent<VideoPlayer>();
         video.renderMode = VideoRenderMode.APIOnly;
         video.source = VideoSource.Url;
-        video.url = Path.Join(Application.streamingAssetsPath, videoName);
+        video.url = Path.Join(Application.streamingAssetsPath, videoFilename);
         video.isLooping = true;
         video.Play();
     }
@@ -167,14 +151,12 @@ public class RunYOLO8n : MonoBehaviour
         }
         else return;
 
-        using var input = TextureConverter.ToTensor(targetRT, imageWidth, imageHeight, 3);
-        engine.Execute(input);
-
-        var output = engine.PeekOutput("output_0") as TensorFloat;
-        var labelIDs = engine.PeekOutput("output_1") as TensorInt;
+        using Tensor<float> inputTensor = new Tensor<float>(new TensorShape(1, 3, imageHeight, imageWidth));
+        TextureConverter.ToTensor(targetRT, inputTensor, default);
+        worker.Schedule(inputTensor);
 
-        output.CompleteOperationsAndDownload();
-        labelIDs.CompleteOperationsAndDownload();
+        using var output = (worker.PeekOutput("output_0") as Tensor<float>).ReadbackAndClone();
+        using var labelIDs = (worker.PeekOutput("output_1") as Tensor<int>).ReadbackAndClone();
 
         float displayWidth = displayImage.rectTransform.rect.width;
         float displayHeight = displayImage.rectTransform.rect.height;
@@ -270,6 +252,6 @@ public class RunYOLO8n : MonoBehaviour
     private void OnDestroy()
     {
         centersToCorners?.Dispose();
-        engine?.Dispose();
+        worker?.Dispose();
     }
-}
+}

info.js

@@ -1,5 +0,0 @@
-{
-      "version" : [
-         "1.4.0-pre.2"
-    ]
-}

info.json

@@ -1,5 +1,14 @@
 {
-      "version" : [
-         "1.3.0-pre.3"
+    "code": [
+        "RunYOLO8n.cs"
+    ],
+    "models": [
+        "yolov8n.onnx"
+    ],
+    "data": [
+        "classes.txt"
+    ],
+    "version": [
+        "2.1.2"
     ]
 }

yolov8n.sentis

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:72c2ccde7dedd160cd8b62907ff2fa06ffe594d4a0fe0d2b13eb270297ca455c
-size 12834028