Upload 3 files

RunBlazeFace.cs

@@ -0,0 +1,355 @@
+using UnityEngine;
+using Unity.Sentis;
+using UnityEngine.Video;
+using UnityEngine.UI;
+using Lays = Unity.Sentis.Layers;
+
+/*
+ *                   Blaze Face Inference
+ *                   ====================
+ *                   
+ * Basic inference script for blaze face
+ * 
+ * Put this script on the Main Camera
+ * Put blazeface.sentis in the Assets/StreamingAssets folder
+ * Create a RawImage of size 320x320 in the scene
+ * Put a link to that image in _previewUI
+ * Put a video in Assets/StreamingAssets folder and put the name of it int videoName
+ * Or put a test image in _inputImage
+ * Set _inputType to appropriate input
+ */
+
+
+public class RunBlazeFace : MonoBehaviour
+{
+    //Drag a link to a raw image here:
+    public RawImage _previewUI = null;
+
+    // Put your bounding box sprite image here
+    public Sprite faceTexture;
+
+    // 6 optional sprite images (left eye, right eye, nose, mouth, left ear, right ear)
+    public Sprite[] markerTextures;
+
+    const string videoName = "chatting.mp4";
+
+    // 
+    public Texture2D _inputImage;
+
+    public InputType _inputType = InputType.Video;
+
+    Vector2Int _resolution = new Vector2Int(640, 640);
+    WebCamTexture _webcam;
+    VideoPlayer _video;
+
+    const BackendType backend = BackendType.GPUCompute;
+
+    RenderTexture _targetTexture;
+    public enum InputType { Image, Video, Webcam };
+   
+    
+    //Some adjustable parameters for the model
+    [SerializeField, Range(0, 1)] float _iouThreshold = 0.5f;
+    [SerializeField, Range(0, 1)] float _scoreThreshold = 0.5f;
+    int _maxOutputBoxes = 64;
+
+    IWorker _worker;
+
+    //Holds image size
+    int _size;
+
+    Ops ops;
+    ITensorAllocator allocator;
+
+    Model _model;
+
+    //webcam device name:
+    const string _deviceName = "";
+
+    bool closing = false;
+
+    public struct BoundingBox
+    {
+        public float centerX;
+        public float centerY;
+        public float width;
+        public float height;
+    }
+
+    void Start()
+    {
+        allocator = new TensorCachingAllocator();
+
+        //(Note: if using a webcam on mobile get permissions here first)
+
+        _targetTexture = new RenderTexture(_resolution.x, _resolution.y, 0);
+
+        SetupInput();
+
+        SetupModel();
+
+        SetupEngine();
+    }
+
+    void SetupInput()
+    {
+        switch (_inputType)
+        {
+            case InputType.Webcam:
+                {
+                    _webcam = new WebCamTexture(_deviceName, _resolution.x, _resolution.y);
+                    _webcam.requestedFPS = 30;
+                    _webcam.Play();
+                    break;
+                }
+            case InputType.Video:
+                {
+                    _video = gameObject.AddComponent<VideoPlayer>();//new VideoPlayer();
+                    _video.renderMode = VideoRenderMode.APIOnly;
+                    _video.source = VideoSource.Url;
+                    _video.url = Application.streamingAssetsPath + "/"+videoName;
+                    _video.isLooping = true;
+                    _video.Play();
+                    break;
+                }
+            default:
+                {
+                    Graphics.Blit(_inputImage, _targetTexture);
+                }
+                break;
+        }
+    }
+
+    void Update()
+    {
+        if (_inputType == InputType.Webcam)
+        {
+            // Format video input
+            if (!_webcam.didUpdateThisFrame) return;
+
+            var aspect1 = (float)_webcam.width / _webcam.height;
+            var aspect2 = (float)_resolution.x / _resolution.y;
+            var gap = aspect2 / aspect1;
+
+            var vflip = _webcam.videoVerticallyMirrored;
+            var scale = new Vector2(gap, vflip ? -1 : 1);
+            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);
+
+            Graphics.Blit(_webcam, _targetTexture, scale, offset);
+        }
+        if (_inputType == InputType.Video)
+        {
+            var aspect1 = (float)_video.width / _video.height;
+            var aspect2 = (float)_resolution.x / _resolution.y;
+            var gap = aspect2 / aspect1;
+
+            var vflip = false;
+            var scale = new Vector2(gap, vflip ? -1 : 1);
+            var offset = new Vector2((1 - gap) / 2, vflip ? 1 : 0);
+            Graphics.Blit(_video.texture, _targetTexture, scale, offset);
+        }
+        if (_inputType == InputType.Image)
+        {
+            Graphics.Blit(_inputImage, _targetTexture);
+        }
+
+        if (Input.GetKeyDown(KeyCode.Escape))
+        {
+            closing = true;
+            Application.Quit();
+        }
+
+        if (Input.GetKeyDown(KeyCode.P))
+        {
+            _previewUI.enabled = !_previewUI.enabled;
+        }
+    }
+
+
+    void LateUpdate()
+    {
+        if (!closing)
+        {
+            RunInference(_targetTexture);
+        }
+    }
+
+    //Calculate the centers of the grid squares for two 16x16 grids and six 8x8 grids
+    float[] GetGridBoxCoords()
+    {
+        var offsets = new float[896 * 4];
+        int n = 0;
+        for (int j = 0; j < 2 * 16 * 16; j++)
+        {
+            offsets[n++] = (j / 2) % 16 - 7.5f;
+            offsets[n++] = (j / 2 / 16) - 7.5f;
+            n += 2;
+        }
+        for (int j = 0; j < 6 * 8 * 8; j++)
+        {
+            offsets[n++] = (j / 6) % 8 - 7f;
+            offsets[n++] = (j / 6 / 8) - 7f;
+            n += 2;
+        }
+        return offsets;
+    }
+
+    void SetupModel()
+    {
+        float[] offsets = GetGridBoxCoords();
+
+        _model = ModelLoader.Load(Application.streamingAssetsPath + "/blazeface.sentis");
+
+        //We need to add extra layers to the model in order to aggregate the box predicions:
+
+        _size = _model.inputs[0].shape.ToTensorShape()[1]; // Input tensor width
+
+        _model.AddConstant(new Lays.Constant("zero", new int[] { 0 }));
+        _model.AddConstant(new Lays.Constant("two", new int[] { 2 }));
+        _model.AddConstant(new Lays.Constant("four", new int[] { 4 }));
+
+        _model.AddLayer(new Lays.Slice("boxes1", "regressors", "zero", "four", "two"));
+
+        _model.AddLayer(new Lays.Transpose("scores", "classificators", new int[] { 0, 2, 1 }));
+
+        _model.AddConstant(new Lays.Constant("eighth", new float[] { 1 / 8f }));
+        _model.AddConstant(new Lays.Constant("offsets",
+            new TensorFloat(new TensorShape(1, 896, 4), offsets)
+         ));
+        _model.AddLayer(new Lays.Mul("boxes1scaled", "boxes1", "eighth"));
+        _model.AddLayer(new Lays.Add("boxCoords", "boxes1scaled", "offsets"));
+        _model.AddOutput("boxCoords");
+
+        _model.AddConstant(new Lays.Constant("maxOutputBoxes", new int[] { _maxOutputBoxes }));
+        _model.AddConstant(new Lays.Constant("iouThreshold", new float[] { _iouThreshold }));
+        _model.AddConstant(new Lays.Constant("scoreThreshold", new float[] { _scoreThreshold }));
+
+        _model.AddLayer(new Lays.NonMaxSuppression("NMS", "boxCoords", "scores",
+            "maxOutputBoxes", "iouThreshold", "scoreThreshold",
+            centerPointBox: Lays.CenterPointBox.Center
+        ));
+        _model.AddOutput("NMS");
+    }
+    public void SetupEngine()
+    {
+        _worker = WorkerFactory.CreateWorker(backend, _model);
+        ops = WorkerFactory.CreateOps(backend, allocator);
+    }
+
+    void DrawFaces(TensorFloat index3, TensorFloat regressors, int NMAX, Vector2 scale)
+    {
+        for (int n = 0; n < NMAX; n++)
+        {
+            //Draw bounding box of face
+            var box = new BoundingBox
+            {
+                centerX = index3[0, n, 0] * scale.x,
+                centerY = index3[0, n, 1] * scale.y,
+                width = index3[0, n, 2] * scale.x,
+                height = index3[0, n, 3] * scale.y
+            };
+            DrawBox(box, faceTexture);
+            if (regressors == null) continue;
+            
+            //Draw markers for eyes, ears, nose, mouth:
+            for (int j = 0; j < 6; j++)
+            {
+                var marker = new BoundingBox
+                {
+                    centerX = box.centerX + (regressors[0, n, 4 + j * 2] - regressors[0, n, 0]) * scale.x / 8,
+                    centerY = box.centerY + (regressors[0, n, 4 + j * 2 + 1] - regressors[0, n, 1]) * scale.y / 8,
+                    width = 1.0f * scale.x,
+                    height = 1.0f * scale.y,
+                };
+                DrawBox(marker, j < markerTextures.Length ? markerTextures[j] : faceTexture);
+            }
+        }
+    }
+
+    void ExecuteML(Texture source)
+    {
+        var transform = new TextureTransform();
+        transform.SetDimensions(_size, _size, 3);
+        transform.SetTensorLayout(0, 3, 1, 2);
+        using var image0 = TextureConverter.ToTensor(source, transform);
+
+        // Pre-process the image to make input in range (-1..1)
+        using var image = ops.Mad(image0, 2f, -1f);
+
+        _worker.Execute(image);
+
+        using var boxCoords = _worker.PeekOutput("boxCoords") as TensorFloat; //face coords
+        using var regressors = _worker.PeekOutput("regressors") as TensorFloat; //contains markers
+
+        var NM1 = _worker.PeekOutput("NMS") as TensorInt;
+
+        using var boxCoords2 = boxCoords.ShallowReshape(
+            new TensorShape(1, boxCoords.shape[0], boxCoords.shape[1], boxCoords.shape[2])) as TensorFloat;
+        var output = ops.GatherND(boxCoords2, NM1, 0);
+
+        using var regressors2 = regressors.ShallowReshape(
+            new TensorShape(1, regressors.shape[0], regressors.shape[1], regressors.shape[2])) as TensorFloat;
+        var markersOutput = ops.GatherND(regressors2, NM1, 0);
+
+        output.MakeReadable();
+        markersOutput.MakeReadable();
+
+        ClearAnnotations();
+
+        Vector2 markerScale = _previewUI.rectTransform.rect.size/ 16;
+
+        DrawFaces(output, markersOutput, output.shape[0], markerScale);
+
+    }
+
+    void RunInference(Texture input)
+    {
+        // Face detection
+        ExecuteML(input);
+
+        _previewUI.texture = input;
+    }
+
+    public void DrawBox(BoundingBox box, Sprite sprite)
+    {
+        var panel = new GameObject("ObjectBox");
+        panel.AddComponent<CanvasRenderer>();
+        panel.AddComponent<Image>();
+        panel.transform.SetParent(_previewUI.transform, false);
+
+        var img = panel.GetComponent<Image>();
+        img.color = Color.white;
+        img.sprite = sprite;
+        img.type = Image.Type.Sliced;
+
+        panel.transform.localPosition = new Vector3(box.centerX, -box.centerY);
+        RectTransform rt = panel.GetComponent<RectTransform>();
+        rt.sizeDelta = new Vector2(box.width, box.height);
+    }
+    public void ClearAnnotations()
+    {
+        foreach (Transform child in _previewUI.transform)
+        {
+            Destroy(child.gameObject);
+        }
+    }
+
+    void CleanUp()
+    {
+        ops?.Dispose();
+        allocator?.Dispose();
+        if (_webcam) Destroy(_webcam);
+        if (_video) Destroy(_video);
+        RenderTexture.active = null;
+        _targetTexture.Release();
+        _worker?.Dispose();
+        _worker = null;
+    }
+
+    void OnDestroy()
+    {
+        CleanUp();
+    }
+
+}
+

blazeface.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:add5ce1bcfba21b2bbd73c65c8d344614c692d0e0ba9800eda48c38bed82522f
+size 418480