Instructions to use Sky-Kim/com.sky.sentis.audio2lipsync with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- unity-sentis
How to use Sky-Kim/com.sky.sentis.audio2lipsync with unity-sentis:
string modelName = "[Your model name here].sentis"; Model model = ModelLoader.Load(Application.streamingAssetsPath + "/" + modelName); IWorker engine = WorkerFactory.CreateWorker(BackendType.GPUCompute, model); // Please see provided C# file for more details
- Notebooks
- Google Colab
- Kaggle
File size: 3,026 Bytes
116878d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | using System.IO;
using Unity.InferenceEngine;
using UnityEngine;
using UnityEngine.Networking;
namespace Audio2Lipsync
{
/// <summary>
/// Locates and loads the packaged model with no setup step:
/// - Editor: straight from the package's Model~ folder (nothing is copied into Assets).
/// - Player: from StreamingAssets/Audio2Lipsync/, where the package's build hook puts it for
/// the duration of each build.
/// </summary>
public static class Audio2LipsyncModel
{
public const string FileName = "audio2lipsync.sentis";
public const string StreamingAssetsFolder = "Audio2Lipsync";
public static string StreamingAssetsPath =>
Path.Combine(Application.streamingAssetsPath, StreamingAssetsFolder, FileName);
#if UNITY_EDITOR
/// <summary>Model~/audio2lipsync.sentis inside the resolved package (editor only).</summary>
public static string PackagedPath
{
get
{
var pkg = UnityEditor.PackageManager.PackageInfo.FindForAssembly(typeof(Audio2LipsyncModel).Assembly);
return pkg == null ? null : Path.Combine(pkg.resolvedPath, "Model~", FileName);
}
}
#endif
/// <summary>
/// Where <see cref="LoadAsync"/> reads the model from: a model placed in
/// StreamingAssets/Audio2Lipsync/ wins (override); otherwise, in the editor, the packaged one.
/// </summary>
public static string DefaultPath
{
get
{
#if UNITY_EDITOR
if (!File.Exists(StreamingAssetsPath) && PackagedPath != null) return PackagedPath;
#endif
return StreamingAssetsPath;
}
}
/// <summary>
/// Loads the default model. Android / WebGL keep StreamingAssets inside the package or on a
/// server, so those are fetched with UnityWebRequest; everywhere else it is a plain file read.
/// </summary>
public static async Awaitable<Model> LoadAsync()
{
var path = DefaultPath;
if (path.Contains("://"))
{
using var req = UnityWebRequest.Get(path);
await req.SendWebRequest();
if (req.result != UnityWebRequest.Result.Success)
throw new FileNotFoundException($"Audio2Lipsync model not loaded from {path}: {req.error}");
using var ms = new MemoryStream(req.downloadHandler.data);
return ModelLoader.Load(ms);
}
if (!File.Exists(path))
throw new FileNotFoundException(
$"Audio2Lipsync model not found: {path}. If the package came from git, install git-lfs and re-resolve it.");
if (new FileInfo(path).Length < 1024 * 1024)
throw new InvalidDataException($"{path} is a Git LFS pointer, not the model. Install git-lfs and re-resolve the package.");
return ModelLoader.Load(path);
}
}
}
|