File size: 6,200 Bytes
116878d | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 | using System;
using UnityEngine;
namespace Audio2Lipsync
{
/// <summary>Post-processing knobs — same names/defaults as server.py LipSyncRequest.</summary>
[Serializable]
public struct LipsyncSettings
{
[Range(1, 15)] public int smoothWindow; // moving-average window (1 = off)
[Range(0.1f, 5f)] public float gain; // centered amplification around the rest pose
[Range(0f, 1f)] public float silenceGate; // RMS mouth gate strength (0 = off)
[Range(-80f, -10f)] public float silenceThresholdDb;
[Range(0.5f, 5f)] public float cfgScale; // classifier-free guidance (1 = off)
public static LipsyncSettings Default => new LipsyncSettings
{
smoothWindow = 3, gain = 1.5f, silenceGate = 0f, silenceThresholdDb = -40f, cfgScale = 1f,
};
}
/// <summary>
/// Pure-C# port of server.py run_inference() post-processing. Frames are row-major [n, 52].
/// </summary>
public static class LipsyncPostProcess
{
const int N = A2LConstants.NumBlendshapes;
const int FadeFrames = 6;
/// <param name="cond">denormalized model output [n*52]</param>
/// <param name="uncond">silent-audio output [n*52] (only read when cfgScale != 1)</param>
public static float[] Apply(float[] cond, float[] uncond, int nFrames, float[] audio16k, LipsyncSettings s)
{
var bs = new float[nFrames * N];
Array.Copy(cond, bs, bs.Length);
if (s.cfgScale != 1f && uncond != null)
for (int i = 0; i < bs.Length; i++)
bs[i] = uncond[i] + s.cfgScale * (bs[i] - uncond[i]);
Clip01(bs);
// Zero non-mouth channels (brows/eyes/nose belong to the upper-face layer).
for (int f = 0; f < nFrames; f++)
for (int c = 0; c < N; c++)
if (!A2LConstants.KeepChannel[c]) bs[f * N + c] = 0f;
if (s.gain != 1f)
{
for (int f = 0; f < nFrames; f++)
for (int c = 0; c < N; c++)
{
if (!A2LConstants.KeepChannel[c]) continue;
float rest = A2LConstants.Mean[c];
bs[f * N + c] = rest + s.gain * (bs[f * N + c] - rest);
}
Clip01(bs);
}
if (s.smoothWindow > 1 && nFrames > s.smoothWindow)
{
bs = UniformFilterFrames(bs, nFrames, s.smoothWindow);
Clip01(bs);
}
if (s.silenceGate > 0f)
{
var gate = SilenceGate(audio16k, nFrames, s.silenceThresholdDb, s.silenceGate);
for (int f = 0; f < nFrames; f++)
for (int c = 0; c < N; c++)
if (A2LConstants.KeepChannel[c]) bs[f * N + c] *= gate[f];
}
// ~0.1 s fade in/out so the mouth opens from and closes to neutral.
for (int f = 0; f < Mathf.Min(FadeFrames, nFrames); f++)
{
float k = (f + 1) / (float)FadeFrames;
for (int c = 0; c < N; c++)
{
bs[f * N + c] *= k;
bs[(nFrames - 1 - f) * N + c] *= k;
}
}
return bs;
}
static void Clip01(float[] a)
{
for (int i = 0; i < a.Length; i++) a[i] = Mathf.Clamp01(a[i]);
}
// scipy.ndimage.uniform_filter1d(axis=0, mode="nearest"): window [i - w/2, i - w/2 + w - 1].
static float[] UniformFilterFrames(float[] src, int nFrames, int w)
{
var dst = new float[src.Length];
int left = w / 2;
for (int f = 0; f < nFrames; f++)
for (int c = 0; c < N; c++)
{
float sum = 0f;
for (int k = 0; k < w; k++)
{
int j = Mathf.Clamp(f - left + k, 0, nFrames - 1);
sum += src[j * N + c];
}
dst[f * N + c] = sum / w;
}
return dst;
}
// server.py compute_silence_gate()
static float[] SilenceGate(float[] audio, int nFrames, float thresholdDb, float strength)
{
var gate = new float[nFrames];
int spf = Mathf.Max(1, A2LConstants.AudioSampleRate / A2LConstants.FaceFps);
int nAudio = audio.Length / spf;
if (nAudio < 1) { for (int i = 0; i < nFrames; i++) gate[i] = 1f; return gate; }
var rms = new float[nAudio];
for (int i = 0; i < nAudio; i++)
{
double acc = 0;
for (int k = 0; k < spf; k++) { float v = audio[i * spf + k]; acc += v * v; }
rms[i] = (float)Math.Sqrt(acc / spf + 1e-12);
}
int smoothN = Mathf.Max(1, (int)(0.25f * A2LConstants.FaceFps));
if (smoothN > 1 && nAudio > smoothN)
{
var sm = new float[nAudio];
int left = smoothN / 2;
for (int i = 0; i < nAudio; i++)
{
float sum = 0f;
for (int k = 0; k < smoothN; k++) sum += rms[Mathf.Clamp(i - left + k, 0, nAudio - 1)];
sm[i] = sum / smoothN;
}
rms = sm;
}
var raw = new float[nAudio];
for (int i = 0; i < nAudio; i++)
{
float db = 20f * Mathf.Log10(Mathf.Max(rms[i], 1e-8f));
raw[i] = 1f / (1f + Mathf.Exp(-(db - thresholdDb) / 4f));
}
for (int f = 0; f < nFrames; f++)
{
float x = nFrames == 1 ? 0f : f / (float)(nFrames - 1) * (nAudio - 1);
int i0 = Mathf.Clamp((int)x, 0, nAudio - 1), i1 = Mathf.Min(i0 + 1, nAudio - 1);
float g = Mathf.Lerp(raw[i0], raw[i1], x - i0);
gate[f] = (1f - strength) + strength * g;
}
return gate;
}
}
}
|