Next-Echo / echo_core.py
Lamapi's picture
Publish Next-Echo model
e187f2f verified
Raw History Blame Contribute Delete
5.45 kB
"""Next Echo: find where a short example sound occurs inside a longer recording.
Shared by the notebook, inference.py and (ported) web/core.js. 16 kHz mono audio.
* logmel: 512-point STFT (periodic Hann, hop 256) -> 64 mel bands -> log
* windows of 64 frames (~1.04 s) every 16 frames (~0.256 s)
* learned: CNN embedding (nlrt) of each window, cosine to the query embedding
* baseline: cosine between per-band mean/std statistics of log-mel windows
* robust_z: per-recording normalisation (median / MAD) so thresholds adapt to each recording's background
* final_scores: Next Echo score = alpha * z(learned) + (1 - alpha) * z(log-mel); baseline = log-mel (raw or z), chosen on validation
* detect: threshold the score curve, merge consecutive windows into intervals
"""
import math
import numpy as np
from nlrt import forward
SR, N_FFT, HOP, N_MELS, WIN, STEP = 16000, 512, 256, 64, 64, 16
WIN_SEC = (WIN - 1) * HOP / SR + N_FFT / SR
STEP_SEC = STEP * HOP / SR
def mel_filterbank(n_mels=N_MELS, n_fft=N_FFT, sr=SR, fmin=50.0, fmax=8000.0):
hz2mel = lambda f: 2595.0 * math.log10(1.0 + f / 700.0) # noqa: E731
mel2hz = lambda m: 700.0 * (10 ** (m / 2595.0) - 1.0) # noqa: E731
mels = np.linspace(hz2mel(fmin), hz2mel(fmax), n_mels + 2)
hz = np.array([mel2hz(m) for m in mels])
bins = np.arange(n_fft // 2 + 1) * sr / n_fft
fb = np.zeros((n_mels, len(bins)))
for i in range(n_mels):
lo, c, hi = hz[i], hz[i + 1], hz[i + 2]
fb[i] = np.maximum(0, np.minimum((bins - lo) / (c - lo), (hi - bins) / (hi - c)))
return fb
def logmel(x, fb):
x = np.asarray(x, dtype=np.float64)
if len(x) < N_FFT:
x = np.concatenate([x, np.zeros(N_FFT - len(x))])
n = 1 + (len(x) - N_FFT) // HOP
w = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(N_FFT) / N_FFT)
idx = np.arange(N_FFT)[None, :] + HOP * np.arange(n)[:, None]
spec = np.fft.rfft(x[idx] * w, axis=1)
power = spec.real ** 2 + spec.imag ** 2
return np.log(fb @ power.T + 1e-6) # (n_mels, frames)
def pad_frames(lm, n):
if lm.shape[1] >= n:
return lm
return np.concatenate([lm, np.full((lm.shape[0], n - lm.shape[1]), lm.min())], axis=1)
def window_starts(n_frames):
return list(range(0, max(n_frames - WIN, 0) + 1, STEP))
def embed_windows(layers, lm, mu, sd, starts):
X = np.stack([(lm[:, s:s + WIN] - mu) / sd for s in starts])[:, None]
E = forward(layers, X)
return E / np.maximum(np.linalg.norm(E, axis=1, keepdims=True), 1e-12)
def stats_windows(lm, mu, sd, starts):
out = []
for s in starts:
z = (lm[:, s:s + WIN] - mu) / sd
v = np.concatenate([z.mean(axis=1), z.std(axis=1)])
v = v - v.mean()
out.append(v / max(float(np.linalg.norm(v)), 1e-12))
return np.stack(out)
def query_starts(lm):
"""Windows of the query that carry the sound: mean log energy within 1.5 of the loudest window."""
lm = pad_frames(lm, WIN)
starts = list(range(0, lm.shape[1] - WIN + 1, 8))
e = [float(lm[:, s:s + WIN].mean()) for s in starts]
top = max(e)
return lm, [s for s, v in zip(starts, e) if v >= top - 1.5]
def query_vectors(model, q_audio):
fb, mu, sd = model["fb"], model["mu"], model["sd"]
lm, starts = query_starts(logmel(q_audio, fb))
qe = embed_windows(model["layers"], lm, mu, sd, starts).mean(axis=0)
qs = stats_windows(lm, mu, sd, starts).mean(axis=0)
return qe / max(float(np.linalg.norm(qe)), 1e-12), qs / max(float(np.linalg.norm(qs)), 1e-12)
def similarity_curves(model, q_audio, rec_audio):
qe, qs = query_vectors(model, q_audio)
lm = pad_frames(logmel(rec_audio, model["fb"]), WIN)
starts = window_starts(lm.shape[1])
learned = embed_windows(model["layers"], lm, model["mu"], model["sd"], starts) @ qe
base = stats_windows(lm, model["mu"], model["sd"], starts) @ qs
times = [s * HOP / SR for s in starts]
return times, learned, base
def robust_z(c):
c = np.asarray(c, dtype=np.float64)
med = float(np.median(c))
mad = float(np.median(np.abs(c - med))) + 1e-6
return (c - med) / (1.4826 * mad)
def final_scores(learned, base, cfg):
"""cfg: {"alpha": float, "baseline_norm": "raw" | "z"} -> (next_echo_curve, baseline_curve)."""
a = cfg["alpha"]
nxt = a * robust_z(learned) + (1.0 - a) * robust_z(base)
bl = robust_z(base) if cfg["baseline_norm"] == "z" else np.asarray(base, dtype=np.float64)
return nxt, bl
def detect(times, sims, threshold):
out, cur = [], None
for t, s in zip(times, sims):
if s >= threshold:
if cur is None:
cur = [t, t + WIN_SEC, float(s)]
else:
cur[1] = t + WIN_SEC
cur[2] = max(cur[2], float(s))
elif cur is not None:
out.append(cur); cur = None
if cur is not None:
out.append(cur)
return out
def match_events(dets, gts, tol=0.5):
used, tp = set(), 0
for g0, g1 in gts:
for j, d in enumerate(dets):
if j not in used and d[0] < g1 + tol and d[1] > g0 - tol:
used.add(j); tp += 1
break
return tp, len(dets) - len(used), len(gts) - tp
def load_model(spec):
t = spec["tensors"]
return {"layers": spec["nets"]["embed"], "fb": t["mel_fb"], "mu": float(spec["meta"]["mu"]), "sd": float(spec["meta"]["sd"]),
"thresholds": spec["meta"]["thresholds"], "score": spec["meta"]["score"]}