ser-inference / scripts /drift_check.py
spacey-cadet's picture
docs: overhaul README and Dockerfile for production pipeline
dddb6a8
Raw History Blame Contribute Delete
3.4 kB
"""
Track 2.2 — serving-gate drift monitoring.
Compares the live feature distribution (logged per-request by
app/logging_pipeline/feature_log.py) against the training distribution using
scipy.stats.ks_2samp. Intended to run periodically — a scheduled GitHub
Action (see .github/workflows/drift-check.yml) or a manual notebook run.
Expect this to fire early and often at first: chat audio characteristics
diverging from RAVDESS/IEMOCAP/CREMA-D is not a bug, it's the exact gap the
cross-corpus numbers already predict.
Usage:
python scripts/drift_check.py
"""
import json
import sys
from scipy.stats import ks_2samp
from app.alerts.webhook import send_alert
from app.config import get_settings
from app.logging_pipeline.feature_log import build_feature_log
from app.logging_pipeline.review_queue import queue_stats
def load_reference_distribution(path: str) -> dict:
"""
Reference distribution built once (e.g. from your RAVDESS/IEMOCAP/CREMA-D
training set) via scripts/build_drift_reference.py — see that script's
docstring for the expected format. Kept separate from this script since
it only needs to run once per training set, not on every drift check.
"""
with open(path) as f:
return json.load(f)
def main():
settings = get_settings()
feature_log = build_feature_log(settings)
if feature_log is None:
print("Feature logging disabled (FEATURE_LOG_BACKEND=none); nothing to check.")
return
live = feature_log.feature_distribution()
try:
reference = load_reference_distribution(settings.drift_reference_path)
except FileNotFoundError:
print(f"No reference distribution at {settings.drift_reference_path}. "
"Run scripts/build_drift_reference.py against your training set first.")
sys.exit(1)
results = {}
fired = False
for feature_name, live_values in live.items():
ref_values = reference.get(feature_name, [])
if len(live_values) < 20 or len(ref_values) < 20:
results[feature_name] = {"skipped": "insufficient samples"}
continue
stat, p_value = ks_2samp(live_values, ref_values)
results[feature_name] = {"ks_statistic": float(stat), "p_value": float(p_value)}
if p_value < settings.drift_p_value_threshold:
fired = True
stats = queue_stats(feature_log)
low_conf_rate = None
total_logged = sum(len(v) for v in live.values()) or 1
if stats.unreviewed_count:
low_conf_rate = stats.unreviewed_count / max(total_logged, 1)
print(json.dumps({"drift": results, "review_queue_unreviewed": stats.unreviewed_count,
"low_confidence_rate": low_conf_rate}, indent=2))
if fired:
send_alert(
settings.alert_webhook_url,
f":rotating_light: SER drift check: KS-test fired (p < {settings.drift_p_value_threshold}) "
f"on one or more features. Details: {json.dumps(results)}",
)
if low_conf_rate is not None and low_conf_rate > settings.low_confidence_rate_threshold:
send_alert(
settings.alert_webhook_url,
f":warning: SER low-confidence rate {low_conf_rate:.1%} exceeds threshold "
f"{settings.low_confidence_rate_threshold:.0%}. Review queue depth: {stats.unreviewed_count}.",
)
if __name__ == "__main__":
main()