Publish Tiny Hinglish Turn Detector development preview
Browse filesrelease inventory sha256: eeddc0bc1ef70abf44ce9f84444751b0b2fb15e2718df215fa3f537cb2a86bfe
- MODEL_CARD.md +1 -1
- README.md +1 -1
- REPORT.md +1 -1
- SOURCE_README.md +31 -7
- app.py +23 -0
- deployment/kaggle/README.md +65 -0
- deployment/kaggle/example_inference.py +30 -0
- deployment/kaggle/requirements.txt +4 -0
- deployment/kaggle/smoke_test.py +29 -0
- deployment/kaggle/turn_detector.py +304 -0
- export_manifest.json +41 -11
- pyproject.toml +13 -2
- release_provenance.json +57 -25
- scripts/build_release.py +77 -1
- scripts/build_upload_folders.py +470 -0
- scripts/export_onnx.py +5 -0
- space/requirements.txt +1 -0
- tests/test_kaggle_runtime.py +64 -0
- tests/test_release_guardrails.py +29 -14
- tests/test_upload_folders.py +66 -0
MODEL_CARD.md
CHANGED
|
@@ -303,7 +303,7 @@ failure aggregates, silence perturbation report, exact environment files, CPU
|
|
| 303 |
benchmarks, and guarded release tooling. Per-example validation predictions and
|
| 304 |
the text-bearing split manifest remain in the private local workspace; the
|
| 305 |
public package contains aggregate reports and hashes, not those sensitive rows.
|
| 306 |
-
The quality gate completed with Ruff 0.15.16 passing and
|
| 307 |
passing via Python `unittest` discovery. The configured Hugging Face owner is
|
| 308 |
`suvradeepp`.
|
| 309 |
|
|
|
|
| 303 |
benchmarks, and guarded release tooling. Per-example validation predictions and
|
| 304 |
the text-bearing split manifest remain in the private local workspace; the
|
| 305 |
public package contains aggregate reports and hashes, not those sensitive rows.
|
| 306 |
+
The quality gate completed with Ruff 0.15.16 passing and 136/136 automated tests
|
| 307 |
passing via Python `unittest` discovery. The configured Hugging Face owner is
|
| 308 |
`suvradeepp`.
|
| 309 |
|
README.md
CHANGED
|
@@ -303,7 +303,7 @@ failure aggregates, silence perturbation report, exact environment files, CPU
|
|
| 303 |
benchmarks, and guarded release tooling. Per-example validation predictions and
|
| 304 |
the text-bearing split manifest remain in the private local workspace; the
|
| 305 |
public package contains aggregate reports and hashes, not those sensitive rows.
|
| 306 |
-
The quality gate completed with Ruff 0.15.16 passing and
|
| 307 |
passing via Python `unittest` discovery. The configured Hugging Face owner is
|
| 308 |
`suvradeepp`.
|
| 309 |
|
|
|
|
| 303 |
benchmarks, and guarded release tooling. Per-example validation predictions and
|
| 304 |
the text-bearing split manifest remain in the private local workspace; the
|
| 305 |
public package contains aggregate reports and hashes, not those sensitive rows.
|
| 306 |
+
The quality gate completed with Ruff 0.15.16 passing and 136/136 automated tests
|
| 307 |
passing via Python `unittest` discovery. The configured Hugging Face owner is
|
| 308 |
`suvradeepp`.
|
| 309 |
|
REPORT.md
CHANGED
|
@@ -525,7 +525,7 @@ calendar days or the missing external data collection occurred.
|
|
| 525 |
| 13 | Integrate controller and demo | replay, three-state controller, Gradio, Hinglish recording prompts | Preview model and synthetic controller integration complete; real sequence replay pending |
|
| 526 |
| 14 | Freeze test, package, publish | sealed-test guard, atomic release builder, exact-mirror publisher | Development packaging path ready; official-test/final gates not satisfied |
|
| 527 |
|
| 528 |
-
The repository quality gate passed Ruff 0.15.16 and all
|
| 529 |
(131 passed via Python `unittest` discovery). That validates the implemented behavior under test; it does not fill
|
| 530 |
the missing empirical datasets or rights review.
|
| 531 |
|
|
|
|
| 525 |
| 13 | Integrate controller and demo | replay, three-state controller, Gradio, Hinglish recording prompts | Preview model and synthetic controller integration complete; real sequence replay pending |
|
| 526 |
| 14 | Freeze test, package, publish | sealed-test guard, atomic release builder, exact-mirror publisher | Development packaging path ready; official-test/final gates not satisfied |
|
| 527 |
|
| 528 |
+
The repository quality gate passed Ruff 0.15.16 and all 136 automated tests
|
| 529 |
(131 passed via Python `unittest` discovery). That validates the implemented behavior under test; it does not fill
|
| 530 |
the missing empirical datasets or rights review.
|
| 531 |
|
SOURCE_README.md
CHANGED
|
@@ -26,9 +26,9 @@ engineering evidence, not a production-readiness claim.
|
|
| 26 |
| Domain-shift stress | Measured, confounded | Source-held-out AP 0.5500/0.5368 for TinyTCN/baseline; both near chance in ranking, with wide nine-source bootstrap intervals |
|
| 27 |
| Runtime/export | Measured | 611 KiB FP32 ONNX; exact PyTorch/ONNX parity; ONNX p95 0.956 ms neural-only and 1.629 ms waveform-to-probability |
|
| 28 |
| Controller integration | Synthetic only | Metadata-bound 8-checkpoint/3-turn fixture emits 3 response edges with 0 duplicates; not model or product-quality evidence |
|
| 29 |
-
| Code quality gate | Passed | Ruff 0.15.16;
|
| 30 |
| Accuracy/generalization | **Not established** | No full-corpus run, collected Hinglish recordings, real conversation replay, or official-test evaluation |
|
| 31 |
-
|
|
| 32 |
|
| 33 |
The official test set remains deliberately sealed. The preview is real
|
| 34 |
development evidence, but validation was reused for threshold selection,
|
|
@@ -99,20 +99,29 @@ model.
|
|
| 99 |
| `data/collection/` | Deterministic 900-recording Hinglish assignment plan; no recordings are included |
|
| 100 |
| `reports/` | Measured one-shard audit and split reports |
|
| 101 |
| `tests/` | Unit and integration coverage across data, model, metrics, runtime, and prompts |
|
|
|
|
|
|
|
| 102 |
|
| 103 |
## Reproduce the pipeline
|
| 104 |
|
| 105 |
Python 3.10–3.12 and [`uv`](https://docs.astral.sh/uv/) are recommended.
|
| 106 |
-
These commands target a full
|
| 107 |
-
|
| 108 |
-
|
| 109 |
-
private/local experiment inputs that are intentionally not redistributed.
|
| 110 |
|
| 111 |
```bash
|
| 112 |
uv sync --extra all
|
| 113 |
uv run pytest
|
| 114 |
```
|
| 115 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 116 |
Place the Hugging Face token in the ignored `.env` file; never commit it:
|
| 117 |
|
| 118 |
```bash
|
|
@@ -224,7 +233,9 @@ must not be treated as a fresh holdout result.
|
|
| 224 |
Only an experiment trained with exact `run.status: final` can be frozen. The
|
| 225 |
freeze manifest hashes the checkpoint, selected config, source tree,
|
| 226 |
preprocessing, controller policy, split manifest, threshold, and pinned test
|
| 227 |
-
identity.
|
|
|
|
|
|
|
| 228 |
|
| 229 |
```bash
|
| 230 |
uv run python scripts/freeze_candidate.py \
|
|
@@ -350,8 +361,21 @@ uv run python scripts/publish_hf.py \
|
|
| 350 |
--username suvradeepp \
|
| 351 |
--release-dir release \
|
| 352 |
--allow-development-release
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 353 |
```
|
| 354 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 355 |
Add `--execute` only after the dry validation succeeds. The publisher verifies
|
| 356 |
the authenticated owner and every packaged hash before creating or updating the
|
| 357 |
remote model repository and Space. Its receipt binds exact remote HEAD commits,
|
|
|
|
| 26 |
| Domain-shift stress | Measured, confounded | Source-held-out AP 0.5500/0.5368 for TinyTCN/baseline; both near chance in ranking, with wide nine-source bootstrap intervals |
|
| 27 |
| Runtime/export | Measured | 611 KiB FP32 ONNX; exact PyTorch/ONNX parity; ONNX p95 0.956 ms neural-only and 1.629 ms waveform-to-probability |
|
| 28 |
| Controller integration | Synthetic only | Metadata-bound 8-checkpoint/3-turn fixture emits 3 response edges with 0 duplicates; not model or product-quality evidence |
|
| 29 |
+
| Code quality gate | Passed | Ruff 0.15.16; 136/136 automated tests passed via `unittest` discovery |
|
| 30 |
| Accuracy/generalization | **Not established** | No full-corpus run, collected Hinglish recordings, real conversation replay, or official-test evaluation |
|
| 31 |
+
| Publication tooling | Guarded | Hash-verified HF publisher plus curated Kaggle/GitHub upload builders; verify each remote runtime separately |
|
| 32 |
|
| 33 |
The official test set remains deliberately sealed. The preview is real
|
| 34 |
development evidence, but validation was reused for threshold selection,
|
|
|
|
| 99 |
| `data/collection/` | Deterministic 900-recording Hinglish assignment plan; no recordings are included |
|
| 100 |
| `reports/` | Measured one-shard audit and split reports |
|
| 101 |
| `tests/` | Unit and integration coverage across data, model, metrics, runtime, and prompts |
|
| 102 |
+
| `deployment/kaggle/` | Flat, standalone ONNX inference templates for Kaggle Models |
|
| 103 |
+
| `upload-ready/` | Generated Kaggle and GitHub upload folders; never a source input |
|
| 104 |
|
| 105 |
## Reproduce the pipeline
|
| 106 |
|
| 107 |
Python 3.10–3.12 and [`uv`](https://docs.astral.sh/uv/) are recommended.
|
| 108 |
+
These commands target a full source checkout. Raw audio, downloaded Parquet,
|
| 109 |
+
processed manifests, and per-example predictions are intentionally not
|
| 110 |
+
redistributed; recreate them with the pinned download/audit commands below.
|
|
|
|
| 111 |
|
| 112 |
```bash
|
| 113 |
uv sync --extra all
|
| 114 |
uv run pytest
|
| 115 |
```
|
| 116 |
|
| 117 |
+
The curated public repository already includes the selected ONNX artifact, so
|
| 118 |
+
the demo resolves it without an environment variable:
|
| 119 |
+
|
| 120 |
+
```bash
|
| 121 |
+
uv sync --extra demo
|
| 122 |
+
uv run python app.py
|
| 123 |
+
```
|
| 124 |
+
|
| 125 |
Place the Hugging Face token in the ignored `.env` file; never commit it:
|
| 126 |
|
| 127 |
```bash
|
|
|
|
| 233 |
Only an experiment trained with exact `run.status: final` can be frozen. The
|
| 234 |
freeze manifest hashes the checkpoint, selected config, source tree,
|
| 235 |
preprocessing, controller policy, split manifest, threshold, and pinned test
|
| 236 |
+
identity. `configs/final.yaml` and `artifacts/final/*` below are intentionally
|
| 237 |
+
future-candidate placeholders; create them only after completing the full-data
|
| 238 |
+
experiment. Only then:
|
| 239 |
|
| 240 |
```bash
|
| 241 |
uv run python scripts/freeze_candidate.py \
|
|
|
|
| 361 |
--username suvradeepp \
|
| 362 |
--release-dir release \
|
| 363 |
--allow-development-release
|
| 364 |
+
|
| 365 |
+
uv run python scripts/build_upload_folders.py \
|
| 366 |
+
--release-dir release \
|
| 367 |
+
--output upload-ready \
|
| 368 |
+
--allow-development-release
|
| 369 |
```
|
| 370 |
|
| 371 |
+
`upload-ready/kaggle-model/` is a flat drag-and-drop ONNX bundle.
|
| 372 |
+
`upload-ready/github-repository/` is a public-repository allowlist containing
|
| 373 |
+
source, CI, tests, aggregate evidence, ONNX weights, and the two checkpoints
|
| 374 |
+
needed to reproduce the selected warm start. It excludes credentials, caches,
|
| 375 |
+
raw/processed data, per-example predictions, stale releases, and publish
|
| 376 |
+
receipts. Keep the Kaggle variation private until the derived-weight rights
|
| 377 |
+
review described in `NOTICE` and `MODEL_CARD.md` is resolved.
|
| 378 |
+
|
| 379 |
Add `--execute` only after the dry validation succeeds. The publisher verifies
|
| 380 |
the authenticated owner and every packaged hash before creating or updating the
|
| 381 |
remote model repository and Space. Its receipt binds exact remote HEAD commits,
|
app.py
CHANGED
|
@@ -12,6 +12,26 @@ from functools import lru_cache
|
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Any
|
| 14 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
PROJECT_ROOT = Path(__file__).resolve().parent
|
| 16 |
SRC_ROOT = PROJECT_ROOT / "src"
|
| 17 |
if str(SRC_ROOT) not in sys.path:
|
|
@@ -23,6 +43,8 @@ from turn_detection.runtime.predictor import load_predictor # noqa: E402
|
|
| 23 |
DEFAULT_MODEL_CANDIDATES = (
|
| 24 |
PROJECT_ROOT / "artifacts" / "model.onnx", # Hugging Face Space layout
|
| 25 |
PROJECT_ROOT / "model.onnx", # Hugging Face model-repository layout
|
|
|
|
|
|
|
| 26 |
)
|
| 27 |
|
| 28 |
|
|
@@ -85,6 +107,7 @@ def _timeline_html(probability: float, threshold: float, state: TurnState) -> st
|
|
| 85 |
"""
|
| 86 |
|
| 87 |
|
|
|
|
| 88 |
def analyze_turn(
|
| 89 |
audio: tuple[int, Any] | None,
|
| 90 |
threshold: float,
|
|
|
|
| 12 |
from pathlib import Path
|
| 13 |
from typing import Any
|
| 14 |
|
| 15 |
+
try:
|
| 16 |
+
import spaces
|
| 17 |
+
except ModuleNotFoundError as exc:
|
| 18 |
+
if exc.name != "spaces":
|
| 19 |
+
raise
|
| 20 |
+
|
| 21 |
+
class _LocalSpaces:
|
| 22 |
+
"""No-op compatibility layer for local installs without ZeroGPU."""
|
| 23 |
+
|
| 24 |
+
@staticmethod
|
| 25 |
+
def GPU(*, duration: int = 60) -> Any: # noqa: N802 - mirrors spaces.GPU
|
| 26 |
+
del duration
|
| 27 |
+
|
| 28 |
+
def decorate(function: Any) -> Any:
|
| 29 |
+
return function
|
| 30 |
+
|
| 31 |
+
return decorate
|
| 32 |
+
|
| 33 |
+
spaces = _LocalSpaces()
|
| 34 |
+
|
| 35 |
PROJECT_ROOT = Path(__file__).resolve().parent
|
| 36 |
SRC_ROOT = PROJECT_ROOT / "src"
|
| 37 |
if str(SRC_ROOT) not in sys.path:
|
|
|
|
| 43 |
DEFAULT_MODEL_CANDIDATES = (
|
| 44 |
PROJECT_ROOT / "artifacts" / "model.onnx", # Hugging Face Space layout
|
| 45 |
PROJECT_ROOT / "model.onnx", # Hugging Face model-repository layout
|
| 46 |
+
# Curated GitHub/source-checkout layout.
|
| 47 |
+
PROJECT_ROOT / "artifacts" / "partial-shard-warmstart-lr3e4-5ep" / "model.onnx",
|
| 48 |
)
|
| 49 |
|
| 50 |
|
|
|
|
| 107 |
"""
|
| 108 |
|
| 109 |
|
| 110 |
+
@spaces.GPU(duration=10)
|
| 111 |
def analyze_turn(
|
| 112 |
audio: tuple[int, Any] | None,
|
| 113 |
threshold: float,
|
deployment/kaggle/README.md
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Tiny Hinglish Turn Detector — ONNX preview
|
| 2 |
+
|
| 3 |
+
This is a flat, dependency-light Kaggle Models bundle for audio-native
|
| 4 |
+
`HOLD`/`END` decisions at VAD pause checkpoints. Upload every file in this
|
| 5 |
+
folder as one ONNX model variation.
|
| 6 |
+
|
| 7 |
+
> **Development preview:** the model was trained on one of 83 upstream training
|
| 8 |
+
> shards. The official test remains sealed, the acoustic logistic baseline is
|
| 9 |
+
> stronger on the current development split, and no verified Hinglish benchmark
|
| 10 |
+
> recordings have been evaluated. Do not claim production or Hinglish accuracy.
|
| 11 |
+
|
| 12 |
+
## Kaggle model settings
|
| 13 |
+
|
| 14 |
+
- Framework: **ONNX**
|
| 15 |
+
- Suggested variation: `tiny-tcn-fp32-preview`
|
| 16 |
+
- Fine-tunable: **No**
|
| 17 |
+
- Visibility: **Private** until upstream-derived-weight redistribution rights
|
| 18 |
+
have been reviewed
|
| 19 |
+
- License: **Other (specified in description)**. Apache-2.0 covers authored
|
| 20 |
+
code, not the upstream data or derived-weight rights.
|
| 21 |
+
|
| 22 |
+
## Files needed for inference
|
| 23 |
+
|
| 24 |
+
- `model.onnx`: 151,812-parameter FP32 TinyTCN
|
| 25 |
+
- `model_metadata.json`: frontend, tensor names, threshold, controller policy
|
| 26 |
+
- `turn_detector.py`: standalone NumPy + ONNX Runtime inference
|
| 27 |
+
- `requirements.txt`: three runtime dependencies
|
| 28 |
+
|
| 29 |
+
`MODEL_CARD.md`, `DATA_CARD.md`, `development_metrics.json`, and
|
| 30 |
+
`benchmark.json` document the limited evidence. `SHA256SUMS` binds the payload.
|
| 31 |
+
|
| 32 |
+
## Use inside a Kaggle Notebook
|
| 33 |
+
|
| 34 |
+
```python
|
| 35 |
+
from pathlib import Path
|
| 36 |
+
import kagglehub
|
| 37 |
+
|
| 38 |
+
model_dir = Path(kagglehub.model_download(
|
| 39 |
+
"YOUR_USERNAME/tiny-hinglish-turn-detector/onnx/tiny-tcn-fp32-preview"
|
| 40 |
+
))
|
| 41 |
+
|
| 42 |
+
import sys
|
| 43 |
+
sys.path.insert(0, str(model_dir))
|
| 44 |
+
from turn_detector import TurnDetector
|
| 45 |
+
|
| 46 |
+
detector = TurnDetector(model_dir)
|
| 47 |
+
result = detector.predict_file("/kaggle/input/your-audio/example.wav", silence_ms=300)
|
| 48 |
+
print(result)
|
| 49 |
+
```
|
| 50 |
+
|
| 51 |
+
For a downloaded folder outside Kaggle:
|
| 52 |
+
|
| 53 |
+
```bash
|
| 54 |
+
python -m pip install -r requirements.txt
|
| 55 |
+
python smoke_test.py
|
| 56 |
+
python example_inference.py path/to/audio.wav --silence-ms 300
|
| 57 |
+
```
|
| 58 |
+
|
| 59 |
+
Input audio may be mono or stereo and is resampled deterministically to 16 kHz.
|
| 60 |
+
The model uses the most recent four seconds. `p_end` is compared with the
|
| 61 |
+
serialized threshold and bounded by the serialized minimum/maximum silence
|
| 62 |
+
policy. This helper makes one stateless checkpoint decision; production callers
|
| 63 |
+
should retain the stateful controller from the full GitHub repository to latch
|
| 64 |
+
`END` and prevent duplicate responses.
|
| 65 |
+
|
deployment/kaggle/example_inference.py
ADDED
|
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Run the Kaggle model bundle on one audio file."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import json
|
| 8 |
+
from pathlib import Path
|
| 9 |
+
|
| 10 |
+
from turn_detector import TurnDetector
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main() -> int:
|
| 14 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 15 |
+
parser.add_argument("audio", help="WAV/FLAC/OGG audio file")
|
| 16 |
+
parser.add_argument(
|
| 17 |
+
"--model-dir",
|
| 18 |
+
default=Path(__file__).resolve().parent,
|
| 19 |
+
type=Path,
|
| 20 |
+
help="directory containing model.onnx and model_metadata.json",
|
| 21 |
+
)
|
| 22 |
+
parser.add_argument("--silence-ms", type=float, default=300.0)
|
| 23 |
+
args = parser.parse_args()
|
| 24 |
+
detector = TurnDetector(args.model_dir)
|
| 25 |
+
print(json.dumps(detector.predict_file(args.audio, silence_ms=args.silence_ms), indent=2))
|
| 26 |
+
return 0
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
if __name__ == "__main__":
|
| 30 |
+
raise SystemExit(main())
|
deployment/kaggle/requirements.txt
ADDED
|
@@ -0,0 +1,4 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
numpy==2.3.5
|
| 2 |
+
onnxruntime==1.26.0
|
| 3 |
+
soundfile==0.14.0
|
| 4 |
+
|
deployment/kaggle/smoke_test.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Dependency and ONNX execution smoke test using generated audio only."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import json
|
| 7 |
+
from pathlib import Path
|
| 8 |
+
|
| 9 |
+
import numpy as np
|
| 10 |
+
from turn_detector import TurnDetector
|
| 11 |
+
|
| 12 |
+
|
| 13 |
+
def main() -> int:
|
| 14 |
+
model_dir = Path(__file__).resolve().parent
|
| 15 |
+
detector = TurnDetector(model_dir)
|
| 16 |
+
sample_rate = 16_000
|
| 17 |
+
timeline = np.arange(sample_rate * 2, dtype=np.float32) / sample_rate
|
| 18 |
+
generated_audio = 0.08 * np.sin(2.0 * np.pi * 220.0 * timeline)
|
| 19 |
+
result = detector.predict(generated_audio, sample_rate, silence_ms=300.0)
|
| 20 |
+
if result["state"] not in {"HOLD", "END"}:
|
| 21 |
+
raise SystemExit("invalid decision state")
|
| 22 |
+
if not 0.0 <= result["p_end"] <= 1.0:
|
| 23 |
+
raise SystemExit("invalid endpoint probability")
|
| 24 |
+
print(json.dumps(result, indent=2))
|
| 25 |
+
return 0
|
| 26 |
+
|
| 27 |
+
|
| 28 |
+
if __name__ == "__main__":
|
| 29 |
+
raise SystemExit(main())
|
deployment/kaggle/turn_detector.py
ADDED
|
@@ -0,0 +1,304 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Standalone ONNX inference for the Kaggle model bundle.
|
| 2 |
+
|
| 3 |
+
This file deliberately has no dependency on the source repository. It mirrors
|
| 4 |
+
the canonical NumPy frontend used during export and reads every serving choice
|
| 5 |
+
from the adjacent ``model_metadata.json`` file.
|
| 6 |
+
"""
|
| 7 |
+
|
| 8 |
+
from __future__ import annotations
|
| 9 |
+
|
| 10 |
+
import json
|
| 11 |
+
import math
|
| 12 |
+
import time
|
| 13 |
+
from dataclasses import dataclass
|
| 14 |
+
from functools import lru_cache
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from typing import Any
|
| 17 |
+
|
| 18 |
+
import numpy as np
|
| 19 |
+
|
| 20 |
+
|
| 21 |
+
@dataclass(frozen=True, slots=True)
|
| 22 |
+
class FrontendConfig:
|
| 23 |
+
sample_rate: int = 16_000
|
| 24 |
+
max_seconds: float = 4.0
|
| 25 |
+
n_fft: int = 400
|
| 26 |
+
win_length: int = 400
|
| 27 |
+
hop_length: int = 160
|
| 28 |
+
n_mels: int = 80
|
| 29 |
+
f_min: float = 0.0
|
| 30 |
+
f_max: float = 8_000.0
|
| 31 |
+
normalization: str = "whisper"
|
| 32 |
+
pad_side: str = "left"
|
| 33 |
+
|
| 34 |
+
def __post_init__(self) -> None:
|
| 35 |
+
if self.sample_rate <= 0 or self.max_seconds <= 0:
|
| 36 |
+
raise ValueError("sample_rate and max_seconds must be positive")
|
| 37 |
+
if self.n_fft <= 0 or self.win_length <= 0 or self.hop_length <= 0:
|
| 38 |
+
raise ValueError("FFT and window sizes must be positive")
|
| 39 |
+
if self.win_length > self.n_fft:
|
| 40 |
+
raise ValueError("win_length cannot exceed n_fft")
|
| 41 |
+
if self.n_mels <= 0:
|
| 42 |
+
raise ValueError("n_mels must be positive")
|
| 43 |
+
if not 0.0 <= self.f_min < self.f_max <= self.sample_rate / 2:
|
| 44 |
+
raise ValueError("mel frequency bounds must lie inside Nyquist")
|
| 45 |
+
if self.normalization not in {"whisper", "log10", "none"}:
|
| 46 |
+
raise ValueError("unsupported normalization")
|
| 47 |
+
if self.pad_side not in {"left", "right"}:
|
| 48 |
+
raise ValueError("pad_side must be left or right")
|
| 49 |
+
|
| 50 |
+
@property
|
| 51 |
+
def max_samples(self) -> int:
|
| 52 |
+
return round(self.sample_rate * self.max_seconds)
|
| 53 |
+
|
| 54 |
+
@property
|
| 55 |
+
def target_frames(self) -> int:
|
| 56 |
+
return self.max_samples // self.hop_length
|
| 57 |
+
|
| 58 |
+
|
| 59 |
+
@dataclass(frozen=True, slots=True)
|
| 60 |
+
class ControllerConfig:
|
| 61 |
+
endpoint_threshold: float
|
| 62 |
+
long_pause_threshold: float
|
| 63 |
+
min_silence_ms: float
|
| 64 |
+
relax_after_ms: float
|
| 65 |
+
max_silence_ms: float
|
| 66 |
+
required_confirmations: int
|
| 67 |
+
|
| 68 |
+
def __post_init__(self) -> None:
|
| 69 |
+
if not 0.0 <= self.long_pause_threshold <= self.endpoint_threshold <= 1.0:
|
| 70 |
+
raise ValueError("controller thresholds are invalid")
|
| 71 |
+
if not self.min_silence_ms <= self.relax_after_ms <= self.max_silence_ms:
|
| 72 |
+
raise ValueError("controller silence bounds are invalid")
|
| 73 |
+
if self.required_confirmations < 1:
|
| 74 |
+
raise ValueError("required_confirmations must be positive")
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def normalize_waveform(audio: Any) -> np.ndarray:
|
| 78 |
+
"""Convert mono/stereo integer/float audio to finite mono float32."""
|
| 79 |
+
|
| 80 |
+
samples = np.asarray(audio)
|
| 81 |
+
if samples.size == 0:
|
| 82 |
+
raise ValueError("audio cannot be empty")
|
| 83 |
+
original_dtype = samples.dtype
|
| 84 |
+
if samples.ndim == 2:
|
| 85 |
+
channel_axis = 1 if samples.shape[1] <= 8 else 0
|
| 86 |
+
samples = samples.astype(np.float32).mean(axis=channel_axis)
|
| 87 |
+
elif samples.ndim != 1:
|
| 88 |
+
raise ValueError(f"expected mono/stereo audio, got shape {samples.shape}")
|
| 89 |
+
if np.issubdtype(original_dtype, np.integer):
|
| 90 |
+
info = np.iinfo(original_dtype)
|
| 91 |
+
samples = samples.astype(np.float32) / float(max(abs(info.min), info.max))
|
| 92 |
+
else:
|
| 93 |
+
samples = samples.astype(np.float32, copy=False)
|
| 94 |
+
samples = np.nan_to_num(samples, nan=0.0, posinf=1.0, neginf=-1.0)
|
| 95 |
+
peak = float(np.max(np.abs(samples)))
|
| 96 |
+
if peak > 1.0:
|
| 97 |
+
samples = samples / peak
|
| 98 |
+
return np.clip(samples, -1.0, 1.0)
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def resample_waveform(audio: Any, source_rate: int, target_rate: int) -> np.ndarray:
|
| 102 |
+
"""Apply the deterministic linear resampler used during training."""
|
| 103 |
+
|
| 104 |
+
if source_rate <= 0 or target_rate <= 0:
|
| 105 |
+
raise ValueError("sample rates must be positive")
|
| 106 |
+
samples = normalize_waveform(audio)
|
| 107 |
+
if source_rate == target_rate:
|
| 108 |
+
return samples
|
| 109 |
+
output_length = max(1, round(len(samples) * target_rate / source_rate))
|
| 110 |
+
old_x = np.linspace(0.0, 1.0, len(samples), endpoint=False)
|
| 111 |
+
new_x = np.linspace(0.0, 1.0, output_length, endpoint=False)
|
| 112 |
+
return np.interp(new_x, old_x, samples).astype(np.float32)
|
| 113 |
+
|
| 114 |
+
|
| 115 |
+
def pad_or_trim(audio: Any, config: FrontendConfig) -> tuple[np.ndarray, int]:
|
| 116 |
+
samples = normalize_waveform(audio)
|
| 117 |
+
if len(samples) >= config.max_samples:
|
| 118 |
+
return samples[-config.max_samples :].copy(), config.max_samples
|
| 119 |
+
padding = config.max_samples - len(samples)
|
| 120 |
+
widths = (padding, 0) if config.pad_side == "left" else (0, padding)
|
| 121 |
+
return np.pad(samples, widths).astype(np.float32), len(samples)
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
def _hz_to_mel(value: Any) -> np.ndarray:
|
| 125 |
+
return 2595.0 * np.log10(1.0 + np.asarray(value) / 700.0)
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
def _mel_to_hz(value: Any) -> np.ndarray:
|
| 129 |
+
return 700.0 * (10.0 ** (np.asarray(value) / 2595.0) - 1.0)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
@lru_cache(maxsize=16)
|
| 133 |
+
def mel_filterbank(config: FrontendConfig) -> np.ndarray:
|
| 134 |
+
mel_points = np.linspace(_hz_to_mel(config.f_min), _hz_to_mel(config.f_max), config.n_mels + 2)
|
| 135 |
+
hz_points = _mel_to_hz(mel_points)
|
| 136 |
+
fft_hz = np.linspace(0.0, config.sample_rate / 2, config.n_fft // 2 + 1)
|
| 137 |
+
filters = np.zeros((config.n_mels, len(fft_hz)), dtype=np.float32)
|
| 138 |
+
for index in range(config.n_mels):
|
| 139 |
+
left, center, right = hz_points[index : index + 3]
|
| 140 |
+
filters[index] = np.maximum(
|
| 141 |
+
0.0,
|
| 142 |
+
np.minimum(
|
| 143 |
+
(fft_hz - left) / max(center - left, 1e-12),
|
| 144 |
+
(right - fft_hz) / max(right - center, 1e-12),
|
| 145 |
+
),
|
| 146 |
+
)
|
| 147 |
+
normalization = 2.0 / np.maximum(
|
| 148 |
+
hz_points[2 : config.n_mels + 2] - hz_points[: config.n_mels],
|
| 149 |
+
1e-12,
|
| 150 |
+
)
|
| 151 |
+
result = filters * normalization[:, None]
|
| 152 |
+
result.flags.writeable = False
|
| 153 |
+
return result
|
| 154 |
+
|
| 155 |
+
|
| 156 |
+
@lru_cache(maxsize=16)
|
| 157 |
+
def _hann_window(length: int) -> np.ndarray:
|
| 158 |
+
window = np.hanning(length).astype(np.float32)
|
| 159 |
+
window.flags.writeable = False
|
| 160 |
+
return window
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
def log_mel_spectrogram(
|
| 164 |
+
audio: Any,
|
| 165 |
+
source_rate: int,
|
| 166 |
+
config: FrontendConfig,
|
| 167 |
+
) -> tuple[np.ndarray, np.ndarray]:
|
| 168 |
+
"""Return canonical ``[80, frames]`` features and valid-frame mask."""
|
| 169 |
+
|
| 170 |
+
if source_rate <= 0:
|
| 171 |
+
raise ValueError("source sample rate must be positive")
|
| 172 |
+
normalized = normalize_waveform(audio)
|
| 173 |
+
source_suffix_samples = max(1, round(config.max_seconds * source_rate))
|
| 174 |
+
normalized = normalized[-source_suffix_samples:]
|
| 175 |
+
resampled = resample_waveform(normalized, source_rate, config.sample_rate)
|
| 176 |
+
fixed, valid_samples = pad_or_trim(resampled, config)
|
| 177 |
+
|
| 178 |
+
padding = config.n_fft // 2
|
| 179 |
+
padded = np.pad(fixed, (padding, padding), mode="reflect")
|
| 180 |
+
frames = np.lib.stride_tricks.sliding_window_view(padded, config.win_length)[
|
| 181 |
+
:: config.hop_length
|
| 182 |
+
]
|
| 183 |
+
frames = frames[: config.target_frames]
|
| 184 |
+
spectrum = np.fft.rfft(
|
| 185 |
+
frames * _hann_window(config.win_length)[None, :],
|
| 186 |
+
n=config.n_fft,
|
| 187 |
+
axis=1,
|
| 188 |
+
)
|
| 189 |
+
power = (spectrum.real**2 + spectrum.imag**2).astype(np.float32)
|
| 190 |
+
mel = np.maximum(mel_filterbank(config) @ power.T, 1e-10)
|
| 191 |
+
features = np.log10(mel)
|
| 192 |
+
if config.normalization == "whisper":
|
| 193 |
+
features = np.maximum(features, float(features.max()) - 8.0)
|
| 194 |
+
features = (features + 4.0) / 4.0
|
| 195 |
+
elif config.normalization == "none":
|
| 196 |
+
features = mel
|
| 197 |
+
|
| 198 |
+
frame_mask = np.zeros(config.target_frames, dtype=np.float32)
|
| 199 |
+
valid_frames = min(
|
| 200 |
+
config.target_frames,
|
| 201 |
+
max(1, (valid_samples + config.hop_length - 1) // config.hop_length),
|
| 202 |
+
)
|
| 203 |
+
if config.pad_side == "left":
|
| 204 |
+
frame_mask[-valid_frames:] = 1.0
|
| 205 |
+
else:
|
| 206 |
+
frame_mask[:valid_frames] = 1.0
|
| 207 |
+
return features.astype(np.float32), frame_mask
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
class TurnDetector:
|
| 211 |
+
"""Load the adjacent ONNX artifact and score VAD pause checkpoints."""
|
| 212 |
+
|
| 213 |
+
def __init__(self, model_directory: str | Path) -> None:
|
| 214 |
+
try:
|
| 215 |
+
import onnxruntime as ort
|
| 216 |
+
except ImportError as exc:
|
| 217 |
+
raise RuntimeError("Install requirements.txt before inference") from exc
|
| 218 |
+
|
| 219 |
+
self.model_directory = Path(model_directory).expanduser().resolve()
|
| 220 |
+
model_path = self.model_directory / "model.onnx"
|
| 221 |
+
metadata_path = self.model_directory / "model_metadata.json"
|
| 222 |
+
if not model_path.is_file() or not metadata_path.is_file():
|
| 223 |
+
raise FileNotFoundError("model.onnx and model_metadata.json must be adjacent")
|
| 224 |
+
self.metadata = json.loads(metadata_path.read_text(encoding="utf-8"))
|
| 225 |
+
self.frontend = FrontendConfig(**self.metadata["frontend"])
|
| 226 |
+
self.controller = ControllerConfig(**self.metadata["controller"])
|
| 227 |
+
|
| 228 |
+
options = ort.SessionOptions()
|
| 229 |
+
options.intra_op_num_threads = 1
|
| 230 |
+
options.inter_op_num_threads = 1
|
| 231 |
+
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
| 232 |
+
self.session = ort.InferenceSession(
|
| 233 |
+
str(model_path),
|
| 234 |
+
sess_options=options,
|
| 235 |
+
providers=["CPUExecutionProvider"],
|
| 236 |
+
)
|
| 237 |
+
expected_inputs = {
|
| 238 |
+
self.metadata["input_features_name"],
|
| 239 |
+
self.metadata["frame_mask_name"],
|
| 240 |
+
}
|
| 241 |
+
actual_inputs = {item.name for item in self.session.get_inputs()}
|
| 242 |
+
if actual_inputs != expected_inputs:
|
| 243 |
+
raise ValueError(f"unexpected ONNX inputs: {sorted(actual_inputs)}")
|
| 244 |
+
expected_output = self.metadata["endpoint_output_name"]
|
| 245 |
+
actual_outputs = [item.name for item in self.session.get_outputs()]
|
| 246 |
+
if actual_outputs != [expected_output]:
|
| 247 |
+
raise ValueError(f"unexpected ONNX outputs: {actual_outputs}")
|
| 248 |
+
|
| 249 |
+
def threshold_for_silence(self, silence_ms: float) -> float:
|
| 250 |
+
if silence_ms <= self.controller.relax_after_ms:
|
| 251 |
+
return self.controller.endpoint_threshold
|
| 252 |
+
span = self.controller.max_silence_ms - self.controller.relax_after_ms
|
| 253 |
+
if span <= 0.0:
|
| 254 |
+
return self.controller.long_pause_threshold
|
| 255 |
+
progress = min(1.0, (silence_ms - self.controller.relax_after_ms) / span)
|
| 256 |
+
delta = self.controller.endpoint_threshold - self.controller.long_pause_threshold
|
| 257 |
+
return self.controller.endpoint_threshold - progress * delta
|
| 258 |
+
|
| 259 |
+
def predict(self, audio: Any, sample_rate: int, *, silence_ms: float = 300.0) -> dict[str, Any]:
|
| 260 |
+
"""Return a stateless HOLD/END decision for one pause checkpoint."""
|
| 261 |
+
|
| 262 |
+
if silence_ms < 0.0:
|
| 263 |
+
raise ValueError("silence_ms cannot be negative")
|
| 264 |
+
started = time.perf_counter_ns()
|
| 265 |
+
features, frame_mask = log_mel_spectrogram(audio, sample_rate, self.frontend)
|
| 266 |
+
raw = self.session.run(
|
| 267 |
+
[self.metadata["endpoint_output_name"]],
|
| 268 |
+
{
|
| 269 |
+
self.metadata["input_features_name"]: features[None, :, :],
|
| 270 |
+
self.metadata["frame_mask_name"]: frame_mask[None, :],
|
| 271 |
+
},
|
| 272 |
+
)[0]
|
| 273 |
+
value = float(np.asarray(raw).reshape(-1)[0])
|
| 274 |
+
probability = (
|
| 275 |
+
1.0 / (1.0 + math.exp(-value)) if self.metadata["output_type"] == "logits" else value
|
| 276 |
+
)
|
| 277 |
+
probability = min(1.0, max(0.0, probability))
|
| 278 |
+
threshold = self.threshold_for_silence(float(silence_ms))
|
| 279 |
+
if silence_ms < self.controller.min_silence_ms:
|
| 280 |
+
state, reason = "HOLD", "minimum_silence_not_reached"
|
| 281 |
+
elif silence_ms >= self.controller.max_silence_ms:
|
| 282 |
+
state, reason = "END", "maximum_timeout"
|
| 283 |
+
elif probability >= threshold:
|
| 284 |
+
state, reason = "END", "model_endpoint"
|
| 285 |
+
else:
|
| 286 |
+
state, reason = "HOLD", "model_hold"
|
| 287 |
+
return {
|
| 288 |
+
"state": state,
|
| 289 |
+
"reason": reason,
|
| 290 |
+
"p_end": probability,
|
| 291 |
+
"threshold": threshold,
|
| 292 |
+
"silence_ms": float(silence_ms),
|
| 293 |
+
"inference_ms": (time.perf_counter_ns() - started) / 1_000_000,
|
| 294 |
+
"model_name": self.metadata["model_name"],
|
| 295 |
+
"development_only": bool(self.metadata.get("development_only", False)),
|
| 296 |
+
}
|
| 297 |
+
|
| 298 |
+
def predict_file(self, audio_path: str | Path, *, silence_ms: float = 300.0) -> dict[str, Any]:
|
| 299 |
+
try:
|
| 300 |
+
import soundfile as sf
|
| 301 |
+
except ImportError as exc:
|
| 302 |
+
raise RuntimeError("Install soundfile to load audio paths") from exc
|
| 303 |
+
samples, sample_rate = sf.read(str(audio_path), dtype="float32", always_2d=False)
|
| 304 |
+
return self.predict(samples, int(sample_rate), silence_ms=silence_ms)
|
export_manifest.json
CHANGED
|
@@ -81,14 +81,39 @@
|
|
| 81 |
},
|
| 82 |
"source_files": [
|
| 83 |
{
|
| 84 |
-
"bytes":
|
| 85 |
"path": "app.py",
|
| 86 |
-
"sha256": "
|
| 87 |
},
|
| 88 |
{
|
| 89 |
-
"bytes":
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
"path": "pyproject.toml",
|
| 91 |
-
"sha256": "
|
| 92 |
},
|
| 93 |
{
|
| 94 |
"bytes": 399,
|
|
@@ -126,9 +151,14 @@
|
|
| 126 |
"sha256": "146763ebbe4d4ca0858c3fda2f56aa235aa9a4b6686b3216bcea58d148945d04"
|
| 127 |
},
|
| 128 |
{
|
| 129 |
-
"bytes":
|
| 130 |
"path": "scripts/build_release.py",
|
| 131 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 132 |
},
|
| 133 |
{
|
| 134 |
"bytes": 10147,
|
|
@@ -151,9 +181,9 @@
|
|
| 151 |
"sha256": "5e97757f74414b2ac003ab5e87a0afeeed88b61eb30baa1801b10a11a57acca4"
|
| 152 |
},
|
| 153 |
{
|
| 154 |
-
"bytes":
|
| 155 |
"path": "scripts/export_onnx.py",
|
| 156 |
-
"sha256": "
|
| 157 |
},
|
| 158 |
{
|
| 159 |
"bytes": 5458,
|
|
@@ -196,9 +226,9 @@
|
|
| 196 |
"sha256": "40569f1e553f0cb340cf81414e072839ca734a7d303c5cb5f1e46c5035590804"
|
| 197 |
},
|
| 198 |
{
|
| 199 |
-
"bytes":
|
| 200 |
"path": "space/requirements.txt",
|
| 201 |
-
"sha256": "
|
| 202 |
},
|
| 203 |
{
|
| 204 |
"bytes": 160,
|
|
@@ -371,7 +401,7 @@
|
|
| 371 |
"sha256": "da0eac829fe43615f7bb149998c2f2762f148790fb0c14d4ba6db4d6b25298f4"
|
| 372 |
}
|
| 373 |
],
|
| 374 |
-
"source_inventory_sha256": "
|
| 375 |
"task": "audio-turn-end-detection",
|
| 376 |
"threshold": 0.741000771522522,
|
| 377 |
"training_data": {
|
|
|
|
| 81 |
},
|
| 82 |
"source_files": [
|
| 83 |
{
|
| 84 |
+
"bytes": 11008,
|
| 85 |
"path": "app.py",
|
| 86 |
+
"sha256": "e0a9ab19732dab255f20b16e4866a04414228b785e72c139c7eadb2f91be5e36"
|
| 87 |
},
|
| 88 |
{
|
| 89 |
+
"bytes": 2359,
|
| 90 |
+
"path": "deployment/kaggle/README.md",
|
| 91 |
+
"sha256": "d107cee49a203172ecbb5b6c531a3898e3634690422d80468693bd2f0ceb333a"
|
| 92 |
+
},
|
| 93 |
+
{
|
| 94 |
+
"bytes": 852,
|
| 95 |
+
"path": "deployment/kaggle/example_inference.py",
|
| 96 |
+
"sha256": "27d34092619a06744fbdd899906e3f3236cfe5dfe391eedcd98f145f6015687c"
|
| 97 |
+
},
|
| 98 |
+
{
|
| 99 |
+
"bytes": 52,
|
| 100 |
+
"path": "deployment/kaggle/requirements.txt",
|
| 101 |
+
"sha256": "0ce722012cff5e44e1fe70632f85689362ed34b2b5c7ef5605e403027089f927"
|
| 102 |
+
},
|
| 103 |
+
{
|
| 104 |
+
"bytes": 892,
|
| 105 |
+
"path": "deployment/kaggle/smoke_test.py",
|
| 106 |
+
"sha256": "726bc853d360d105991563e94115bd4bbdb03fe9f5b3c007d712898ff0b5795b"
|
| 107 |
+
},
|
| 108 |
+
{
|
| 109 |
+
"bytes": 12363,
|
| 110 |
+
"path": "deployment/kaggle/turn_detector.py",
|
| 111 |
+
"sha256": "83de2bea2eb31b0fa123441a28c7238e122ba4e7986aa87859a2b94bdeab4258"
|
| 112 |
+
},
|
| 113 |
+
{
|
| 114 |
+
"bytes": 2601,
|
| 115 |
"path": "pyproject.toml",
|
| 116 |
+
"sha256": "fab1e0c093a89d5819a7109b9278332c61709e8bf61149486e31399c57a25f12"
|
| 117 |
},
|
| 118 |
{
|
| 119 |
"bytes": 399,
|
|
|
|
| 151 |
"sha256": "146763ebbe4d4ca0858c3fda2f56aa235aa9a4b6686b3216bcea58d148945d04"
|
| 152 |
},
|
| 153 |
{
|
| 154 |
+
"bytes": 43576,
|
| 155 |
"path": "scripts/build_release.py",
|
| 156 |
+
"sha256": "6bcab53184466e3ee5870e87bf85b606db62ddcd06956cd03679502958ea589b"
|
| 157 |
+
},
|
| 158 |
+
{
|
| 159 |
+
"bytes": 17830,
|
| 160 |
+
"path": "scripts/build_upload_folders.py",
|
| 161 |
+
"sha256": "0da4af7974df001c27b6fe78d9e2a60ac1fd841a8d48e27a2e499996c58060a7"
|
| 162 |
},
|
| 163 |
{
|
| 164 |
"bytes": 10147,
|
|
|
|
| 181 |
"sha256": "5e97757f74414b2ac003ab5e87a0afeeed88b61eb30baa1801b10a11a57acca4"
|
| 182 |
},
|
| 183 |
{
|
| 184 |
+
"bytes": 17939,
|
| 185 |
"path": "scripts/export_onnx.py",
|
| 186 |
+
"sha256": "6b6519191096614da193ebe07d553b4a422be5dd1457f803cb11f53530e88d3b"
|
| 187 |
},
|
| 188 |
{
|
| 189 |
"bytes": 5458,
|
|
|
|
| 226 |
"sha256": "40569f1e553f0cb340cf81414e072839ca734a7d303c5cb5f1e46c5035590804"
|
| 227 |
},
|
| 228 |
{
|
| 229 |
+
"bytes": 129,
|
| 230 |
"path": "space/requirements.txt",
|
| 231 |
+
"sha256": "e4fbc8a9d7c9d5bf0955e128a4cb1ba73c98b143daac0ebf5cc9afe79f2016f1"
|
| 232 |
},
|
| 233 |
{
|
| 234 |
"bytes": 160,
|
|
|
|
| 401 |
"sha256": "da0eac829fe43615f7bb149998c2f2762f148790fb0c14d4ba6db4d6b25298f4"
|
| 402 |
}
|
| 403 |
],
|
| 404 |
+
"source_inventory_sha256": "3a4b9eb2525f25f50d302cd657f6890e0ed82931c23fdb338e43d00b58597d78",
|
| 405 |
"task": "audio-turn-end-detection",
|
| 406 |
"threshold": 0.741000771522522,
|
| 407 |
"training_data": {
|
pyproject.toml
CHANGED
|
@@ -51,6 +51,7 @@ export = [
|
|
| 51 |
demo = [
|
| 52 |
"gradio>=5,<7",
|
| 53 |
"onnxruntime>=1.20,<2",
|
|
|
|
| 54 |
]
|
| 55 |
dev = [
|
| 56 |
"pytest>=8,<10",
|
|
@@ -69,6 +70,7 @@ all = [
|
|
| 69 |
"PyYAML>=6,<7",
|
| 70 |
"ruff>=0.9,<1",
|
| 71 |
"scikit-learn>=1.5,<2",
|
|
|
|
| 72 |
"torch>=2.4,<3",
|
| 73 |
"torchaudio>=2.4,<3",
|
| 74 |
"torchcodec>=0.7,<1",
|
|
@@ -77,7 +79,7 @@ all = [
|
|
| 77 |
]
|
| 78 |
|
| 79 |
[project.urls]
|
| 80 |
-
|
| 81 |
Demo = "https://huggingface.co/spaces/suvradeepp/tiny-hinglish-turn-detector"
|
| 82 |
|
| 83 |
[tool.hatch.build.targets.wheel]
|
|
@@ -95,7 +97,16 @@ markers = [
|
|
| 95 |
[tool.ruff]
|
| 96 |
target-version = "py310"
|
| 97 |
line-length = 100
|
| 98 |
-
extend-exclude = [
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 99 |
|
| 100 |
[tool.ruff.lint]
|
| 101 |
select = ["E", "F", "I", "B", "UP", "SIM"]
|
|
|
|
| 51 |
demo = [
|
| 52 |
"gradio>=5,<7",
|
| 53 |
"onnxruntime>=1.20,<2",
|
| 54 |
+
"spaces>=0.51,<1",
|
| 55 |
]
|
| 56 |
dev = [
|
| 57 |
"pytest>=8,<10",
|
|
|
|
| 70 |
"PyYAML>=6,<7",
|
| 71 |
"ruff>=0.9,<1",
|
| 72 |
"scikit-learn>=1.5,<2",
|
| 73 |
+
"spaces>=0.51,<1",
|
| 74 |
"torch>=2.4,<3",
|
| 75 |
"torchaudio>=2.4,<3",
|
| 76 |
"torchcodec>=0.7,<1",
|
|
|
|
| 79 |
]
|
| 80 |
|
| 81 |
[project.urls]
|
| 82 |
+
Model = "https://huggingface.co/suvradeepp/tiny-hinglish-turn-detector"
|
| 83 |
Demo = "https://huggingface.co/spaces/suvradeepp/tiny-hinglish-turn-detector"
|
| 84 |
|
| 85 |
[tool.hatch.build.targets.wheel]
|
|
|
|
| 97 |
[tool.ruff]
|
| 98 |
target-version = "py310"
|
| 99 |
line-length = 100
|
| 100 |
+
extend-exclude = [
|
| 101 |
+
".cache",
|
| 102 |
+
".venv",
|
| 103 |
+
"artifacts",
|
| 104 |
+
"checkpoints",
|
| 105 |
+
"data",
|
| 106 |
+
"release",
|
| 107 |
+
"upload-ready",
|
| 108 |
+
"upload-ready-*",
|
| 109 |
+
]
|
| 110 |
|
| 111 |
[tool.ruff.lint]
|
| 112 |
select = ["E", "F", "I", "B", "UP", "SIM"]
|
release_provenance.json
CHANGED
|
@@ -16,7 +16,7 @@
|
|
| 16 |
},
|
| 17 |
"MODEL_CARD.md": {
|
| 18 |
"bytes": 14805,
|
| 19 |
-
"sha256": "
|
| 20 |
},
|
| 21 |
"Makefile": {
|
| 22 |
"bytes": 777,
|
|
@@ -28,19 +28,19 @@
|
|
| 28 |
},
|
| 29 |
"README.md": {
|
| 30 |
"bytes": 14805,
|
| 31 |
-
"sha256": "
|
| 32 |
},
|
| 33 |
"REPORT.md": {
|
| 34 |
"bytes": 32272,
|
| 35 |
-
"sha256": "
|
| 36 |
},
|
| 37 |
"SOURCE_README.md": {
|
| 38 |
-
"bytes":
|
| 39 |
-
"sha256": "
|
| 40 |
},
|
| 41 |
"app.py": {
|
| 42 |
-
"bytes":
|
| 43 |
-
"sha256": "
|
| 44 |
},
|
| 45 |
"configs/datasets.json": {
|
| 46 |
"bytes": 635,
|
|
@@ -86,6 +86,26 @@
|
|
| 86 |
"bytes": 1115,
|
| 87 |
"sha256": "a96626ce8e93bff0a88f00f652932c5ac806377e67fc4fec3b03ae34658f9a04"
|
| 88 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 89 |
"development_metrics.json": {
|
| 90 |
"bytes": 18041,
|
| 91 |
"sha256": "ee94447afe6c6027089f0b9f091d4ae0864c22a25a809f3c9303e541d2bcf076"
|
|
@@ -107,8 +127,8 @@
|
|
| 107 |
"sha256": "d57a8b0b7e11eebce8b76812548d7e12bcc63c5880ef01633fef4914c8633b02"
|
| 108 |
},
|
| 109 |
"export_manifest.json": {
|
| 110 |
-
"bytes":
|
| 111 |
-
"sha256": "
|
| 112 |
},
|
| 113 |
"model.onnx": {
|
| 114 |
"bytes": 625431,
|
|
@@ -119,8 +139,8 @@
|
|
| 119 |
"sha256": "43636ad5efb642138c281a4582e85f19b5aae2f4f6cca69211169341ae4399b4"
|
| 120 |
},
|
| 121 |
"pyproject.toml": {
|
| 122 |
-
"bytes":
|
| 123 |
-
"sha256": "
|
| 124 |
},
|
| 125 |
"reference_models/acoustic_baseline.json": {
|
| 126 |
"bytes": 1240,
|
|
@@ -247,8 +267,12 @@
|
|
| 247 |
"sha256": "146763ebbe4d4ca0858c3fda2f56aa235aa9a4b6686b3216bcea58d148945d04"
|
| 248 |
},
|
| 249 |
"scripts/build_release.py": {
|
| 250 |
-
"bytes":
|
| 251 |
-
"sha256": "
|
|
|
|
|
|
|
|
|
|
|
|
|
| 252 |
},
|
| 253 |
"scripts/compare_models.py": {
|
| 254 |
"bytes": 10147,
|
|
@@ -267,8 +291,8 @@
|
|
| 267 |
"sha256": "5e97757f74414b2ac003ab5e87a0afeeed88b61eb30baa1801b10a11a57acca4"
|
| 268 |
},
|
| 269 |
"scripts/export_onnx.py": {
|
| 270 |
-
"bytes":
|
| 271 |
-
"sha256": "
|
| 272 |
},
|
| 273 |
"scripts/freeze_candidate.py": {
|
| 274 |
"bytes": 5458,
|
|
@@ -303,8 +327,8 @@
|
|
| 303 |
"sha256": "40569f1e553f0cb340cf81414e072839ca734a7d303c5cb5f1e46c5035590804"
|
| 304 |
},
|
| 305 |
"space/requirements.txt": {
|
| 306 |
-
"bytes":
|
| 307 |
-
"sha256": "
|
| 308 |
},
|
| 309 |
"src/turn_detection/__init__.py": {
|
| 310 |
"bytes": 160,
|
|
@@ -498,6 +522,10 @@
|
|
| 498 |
"bytes": 2378,
|
| 499 |
"sha256": "eda48e48a4eea66f4fe3f97b194e8a542d323a5d581ddcdcf4054fb3372bdbea"
|
| 500 |
},
|
|
|
|
|
|
|
|
|
|
|
|
|
| 501 |
"tests/test_metrics.py": {
|
| 502 |
"bytes": 5291,
|
| 503 |
"sha256": "c45739a050fcc1d5611d8dd228bdcc53ff737b0ff71da2aab26490945996d55c"
|
|
@@ -531,8 +559,8 @@
|
|
| 531 |
"sha256": "5af81ad7995b6bfd3816b7d97ee1226027659dc5b8a7cb4fe4376fa850dcd14e"
|
| 532 |
},
|
| 533 |
"tests/test_release_guardrails.py": {
|
| 534 |
-
"bytes":
|
| 535 |
-
"sha256": "
|
| 536 |
},
|
| 537 |
"tests/test_runtime_controller.py": {
|
| 538 |
"bytes": 3086,
|
|
@@ -553,6 +581,10 @@
|
|
| 553 |
"tests/test_train_warmstart.py": {
|
| 554 |
"bytes": 1785,
|
| 555 |
"sha256": "afad22048c9a8a84ec0fa4418596873ddf1d1dfd43e34301bc5ea98da659279e"
|
|
|
|
|
|
|
|
|
|
|
|
|
| 556 |
}
|
| 557 |
},
|
| 558 |
"space": {
|
|
@@ -566,7 +598,7 @@
|
|
| 566 |
},
|
| 567 |
"MODEL_CARD.md": {
|
| 568 |
"bytes": 14805,
|
| 569 |
-
"sha256": "
|
| 570 |
},
|
| 571 |
"NOTICE": {
|
| 572 |
"bytes": 563,
|
|
@@ -577,8 +609,8 @@
|
|
| 577 |
"sha256": "a604dbde1c5d837eea5a4a924747df34c0d3af653a8c2e7a5703a9816898ca3f"
|
| 578 |
},
|
| 579 |
"app.py": {
|
| 580 |
-
"bytes":
|
| 581 |
-
"sha256": "
|
| 582 |
},
|
| 583 |
"artifacts/model.onnx": {
|
| 584 |
"bytes": 625431,
|
|
@@ -589,8 +621,8 @@
|
|
| 589 |
"sha256": "43636ad5efb642138c281a4582e85f19b5aae2f4f6cca69211169341ae4399b4"
|
| 590 |
},
|
| 591 |
"requirements.txt": {
|
| 592 |
-
"bytes":
|
| 593 |
-
"sha256": "
|
| 594 |
},
|
| 595 |
"src/turn_detection/__init__.py": {
|
| 596 |
"bytes": 160,
|
|
@@ -626,5 +658,5 @@
|
|
| 626 |
"metrics_file": "development_metrics.json",
|
| 627 |
"metrics_scope": "development",
|
| 628 |
"model_file": "model.onnx",
|
| 629 |
-
"source_and_artifact_inventory_sha256": "
|
| 630 |
}
|
|
|
|
| 16 |
},
|
| 17 |
"MODEL_CARD.md": {
|
| 18 |
"bytes": 14805,
|
| 19 |
+
"sha256": "236a5a2d85be05295459fdd5fd453ed29197b74a9e320ad560f98c2454913c36"
|
| 20 |
},
|
| 21 |
"Makefile": {
|
| 22 |
"bytes": 777,
|
|
|
|
| 28 |
},
|
| 29 |
"README.md": {
|
| 30 |
"bytes": 14805,
|
| 31 |
+
"sha256": "236a5a2d85be05295459fdd5fd453ed29197b74a9e320ad560f98c2454913c36"
|
| 32 |
},
|
| 33 |
"REPORT.md": {
|
| 34 |
"bytes": 32272,
|
| 35 |
+
"sha256": "26daf79685188e8fcbb36c967ec8ddb51841f7263fbb35aab6242a0ea52b6bee"
|
| 36 |
},
|
| 37 |
"SOURCE_README.md": {
|
| 38 |
+
"bytes": 21628,
|
| 39 |
+
"sha256": "7cd1105e43f17a658005e56904665ab18009eea9dfc0c633c740ac9aae8fe779"
|
| 40 |
},
|
| 41 |
"app.py": {
|
| 42 |
+
"bytes": 11008,
|
| 43 |
+
"sha256": "e0a9ab19732dab255f20b16e4866a04414228b785e72c139c7eadb2f91be5e36"
|
| 44 |
},
|
| 45 |
"configs/datasets.json": {
|
| 46 |
"bytes": 635,
|
|
|
|
| 86 |
"bytes": 1115,
|
| 87 |
"sha256": "a96626ce8e93bff0a88f00f652932c5ac806377e67fc4fec3b03ae34658f9a04"
|
| 88 |
},
|
| 89 |
+
"deployment/kaggle/README.md": {
|
| 90 |
+
"bytes": 2359,
|
| 91 |
+
"sha256": "d107cee49a203172ecbb5b6c531a3898e3634690422d80468693bd2f0ceb333a"
|
| 92 |
+
},
|
| 93 |
+
"deployment/kaggle/example_inference.py": {
|
| 94 |
+
"bytes": 852,
|
| 95 |
+
"sha256": "27d34092619a06744fbdd899906e3f3236cfe5dfe391eedcd98f145f6015687c"
|
| 96 |
+
},
|
| 97 |
+
"deployment/kaggle/requirements.txt": {
|
| 98 |
+
"bytes": 52,
|
| 99 |
+
"sha256": "0ce722012cff5e44e1fe70632f85689362ed34b2b5c7ef5605e403027089f927"
|
| 100 |
+
},
|
| 101 |
+
"deployment/kaggle/smoke_test.py": {
|
| 102 |
+
"bytes": 892,
|
| 103 |
+
"sha256": "726bc853d360d105991563e94115bd4bbdb03fe9f5b3c007d712898ff0b5795b"
|
| 104 |
+
},
|
| 105 |
+
"deployment/kaggle/turn_detector.py": {
|
| 106 |
+
"bytes": 12363,
|
| 107 |
+
"sha256": "83de2bea2eb31b0fa123441a28c7238e122ba4e7986aa87859a2b94bdeab4258"
|
| 108 |
+
},
|
| 109 |
"development_metrics.json": {
|
| 110 |
"bytes": 18041,
|
| 111 |
"sha256": "ee94447afe6c6027089f0b9f091d4ae0864c22a25a809f3c9303e541d2bcf076"
|
|
|
|
| 127 |
"sha256": "d57a8b0b7e11eebce8b76812548d7e12bcc63c5880ef01633fef4914c8633b02"
|
| 128 |
},
|
| 129 |
"export_manifest.json": {
|
| 130 |
+
"bytes": 13434,
|
| 131 |
+
"sha256": "be4953e55580d453992b2590fb85e26e336bbcee66b56a0f3e5198551813cf7d"
|
| 132 |
},
|
| 133 |
"model.onnx": {
|
| 134 |
"bytes": 625431,
|
|
|
|
| 139 |
"sha256": "43636ad5efb642138c281a4582e85f19b5aae2f4f6cca69211169341ae4399b4"
|
| 140 |
},
|
| 141 |
"pyproject.toml": {
|
| 142 |
+
"bytes": 2601,
|
| 143 |
+
"sha256": "fab1e0c093a89d5819a7109b9278332c61709e8bf61149486e31399c57a25f12"
|
| 144 |
},
|
| 145 |
"reference_models/acoustic_baseline.json": {
|
| 146 |
"bytes": 1240,
|
|
|
|
| 267 |
"sha256": "146763ebbe4d4ca0858c3fda2f56aa235aa9a4b6686b3216bcea58d148945d04"
|
| 268 |
},
|
| 269 |
"scripts/build_release.py": {
|
| 270 |
+
"bytes": 43576,
|
| 271 |
+
"sha256": "6bcab53184466e3ee5870e87bf85b606db62ddcd06956cd03679502958ea589b"
|
| 272 |
+
},
|
| 273 |
+
"scripts/build_upload_folders.py": {
|
| 274 |
+
"bytes": 17830,
|
| 275 |
+
"sha256": "0da4af7974df001c27b6fe78d9e2a60ac1fd841a8d48e27a2e499996c58060a7"
|
| 276 |
},
|
| 277 |
"scripts/compare_models.py": {
|
| 278 |
"bytes": 10147,
|
|
|
|
| 291 |
"sha256": "5e97757f74414b2ac003ab5e87a0afeeed88b61eb30baa1801b10a11a57acca4"
|
| 292 |
},
|
| 293 |
"scripts/export_onnx.py": {
|
| 294 |
+
"bytes": 17939,
|
| 295 |
+
"sha256": "6b6519191096614da193ebe07d553b4a422be5dd1457f803cb11f53530e88d3b"
|
| 296 |
},
|
| 297 |
"scripts/freeze_candidate.py": {
|
| 298 |
"bytes": 5458,
|
|
|
|
| 327 |
"sha256": "40569f1e553f0cb340cf81414e072839ca734a7d303c5cb5f1e46c5035590804"
|
| 328 |
},
|
| 329 |
"space/requirements.txt": {
|
| 330 |
+
"bytes": 129,
|
| 331 |
+
"sha256": "e4fbc8a9d7c9d5bf0955e128a4cb1ba73c98b143daac0ebf5cc9afe79f2016f1"
|
| 332 |
},
|
| 333 |
"src/turn_detection/__init__.py": {
|
| 334 |
"bytes": 160,
|
|
|
|
| 522 |
"bytes": 2378,
|
| 523 |
"sha256": "eda48e48a4eea66f4fe3f97b194e8a542d323a5d581ddcdcf4054fb3372bdbea"
|
| 524 |
},
|
| 525 |
+
"tests/test_kaggle_runtime.py": {
|
| 526 |
+
"bytes": 2566,
|
| 527 |
+
"sha256": "768de85d1fc1c123e71928c250b91b60b64624a5ec7ceb5787b50b29fbe0c9eb"
|
| 528 |
+
},
|
| 529 |
"tests/test_metrics.py": {
|
| 530 |
"bytes": 5291,
|
| 531 |
"sha256": "c45739a050fcc1d5611d8dd228bdcc53ff737b0ff71da2aab26490945996d55c"
|
|
|
|
| 559 |
"sha256": "5af81ad7995b6bfd3816b7d97ee1226027659dc5b8a7cb4fe4376fa850dcd14e"
|
| 560 |
},
|
| 561 |
"tests/test_release_guardrails.py": {
|
| 562 |
+
"bytes": 19481,
|
| 563 |
+
"sha256": "089c87b9e98fbc9b0aab45e29245433a910952c13a854693ce077d170b3c2f2f"
|
| 564 |
},
|
| 565 |
"tests/test_runtime_controller.py": {
|
| 566 |
"bytes": 3086,
|
|
|
|
| 581 |
"tests/test_train_warmstart.py": {
|
| 582 |
"bytes": 1785,
|
| 583 |
"sha256": "afad22048c9a8a84ec0fa4418596873ddf1d1dfd43e34301bc5ea98da659279e"
|
| 584 |
+
},
|
| 585 |
+
"tests/test_upload_folders.py": {
|
| 586 |
+
"bytes": 2736,
|
| 587 |
+
"sha256": "0d14aa506b66f42e5c61c8f60b426f2ae7a294245a09627e35b477f213fd2359"
|
| 588 |
}
|
| 589 |
},
|
| 590 |
"space": {
|
|
|
|
| 598 |
},
|
| 599 |
"MODEL_CARD.md": {
|
| 600 |
"bytes": 14805,
|
| 601 |
+
"sha256": "236a5a2d85be05295459fdd5fd453ed29197b74a9e320ad560f98c2454913c36"
|
| 602 |
},
|
| 603 |
"NOTICE": {
|
| 604 |
"bytes": 563,
|
|
|
|
| 609 |
"sha256": "a604dbde1c5d837eea5a4a924747df34c0d3af653a8c2e7a5703a9816898ca3f"
|
| 610 |
},
|
| 611 |
"app.py": {
|
| 612 |
+
"bytes": 11008,
|
| 613 |
+
"sha256": "e0a9ab19732dab255f20b16e4866a04414228b785e72c139c7eadb2f91be5e36"
|
| 614 |
},
|
| 615 |
"artifacts/model.onnx": {
|
| 616 |
"bytes": 625431,
|
|
|
|
| 621 |
"sha256": "43636ad5efb642138c281a4582e85f19b5aae2f4f6cca69211169341ae4399b4"
|
| 622 |
},
|
| 623 |
"requirements.txt": {
|
| 624 |
+
"bytes": 129,
|
| 625 |
+
"sha256": "e4fbc8a9d7c9d5bf0955e128a4cb1ba73c98b143daac0ebf5cc9afe79f2016f1"
|
| 626 |
},
|
| 627 |
"src/turn_detection/__init__.py": {
|
| 628 |
"bytes": 160,
|
|
|
|
| 658 |
"metrics_file": "development_metrics.json",
|
| 659 |
"metrics_scope": "development",
|
| 660 |
"model_file": "model.onnx",
|
| 661 |
+
"source_and_artifact_inventory_sha256": "29c841cda437d73f0d1a401424a0f8401f6793f358bffae9b12edc2d0db1776a"
|
| 662 |
}
|
scripts/build_release.py
CHANGED
|
@@ -4,6 +4,7 @@
|
|
| 4 |
from __future__ import annotations
|
| 5 |
|
| 6 |
import argparse
|
|
|
|
| 7 |
import errno
|
| 8 |
import hashlib
|
| 9 |
import json
|
|
@@ -24,7 +25,12 @@ _SPACE_CARD_COLORS = frozenset(
|
|
| 24 |
)
|
| 25 |
_SPACE_SHORT_DESCRIPTION_MAX_LENGTH = 60
|
| 26 |
_SPACE_RUNTIME_REQUIREMENTS = frozenset(
|
| 27 |
-
{
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
)
|
| 29 |
|
| 30 |
|
|
@@ -139,6 +145,62 @@ def _validate_space_requirements(path: Path) -> None:
|
|
| 139 |
)
|
| 140 |
|
| 141 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 142 |
def _copy_runtime(destination: Path) -> None:
|
| 143 |
package_root = destination / "src" / "turn_detection"
|
| 144 |
package_root.mkdir(parents=True, exist_ok=True)
|
|
@@ -198,6 +260,14 @@ def _copy_project_source(destination: Path, *, include_synthetic_replay: bool) -
|
|
| 198 |
dirs_exist_ok=True,
|
| 199 |
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
| 200 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 201 |
safe_reports = {
|
| 202 |
"dataset_snapshot_status.json",
|
| 203 |
"environment_snapshot.json",
|
|
@@ -285,6 +355,11 @@ def _deployment_source_paths() -> list[Path]:
|
|
| 285 |
*sorted((ROOT / "src" / "turn_detection").rglob("*.py")),
|
| 286 |
*sorted((ROOT / "scripts").glob("*.py")),
|
| 287 |
*sorted((ROOT / "scripts").glob("*.sh")),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 288 |
ROOT / "app.py",
|
| 289 |
ROOT / "pyproject.toml",
|
| 290 |
ROOT / "space" / "requirements.txt",
|
|
@@ -774,6 +849,7 @@ def main() -> int:
|
|
| 774 |
)
|
| 775 |
_validate_space_card(ROOT / "space" / "README.md")
|
| 776 |
_validate_space_requirements(ROOT / "space" / "requirements.txt")
|
|
|
|
| 777 |
|
| 778 |
frozen_manifest: dict | None = None
|
| 779 |
if not metadata_is_development:
|
|
|
|
| 4 |
from __future__ import annotations
|
| 5 |
|
| 6 |
import argparse
|
| 7 |
+
import ast
|
| 8 |
import errno
|
| 9 |
import hashlib
|
| 10 |
import json
|
|
|
|
| 25 |
)
|
| 26 |
_SPACE_SHORT_DESCRIPTION_MAX_LENGTH = 60
|
| 27 |
_SPACE_RUNTIME_REQUIREMENTS = frozenset(
|
| 28 |
+
{
|
| 29 |
+
"numpy==2.3.5",
|
| 30 |
+
"onnxruntime==1.26.0",
|
| 31 |
+
"soundfile==0.14.0",
|
| 32 |
+
"spaces==0.51.1",
|
| 33 |
+
}
|
| 34 |
)
|
| 35 |
|
| 36 |
|
|
|
|
| 145 |
)
|
| 146 |
|
| 147 |
|
| 148 |
+
def _validate_space_app(path: Path) -> None:
|
| 149 |
+
"""Require ZeroGPU to wrap the real Gradio inference callback."""
|
| 150 |
+
|
| 151 |
+
try:
|
| 152 |
+
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
| 153 |
+
except (OSError, SyntaxError) as exc:
|
| 154 |
+
raise SystemExit(f"Space app is unreadable or invalid Python: {path}") from exc
|
| 155 |
+
|
| 156 |
+
imports_spaces = any(
|
| 157 |
+
isinstance(node, ast.Import)
|
| 158 |
+
and any(alias.name == "spaces" and alias.asname in {None, "spaces"} for alias in node.names)
|
| 159 |
+
for node in ast.walk(tree)
|
| 160 |
+
)
|
| 161 |
+
if not imports_spaces:
|
| 162 |
+
raise SystemExit("Space app must import spaces for ZeroGPU")
|
| 163 |
+
|
| 164 |
+
analyze_function = next(
|
| 165 |
+
(
|
| 166 |
+
node
|
| 167 |
+
for node in tree.body
|
| 168 |
+
if isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef)
|
| 169 |
+
and node.name == "analyze_turn"
|
| 170 |
+
),
|
| 171 |
+
None,
|
| 172 |
+
)
|
| 173 |
+
if analyze_function is None:
|
| 174 |
+
raise SystemExit("Space app must define the analyze_turn callback")
|
| 175 |
+
|
| 176 |
+
def is_spaces_gpu(decorator: ast.expr) -> bool:
|
| 177 |
+
target = decorator.func if isinstance(decorator, ast.Call) else decorator
|
| 178 |
+
return (
|
| 179 |
+
isinstance(target, ast.Attribute)
|
| 180 |
+
and target.attr == "GPU"
|
| 181 |
+
and isinstance(target.value, ast.Name)
|
| 182 |
+
and target.value.id == "spaces"
|
| 183 |
+
)
|
| 184 |
+
|
| 185 |
+
if not any(is_spaces_gpu(item) for item in analyze_function.decorator_list):
|
| 186 |
+
raise SystemExit("Space analyze_turn callback must use @spaces.GPU")
|
| 187 |
+
|
| 188 |
+
callback_is_bound = any(
|
| 189 |
+
isinstance(node, ast.Call)
|
| 190 |
+
and isinstance(node.func, ast.Attribute)
|
| 191 |
+
and node.func.attr == "click"
|
| 192 |
+
and any(
|
| 193 |
+
keyword.arg == "fn"
|
| 194 |
+
and isinstance(keyword.value, ast.Name)
|
| 195 |
+
and keyword.value.id == "analyze_turn"
|
| 196 |
+
for keyword in node.keywords
|
| 197 |
+
)
|
| 198 |
+
for node in ast.walk(tree)
|
| 199 |
+
)
|
| 200 |
+
if not callback_is_bound:
|
| 201 |
+
raise SystemExit("Space Analyze button must bind the decorated analyze_turn callback")
|
| 202 |
+
|
| 203 |
+
|
| 204 |
def _copy_runtime(destination: Path) -> None:
|
| 205 |
package_root = destination / "src" / "turn_detection"
|
| 206 |
package_root.mkdir(parents=True, exist_ok=True)
|
|
|
|
| 260 |
dirs_exist_ok=True,
|
| 261 |
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
| 262 |
)
|
| 263 |
+
if (ROOT / "deployment").is_dir():
|
| 264 |
+
_reject_symlinks(ROOT / "deployment")
|
| 265 |
+
shutil.copytree(
|
| 266 |
+
ROOT / "deployment",
|
| 267 |
+
destination / "deployment",
|
| 268 |
+
dirs_exist_ok=True,
|
| 269 |
+
ignore=shutil.ignore_patterns("__pycache__", "*.pyc"),
|
| 270 |
+
)
|
| 271 |
safe_reports = {
|
| 272 |
"dataset_snapshot_status.json",
|
| 273 |
"environment_snapshot.json",
|
|
|
|
| 355 |
*sorted((ROOT / "src" / "turn_detection").rglob("*.py")),
|
| 356 |
*sorted((ROOT / "scripts").glob("*.py")),
|
| 357 |
*sorted((ROOT / "scripts").glob("*.sh")),
|
| 358 |
+
*(
|
| 359 |
+
path
|
| 360 |
+
for path in sorted((ROOT / "deployment").rglob("*"))
|
| 361 |
+
if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc"
|
| 362 |
+
),
|
| 363 |
ROOT / "app.py",
|
| 364 |
ROOT / "pyproject.toml",
|
| 365 |
ROOT / "space" / "requirements.txt",
|
|
|
|
| 849 |
)
|
| 850 |
_validate_space_card(ROOT / "space" / "README.md")
|
| 851 |
_validate_space_requirements(ROOT / "space" / "requirements.txt")
|
| 852 |
+
_validate_space_app(ROOT / "app.py")
|
| 853 |
|
| 854 |
frozen_manifest: dict | None = None
|
| 855 |
if not metadata_is_development:
|
scripts/build_upload_folders.py
ADDED
|
@@ -0,0 +1,470 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
"""Build curated, integrity-checked Kaggle and GitHub upload folders."""
|
| 3 |
+
|
| 4 |
+
from __future__ import annotations
|
| 5 |
+
|
| 6 |
+
import argparse
|
| 7 |
+
import hashlib
|
| 8 |
+
import json
|
| 9 |
+
import os
|
| 10 |
+
import re
|
| 11 |
+
import shutil
|
| 12 |
+
import sys
|
| 13 |
+
import tempfile
|
| 14 |
+
from collections import defaultdict
|
| 15 |
+
from pathlib import Path
|
| 16 |
+
from urllib.parse import unquote, urlsplit
|
| 17 |
+
|
| 18 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 19 |
+
if str(ROOT) not in sys.path:
|
| 20 |
+
sys.path.insert(0, str(ROOT))
|
| 21 |
+
|
| 22 |
+
from scripts.publish_hf import ( # noqa: E402
|
| 23 |
+
_validate_inventory,
|
| 24 |
+
_validate_provenance,
|
| 25 |
+
_validated_release_status,
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
_KAGGLE_TEMPLATE_FILES = (
|
| 29 |
+
"README.md",
|
| 30 |
+
"example_inference.py",
|
| 31 |
+
"requirements.txt",
|
| 32 |
+
"smoke_test.py",
|
| 33 |
+
"turn_detector.py",
|
| 34 |
+
)
|
| 35 |
+
_ROOT_GITHUB_FILES = (
|
| 36 |
+
".env.example",
|
| 37 |
+
".gitignore",
|
| 38 |
+
".python-version",
|
| 39 |
+
"DATA_CARD.md",
|
| 40 |
+
"LICENSE",
|
| 41 |
+
"MODEL_CARD.md",
|
| 42 |
+
"Makefile",
|
| 43 |
+
"NOTICE",
|
| 44 |
+
"README.md",
|
| 45 |
+
"REPORT.md",
|
| 46 |
+
"app.py",
|
| 47 |
+
"pyproject.toml",
|
| 48 |
+
"requirements-export.txt",
|
| 49 |
+
"requirements-publish.txt",
|
| 50 |
+
"requirements-training.txt",
|
| 51 |
+
)
|
| 52 |
+
_GITHUB_ARTIFACTS = (
|
| 53 |
+
"artifacts/partial-baseline/model.json",
|
| 54 |
+
"artifacts/partial-shard-preview-tinytcn-4s/best.pt",
|
| 55 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/best.pt",
|
| 56 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/cpu_benchmark.json",
|
| 57 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/export_manifest.json",
|
| 58 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/history.json",
|
| 59 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/model.onnx",
|
| 60 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/model_metadata.json",
|
| 61 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/onnx_benchmark.json",
|
| 62 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/onnx_e2e_benchmark.json",
|
| 63 |
+
"artifacts/partial-shard-warmstart-lr3e4-5ep/resolved_config.json",
|
| 64 |
+
)
|
| 65 |
+
_COLLECTION_FILES = (
|
| 66 |
+
"assignments.jsonl",
|
| 67 |
+
"assignments.summary.json",
|
| 68 |
+
"controller_replay_fixture.jsonl",
|
| 69 |
+
)
|
| 70 |
+
_PUBLIC_REPORTS = (
|
| 71 |
+
"controller_replay_integration.jsonl",
|
| 72 |
+
"controller_replay_integration.summary.json",
|
| 73 |
+
"dataset_snapshot_status.json",
|
| 74 |
+
"environment_snapshot.json",
|
| 75 |
+
"partial_baseline_metrics.json",
|
| 76 |
+
"partial_iid_split.json",
|
| 77 |
+
"partial_shard_audit.json",
|
| 78 |
+
"partial_source_holdout_baseline_metrics.json",
|
| 79 |
+
"partial_source_holdout_comparison.json",
|
| 80 |
+
"partial_source_holdout_split.json",
|
| 81 |
+
"partial_source_holdout_tinytcn_metrics.json",
|
| 82 |
+
"partial_tinytcn_comparison.json",
|
| 83 |
+
"partial_tinytcn_e2e_benchmark.json",
|
| 84 |
+
"partial_tinytcn_failures.json",
|
| 85 |
+
"partial_tinytcn_metrics.json",
|
| 86 |
+
"partial_tinytcn_onnx_benchmark.json",
|
| 87 |
+
"partial_tinytcn_pytorch_benchmark.json",
|
| 88 |
+
"partial_tinytcn_silence_sensitivity.json",
|
| 89 |
+
)
|
| 90 |
+
_SECRET_PATTERNS = {
|
| 91 |
+
"Hugging Face token": re.compile(r"\bhf_[A-Za-z0-9]{20,}\b"),
|
| 92 |
+
"GitHub token": re.compile(r"\b(?:ghp|github_pat)_[A-Za-z0-9_]{20,}\b"),
|
| 93 |
+
"OpenAI-style token": re.compile(r"\bsk-[A-Za-z0-9_-]{20,}\b"),
|
| 94 |
+
"private key": re.compile(r"-----BEGIN (?:RSA |EC |OPENSSH )?PRIVATE KEY-----"),
|
| 95 |
+
"absolute macOS user path": re.compile("/" + r"Users/[^/\s]+/"),
|
| 96 |
+
}
|
| 97 |
+
_MARKDOWN_LINK = re.compile(r"!?\[[^\]]*\]\(([^)]+)\)")
|
| 98 |
+
|
| 99 |
+
|
| 100 |
+
def parse_args() -> argparse.Namespace:
|
| 101 |
+
parser = argparse.ArgumentParser(description=__doc__)
|
| 102 |
+
parser.add_argument("--release-dir", default="release")
|
| 103 |
+
parser.add_argument("--output", default="upload-ready")
|
| 104 |
+
parser.add_argument(
|
| 105 |
+
"--allow-development-release",
|
| 106 |
+
action="store_true",
|
| 107 |
+
help="package the visibly labelled development preview",
|
| 108 |
+
)
|
| 109 |
+
return parser.parse_args()
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def _resolve_inside_root(value: str, *, purpose: str) -> Path:
|
| 113 |
+
unresolved = Path(value)
|
| 114 |
+
path = (unresolved if unresolved.is_absolute() else ROOT / unresolved).resolve()
|
| 115 |
+
try:
|
| 116 |
+
path.relative_to(ROOT)
|
| 117 |
+
except ValueError as exc:
|
| 118 |
+
raise SystemExit(f"{purpose} must stay inside the project directory") from exc
|
| 119 |
+
return path
|
| 120 |
+
|
| 121 |
+
|
| 122 |
+
def _sha256(path: Path) -> str:
|
| 123 |
+
digest = hashlib.sha256()
|
| 124 |
+
with path.open("rb") as stream:
|
| 125 |
+
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
| 126 |
+
digest.update(block)
|
| 127 |
+
return digest.hexdigest()
|
| 128 |
+
|
| 129 |
+
|
| 130 |
+
def _copy_file(source: Path, destination: Path) -> None:
|
| 131 |
+
if source.is_symlink() or not source.is_file():
|
| 132 |
+
raise SystemExit(f"refusing non-regular upload source: {source}")
|
| 133 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 134 |
+
shutil.copy2(source, destination)
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def _copy_tree(source: Path, destination: Path, *, suffixes: set[str] | None = None) -> None:
|
| 138 |
+
if source.is_symlink() or not source.is_dir():
|
| 139 |
+
raise SystemExit(f"upload source directory is missing or unsafe: {source}")
|
| 140 |
+
for path in sorted(source.rglob("*")):
|
| 141 |
+
if path.is_symlink():
|
| 142 |
+
raise SystemExit(f"refusing symlink in upload source: {path}")
|
| 143 |
+
if not path.is_file():
|
| 144 |
+
continue
|
| 145 |
+
if "__pycache__" in path.parts or path.suffix == ".pyc":
|
| 146 |
+
continue
|
| 147 |
+
if suffixes is not None and path.suffix not in suffixes:
|
| 148 |
+
continue
|
| 149 |
+
_copy_file(path, destination / path.relative_to(source))
|
| 150 |
+
|
| 151 |
+
|
| 152 |
+
def _load_validated_release(release_dir: Path) -> tuple[dict, bool]:
|
| 153 |
+
manifest_path = release_dir / "release_manifest.json"
|
| 154 |
+
try:
|
| 155 |
+
manifest = json.loads(manifest_path.read_text(encoding="utf-8"))
|
| 156 |
+
except (OSError, json.JSONDecodeError) as exc:
|
| 157 |
+
raise SystemExit("release_manifest.json is missing or invalid; rebuild release") from exc
|
| 158 |
+
if not isinstance(manifest, dict):
|
| 159 |
+
raise SystemExit("release_manifest.json must contain an object")
|
| 160 |
+
_validate_inventory(release_dir, manifest)
|
| 161 |
+
_validate_provenance(release_dir)
|
| 162 |
+
development_only = _validated_release_status(release_dir, manifest)
|
| 163 |
+
return manifest, development_only
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _require_identical(left: Path, right: Path, label: str) -> None:
|
| 167 |
+
if left.stat().st_size != right.stat().st_size or _sha256(left) != _sha256(right):
|
| 168 |
+
raise SystemExit(f"{label} differs from the validated Hugging Face release")
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def _build_kaggle_folder(release_dir: Path, destination: Path, metrics_name: str) -> None:
|
| 172 |
+
model_release = release_dir / "model"
|
| 173 |
+
for name in _KAGGLE_TEMPLATE_FILES:
|
| 174 |
+
_copy_file(ROOT / "deployment" / "kaggle" / name, destination / name)
|
| 175 |
+
for source_name, destination_name in (
|
| 176 |
+
("model.onnx", "model.onnx"),
|
| 177 |
+
("model_metadata.json", "model_metadata.json"),
|
| 178 |
+
("export_manifest.json", "export_manifest.json"),
|
| 179 |
+
(
|
| 180 |
+
metrics_name,
|
| 181 |
+
"development_metrics.json" if "development" in metrics_name else "test_metrics.json",
|
| 182 |
+
),
|
| 183 |
+
("MODEL_CARD.md", "MODEL_CARD.md"),
|
| 184 |
+
("DATA_CARD.md", "DATA_CARD.md"),
|
| 185 |
+
("LICENSE", "LICENSE"),
|
| 186 |
+
("NOTICE", "NOTICE"),
|
| 187 |
+
):
|
| 188 |
+
_copy_file(model_release / source_name, destination / destination_name)
|
| 189 |
+
_copy_file(
|
| 190 |
+
model_release / "run_artifacts" / "onnx_e2e_benchmark.json",
|
| 191 |
+
destination / "benchmark.json",
|
| 192 |
+
)
|
| 193 |
+
|
| 194 |
+
|
| 195 |
+
def _github_gitignore() -> str:
|
| 196 |
+
base = (ROOT / ".gitignore").read_text(encoding="utf-8").rstrip()
|
| 197 |
+
by_parent: dict[Path, list[Path]] = defaultdict(list)
|
| 198 |
+
for relative in map(Path, _GITHUB_ARTIFACTS):
|
| 199 |
+
by_parent[relative.parent].append(relative)
|
| 200 |
+
lines = [
|
| 201 |
+
base,
|
| 202 |
+
"",
|
| 203 |
+
"# Exact public-preview artifacts included by build_upload_folders.py",
|
| 204 |
+
"!artifacts/",
|
| 205 |
+
"artifacts/*",
|
| 206 |
+
]
|
| 207 |
+
for parent in sorted(by_parent, key=lambda item: item.as_posix()):
|
| 208 |
+
lines.append(f"!{parent.as_posix()}/")
|
| 209 |
+
lines.append(f"{parent.as_posix()}/*")
|
| 210 |
+
for relative in sorted(by_parent[parent], key=lambda item: item.as_posix()):
|
| 211 |
+
lines.append(f"!{relative.as_posix()}")
|
| 212 |
+
return "\n".join(lines) + "\n"
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def _build_github_folder(destination: Path) -> None:
|
| 216 |
+
for name in _ROOT_GITHUB_FILES:
|
| 217 |
+
_copy_file(ROOT / name, destination / name)
|
| 218 |
+
(destination / ".gitignore").write_text(_github_gitignore(), encoding="utf-8")
|
| 219 |
+
|
| 220 |
+
_copy_tree(ROOT / ".github", destination / ".github")
|
| 221 |
+
_copy_tree(ROOT / "src", destination / "src", suffixes={".py", ".typed"})
|
| 222 |
+
_copy_tree(ROOT / "scripts", destination / "scripts", suffixes={".py", ".sh"})
|
| 223 |
+
_copy_tree(ROOT / "configs", destination / "configs", suffixes={".json", ".yaml", ".yml"})
|
| 224 |
+
_copy_tree(ROOT / "tests", destination / "tests", suffixes={".py"})
|
| 225 |
+
_copy_tree(ROOT / "docs", destination / "docs", suffixes={".md"})
|
| 226 |
+
_copy_tree(ROOT / "deployment", destination / "deployment", suffixes={".md", ".py", ".txt"})
|
| 227 |
+
|
| 228 |
+
for name in ("README.md", "requirements.txt"):
|
| 229 |
+
_copy_file(ROOT / "space" / name, destination / "space" / name)
|
| 230 |
+
for name in _COLLECTION_FILES:
|
| 231 |
+
_copy_file(
|
| 232 |
+
ROOT / "data" / "collection" / name,
|
| 233 |
+
destination / "data" / "collection" / name,
|
| 234 |
+
)
|
| 235 |
+
for name in _PUBLIC_REPORTS:
|
| 236 |
+
_copy_file(ROOT / "reports" / name, destination / "reports" / name)
|
| 237 |
+
for relative in _GITHUB_ARTIFACTS:
|
| 238 |
+
_copy_file(ROOT / relative, destination / relative)
|
| 239 |
+
_copy_file(
|
| 240 |
+
ROOT / "artifacts/partial-baseline/model.json",
|
| 241 |
+
destination / "reference_models/acoustic_baseline.json",
|
| 242 |
+
)
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
def _inventory(
|
| 246 |
+
folder: Path, *, excluded: set[str] | None = None
|
| 247 |
+
) -> dict[str, dict[str, int | str]]:
|
| 248 |
+
excluded = excluded or set()
|
| 249 |
+
return {
|
| 250 |
+
path.relative_to(folder).as_posix(): {
|
| 251 |
+
"bytes": path.stat().st_size,
|
| 252 |
+
"sha256": _sha256(path),
|
| 253 |
+
}
|
| 254 |
+
for path in sorted(folder.rglob("*"))
|
| 255 |
+
if path.is_file() and path.relative_to(folder).as_posix() not in excluded
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def _write_package_evidence(
|
| 260 |
+
folder: Path,
|
| 261 |
+
*,
|
| 262 |
+
kind: str,
|
| 263 |
+
development_only: bool,
|
| 264 |
+
release_inventory_sha256: str,
|
| 265 |
+
) -> None:
|
| 266 |
+
manifest_path = folder / "UPLOAD_MANIFEST.json"
|
| 267 |
+
checksums_path = folder / "SHA256SUMS"
|
| 268 |
+
payload = {
|
| 269 |
+
"format_version": 1,
|
| 270 |
+
"package_kind": kind,
|
| 271 |
+
"development_only": development_only,
|
| 272 |
+
"source_release_inventory_sha256": release_inventory_sha256,
|
| 273 |
+
"files": _inventory(folder),
|
| 274 |
+
}
|
| 275 |
+
manifest_path.write_text(
|
| 276 |
+
json.dumps(payload, indent=2, sort_keys=True) + "\n",
|
| 277 |
+
encoding="utf-8",
|
| 278 |
+
)
|
| 279 |
+
checksums = _inventory(folder, excluded={checksums_path.name})
|
| 280 |
+
checksums_path.write_text(
|
| 281 |
+
"".join(f"{evidence['sha256']} {relative}\n" for relative, evidence in checksums.items()),
|
| 282 |
+
encoding="utf-8",
|
| 283 |
+
)
|
| 284 |
+
|
| 285 |
+
|
| 286 |
+
def _secret_findings(folder: Path) -> list[str]:
|
| 287 |
+
findings: list[str] = []
|
| 288 |
+
for path in sorted(folder.rglob("*")):
|
| 289 |
+
if path.is_symlink():
|
| 290 |
+
findings.append(f"symlink: {path.relative_to(folder)}")
|
| 291 |
+
continue
|
| 292 |
+
if not path.is_file() or path.suffix in {".onnx", ".pt"}:
|
| 293 |
+
continue
|
| 294 |
+
if path.name == ".env" or path.name == "hf_publish_receipt.json":
|
| 295 |
+
findings.append(f"forbidden file: {path.relative_to(folder)}")
|
| 296 |
+
continue
|
| 297 |
+
try:
|
| 298 |
+
text = path.read_text(encoding="utf-8")
|
| 299 |
+
except UnicodeDecodeError:
|
| 300 |
+
continue
|
| 301 |
+
for label, pattern in _SECRET_PATTERNS.items():
|
| 302 |
+
if pattern.search(text):
|
| 303 |
+
findings.append(f"{label}: {path.relative_to(folder)}")
|
| 304 |
+
return findings
|
| 305 |
+
|
| 306 |
+
|
| 307 |
+
def _broken_local_markdown_links(folder: Path) -> list[str]:
|
| 308 |
+
broken: list[str] = []
|
| 309 |
+
for markdown in sorted(folder.rglob("*.md")):
|
| 310 |
+
text = markdown.read_text(encoding="utf-8")
|
| 311 |
+
for match in _MARKDOWN_LINK.finditer(text):
|
| 312 |
+
raw = match.group(1).strip().strip("<>")
|
| 313 |
+
raw = raw.split(maxsplit=1)[0].strip("\"'")
|
| 314 |
+
parsed = urlsplit(raw)
|
| 315 |
+
if parsed.scheme or parsed.netloc or not parsed.path:
|
| 316 |
+
continue
|
| 317 |
+
target = (markdown.parent / unquote(parsed.path)).resolve()
|
| 318 |
+
try:
|
| 319 |
+
target.relative_to(folder.resolve())
|
| 320 |
+
except ValueError:
|
| 321 |
+
broken.append(f"{markdown.relative_to(folder)} -> {raw}")
|
| 322 |
+
continue
|
| 323 |
+
if not target.exists():
|
| 324 |
+
broken.append(f"{markdown.relative_to(folder)} -> {raw}")
|
| 325 |
+
return broken
|
| 326 |
+
|
| 327 |
+
|
| 328 |
+
def _verify_sha256sums(folder: Path) -> None:
|
| 329 |
+
path = folder / "SHA256SUMS"
|
| 330 |
+
for line in path.read_text(encoding="utf-8").splitlines():
|
| 331 |
+
expected, relative = line.split(" ", 1)
|
| 332 |
+
target = folder / relative
|
| 333 |
+
if not target.is_file() or _sha256(target) != expected:
|
| 334 |
+
raise SystemExit(f"generated checksum failed validation: {folder.name}/{relative}")
|
| 335 |
+
|
| 336 |
+
|
| 337 |
+
def _validate_output(kaggle: Path, github: Path) -> None:
|
| 338 |
+
nested_kaggle = [path for path in kaggle.rglob("*") if path.is_file() and path.parent != kaggle]
|
| 339 |
+
if nested_kaggle:
|
| 340 |
+
raise SystemExit(f"Kaggle drag-and-drop package is not flat: {nested_kaggle[0]}")
|
| 341 |
+
for folder in (kaggle, github):
|
| 342 |
+
findings = _secret_findings(folder)
|
| 343 |
+
if findings:
|
| 344 |
+
raise SystemExit(f"unsafe generated upload package: {findings[0]}")
|
| 345 |
+
_verify_sha256sums(folder)
|
| 346 |
+
broken = _broken_local_markdown_links(github)
|
| 347 |
+
if broken:
|
| 348 |
+
raise SystemExit(f"generated GitHub package has a broken local link: {broken[0]}")
|
| 349 |
+
oversized = [
|
| 350 |
+
path for path in github.rglob("*") if path.is_file() and path.stat().st_size >= 100_000_000
|
| 351 |
+
]
|
| 352 |
+
if oversized:
|
| 353 |
+
raise SystemExit(f"generated GitHub file exceeds 100 MB: {oversized[0]}")
|
| 354 |
+
|
| 355 |
+
|
| 356 |
+
def _replace_directory(staged: Path, destination: Path) -> None:
|
| 357 |
+
backup: Path | None = None
|
| 358 |
+
if destination.exists():
|
| 359 |
+
backup = Path(
|
| 360 |
+
tempfile.mkdtemp(prefix=f".{destination.name}.previous-", dir=destination.parent)
|
| 361 |
+
)
|
| 362 |
+
backup.rmdir()
|
| 363 |
+
os.replace(destination, backup)
|
| 364 |
+
try:
|
| 365 |
+
os.replace(staged, destination)
|
| 366 |
+
except Exception:
|
| 367 |
+
if backup is not None and backup.exists() and not destination.exists():
|
| 368 |
+
os.replace(backup, destination)
|
| 369 |
+
raise
|
| 370 |
+
if backup is not None:
|
| 371 |
+
shutil.rmtree(backup)
|
| 372 |
+
|
| 373 |
+
|
| 374 |
+
def main() -> int:
|
| 375 |
+
args = parse_args()
|
| 376 |
+
release_dir = _resolve_inside_root(args.release_dir, purpose="--release-dir")
|
| 377 |
+
destination = _resolve_inside_root(args.output, purpose="--output")
|
| 378 |
+
relative_output = destination.relative_to(ROOT)
|
| 379 |
+
valid_output_root = bool(relative_output.parts) and (
|
| 380 |
+
relative_output.parts[0] == "upload-ready"
|
| 381 |
+
or relative_output.parts[0].startswith("upload-ready-")
|
| 382 |
+
)
|
| 383 |
+
if not valid_output_root:
|
| 384 |
+
raise SystemExit("--output must be upload-ready/ or upload-ready-*/")
|
| 385 |
+
if (
|
| 386 |
+
destination == ROOT
|
| 387 |
+
or release_dir == destination
|
| 388 |
+
or destination in release_dir.parents
|
| 389 |
+
or release_dir in destination.parents
|
| 390 |
+
):
|
| 391 |
+
raise SystemExit("upload output overlaps the project or release input")
|
| 392 |
+
if Path(args.output).is_symlink():
|
| 393 |
+
raise SystemExit("refusing a symlink upload destination")
|
| 394 |
+
|
| 395 |
+
manifest, development_only = _load_validated_release(release_dir)
|
| 396 |
+
if development_only and not args.allow_development_release:
|
| 397 |
+
raise SystemExit("development preview requires --allow-development-release")
|
| 398 |
+
model_release = release_dir / "model"
|
| 399 |
+
_require_identical(
|
| 400 |
+
ROOT / "artifacts/partial-shard-warmstart-lr3e4-5ep/model.onnx",
|
| 401 |
+
model_release / "model.onnx",
|
| 402 |
+
"ONNX artifact",
|
| 403 |
+
)
|
| 404 |
+
_require_identical(
|
| 405 |
+
ROOT / "artifacts/partial-shard-warmstart-lr3e4-5ep/model_metadata.json",
|
| 406 |
+
model_release / "model_metadata.json",
|
| 407 |
+
"model metadata",
|
| 408 |
+
)
|
| 409 |
+
_require_identical(
|
| 410 |
+
ROOT / "artifacts/partial-shard-warmstart-lr3e4-5ep/export_manifest.json",
|
| 411 |
+
model_release / "export_manifest.json",
|
| 412 |
+
"export manifest",
|
| 413 |
+
)
|
| 414 |
+
|
| 415 |
+
destination.parent.mkdir(parents=True, exist_ok=True)
|
| 416 |
+
staged = Path(tempfile.mkdtemp(prefix=f".{destination.name}.staging-", dir=destination.parent))
|
| 417 |
+
kaggle = staged / "kaggle-model"
|
| 418 |
+
github = staged / "github-repository"
|
| 419 |
+
kaggle.mkdir()
|
| 420 |
+
github.mkdir()
|
| 421 |
+
metrics_name = "development_metrics.json" if development_only else "test_metrics.json"
|
| 422 |
+
_build_kaggle_folder(release_dir, kaggle, metrics_name)
|
| 423 |
+
_build_github_folder(github)
|
| 424 |
+
release_digest = str(manifest["release_inventory_sha256"])
|
| 425 |
+
_write_package_evidence(
|
| 426 |
+
kaggle,
|
| 427 |
+
kind="kaggle_model_drag_and_drop",
|
| 428 |
+
development_only=development_only,
|
| 429 |
+
release_inventory_sha256=release_digest,
|
| 430 |
+
)
|
| 431 |
+
_write_package_evidence(
|
| 432 |
+
github,
|
| 433 |
+
kind="github_repository",
|
| 434 |
+
development_only=development_only,
|
| 435 |
+
release_inventory_sha256=release_digest,
|
| 436 |
+
)
|
| 437 |
+
_validate_output(kaggle, github)
|
| 438 |
+
summary = {
|
| 439 |
+
"format_version": 1,
|
| 440 |
+
"development_only": development_only,
|
| 441 |
+
"source_release_inventory_sha256": release_digest,
|
| 442 |
+
"folders": {
|
| 443 |
+
"kaggle-model": {
|
| 444 |
+
"files": len(_inventory(kaggle)),
|
| 445 |
+
"bytes": sum(path.stat().st_size for path in kaggle.iterdir() if path.is_file()),
|
| 446 |
+
},
|
| 447 |
+
"github-repository": {
|
| 448 |
+
"files": len(_inventory(github)),
|
| 449 |
+
"bytes": sum(path.stat().st_size for path in github.rglob("*") if path.is_file()),
|
| 450 |
+
},
|
| 451 |
+
},
|
| 452 |
+
}
|
| 453 |
+
(staged / "README.md").write_text(
|
| 454 |
+
"# Upload-ready folders\n\n"
|
| 455 |
+
"- Upload the **contents** of `kaggle-model/` as one Kaggle ONNX model variation.\n"
|
| 456 |
+
"- Upload the **contents** of `github-repository/` to a new GitHub repository.\n"
|
| 457 |
+
"- Both packages are development previews; read their cards before changing visibility.\n",
|
| 458 |
+
encoding="utf-8",
|
| 459 |
+
)
|
| 460 |
+
(staged / "UPLOAD_SUMMARY.json").write_text(
|
| 461 |
+
json.dumps(summary, indent=2, sort_keys=True) + "\n",
|
| 462 |
+
encoding="utf-8",
|
| 463 |
+
)
|
| 464 |
+
_replace_directory(staged, destination)
|
| 465 |
+
print(json.dumps(summary, indent=2, sort_keys=True))
|
| 466 |
+
return 0
|
| 467 |
+
|
| 468 |
+
|
| 469 |
+
if __name__ == "__main__":
|
| 470 |
+
raise SystemExit(main())
|
scripts/export_onnx.py
CHANGED
|
@@ -67,6 +67,11 @@ def _deployment_source_paths() -> list[Path]:
|
|
| 67 |
*sorted((REPOSITORY_ROOT / "src" / "turn_detection").rglob("*.py")),
|
| 68 |
*sorted((REPOSITORY_ROOT / "scripts").glob("*.py")),
|
| 69 |
*sorted((REPOSITORY_ROOT / "scripts").glob("*.sh")),
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 70 |
REPOSITORY_ROOT / "app.py",
|
| 71 |
REPOSITORY_ROOT / "pyproject.toml",
|
| 72 |
REPOSITORY_ROOT / "space" / "requirements.txt",
|
|
|
|
| 67 |
*sorted((REPOSITORY_ROOT / "src" / "turn_detection").rglob("*.py")),
|
| 68 |
*sorted((REPOSITORY_ROOT / "scripts").glob("*.py")),
|
| 69 |
*sorted((REPOSITORY_ROOT / "scripts").glob("*.sh")),
|
| 70 |
+
*(
|
| 71 |
+
path
|
| 72 |
+
for path in sorted((REPOSITORY_ROOT / "deployment").rglob("*"))
|
| 73 |
+
if path.is_file() and "__pycache__" not in path.parts and path.suffix != ".pyc"
|
| 74 |
+
),
|
| 75 |
REPOSITORY_ROOT / "app.py",
|
| 76 |
REPOSITORY_ROOT / "pyproject.toml",
|
| 77 |
REPOSITORY_ROOT / "space" / "requirements.txt",
|
space/requirements.txt
CHANGED
|
@@ -2,3 +2,4 @@
|
|
| 2 |
numpy==2.3.5
|
| 3 |
onnxruntime==1.26.0
|
| 4 |
soundfile==0.14.0
|
|
|
|
|
|
| 2 |
numpy==2.3.5
|
| 3 |
onnxruntime==1.26.0
|
| 4 |
soundfile==0.14.0
|
| 5 |
+
spaces==0.51.1
|
tests/test_kaggle_runtime.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import importlib.util
|
| 4 |
+
import sys
|
| 5 |
+
import unittest
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
import numpy as np
|
| 9 |
+
|
| 10 |
+
from turn_detection.runtime.features import (
|
| 11 |
+
FrontendConfig as CanonicalFrontendConfig,
|
| 12 |
+
)
|
| 13 |
+
from turn_detection.runtime.features import (
|
| 14 |
+
log_mel_spectrogram as canonical_log_mel_spectrogram,
|
| 15 |
+
)
|
| 16 |
+
|
| 17 |
+
ROOT = Path(__file__).resolve().parents[1]
|
| 18 |
+
MODULE_PATH = ROOT / "deployment" / "kaggle" / "turn_detector.py"
|
| 19 |
+
SPEC = importlib.util.spec_from_file_location("kaggle_turn_detector", MODULE_PATH)
|
| 20 |
+
if SPEC is None or SPEC.loader is None: # pragma: no cover - import machinery failure
|
| 21 |
+
raise RuntimeError(f"cannot load {MODULE_PATH}")
|
| 22 |
+
KAGGLE_RUNTIME = importlib.util.module_from_spec(SPEC)
|
| 23 |
+
sys.modules[SPEC.name] = KAGGLE_RUNTIME
|
| 24 |
+
SPEC.loader.exec_module(KAGGLE_RUNTIME)
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
class KaggleRuntimeParityTest(unittest.TestCase):
|
| 28 |
+
def test_flat_bundle_frontend_matches_canonical_runtime(self) -> None:
|
| 29 |
+
canonical_config = CanonicalFrontendConfig(max_seconds=4.0)
|
| 30 |
+
kaggle_config = KAGGLE_RUNTIME.FrontendConfig(**canonical_config.to_dict())
|
| 31 |
+
for sample_rate in (8_000, 16_000, 48_000):
|
| 32 |
+
timeline = np.arange(round(sample_rate * 1.37), dtype=np.float32) / sample_rate
|
| 33 |
+
signal = 0.13 * np.sin(2.0 * np.pi * 337.0 * timeline)
|
| 34 |
+
stereo = np.stack((signal, signal * 0.7), axis=1)
|
| 35 |
+
expected_features, expected_mask = canonical_log_mel_spectrogram(
|
| 36 |
+
stereo,
|
| 37 |
+
sample_rate,
|
| 38 |
+
canonical_config,
|
| 39 |
+
)
|
| 40 |
+
actual_features, actual_mask = KAGGLE_RUNTIME.log_mel_spectrogram(
|
| 41 |
+
stereo,
|
| 42 |
+
sample_rate,
|
| 43 |
+
kaggle_config,
|
| 44 |
+
)
|
| 45 |
+
np.testing.assert_array_equal(actual_features, expected_features)
|
| 46 |
+
np.testing.assert_array_equal(actual_mask, expected_mask)
|
| 47 |
+
|
| 48 |
+
def test_controller_threshold_relaxation_matches_serialized_policy(self) -> None:
|
| 49 |
+
detector = object.__new__(KAGGLE_RUNTIME.TurnDetector)
|
| 50 |
+
detector.controller = KAGGLE_RUNTIME.ControllerConfig(
|
| 51 |
+
endpoint_threshold=0.74,
|
| 52 |
+
long_pause_threshold=0.56,
|
| 53 |
+
min_silence_ms=200.0,
|
| 54 |
+
relax_after_ms=800.0,
|
| 55 |
+
max_silence_ms=1_800.0,
|
| 56 |
+
required_confirmations=1,
|
| 57 |
+
)
|
| 58 |
+
self.assertEqual(detector.threshold_for_silence(300.0), 0.74)
|
| 59 |
+
self.assertAlmostEqual(detector.threshold_for_silence(1_300.0), 0.65)
|
| 60 |
+
self.assertEqual(detector.threshold_for_silence(2_000.0), 0.56)
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
if __name__ == "__main__":
|
| 64 |
+
unittest.main()
|
tests/test_release_guardrails.py
CHANGED
|
@@ -86,7 +86,11 @@ class ReleaseGuardrailTest(unittest.TestCase):
|
|
| 86 |
self.assertFalse((destination / "old.txt").exists())
|
| 87 |
|
| 88 |
def test_space_card_rejects_server_invalid_frontmatter(self) -> None:
|
| 89 |
-
from scripts.build_release import
|
|
|
|
|
|
|
|
|
|
|
|
|
| 90 |
|
| 91 |
valid = """---
|
| 92 |
title: Test Space
|
|
@@ -123,7 +127,8 @@ short_description: Tiny endpoint detector
|
|
| 123 |
|
| 124 |
requirements = Path(directory) / "requirements.txt"
|
| 125 |
requirements.write_text(
|
| 126 |
-
"# ZeroGPU CPython 3.12\nnumpy==2.3.5\nonnxruntime==1.26.0\
|
|
|
|
| 127 |
encoding="utf-8",
|
| 128 |
)
|
| 129 |
_validate_space_requirements(requirements)
|
|
@@ -131,6 +136,25 @@ short_description: Tiny endpoint detector
|
|
| 131 |
with self.assertRaisesRegex(SystemExit, "CPython-3.12-compatible"):
|
| 132 |
_validate_space_requirements(requirements)
|
| 133 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 134 |
def test_synthetic_replay_must_be_labelled_and_hash_bound(self) -> None:
|
| 135 |
from scripts.build_release import _validate_synthetic_replay
|
| 136 |
|
|
@@ -253,6 +277,8 @@ short_description: Tiny endpoint detector
|
|
| 253 |
)
|
| 254 |
|
| 255 |
def test_development_metrics_are_never_labelled_as_test_metrics(self) -> None:
|
|
|
|
|
|
|
| 256 |
runtime_python = self._onnx_runtime_python()
|
| 257 |
fixture = ROOT / "artifacts" / "smoke" / "model.onnx"
|
| 258 |
if runtime_python is None or not fixture.is_file():
|
|
@@ -312,18 +338,7 @@ short_description: Tiny endpoint detector
|
|
| 312 |
),
|
| 313 |
encoding="utf-8",
|
| 314 |
)
|
| 315 |
-
source_files = [
|
| 316 |
-
evidence(path)
|
| 317 |
-
for path in [
|
| 318 |
-
*sorted((ROOT / "src" / "turn_detection").rglob("*.py")),
|
| 319 |
-
*sorted((ROOT / "scripts").glob("*.py")),
|
| 320 |
-
*sorted((ROOT / "scripts").glob("*.sh")),
|
| 321 |
-
ROOT / "app.py",
|
| 322 |
-
ROOT / "pyproject.toml",
|
| 323 |
-
ROOT / "space" / "requirements.txt",
|
| 324 |
-
*sorted(ROOT.glob("requirements-*.txt")),
|
| 325 |
-
]
|
| 326 |
-
]
|
| 327 |
source_files = sorted(source_files, key=lambda item: str(item["path"]))
|
| 328 |
source_inventory_sha256 = hashlib.sha256(
|
| 329 |
json.dumps(source_files, sort_keys=True, separators=(",", ":")).encode()
|
|
|
|
| 86 |
self.assertFalse((destination / "old.txt").exists())
|
| 87 |
|
| 88 |
def test_space_card_rejects_server_invalid_frontmatter(self) -> None:
|
| 89 |
+
from scripts.build_release import (
|
| 90 |
+
_validate_space_app,
|
| 91 |
+
_validate_space_card,
|
| 92 |
+
_validate_space_requirements,
|
| 93 |
+
)
|
| 94 |
|
| 95 |
valid = """---
|
| 96 |
title: Test Space
|
|
|
|
| 127 |
|
| 128 |
requirements = Path(directory) / "requirements.txt"
|
| 129 |
requirements.write_text(
|
| 130 |
+
"# ZeroGPU CPython 3.12\nnumpy==2.3.5\nonnxruntime==1.26.0\n"
|
| 131 |
+
"soundfile==0.14.0\nspaces==0.51.1\n",
|
| 132 |
encoding="utf-8",
|
| 133 |
)
|
| 134 |
_validate_space_requirements(requirements)
|
|
|
|
| 136 |
with self.assertRaisesRegex(SystemExit, "CPython-3.12-compatible"):
|
| 137 |
_validate_space_requirements(requirements)
|
| 138 |
|
| 139 |
+
app = Path(directory) / "app.py"
|
| 140 |
+
valid_app = """
|
| 141 |
+
try:
|
| 142 |
+
import spaces
|
| 143 |
+
except ModuleNotFoundError:
|
| 144 |
+
spaces = None
|
| 145 |
+
|
| 146 |
+
@spaces.GPU(duration=10)
|
| 147 |
+
def analyze_turn(audio):
|
| 148 |
+
return audio
|
| 149 |
+
|
| 150 |
+
analyze.click(fn=analyze_turn, inputs=[], outputs=[])
|
| 151 |
+
"""
|
| 152 |
+
app.write_text(valid_app, encoding="utf-8")
|
| 153 |
+
_validate_space_app(app)
|
| 154 |
+
app.write_text(valid_app.replace("@spaces.GPU(duration=10)\n", ""), encoding="utf-8")
|
| 155 |
+
with self.assertRaisesRegex(SystemExit, "@spaces.GPU"):
|
| 156 |
+
_validate_space_app(app)
|
| 157 |
+
|
| 158 |
def test_synthetic_replay_must_be_labelled_and_hash_bound(self) -> None:
|
| 159 |
from scripts.build_release import _validate_synthetic_replay
|
| 160 |
|
|
|
|
| 277 |
)
|
| 278 |
|
| 279 |
def test_development_metrics_are_never_labelled_as_test_metrics(self) -> None:
|
| 280 |
+
from scripts.build_release import _deployment_source_paths
|
| 281 |
+
|
| 282 |
runtime_python = self._onnx_runtime_python()
|
| 283 |
fixture = ROOT / "artifacts" / "smoke" / "model.onnx"
|
| 284 |
if runtime_python is None or not fixture.is_file():
|
|
|
|
| 338 |
),
|
| 339 |
encoding="utf-8",
|
| 340 |
)
|
| 341 |
+
source_files = [evidence(path) for path in _deployment_source_paths()]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 342 |
source_files = sorted(source_files, key=lambda item: str(item["path"]))
|
| 343 |
source_inventory_sha256 = hashlib.sha256(
|
| 344 |
json.dumps(source_files, sort_keys=True, separators=(",", ":")).encode()
|
tests/test_upload_folders.py
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import subprocess
|
| 4 |
+
import tempfile
|
| 5 |
+
import unittest
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
from scripts.build_upload_folders import (
|
| 9 |
+
_GITHUB_ARTIFACTS,
|
| 10 |
+
_broken_local_markdown_links,
|
| 11 |
+
_github_gitignore,
|
| 12 |
+
_secret_findings,
|
| 13 |
+
)
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
class UploadFolderGuardrailTest(unittest.TestCase):
|
| 17 |
+
def test_secret_scanner_allows_placeholder_and_rejects_real_token_shape(self) -> None:
|
| 18 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 19 |
+
root = Path(directory)
|
| 20 |
+
(root / ".env.example").write_text("HF_TOKEN=hf_replace_me\n", encoding="utf-8")
|
| 21 |
+
self.assertEqual(_secret_findings(root), [])
|
| 22 |
+
(root / "bad.txt").write_text("HF_TOKEN=hf_" + "A" * 30, encoding="utf-8")
|
| 23 |
+
self.assertIn("Hugging Face token: bad.txt", _secret_findings(root))
|
| 24 |
+
|
| 25 |
+
def test_markdown_guard_rejects_only_broken_local_links(self) -> None:
|
| 26 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 27 |
+
root = Path(directory)
|
| 28 |
+
(root / "present.md").write_text("ok\n", encoding="utf-8")
|
| 29 |
+
readme = root / "README.md"
|
| 30 |
+
readme.write_text(
|
| 31 |
+
"[local](present.md) [anchor](#section) [web](https://example.com)\n",
|
| 32 |
+
encoding="utf-8",
|
| 33 |
+
)
|
| 34 |
+
self.assertEqual(_broken_local_markdown_links(root), [])
|
| 35 |
+
readme.write_text("[missing](absent.md)\n", encoding="utf-8")
|
| 36 |
+
self.assertEqual(_broken_local_markdown_links(root), ["README.md -> absent.md"])
|
| 37 |
+
|
| 38 |
+
def test_generated_gitignore_tracks_only_curated_artifacts(self) -> None:
|
| 39 |
+
with tempfile.TemporaryDirectory() as directory:
|
| 40 |
+
root = Path(directory)
|
| 41 |
+
(root / ".gitignore").write_text(_github_gitignore(), encoding="utf-8")
|
| 42 |
+
subprocess.run(["git", "init", "-q"], cwd=root, check=True)
|
| 43 |
+
for relative in _GITHUB_ARTIFACTS:
|
| 44 |
+
path = root / relative
|
| 45 |
+
path.parent.mkdir(parents=True, exist_ok=True)
|
| 46 |
+
path.write_bytes(b"curated")
|
| 47 |
+
completed = subprocess.run(
|
| 48 |
+
["git", "check-ignore", "--quiet", relative],
|
| 49 |
+
cwd=root,
|
| 50 |
+
check=False,
|
| 51 |
+
)
|
| 52 |
+
self.assertEqual(completed.returncode, 1, relative)
|
| 53 |
+
|
| 54 |
+
excluded = root / "artifacts" / "debug-run" / "model.pt"
|
| 55 |
+
excluded.parent.mkdir(parents=True)
|
| 56 |
+
excluded.write_bytes(b"excluded")
|
| 57 |
+
completed = subprocess.run(
|
| 58 |
+
["git", "check-ignore", "--quiet", str(excluded.relative_to(root))],
|
| 59 |
+
cwd=root,
|
| 60 |
+
check=False,
|
| 61 |
+
)
|
| 62 |
+
self.assertEqual(completed.returncode, 0)
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
if __name__ == "__main__":
|
| 66 |
+
unittest.main()
|