side-r3-ettin-150m-s0: ONNX export, model card and training rows
Browse files- MANIFEST.json +67 -0
- README.md +101 -0
- config.json +38 -0
- data/dev.jsonl.gz +3 -0
- data/train.jsonl.gz +3 -0
- model-int8.onnx +3 -0
- model.onnx +3 -0
- tokenizer.json +0 -0
MANIFEST.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_id": "side-r3-ettin-150m-s0",
|
| 3 |
+
"files": {
|
| 4 |
+
"README.md": {
|
| 5 |
+
"bytes": 4074,
|
| 6 |
+
"sha256": "8f10a960bc8a2e6b1d0f339e41182633aa0c53d7d5587f4f371fce2227937fe8"
|
| 7 |
+
},
|
| 8 |
+
"config.json": {
|
| 9 |
+
"bytes": 758,
|
| 10 |
+
"sha256": "20ad58370dc33d8a1e6797423403aeef5c832a83f97de762e13b79820acce8ab"
|
| 11 |
+
},
|
| 12 |
+
"data/dev.jsonl.gz": {
|
| 13 |
+
"bytes": 1784536,
|
| 14 |
+
"sha256": "348b35bc4b2579d3dc29b90cf6a41700792dfa54ff916616fd1e1b1f1e943dd2"
|
| 15 |
+
},
|
| 16 |
+
"data/train.jsonl.gz": {
|
| 17 |
+
"bytes": 17792467,
|
| 18 |
+
"sha256": "a2258ce78a91f77733b42fead8651d3f2ad206e2cb959515296169cbccbbe450"
|
| 19 |
+
},
|
| 20 |
+
"model-int8.onnx": {
|
| 21 |
+
"bytes": 267161332,
|
| 22 |
+
"sha256": "f2df01639ac45f447017ee5061eb6c7d4f55330e0cc4c010a5535208c1990fe4"
|
| 23 |
+
},
|
| 24 |
+
"model.onnx": {
|
| 25 |
+
"bytes": 598295149,
|
| 26 |
+
"sha256": "b5bae71649a5f0ccbd082f0b5cccb3a8097bc12ea9be89e2145708250e48e3c4"
|
| 27 |
+
},
|
| 28 |
+
"tokenizer.json": {
|
| 29 |
+
"bytes": 3583485,
|
| 30 |
+
"sha256": "65a48126bc4c87cba0b2da66c980be3c274997dff7aed9cbef75f06fbe112192"
|
| 31 |
+
}
|
| 32 |
+
},
|
| 33 |
+
"rows": {
|
| 34 |
+
"train": {
|
| 35 |
+
"rows": 235069,
|
| 36 |
+
"by_set": {
|
| 37 |
+
"gen-0.1": 83732,
|
| 38 |
+
"gen-pilot": 6263,
|
| 39 |
+
"gen-0.2": 129798,
|
| 40 |
+
"ami-silent-0.1": 15276
|
| 41 |
+
},
|
| 42 |
+
"by_respond": {
|
| 43 |
+
"0.0": 143613,
|
| 44 |
+
"1.0": 55098,
|
| 45 |
+
"0.75": 5519,
|
| 46 |
+
"0.5": 29275,
|
| 47 |
+
"0.25": 1564
|
| 48 |
+
}
|
| 49 |
+
},
|
| 50 |
+
"dev": {
|
| 51 |
+
"rows": 25631,
|
| 52 |
+
"by_set": {
|
| 53 |
+
"gen-0.1": 8346,
|
| 54 |
+
"gen-pilot": 656,
|
| 55 |
+
"gen-0.2": 12443,
|
| 56 |
+
"ami-silent-0.1": 4186
|
| 57 |
+
},
|
| 58 |
+
"by_respond": {
|
| 59 |
+
"0.0": 16809,
|
| 60 |
+
"0.5": 2780,
|
| 61 |
+
"1.0": 5372,
|
| 62 |
+
"0.75": 526,
|
| 63 |
+
"0.25": 144
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
}
|
README.md
ADDED
|
@@ -0,0 +1,101 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
---
|
| 2 |
+
license: bsd-2-clause
|
| 3 |
+
language: en
|
| 4 |
+
library_name: onnx
|
| 5 |
+
pipeline_tag: text-classification
|
| 6 |
+
base_model: jhu-clsp/ettin-encoder-150m
|
| 7 |
+
datasets:
|
| 8 |
+
- ACloudCenter/ami-addressee
|
| 9 |
+
tags:
|
| 10 |
+
- pipecat
|
| 11 |
+
- onnx
|
| 12 |
+
- addressee-detection
|
| 13 |
+
- multi-party
|
| 14 |
+
- turn-taking
|
| 15 |
+
---
|
| 16 |
+
|
| 17 |
+
# Pipecat Sidechat (experimental)
|
| 18 |
+
|
| 19 |
+
> **⚠️ Experimental.** The first trained release, `side-r3-ettin-150m-s0`. Outputs, names and calibration may
|
| 20 |
+
> change between releases.
|
| 21 |
+
|
| 22 |
+
A small text classifier for a voice agent in a conversation with several people. For each line it gives the
|
| 23 |
+
probability that the agent should speak now, and whether the line was for it, for the room or about it. Speaker
|
| 24 |
+
labels can be anonymous ("Speaker 1"); the agent's name and the rhythm of the last turns carry the signal.
|
| 25 |
+
|
| 26 |
+
Code: [github.com/pipecat-ai/sidechat](https://github.com/pipecat-ai/sidechat)
|
| 27 |
+
|
| 28 |
+
## Use
|
| 29 |
+
|
| 30 |
+
```bash
|
| 31 |
+
pip install git+https://github.com/pipecat-ai/sidechat
|
| 32 |
+
```
|
| 33 |
+
|
| 34 |
+
```python
|
| 35 |
+
from huggingface_hub import snapshot_download
|
| 36 |
+
from sidechat.runtime import load
|
| 37 |
+
from sidechat.schema import Agent, SidechatRequest, Turn
|
| 38 |
+
|
| 39 |
+
model = load(snapshot_download("pipecat-ai/sidechat", allow_patterns=["*.json", "model.onnx"]))
|
| 40 |
+
a = model.answer(SidechatRequest(
|
| 41 |
+
agent=Agent(name="Atlas"),
|
| 42 |
+
speaker="Speaker 1",
|
| 43 |
+
text="Can you check that for us?",
|
| 44 |
+
history=[Turn(speaker="Atlas", text="Want me to look up flights?", agent=True)],
|
| 45 |
+
))
|
| 46 |
+
print(a.respond, model.decide(a)) # 0.99 respond
|
| 47 |
+
```
|
| 48 |
+
|
| 49 |
+
| Output | Meaning |
|
| 50 |
+
|---|---|
|
| 51 |
+
| `respond` | probability the agent should speak now; about 0.5 for a question left open to the room |
|
| 52 |
+
| `intent` | for the agent directly or indirectly, open to the room, about the agent, or other |
|
| 53 |
+
| `unclear` | the line fits the agent and someone else equally |
|
| 54 |
+
|
| 55 |
+
`decide` gives `respond`, `wait` (speak only if nobody else does) or `ignore`, at the thresholds in `config.json`.
|
| 56 |
+
Pass `int8=True` to `load` for `model-int8.onnx`, which has thresholds of its own.
|
| 57 |
+
|
| 58 |
+
## Files
|
| 59 |
+
|
| 60 |
+
| File | |
|
| 61 |
+
|---|---|
|
| 62 |
+
| `model.onnx` | fp32, 598 MB |
|
| 63 |
+
| `model-int8.onnx` | int8, 267 MB |
|
| 64 |
+
| `tokenizer.json`, `config.json` | tokenizer, calibration and thresholds |
|
| 65 |
+
| `data/train.jsonl.gz`, `data/dev.jsonl.gz` | the rows it was trained on |
|
| 66 |
+
| `MANIFEST.json` | sha256 of every file, row counts |
|
| 67 |
+
|
| 68 |
+
## Results against Jev
|
| 69 |
+
|
| 70 |
+
Anonymous speakers. Jev is asked as Pipecat's daily-audience demo asks it.
|
| 71 |
+
|
| 72 |
+
| Set | Sidechat | Jev |
|
| 73 |
+
|---|---|---|
|
| 74 |
+
| [Duplex-MPE](https://huggingface.co/datasets/ChengqianMa/Duplex-MPE), 52,336 lines | 96.7% | 83.1% |
|
| 75 |
+
| … lines for others, answered outright | 0.2% | 0.4% |
|
| 76 |
+
| … request to the agent without its name, answered | 38.8% | 26.6% |
|
| 77 |
+
| [Social AI Ambient Bench](https://huggingface.co/datasets/text-ai/social-ai-ambient-bench), 15,793 lines | 98.7% | 85.4% |
|
| 78 |
+
| [AMI](https://huggingface.co/datasets/ACloudCenter/ami-addressee) labelled meetings, 8,875 lines | 99.9% | 85.2% |
|
| 79 |
+
| 270 hand-written lines | 87.4% | 77.8% |
|
| 80 |
+
|
| 81 |
+
These are baselines, not proper benchmarks: labels follow the project's own guide, and the choice between
|
| 82 |
+
training rounds looked at these sets.
|
| 83 |
+
|
| 84 |
+
## Speed
|
| 85 |
+
|
| 86 |
+
Median per line, short history / 8 earlier lines, one thread: 10 / 52 ms on an M5 Max, 66 / 253 ms on 1 vCPU,
|
| 87 |
+
507 / 648 ms on 0.5 vCPU (int8: 103 / 532 ms). On a small cloud CPU it is slower than an API call to Jev.
|
| 88 |
+
|
| 89 |
+
## Limitations
|
| 90 |
+
|
| 91 |
+
- A request without the agent's name, before the agent has spoken, is answered less than half the time.
|
| 92 |
+
- English text only. Most training lines are generated; only AMI is real speech.
|
| 93 |
+
|
| 94 |
+
## Training
|
| 95 |
+
|
| 96 |
+
The [Ettin-150M encoder](https://huggingface.co/jhu-clsp/ettin-encoder-150m) (MIT) with intent, respond and
|
| 97 |
+
unclear heads, 2 epochs. 235,069 training lines: 219,793 from generated multi-party conversations, labelled by
|
| 98 |
+
kind under the project's guide, and 15,276 lines for people from AMI meetings
|
| 99 |
+
([ACloudCenter/ami-addressee](https://huggingface.co/datasets/ACloudCenter/ami-addressee), CC BY 4.0) outside
|
| 100 |
+
every test meeting. Speakers were anonymous in 60% of training lines, named in 25% and unlabelled in 15%.
|
| 101 |
+
Duplex-MPE and the Social AI Ambient Bench were used for evaluation only.
|
config.json
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{
|
| 2 |
+
"run_id": "side-r3-ettin-150m-s0",
|
| 3 |
+
"encoder": "ettin-150m",
|
| 4 |
+
"input_version": "sidechat-input-0.2",
|
| 5 |
+
"max_len": 320,
|
| 6 |
+
"temperatures": {
|
| 7 |
+
"intent": 1.55,
|
| 8 |
+
"respond": 1.55,
|
| 9 |
+
"unclear": 1.35
|
| 10 |
+
},
|
| 11 |
+
"thresholds": {
|
| 12 |
+
"respond": 0.6,
|
| 13 |
+
"wait": 0.4,
|
| 14 |
+
"open": 0.3,
|
| 15 |
+
"dev_right": 20625,
|
| 16 |
+
"dev_n": 20841,
|
| 17 |
+
"dev_wrong_now": 30,
|
| 18 |
+
"dev_quiet": 12767,
|
| 19 |
+
"max_wrong_now": 0.01,
|
| 20 |
+
"real_wrong_now": 0,
|
| 21 |
+
"real_quiet": 4186,
|
| 22 |
+
"max_wrong_now_real": 0.004
|
| 23 |
+
},
|
| 24 |
+
"exported_at": "2026-11-19T01:58:38+00:00",
|
| 25 |
+
"thresholds_int8": {
|
| 26 |
+
"respond": 0.55,
|
| 27 |
+
"wait": 0.4,
|
| 28 |
+
"open": 0.3,
|
| 29 |
+
"dev_right": 20544,
|
| 30 |
+
"dev_n": 20841,
|
| 31 |
+
"dev_wrong_now": 33,
|
| 32 |
+
"dev_quiet": 12767,
|
| 33 |
+
"max_wrong_now": 0.01,
|
| 34 |
+
"real_wrong_now": 0,
|
| 35 |
+
"real_quiet": 4186,
|
| 36 |
+
"max_wrong_now_real": 0.004
|
| 37 |
+
}
|
| 38 |
+
}
|
data/dev.jsonl.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:348b35bc4b2579d3dc29b90cf6a41700792dfa54ff916616fd1e1b1f1e943dd2
|
| 3 |
+
size 1784536
|
data/train.jsonl.gz
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:a2258ce78a91f77733b42fead8651d3f2ad206e2cb959515296169cbccbbe450
|
| 3 |
+
size 17792467
|
model-int8.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:f2df01639ac45f447017ee5061eb6c7d4f55330e0cc4c010a5535208c1990fe4
|
| 3 |
+
size 267161332
|
model.onnx
ADDED
|
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
version https://git-lfs.github.com/spec/v1
|
| 2 |
+
oid sha256:b5bae71649a5f0ccbd082f0b5cccb3a8097bc12ea9be89e2145708250e48e3c4
|
| 3 |
+
size 598295149
|
tokenizer.json
ADDED
|
The diff for this file is too large to render.
See raw diff
|
|
|