Next-Echo / evaluation.json
Lamapi's picture
Publish Next-Echo model
e187f2f verified
Raw History Blame Contribute Delete
12.6 kB
{
"project": "Next Echo",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"name": "ESC-50",
"source": "https://github.com/karoldvl/ESC-50/archive/master.zip",
"download": {
"url": "https://github.com/karoldvl/ESC-50/archive/master.zip",
"file": "esc50.zip",
"bytes": 645695005,
"sha256": "805afc618aff80eff0641d51311647871fead5807da68bb18e43a27db47a79ce",
"verified_against_pinned_sha256": false
},
"license": "CC BY-NC 3.0 (dataset); per-clip Freesound licenses in the archive",
"split": {
"embedding_training": "folds 1-3, 40 seen classes",
"validation_recordings": "fold 4, seen classes",
"test_recordings": "fold 5, seen and held-out classes"
},
"held_out_classes": [
"clapping",
"door_wood_knock",
"rooster",
"church_bells",
"glass_breaking",
"keyboard_typing",
"siren",
"cow",
"sneezing",
"can_opening"
],
"resampling": "44.1 kHz -> 16 kHz (scipy resample_poly 160/441)"
},
"training": {
"steps": 5000,
"seconds": 1473.6,
"final_loss": 2.7861027669906617,
"clips": 960,
"mu": -3.852266374845604,
"sd": 5.0395802756944885
},
"thresholds": {
"learned": 1.553792940244492,
"baseline": 0.5790993811336966
},
"validation": {
"learned": {
"f1": 0.6113207547169811,
"precision": 0.54,
"recall": 0.7043478260869566,
"false_alarms_per_min": 2.3,
"tp": 81,
"fp": 69,
"fn": 34
},
"baseline": {
"f1": 0.5882352941176471,
"precision": 0.5357142857142857,
"recall": 0.6521739130434783,
"false_alarms_per_min": 2.1666666666666665,
"tp": 75,
"fp": 65,
"fn": 40
}
},
"protocol": {
"recording_seconds": 30,
"per_cell": 40,
"tolerance_s": 0.5,
"conditions": {
"clean": "30 dB SNR brown noise",
"noisy": "5 dB SNR",
"overlap": "distractors overlap target events",
"absent": "no target event"
},
"construction": "real ESC-50 event crops mixed at random times with 3-5 distractor clips"
},
"results": {
"seen/clean": {
"learned": {
"f1": 0.6186440677966102,
"precision": 0.5289855072463768,
"recall": 0.7448979591836735,
"false_alarms_per_min": 3.25,
"tp": 73,
"fp": 65,
"fn": 25
},
"baseline": {
"f1": 0.676056338028169,
"precision": 0.6260869565217392,
"recall": 0.7346938775510204,
"false_alarms_per_min": 2.15,
"tp": 72,
"fp": 43,
"fn": 26
}
},
"seen/noisy": {
"learned": {
"f1": 0.6296296296296297,
"precision": 0.5964912280701754,
"recall": 0.6666666666666666,
"false_alarms_per_min": 2.3,
"tp": 68,
"fp": 46,
"fn": 34
},
"baseline": {
"f1": 0.6063829787234043,
"precision": 0.6627906976744186,
"recall": 0.5588235294117647,
"false_alarms_per_min": 1.45,
"tp": 57,
"fp": 29,
"fn": 45
}
},
"seen/overlap": {
"learned": {
"f1": 0.7487179487179487,
"precision": 0.776595744680851,
"recall": 0.7227722772277227,
"false_alarms_per_min": 1.05,
"tp": 73,
"fp": 21,
"fn": 28
},
"baseline": {
"f1": 0.7875647668393783,
"precision": 0.8260869565217391,
"recall": 0.7524752475247525,
"false_alarms_per_min": 0.8,
"tp": 76,
"fp": 16,
"fn": 25
}
},
"seen/absent": {
"learned": {
"f1": 0.0,
"precision": 0.0,
"recall": 0.0,
"false_alarms_per_min": 4.95,
"tp": 0,
"fp": 99,
"fn": 0
},
"baseline": {
"f1": 0.0,
"precision": 0.0,
"recall": 0.0,
"false_alarms_per_min": 4.0,
"tp": 0,
"fp": 80,
"fn": 0
}
},
"unseen/clean": {
"learned": {
"f1": 0.6584362139917695,
"precision": 0.547945205479452,
"recall": 0.8247422680412371,
"false_alarms_per_min": 3.3,
"tp": 80,
"fp": 66,
"fn": 17
},
"baseline": {
"f1": 0.6169154228855721,
"precision": 0.5961538461538461,
"recall": 0.6391752577319587,
"false_alarms_per_min": 2.1,
"tp": 62,
"fp": 42,
"fn": 35
}
},
"unseen/noisy": {
"learned": {
"f1": 0.7168141592920354,
"precision": 0.6639344262295082,
"recall": 0.7788461538461539,
"false_alarms_per_min": 2.05,
"tp": 81,
"fp": 41,
"fn": 23
},
"baseline": {
"f1": 0.6666666666666666,
"precision": 0.7411764705882353,
"recall": 0.6057692307692307,
"false_alarms_per_min": 1.1,
"tp": 63,
"fp": 22,
"fn": 41
}
},
"unseen/overlap": {
"learned": {
"f1": 0.7671232876712328,
"precision": 0.711864406779661,
"recall": 0.8316831683168316,
"false_alarms_per_min": 1.7,
"tp": 84,
"fp": 34,
"fn": 17
},
"baseline": {
"f1": 0.7897435897435897,
"precision": 0.8191489361702128,
"recall": 0.7623762376237624,
"false_alarms_per_min": 0.85,
"tp": 77,
"fp": 17,
"fn": 24
}
},
"unseen/absent": {
"learned": {
"f1": 0.0,
"precision": 0.0,
"recall": 0.0,
"false_alarms_per_min": 5.6,
"tp": 0,
"fp": 112,
"fn": 0
},
"baseline": {
"f1": 0.0,
"precision": 0.0,
"recall": 0.0,
"false_alarms_per_min": 3.65,
"tp": 0,
"fp": 73,
"fn": 0
}
},
"seen/all-present": {
"learned": {
"f1": 0.6615146831530139,
"precision": 0.6184971098265896,
"recall": 0.7109634551495017,
"false_alarms_per_min": 2.2,
"tp": 214,
"fp": 132,
"fn": 87
},
"baseline": {
"f1": 0.6902356902356902,
"precision": 0.6996587030716723,
"recall": 0.6810631229235881,
"false_alarms_per_min": 1.4666666666666666,
"tp": 205,
"fp": 88,
"fn": 96
}
},
"unseen/all-present": {
"learned": {
"f1": 0.7122093023255814,
"precision": 0.6347150259067358,
"recall": 0.8112582781456954,
"false_alarms_per_min": 2.35,
"tp": 245,
"fp": 141,
"fn": 57
},
"baseline": {
"f1": 0.6905982905982906,
"precision": 0.7137809187279152,
"recall": 0.6688741721854304,
"false_alarms_per_min": 1.35,
"tp": 202,
"fp": 81,
"fn": 100
}
}
},
"failures_learned": [
{
"group": "seen",
"condition": "clean",
"target": "door_wood_creaks",
"missed": 2,
"false_alarms": 0,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "fireworks",
"missed": 1,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "crow",
"missed": 1,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "cat",
"missed": 1,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "door_wood_creaks",
"missed": 2,
"false_alarms": 0,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "sea_waves",
"missed": 0,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "snoring",
"missed": 0,
"false_alarms": 3,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "thunderstorm",
"missed": 1,
"false_alarms": 0,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "crow",
"missed": 1,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "drinking_sipping",
"missed": 1,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "drinking_sipping",
"missed": 1,
"false_alarms": 1,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "drinking_sipping",
"missed": 2,
"false_alarms": 0,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "water_drops",
"missed": 1,
"false_alarms": 0,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "breathing",
"missed": 0,
"false_alarms": 4,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "brushing_teeth",
"missed": 0,
"false_alarms": 2,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "thunderstorm",
"missed": 1,
"false_alarms": 1,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "breathing",
"missed": 0,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "dog",
"missed": 0,
"false_alarms": 2,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "rain",
"missed": 0,
"false_alarms": 1,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "mouse_click",
"missed": 2,
"false_alarms": 0,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "crickets",
"missed": 1,
"false_alarms": 2,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "rain",
"missed": 1,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "breathing",
"missed": 0,
"false_alarms": 6,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "crow",
"missed": 0,
"false_alarms": 2,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "engine",
"missed": 1,
"false_alarms": 0,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "car_horn",
"missed": 0,
"false_alarms": 4,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "insects",
"missed": 0,
"false_alarms": 1,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "toilet_flush",
"missed": 0,
"false_alarms": 1,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "crackling_fire",
"missed": 1,
"false_alarms": 3,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "hand_saw",
"missed": 1,
"false_alarms": 3,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "chainsaw",
"missed": 0,
"false_alarms": 3,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "crying_baby",
"missed": 0,
"false_alarms": 4,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "sea_waves",
"missed": 1,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "engine",
"missed": 1,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "helicopter",
"missed": 1,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "clean",
"target": "hand_saw",
"missed": 0,
"false_alarms": 2,
"true_events": 2
},
{
"group": "seen",
"condition": "clean",
"target": "sheep",
"missed": 0,
"false_alarms": 3,
"true_events": 2
},
{
"group": "seen",
"condition": "noisy",
"target": "airplane",
"missed": 0,
"false_alarms": 2,
"true_events": 3
},
{
"group": "seen",
"condition": "noisy",
"target": "washing_machine",
"missed": 0,
"false_alarms": 1,
"true_events": 2
},
{
"group": "seen",
"condition": "noisy",
"target": "toilet_flush",
"missed": 0,
"false_alarms": 1,
"true_events": 3
}
],
"n_recordings_with_errors_learned": 272,
"tests": {
"reload": {
"returncode": 0,
"seconds": 0.42,
"process": "fresh python -E subprocess, cwd = temporary copy of the model folder",
"passed": true,
"max_abs_diff": 0.0,
"windows": 28
},
"node_parity": {
"cases": 3,
"failed": 0,
"maxDiff": 7.105427357601002e-15,
"results": [
{
"name": "snippet0",
"maxDiff": 7.105427357601002e-15
},
{
"name": "snippet1",
"maxDiff": 7.771561172376096e-16
},
{
"name": "snippet2",
"maxDiff": 1.6653345369377348e-15
}
],
"ms": 11322,
"runtime": "node v22.23.4",
"status": "passed"
},
"browser": {
"status": "skipped",
"reason": "playwright not installed"
}
},
"not_run": [
"No evaluation on naturally recorded long scenes; recordings are constructed mixtures."
]
}