firstread-models / metrics.json
Unmesh30's picture
Release v0.2.0
099201f verified
Raw History Blame Contribute Delete
13.3 kB
{
"bootstrap": {
"baseline": "mean_history",
"metric": "ndcg@10",
"slices": {
"cold_candidates": {
"bootstrap_seed": 12345,
"ci95_high": 0.01878891816194426,
"ci95_low": 0.015859448836582184,
"gain_mean": 0.01736395319661053,
"n_impressions": 125472,
"per_seed_gain": [
0.018245716935635255,
0.016926750348847792,
0.016919392305348543
],
"resamples": 1000,
"share_resamples_gain_le_0": 0.0
},
"overall": {
"bootstrap_seed": 12345,
"ci95_high": 0.024816258579701853,
"ci95_low": 0.023167105190676558,
"gain_mean": 0.023932217960012436,
"n_impressions": 376471,
"per_seed_gain": [
0.024344917862921273,
0.023524082491960308,
0.02392765352515573
],
"resamples": 1000,
"share_resamples_gain_le_0": 0.0
}
}
},
"classifier": {
"dataset": "MIND-large, articles only in dev news.tsv",
"gate": {
"failures": [],
"max_macro_f1_gap": 0.1,
"min_topic_f1_title_only": 0.3,
"passed": true
},
"kind": "topics",
"labels": [
"us",
"world",
"politics",
"business",
"tech_science",
"health",
"sports",
"entertainment",
"lifestyle",
"food_drink",
"travel",
"autos",
"weather"
],
"n_test": 2457,
"threshold": 0.5,
"title_abstract": {
"accuracy": 0.7346357346357346,
"macro_f1": 0.6371489968601759,
"majority_class_accuracy": 0.3268213268213268,
"per_class": [
{
"f1": 0.6606170598911071,
"label": "us",
"precision": 0.6201022146507666,
"recall": 0.7067961165048544,
"support": 515
},
{
"f1": 0.45977011494252873,
"label": "world",
"precision": 0.6666666666666666,
"recall": 0.3508771929824561,
"support": 57
},
{
"f1": 0.815,
"label": "politics",
"precision": 0.7688679245283019,
"recall": 0.8670212765957447,
"support": 188
},
{
"f1": 0.5623003194888179,
"label": "business",
"precision": 0.5398773006134969,
"recall": 0.5866666666666667,
"support": 150
},
{
"f1": 0.6511627906976745,
"label": "tech_science",
"precision": 0.6829268292682927,
"recall": 0.6222222222222222,
"support": 90
},
{
"f1": 0.628099173553719,
"label": "health",
"precision": 0.6031746031746031,
"recall": 0.6551724137931034,
"support": 58
},
{
"f1": 0.9357682619647355,
"label": "sports",
"precision": 0.9464968152866242,
"recall": 0.925280199252802,
"support": 803
},
{
"f1": 0.6628571428571428,
"label": "entertainment",
"precision": 0.7160493827160493,
"recall": 0.6170212765957447,
"support": 94
},
{
"f1": 0.5610859728506787,
"label": "lifestyle",
"precision": 0.6262626262626263,
"recall": 0.5081967213114754,
"support": 122
},
{
"f1": 0.6605504587155964,
"label": "food_drink",
"precision": 0.6260869565217392,
"recall": 0.6990291262135923,
"support": 103
},
{
"f1": 0.4691358024691358,
"label": "travel",
"precision": 0.5229357798165137,
"recall": 0.4253731343283582,
"support": 134
},
{
"f1": 0.5714285714285714,
"label": "autos",
"precision": 0.5862068965517241,
"recall": 0.5573770491803278,
"support": 61
},
{
"f1": 0.6451612903225806,
"label": "weather",
"precision": 0.684931506849315,
"recall": 0.6097560975609756,
"support": 82
}
]
},
"title_only": {
"accuracy": 0.7183557183557183,
"macro_f1": 0.624372144861339,
"majority_class_accuracy": 0.3268213268213268,
"per_class": [
{
"f1": 0.6268939393939394,
"label": "us",
"precision": 0.6118299445471349,
"recall": 0.6427184466019418,
"support": 515
},
{
"f1": 0.43956043956043955,
"label": "world",
"precision": 0.5882352941176471,
"recall": 0.3508771929824561,
"support": 57
},
{
"f1": 0.8136482939632546,
"label": "politics",
"precision": 0.8031088082901554,
"recall": 0.824468085106383,
"support": 188
},
{
"f1": 0.5625,
"label": "business",
"precision": 0.5294117647058824,
"recall": 0.6,
"support": 150
},
{
"f1": 0.6545454545454545,
"label": "tech_science",
"precision": 0.72,
"recall": 0.6,
"support": 90
},
{
"f1": 0.5833333333333334,
"label": "health",
"precision": 0.5645161290322581,
"recall": 0.603448275862069,
"support": 58
},
{
"f1": 0.9229805886036319,
"label": "sports",
"precision": 0.9282115869017632,
"recall": 0.9178082191780822,
"support": 803
},
{
"f1": 0.6413043478260869,
"label": "entertainment",
"precision": 0.6555555555555556,
"recall": 0.6276595744680851,
"support": 94
},
{
"f1": 0.575,
"label": "lifestyle",
"precision": 0.5847457627118644,
"recall": 0.5655737704918032,
"support": 122
},
{
"f1": 0.6278026905829597,
"label": "food_drink",
"precision": 0.5833333333333334,
"recall": 0.6796116504854369,
"support": 103
},
{
"f1": 0.44881889763779526,
"label": "travel",
"precision": 0.475,
"recall": 0.4253731343283582,
"support": 134
},
{
"f1": 0.5333333333333333,
"label": "autos",
"precision": 0.5423728813559322,
"recall": 0.5245901639344263,
"support": 61
},
{
"f1": 0.6871165644171779,
"label": "weather",
"precision": 0.691358024691358,
"recall": 0.6829268292682927,
"support": 82
}
]
},
"topic_map_sha256": "ed1b13889e9ed2f6964b98af94fa4b5c3a32cb224cbbadaa4c7be387e2f73220"
},
"dataset": "MIND-large dev",
"eval_gate_real": {
"meta": {
"dataset_revision": "2115068f457b577f93a268fffe22373e45727d89",
"split": "dev",
"subset_seed": 0,
"subset_size": 5000,
"variant": "small"
},
"ndcg@10": {
"mean_history": {
"cold_candidates": 0.432943,
"cold_users": 0.348226,
"overall": 0.39458
},
"onnx_release": {
"cold_candidates": 0.460018,
"cold_users": 0.366311,
"overall": 0.421722
}
},
"passed": true,
"slices": {
"category:autos": 149,
"category:entertainment": 167,
"category:finance": 231,
"category:foodanddrink": 338,
"category:games": 0,
"category:health": 219,
"category:kids": 0,
"category:lifestyle": 933,
"category:middleeast": 0,
"category:movies": 143,
"category:music": 104,
"category:news": 1191,
"category:northamerica": 0,
"category:sports": 1432,
"category:travel": 106,
"category:tv": 286,
"category:video": 158,
"category:weather": 37,
"cold_candidates": 2745,
"cold_users": 340,
"overall": 5000
}
},
"latency": {
"machine": {
"chip": "Apple M4",
"cpus": "10",
"os": "macOS-26.3.1-arm64-arm-64bit"
},
"onnxruntime": "1.31.0",
"rows": [
{
"batch": 1,
"mean_ms": 0.178,
"p50_ms": 0.176,
"p95_ms": 0.186,
"runs": 1000,
"threads": 1
},
{
"batch": 32,
"mean_ms": 4.715,
"p50_ms": 4.407,
"p95_ms": 5.708,
"runs": 1000,
"threads": 1
},
{
"batch": 1,
"mean_ms": 0.211,
"p50_ms": 0.181,
"p95_ms": 0.38,
"runs": 1000,
"threads": 4
},
{
"batch": 32,
"mean_ms": 3.671,
"p50_ms": 3.484,
"p95_ms": 4.568,
"runs": 1000,
"threads": 4
}
]
},
"parity": {
"classifier": {
"label_mismatches": 0,
"max_abs_diff": 3.5762786865234375e-07,
"n": 2048,
"ok": true,
"tolerance": 0.0001
},
"user_encoder": {
"inherited_from": "v0.1.0",
"note": "byte-identical copy; parity was checked when that release was built"
}
},
"results": {
"mean_history": {
"cold_candidates": {
"auc": 0.6324316689136716,
"mrr": 0.42461835559498284,
"ndcg@10": 0.5332137084239148,
"ndcg@5": 0.4780115575532085
},
"cold_users": {
"auc": 0.5581429913735874,
"mrr": 0.2729486864021774,
"ndcg@10": 0.3551670278023427,
"ndcg@5": 0.29362105050668746
},
"overall": {
"auc": 0.6246578761530922,
"mrr": 0.2999019545432084,
"ndcg@10": 0.3906026055566603,
"ndcg@5": 0.3291824815400495
}
},
"nrms_lite_s0": {
"cold_candidates": {
"auc": 0.6591007517080358,
"mrr": 0.4406796580760931,
"ndcg@10": 0.55145942535955,
"ndcg@5": 0.498055574055922
},
"cold_users": {
"auc": 0.5716962819371971,
"mrr": 0.2790963861247484,
"ndcg@10": 0.3624854473252031,
"ndcg@5": 0.30122273529552634
},
"overall": {
"auc": 0.6623871500247188,
"mrr": 0.3189863291171016,
"ndcg@10": 0.4149475234195816,
"ndcg@5": 0.3536017233878056
}
},
"nrms_lite_s1": {
"cold_candidates": {
"auc": 0.6581422557535548,
"mrr": 0.43927381767112933,
"ndcg@10": 0.5501404587727624,
"ndcg@5": 0.4966467644259163
},
"cold_users": {
"auc": 0.5712849029040606,
"mrr": 0.2788251954731917,
"ndcg@10": 0.3622156275549781,
"ndcg@5": 0.30126478240214766
},
"overall": {
"auc": 0.6617699420349328,
"mrr": 0.3180469168821803,
"ndcg@10": 0.41412668804862063,
"ndcg@5": 0.35259653822267617
}
},
"nrms_lite_s2": {
"cold_candidates": {
"auc": 0.6576318646482496,
"mrr": 0.43916890011004145,
"ndcg@10": 0.5501331007292632,
"ndcg@5": 0.49657430842303923
},
"cold_users": {
"auc": 0.5710677317440604,
"mrr": 0.2782410363090575,
"ndcg@10": 0.36224011028274966,
"ndcg@5": 0.30046827807797755
},
"overall": {
"auc": 0.6620545936191268,
"mrr": 0.31856653726011797,
"ndcg@10": 0.41453025908181607,
"ndcg@5": 0.3530209299323185
}
},
"popularity": {
"cold_candidates": {
"auc": 0.5,
"mrr": 0.3261811083043824,
"ndcg@10": 0.4302939392596537,
"ndcg@5": 0.36316005040780136
},
"cold_users": {
"auc": 0.5439906748380747,
"mrr": 0.25652283764954836,
"ndcg@10": 0.3347620826953331,
"ndcg@5": 0.27220816832688705
},
"overall": {
"auc": 0.5385269910412536,
"mrr": 0.23371154987845752,
"ndcg@10": 0.30790251301600663,
"ndcg@5": 0.24538664136262714
}
},
"random": {
"cold_candidates": {
"auc": 0.4990773252887991,
"mrr": 0.3256328223096701,
"ndcg@10": 0.43018173738832904,
"ndcg@5": 0.36241606893915296
},
"cold_users": {
"auc": 0.501667122371972,
"mrr": 0.24186796156568705,
"ndcg@10": 0.31191727277713,
"ndcg@5": 0.2476318560164239
},
"overall": {
"auc": 0.500015764538092,
"mrr": 0.21873468315228298,
"ndcg@10": 0.2863806054251861,
"ndcg@5": 0.22332871696768097
}
},
"recency": {
"cold_candidates": {
"auc": 0.5717780181218608,
"mrr": 0.3756080602430753,
"ndcg@10": 0.483022817009282,
"ndcg@5": 0.41937461292752437
},
"cold_users": {
"auc": 0.4908663230522278,
"mrr": 0.23064831084833823,
"ndcg@10": 0.30751459622304883,
"ndcg@5": 0.2474741348776389
},
"overall": {
"auc": 0.5116666689745792,
"mrr": 0.21875781691365018,
"ndcg@10": 0.29458689008731354,
"ndcg@5": 0.23305307154273616
}
}
},
"seed_summary": {
"cold_candidates": {
"auc": {
"max": 0.6591007517080358,
"mean": 0.6582916240366133,
"min": 0.6576318646482496,
"std": 0.0007457482561761403
},
"mrr": {
"max": 0.4406796580760931,
"mean": 0.43970745861908794,
"min": 0.43916890011004145,
"std": 0.0008435821014026912
},
"ndcg@10": {
"max": 0.55145942535955,
"mean": 0.5505776616205252,
"min": 0.5501331007292632,
"std": 0.0007636386604901632
},
"ndcg@5": {
"max": 0.498055574055922,
"mean": 0.49709221563495914,
"min": 0.49657430842303923,
"std": 0.0008350790702192205
}
},
"cold_users": {
"auc": {
"max": 0.5716962819371971,
"mean": 0.5713496388617727,
"min": 0.5710677317440604,
"std": 0.00031923642412593796
},
"mrr": {
"max": 0.2790963861247484,
"mean": 0.27872087263566586,
"min": 0.2782410363090575,
"std": 0.0004371135637596898
},
"ndcg@10": {
"max": 0.3624854473252031,
"mean": 0.36231372838764364,
"min": 0.3622156275549781,
"std": 0.0001492159379128168
},
"ndcg@5": {
"max": 0.30126478240214766,
"mean": 0.3009852652585505,
"min": 0.30046827807797755,
"std": 0.0004482173562585426
}
},
"overall": {
"auc": {
"max": 0.6623871500247188,
"mean": 0.662070561892926,
"min": 0.6617699420349328,
"std": 0.00030891368533949605
},
"mrr": {
"max": 0.3189863291171016,
"mean": 0.3185332610864666,
"min": 0.3180469168821803,
"std": 0.00047058932688664876
},
"ndcg@10": {
"max": 0.4149475234195816,
"mean": 0.41453482351667276,
"min": 0.41412668804862063,
"std": 0.0004104367211938037
},
"ndcg@5": {
"max": 0.3536017233878056,
"mean": 0.3530730638476001,
"min": 0.35259653822267617,
"std": 0.0005046164512686953
}
}
},
"shipped_model": "nrms_lite_s0",
"shipped_seed": 0,
"slices": {
"cold_candidates": 125472,
"cold_users": 25688,
"overall": 376471
}
}