Download metrics.json from Unmesh30/firstread-models: direct link, hf CLI and curl.
- Browser
- Download file 13.3 kB
-
https://huggingface.co/Unmesh30/firstread-models/resolve/main/metrics.json
- Command line
-
hf download hf://Unmesh30/firstread-models/metrics.json
-
curl -L -o metrics.json https://huggingface.co/Unmesh30/firstread-models/resolve/main/metrics.json
13.3 kB
| { | |
| "bootstrap": { | |
| "baseline": "mean_history", | |
| "metric": "ndcg@10", | |
| "slices": { | |
| "cold_candidates": { | |
| "bootstrap_seed": 12345, | |
| "ci95_high": 0.01878891816194426, | |
| "ci95_low": 0.015859448836582184, | |
| "gain_mean": 0.01736395319661053, | |
| "n_impressions": 125472, | |
| "per_seed_gain": [ | |
| 0.018245716935635255, | |
| 0.016926750348847792, | |
| 0.016919392305348543 | |
| ], | |
| "resamples": 1000, | |
| "share_resamples_gain_le_0": 0.0 | |
| }, | |
| "overall": { | |
| "bootstrap_seed": 12345, | |
| "ci95_high": 0.024816258579701853, | |
| "ci95_low": 0.023167105190676558, | |
| "gain_mean": 0.023932217960012436, | |
| "n_impressions": 376471, | |
| "per_seed_gain": [ | |
| 0.024344917862921273, | |
| 0.023524082491960308, | |
| 0.02392765352515573 | |
| ], | |
| "resamples": 1000, | |
| "share_resamples_gain_le_0": 0.0 | |
| } | |
| } | |
| }, | |
| "classifier": { | |
| "dataset": "MIND-large, articles only in dev news.tsv", | |
| "gate": { | |
| "failures": [], | |
| "max_macro_f1_gap": 0.1, | |
| "min_topic_f1_title_only": 0.3, | |
| "passed": true | |
| }, | |
| "kind": "topics", | |
| "labels": [ | |
| "us", | |
| "world", | |
| "politics", | |
| "business", | |
| "tech_science", | |
| "health", | |
| "sports", | |
| "entertainment", | |
| "lifestyle", | |
| "food_drink", | |
| "travel", | |
| "autos", | |
| "weather" | |
| ], | |
| "n_test": 2457, | |
| "threshold": 0.5, | |
| "title_abstract": { | |
| "accuracy": 0.7346357346357346, | |
| "macro_f1": 0.6371489968601759, | |
| "majority_class_accuracy": 0.3268213268213268, | |
| "per_class": [ | |
| { | |
| "f1": 0.6606170598911071, | |
| "label": "us", | |
| "precision": 0.6201022146507666, | |
| "recall": 0.7067961165048544, | |
| "support": 515 | |
| }, | |
| { | |
| "f1": 0.45977011494252873, | |
| "label": "world", | |
| "precision": 0.6666666666666666, | |
| "recall": 0.3508771929824561, | |
| "support": 57 | |
| }, | |
| { | |
| "f1": 0.815, | |
| "label": "politics", | |
| "precision": 0.7688679245283019, | |
| "recall": 0.8670212765957447, | |
| "support": 188 | |
| }, | |
| { | |
| "f1": 0.5623003194888179, | |
| "label": "business", | |
| "precision": 0.5398773006134969, | |
| "recall": 0.5866666666666667, | |
| "support": 150 | |
| }, | |
| { | |
| "f1": 0.6511627906976745, | |
| "label": "tech_science", | |
| "precision": 0.6829268292682927, | |
| "recall": 0.6222222222222222, | |
| "support": 90 | |
| }, | |
| { | |
| "f1": 0.628099173553719, | |
| "label": "health", | |
| "precision": 0.6031746031746031, | |
| "recall": 0.6551724137931034, | |
| "support": 58 | |
| }, | |
| { | |
| "f1": 0.9357682619647355, | |
| "label": "sports", | |
| "precision": 0.9464968152866242, | |
| "recall": 0.925280199252802, | |
| "support": 803 | |
| }, | |
| { | |
| "f1": 0.6628571428571428, | |
| "label": "entertainment", | |
| "precision": 0.7160493827160493, | |
| "recall": 0.6170212765957447, | |
| "support": 94 | |
| }, | |
| { | |
| "f1": 0.5610859728506787, | |
| "label": "lifestyle", | |
| "precision": 0.6262626262626263, | |
| "recall": 0.5081967213114754, | |
| "support": 122 | |
| }, | |
| { | |
| "f1": 0.6605504587155964, | |
| "label": "food_drink", | |
| "precision": 0.6260869565217392, | |
| "recall": 0.6990291262135923, | |
| "support": 103 | |
| }, | |
| { | |
| "f1": 0.4691358024691358, | |
| "label": "travel", | |
| "precision": 0.5229357798165137, | |
| "recall": 0.4253731343283582, | |
| "support": 134 | |
| }, | |
| { | |
| "f1": 0.5714285714285714, | |
| "label": "autos", | |
| "precision": 0.5862068965517241, | |
| "recall": 0.5573770491803278, | |
| "support": 61 | |
| }, | |
| { | |
| "f1": 0.6451612903225806, | |
| "label": "weather", | |
| "precision": 0.684931506849315, | |
| "recall": 0.6097560975609756, | |
| "support": 82 | |
| } | |
| ] | |
| }, | |
| "title_only": { | |
| "accuracy": 0.7183557183557183, | |
| "macro_f1": 0.624372144861339, | |
| "majority_class_accuracy": 0.3268213268213268, | |
| "per_class": [ | |
| { | |
| "f1": 0.6268939393939394, | |
| "label": "us", | |
| "precision": 0.6118299445471349, | |
| "recall": 0.6427184466019418, | |
| "support": 515 | |
| }, | |
| { | |
| "f1": 0.43956043956043955, | |
| "label": "world", | |
| "precision": 0.5882352941176471, | |
| "recall": 0.3508771929824561, | |
| "support": 57 | |
| }, | |
| { | |
| "f1": 0.8136482939632546, | |
| "label": "politics", | |
| "precision": 0.8031088082901554, | |
| "recall": 0.824468085106383, | |
| "support": 188 | |
| }, | |
| { | |
| "f1": 0.5625, | |
| "label": "business", | |
| "precision": 0.5294117647058824, | |
| "recall": 0.6, | |
| "support": 150 | |
| }, | |
| { | |
| "f1": 0.6545454545454545, | |
| "label": "tech_science", | |
| "precision": 0.72, | |
| "recall": 0.6, | |
| "support": 90 | |
| }, | |
| { | |
| "f1": 0.5833333333333334, | |
| "label": "health", | |
| "precision": 0.5645161290322581, | |
| "recall": 0.603448275862069, | |
| "support": 58 | |
| }, | |
| { | |
| "f1": 0.9229805886036319, | |
| "label": "sports", | |
| "precision": 0.9282115869017632, | |
| "recall": 0.9178082191780822, | |
| "support": 803 | |
| }, | |
| { | |
| "f1": 0.6413043478260869, | |
| "label": "entertainment", | |
| "precision": 0.6555555555555556, | |
| "recall": 0.6276595744680851, | |
| "support": 94 | |
| }, | |
| { | |
| "f1": 0.575, | |
| "label": "lifestyle", | |
| "precision": 0.5847457627118644, | |
| "recall": 0.5655737704918032, | |
| "support": 122 | |
| }, | |
| { | |
| "f1": 0.6278026905829597, | |
| "label": "food_drink", | |
| "precision": 0.5833333333333334, | |
| "recall": 0.6796116504854369, | |
| "support": 103 | |
| }, | |
| { | |
| "f1": 0.44881889763779526, | |
| "label": "travel", | |
| "precision": 0.475, | |
| "recall": 0.4253731343283582, | |
| "support": 134 | |
| }, | |
| { | |
| "f1": 0.5333333333333333, | |
| "label": "autos", | |
| "precision": 0.5423728813559322, | |
| "recall": 0.5245901639344263, | |
| "support": 61 | |
| }, | |
| { | |
| "f1": 0.6871165644171779, | |
| "label": "weather", | |
| "precision": 0.691358024691358, | |
| "recall": 0.6829268292682927, | |
| "support": 82 | |
| } | |
| ] | |
| }, | |
| "topic_map_sha256": "ed1b13889e9ed2f6964b98af94fa4b5c3a32cb224cbbadaa4c7be387e2f73220" | |
| }, | |
| "dataset": "MIND-large dev", | |
| "eval_gate_real": { | |
| "meta": { | |
| "dataset_revision": "2115068f457b577f93a268fffe22373e45727d89", | |
| "split": "dev", | |
| "subset_seed": 0, | |
| "subset_size": 5000, | |
| "variant": "small" | |
| }, | |
| "ndcg@10": { | |
| "mean_history": { | |
| "cold_candidates": 0.432943, | |
| "cold_users": 0.348226, | |
| "overall": 0.39458 | |
| }, | |
| "onnx_release": { | |
| "cold_candidates": 0.460018, | |
| "cold_users": 0.366311, | |
| "overall": 0.421722 | |
| } | |
| }, | |
| "passed": true, | |
| "slices": { | |
| "category:autos": 149, | |
| "category:entertainment": 167, | |
| "category:finance": 231, | |
| "category:foodanddrink": 338, | |
| "category:games": 0, | |
| "category:health": 219, | |
| "category:kids": 0, | |
| "category:lifestyle": 933, | |
| "category:middleeast": 0, | |
| "category:movies": 143, | |
| "category:music": 104, | |
| "category:news": 1191, | |
| "category:northamerica": 0, | |
| "category:sports": 1432, | |
| "category:travel": 106, | |
| "category:tv": 286, | |
| "category:video": 158, | |
| "category:weather": 37, | |
| "cold_candidates": 2745, | |
| "cold_users": 340, | |
| "overall": 5000 | |
| } | |
| }, | |
| "latency": { | |
| "machine": { | |
| "chip": "Apple M4", | |
| "cpus": "10", | |
| "os": "macOS-26.3.1-arm64-arm-64bit" | |
| }, | |
| "onnxruntime": "1.31.0", | |
| "rows": [ | |
| { | |
| "batch": 1, | |
| "mean_ms": 0.178, | |
| "p50_ms": 0.176, | |
| "p95_ms": 0.186, | |
| "runs": 1000, | |
| "threads": 1 | |
| }, | |
| { | |
| "batch": 32, | |
| "mean_ms": 4.715, | |
| "p50_ms": 4.407, | |
| "p95_ms": 5.708, | |
| "runs": 1000, | |
| "threads": 1 | |
| }, | |
| { | |
| "batch": 1, | |
| "mean_ms": 0.211, | |
| "p50_ms": 0.181, | |
| "p95_ms": 0.38, | |
| "runs": 1000, | |
| "threads": 4 | |
| }, | |
| { | |
| "batch": 32, | |
| "mean_ms": 3.671, | |
| "p50_ms": 3.484, | |
| "p95_ms": 4.568, | |
| "runs": 1000, | |
| "threads": 4 | |
| } | |
| ] | |
| }, | |
| "parity": { | |
| "classifier": { | |
| "label_mismatches": 0, | |
| "max_abs_diff": 3.5762786865234375e-07, | |
| "n": 2048, | |
| "ok": true, | |
| "tolerance": 0.0001 | |
| }, | |
| "user_encoder": { | |
| "inherited_from": "v0.1.0", | |
| "note": "byte-identical copy; parity was checked when that release was built" | |
| } | |
| }, | |
| "results": { | |
| "mean_history": { | |
| "cold_candidates": { | |
| "auc": 0.6324316689136716, | |
| "mrr": 0.42461835559498284, | |
| "ndcg@10": 0.5332137084239148, | |
| "ndcg@5": 0.4780115575532085 | |
| }, | |
| "cold_users": { | |
| "auc": 0.5581429913735874, | |
| "mrr": 0.2729486864021774, | |
| "ndcg@10": 0.3551670278023427, | |
| "ndcg@5": 0.29362105050668746 | |
| }, | |
| "overall": { | |
| "auc": 0.6246578761530922, | |
| "mrr": 0.2999019545432084, | |
| "ndcg@10": 0.3906026055566603, | |
| "ndcg@5": 0.3291824815400495 | |
| } | |
| }, | |
| "nrms_lite_s0": { | |
| "cold_candidates": { | |
| "auc": 0.6591007517080358, | |
| "mrr": 0.4406796580760931, | |
| "ndcg@10": 0.55145942535955, | |
| "ndcg@5": 0.498055574055922 | |
| }, | |
| "cold_users": { | |
| "auc": 0.5716962819371971, | |
| "mrr": 0.2790963861247484, | |
| "ndcg@10": 0.3624854473252031, | |
| "ndcg@5": 0.30122273529552634 | |
| }, | |
| "overall": { | |
| "auc": 0.6623871500247188, | |
| "mrr": 0.3189863291171016, | |
| "ndcg@10": 0.4149475234195816, | |
| "ndcg@5": 0.3536017233878056 | |
| } | |
| }, | |
| "nrms_lite_s1": { | |
| "cold_candidates": { | |
| "auc": 0.6581422557535548, | |
| "mrr": 0.43927381767112933, | |
| "ndcg@10": 0.5501404587727624, | |
| "ndcg@5": 0.4966467644259163 | |
| }, | |
| "cold_users": { | |
| "auc": 0.5712849029040606, | |
| "mrr": 0.2788251954731917, | |
| "ndcg@10": 0.3622156275549781, | |
| "ndcg@5": 0.30126478240214766 | |
| }, | |
| "overall": { | |
| "auc": 0.6617699420349328, | |
| "mrr": 0.3180469168821803, | |
| "ndcg@10": 0.41412668804862063, | |
| "ndcg@5": 0.35259653822267617 | |
| } | |
| }, | |
| "nrms_lite_s2": { | |
| "cold_candidates": { | |
| "auc": 0.6576318646482496, | |
| "mrr": 0.43916890011004145, | |
| "ndcg@10": 0.5501331007292632, | |
| "ndcg@5": 0.49657430842303923 | |
| }, | |
| "cold_users": { | |
| "auc": 0.5710677317440604, | |
| "mrr": 0.2782410363090575, | |
| "ndcg@10": 0.36224011028274966, | |
| "ndcg@5": 0.30046827807797755 | |
| }, | |
| "overall": { | |
| "auc": 0.6620545936191268, | |
| "mrr": 0.31856653726011797, | |
| "ndcg@10": 0.41453025908181607, | |
| "ndcg@5": 0.3530209299323185 | |
| } | |
| }, | |
| "popularity": { | |
| "cold_candidates": { | |
| "auc": 0.5, | |
| "mrr": 0.3261811083043824, | |
| "ndcg@10": 0.4302939392596537, | |
| "ndcg@5": 0.36316005040780136 | |
| }, | |
| "cold_users": { | |
| "auc": 0.5439906748380747, | |
| "mrr": 0.25652283764954836, | |
| "ndcg@10": 0.3347620826953331, | |
| "ndcg@5": 0.27220816832688705 | |
| }, | |
| "overall": { | |
| "auc": 0.5385269910412536, | |
| "mrr": 0.23371154987845752, | |
| "ndcg@10": 0.30790251301600663, | |
| "ndcg@5": 0.24538664136262714 | |
| } | |
| }, | |
| "random": { | |
| "cold_candidates": { | |
| "auc": 0.4990773252887991, | |
| "mrr": 0.3256328223096701, | |
| "ndcg@10": 0.43018173738832904, | |
| "ndcg@5": 0.36241606893915296 | |
| }, | |
| "cold_users": { | |
| "auc": 0.501667122371972, | |
| "mrr": 0.24186796156568705, | |
| "ndcg@10": 0.31191727277713, | |
| "ndcg@5": 0.2476318560164239 | |
| }, | |
| "overall": { | |
| "auc": 0.500015764538092, | |
| "mrr": 0.21873468315228298, | |
| "ndcg@10": 0.2863806054251861, | |
| "ndcg@5": 0.22332871696768097 | |
| } | |
| }, | |
| "recency": { | |
| "cold_candidates": { | |
| "auc": 0.5717780181218608, | |
| "mrr": 0.3756080602430753, | |
| "ndcg@10": 0.483022817009282, | |
| "ndcg@5": 0.41937461292752437 | |
| }, | |
| "cold_users": { | |
| "auc": 0.4908663230522278, | |
| "mrr": 0.23064831084833823, | |
| "ndcg@10": 0.30751459622304883, | |
| "ndcg@5": 0.2474741348776389 | |
| }, | |
| "overall": { | |
| "auc": 0.5116666689745792, | |
| "mrr": 0.21875781691365018, | |
| "ndcg@10": 0.29458689008731354, | |
| "ndcg@5": 0.23305307154273616 | |
| } | |
| } | |
| }, | |
| "seed_summary": { | |
| "cold_candidates": { | |
| "auc": { | |
| "max": 0.6591007517080358, | |
| "mean": 0.6582916240366133, | |
| "min": 0.6576318646482496, | |
| "std": 0.0007457482561761403 | |
| }, | |
| "mrr": { | |
| "max": 0.4406796580760931, | |
| "mean": 0.43970745861908794, | |
| "min": 0.43916890011004145, | |
| "std": 0.0008435821014026912 | |
| }, | |
| "ndcg@10": { | |
| "max": 0.55145942535955, | |
| "mean": 0.5505776616205252, | |
| "min": 0.5501331007292632, | |
| "std": 0.0007636386604901632 | |
| }, | |
| "ndcg@5": { | |
| "max": 0.498055574055922, | |
| "mean": 0.49709221563495914, | |
| "min": 0.49657430842303923, | |
| "std": 0.0008350790702192205 | |
| } | |
| }, | |
| "cold_users": { | |
| "auc": { | |
| "max": 0.5716962819371971, | |
| "mean": 0.5713496388617727, | |
| "min": 0.5710677317440604, | |
| "std": 0.00031923642412593796 | |
| }, | |
| "mrr": { | |
| "max": 0.2790963861247484, | |
| "mean": 0.27872087263566586, | |
| "min": 0.2782410363090575, | |
| "std": 0.0004371135637596898 | |
| }, | |
| "ndcg@10": { | |
| "max": 0.3624854473252031, | |
| "mean": 0.36231372838764364, | |
| "min": 0.3622156275549781, | |
| "std": 0.0001492159379128168 | |
| }, | |
| "ndcg@5": { | |
| "max": 0.30126478240214766, | |
| "mean": 0.3009852652585505, | |
| "min": 0.30046827807797755, | |
| "std": 0.0004482173562585426 | |
| } | |
| }, | |
| "overall": { | |
| "auc": { | |
| "max": 0.6623871500247188, | |
| "mean": 0.662070561892926, | |
| "min": 0.6617699420349328, | |
| "std": 0.00030891368533949605 | |
| }, | |
| "mrr": { | |
| "max": 0.3189863291171016, | |
| "mean": 0.3185332610864666, | |
| "min": 0.3180469168821803, | |
| "std": 0.00047058932688664876 | |
| }, | |
| "ndcg@10": { | |
| "max": 0.4149475234195816, | |
| "mean": 0.41453482351667276, | |
| "min": 0.41412668804862063, | |
| "std": 0.0004104367211938037 | |
| }, | |
| "ndcg@5": { | |
| "max": 0.3536017233878056, | |
| "mean": 0.3530730638476001, | |
| "min": 0.35259653822267617, | |
| "std": 0.0005046164512686953 | |
| } | |
| } | |
| }, | |
| "shipped_model": "nrms_lite_s0", | |
| "shipped_seed": 0, | |
| "slices": { | |
| "cold_candidates": 125472, | |
| "cold_users": 25688, | |
| "overall": 376471 | |
| } | |
| } | |