Next-Ask / evaluation.json
Lamapi's picture
Publish Next-Ask model
d14d984 verified
Raw History Blame Contribute Delete
9.46 kB
{
"project": "Next Ask",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"name": "Optical Recognition of Handwritten Digits (8x8)",
"source": "UCI ML Repository id 80 via sklearn.datasets.load_digits",
"license": "CC BY 4.0",
"sklearn_version": "1.6.1",
"n_total": 1797,
"split": {
"pool": 1237,
"validation": 200,
"test": 360,
"method": "stratified train_test_split, random_state=73; test only for final reporting"
}
},
"classifier": {
"type": "multinomial logistic regression, full-batch GD from zeros",
"hyper": {
"steps": 150,
"lr": 1.0,
"l2": 0.001
},
"hyper_selection": {
"split": "validation",
"grid": [
{
"hyper": {
"steps": 100,
"lr": 0.5,
"l2": 0.001
},
"val_accuracy": 0.7816666666666666
},
{
"hyper": {
"steps": 100,
"lr": 0.5,
"l2": 0.01
},
"val_accuracy": 0.775
},
{
"hyper": {
"steps": 100,
"lr": 1.0,
"l2": 0.001
},
"val_accuracy": 0.7966666666666667
},
{
"hyper": {
"steps": 100,
"lr": 1.0,
"l2": 0.01
},
"val_accuracy": 0.79
},
{
"hyper": {
"steps": 150,
"lr": 0.5,
"l2": 0.001
},
"val_accuracy": 0.7949999999999999
},
{
"hyper": {
"steps": 150,
"lr": 0.5,
"l2": 0.01
},
"val_accuracy": 0.7833333333333333
},
{
"hyper": {
"steps": 150,
"lr": 1.0,
"l2": 0.001
},
"val_accuracy": 0.8083333333333335
},
{
"hyper": {
"steps": 150,
"lr": 1.0,
"l2": 0.01
},
"val_accuracy": 0.7949999999999999
}
]
}
},
"protocol": {
"seeds": [
0,
1,
2,
3,
4,
5,
6,
7,
8,
9
],
"initial_labels": 10,
"batch": 10,
"budget": 200,
"pairing": "all strategies share the same initial labeled set per seed"
},
"label_axis": [
10,
20,
30,
40,
50,
60,
70,
80,
90,
100,
110,
120,
130,
140,
150,
160,
170,
180,
190,
200
],
"results": {
"random": {
"mean_curve": [
0.4972222222222221,
0.6508333333333334,
0.7319444444444445,
0.8058333333333334,
0.8386111111111111,
0.8577777777777778,
0.875,
0.8733333333333334,
0.8822222222222222,
0.8863888888888889,
0.891388888888889,
0.8952777777777777,
0.9038888888888887,
0.9038888888888892,
0.9130555555555556,
0.9152777777777779,
0.9175000000000001,
0.9233333333333335,
0.9252777777777779,
0.9283333333333335
],
"std_curve": [
0.059004080210452226,
0.06322175844391503,
0.04262327799642462,
0.04171201236225172,
0.03069287326295477,
0.020927564668708192,
0.024438130497691987,
0.025645975305741767,
0.02351122663277648,
0.019580870609530135,
0.01654819000956276,
0.016622627000419093,
0.016536528962374692,
0.015020562038439754,
0.010974859036188004,
0.012187450538044603,
0.015518308581626014,
0.013217085837078126,
0.010573814617041271,
0.009922539499717643
],
"aulc_mean": 0.8508194444444446,
"aulc_std": 0.011463921196411408,
"final_accuracy_mean": 0.9283333333333335,
"labels_to_90pct_per_seed": [
120,
90,
130,
110,
80,
100,
170,
70,
130,
150
]
},
"uncertainty": {
"mean_curve": [
0.4972222222222221,
0.6608333333333334,
0.7691666666666667,
0.836111111111111,
0.865,
0.8869444444444443,
0.9080555555555556,
0.9172222222222223,
0.9241666666666666,
0.9294444444444444,
0.9358333333333333,
0.9411111111111111,
0.9413888888888889,
0.943611111111111,
0.9447222222222222,
0.9497222222222221,
0.9533333333333334,
0.9541666666666666,
0.9588888888888889,
0.9569444444444445
],
"std_curve": [
0.059004080210452226,
0.04963524360177361,
0.06437556184580423,
0.04897089082453114,
0.032231799343022866,
0.02629515516564194,
0.012695994313362711,
0.010815512407739881,
0.01061750825200912,
0.011466537466972383,
0.009972957261675673,
0.007222222222222197,
0.008093779046851896,
0.007762882562490039,
0.010277777777777776,
0.006854979266251711,
0.006781419786518746,
0.006364688465216471,
0.004937885787397574,
0.006242279181734529
],
"aulc_mean": 0.8836944444444444,
"aulc_std": 0.013546217184144036,
"final_accuracy_mean": 0.9569444444444443,
"labels_to_90pct_per_seed": [
60,
70,
70,
70,
60,
80,
60,
80,
70,
60
]
},
"hybrid": {
"mean_curve": [
0.4972222222222221,
0.7036111111111111,
0.7972222222222223,
0.8483333333333333,
0.8736111111111111,
0.8933333333333333,
0.9052777777777778,
0.9175000000000001,
0.9244444444444445,
0.931388888888889,
0.9349999999999999,
0.9375,
0.9422222222222221,
0.9480555555555554,
0.9491666666666665,
0.9533333333333334,
0.9574999999999999,
0.9605555555555554,
0.9594444444444443,
0.9583333333333333
],
"std_curve": [
0.059004080210452226,
0.04441057737427031,
0.027943947422131746,
0.03218388523991164,
0.011197580205970207,
0.006712803318663668,
0.012877029984999185,
0.013031893450822117,
0.007535922203472533,
0.008962369891612037,
0.009312808119022332,
0.011266277661485132,
0.009362388636862604,
0.007458734212276422,
0.006579288490181691,
0.006060395619242084,
0.006579288490181691,
0.0049378857873975624,
0.0033333333333333435,
0.005555555555555575
],
"aulc_mean": 0.8896527777777777,
"aulc_std": 0.0066435507806675955,
"final_accuracy_mean": 0.9583333333333333,
"labels_to_90pct_per_seed": [
60,
70,
60,
70,
70,
80,
80,
40,
80,
80
]
}
},
"paired_aulc": {
"hybrid_minus_random": {
"aulc_diff_mean": 0.038833333333333366,
"aulc_diff_std": 0.011321139954816477,
"hybrid_wins": 10,
"seeds": 10
},
"hybrid_minus_uncertainty": {
"aulc_diff_mean": 0.005958333333333366,
"aulc_diff_std": 0.010409525329840541,
"hybrid_wins": 6,
"seeds": 10
}
},
"failure_examples": {
"model": "hybrid seed 0 at full budget",
"n_wrong": 17,
"first": [
{
"test_index": 648,
"true": 2,
"predicted": 3,
"confidence": 0.4906110093411642
},
{
"test_index": 1149,
"true": 8,
"predicted": 1,
"confidence": 0.279524654801159
},
{
"test_index": 1264,
"true": 1,
"predicted": 8,
"confidence": 0.4017349208333158
},
{
"test_index": 901,
"true": 9,
"predicted": 8,
"confidence": 0.39549887000865813
},
{
"test_index": 1595,
"true": 7,
"predicted": 4,
"confidence": 0.3269425499797302
},
{
"test_index": 498,
"true": 7,
"predicted": 9,
"confidence": 0.3155363201045011
},
{
"test_index": 492,
"true": 6,
"predicted": 8,
"confidence": 0.4289399503435557
},
{
"test_index": 746,
"true": 4,
"predicted": 7,
"confidence": 0.513914544250967
},
{
"test_index": 447,
"true": 7,
"predicted": 5,
"confidence": 0.4440512466278117
},
{
"test_index": 457,
"true": 5,
"predicted": 9,
"confidence": 0.5211896895367181
},
{
"test_index": 905,
"true": 8,
"predicted": 1,
"confidence": 0.5462623831942506
},
{
"test_index": 1095,
"true": 4,
"predicted": 9,
"confidence": 0.38364309630663607
},
{
"test_index": 857,
"true": 7,
"predicted": 3,
"confidence": 0.5475465941519988
},
{
"test_index": 719,
"true": 9,
"predicted": 1,
"confidence": 0.5353181203934337
},
{
"test_index": 1384,
"true": 4,
"predicted": 6,
"confidence": 0.4296230000789044
},
{
"test_index": 769,
"true": 8,
"predicted": 2,
"confidence": 0.3883439380665226
},
{
"test_index": 794,
"true": 8,
"predicted": 1,
"confidence": 0.4823884399122324
}
]
},
"tests": {
"reload": {
"returncode": 0,
"seconds": 0.18,
"process": "fresh python -E subprocess, cwd = temporary copy of the model folder",
"passed": true,
"max_abs_diff": 5.220997367771929e-08,
"picks": [
96,
36,
26,
7,
105
],
"expected_picks": [
96,
36,
26,
7,
105
],
"n_samples": 16
},
"node_parity": {
"cases": 4,
"failed": 0,
"maxDiff": 1.1102230246251565e-16,
"results": [
{
"name": "hybrid-10",
"maxDiff": 1.1102230246251565e-16
},
{
"name": "uncertainty-25",
"maxDiff": 5.551115123125783e-17
},
{
"name": "random-40",
"maxDiff": 1.1102230246251565e-16
},
{
"name": "hybrid-60",
"maxDiff": 4.163336342344337e-17
}
],
"ms": 114,
"runtime": "node v22.23.4",
"status": "passed"
},
"browser": {
"status": "skipped",
"reason": "playwright not installed"
}
},
"not_run": [
"Human annotation study: the Space accepts user labels, but no user study was run."
]
}