Buckets:

davanstrien
/

iconclass-smoke-evals

Files

xet

davanstrien/iconclass-smoke-evals / v4.json

davanstrien

5 days ago

download

raw

21.3 kB

	{
	"summary": {
	"model": "davanstrien/qwen35-4b-iconclass-grpo-v4",
	"dataset": "davanstrien/iconclass-vlm-sft",
	"split": "test",
	"num_evaluated": 40,
	"num_skipped": 0,
	"elapsed_seconds": 253.9,
	"exact_match_accuracy": 0.05,
	"hierarchical_f1": 0.5084,
	"code_precision": 0.1929,
	"code_recall": 0.2033,
	"code_f1": 0.198,
	"valid_json_pct": 1.0,
	"valid_code_pct": 1.0
	},
	"results": [
	{
	"index": 0,
	"gt_codes": [
	"92B361",
	"12B13",
	"12EE13",
	"25F33(EAGLE)",
	"31E3"
	],
	"predicted_codes": [
	"73",
	"73D721",
	"73D721(+3)"
	],
	"generated_text": "{\"iconclass-codes\": [\"73\", \"73D721\", \"73D721(+3)\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 1,
	"gt_codes": [
	"48C27(+321)",
	"92F2"
	],
	"predicted_codes": [
	"48C24"
	],
	"generated_text": "{\"iconclass-codes\": [\"48C24\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.5333333333333333,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 2,
	"gt_codes": [
	"86(MONSTRATUR IN UNDIS)",
	"22C5:24A6",
	"46C24",
	"25H214",
	"25I1"
	],
	"predicted_codes": [
	"25H13",
	"25H2",
	"25H23",
	"25I",
	"86(AMOR OMNIA VINCIT)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H13\", \"25H2\", \"25H23\", \"25I\", \"86(AMOR OMNIA VINCIT)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.5969708737864078,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 3,
	"gt_codes": [
	"11I613",
	"11I1"
	],
	"predicted_codes": [
	"61B2",
	"61B(+51)"
	],
	"generated_text": "{\"iconclass-codes\": [\"61B2\", \"61B(+51)\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 4,
	"gt_codes": [
	"25H11",
	"25H213",
	"25I",
	"46C112"
	],
	"predicted_codes": [
	"25H11",
	"25H",
	"46C11",
	"46C131",
	"25G3"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H11\", \"25H\", \"46C11\", \"46C131\", \"25G3\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.713548782623927,
	"code_precision": 0.2,
	"code_recall": 0.25
	},
	{
	"index": 5,
	"gt_codes": [
	"25G31",
	"31D15",
	"25H11",
	"46C13142",
	"25H"
	],
	"predicted_codes": [
	"25H",
	"25H11",
	"25H213",
	"46C11",
	"46C131"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H\", \"25H11\", \"25H213\", \"46C11\", \"46C131\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.7420215660252156,
	"code_precision": 0.4,
	"code_recall": 0.4
	},
	{
	"index": 6,
	"gt_codes": [
	"25G41(CARNATION)",
	"31A2421",
	"31D15:61BB",
	"41D221(SLUIERKAP)",
	"61B(+53)"
	],
	"predicted_codes": [
	"31A2211",
	"31A2421",
	"31D15:61BB",
	"61B(+52)",
	"61B:31A2212(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2211\", \"31A2421\", \"31D15:61BB\", \"61B(+52)\", \"61B:31A2212(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.7115332428765264,
	"code_precision": 0.4,
	"code_recall": 0.4
	},
	{
	"index": 7,
	"gt_codes": [
	"61B(+51)",
	"41D221(TURBAN)",
	"73",
	"32B321",
	"61B:31D14"
	],
	"predicted_codes": [
	"31A221",
	"31D14",
	"61B(+2)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A221\", \"31D14\", \"61B(+2)\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.4013539967373572,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 8,
	"gt_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(V)",
	"49L12(E)"
	],
	"predicted_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(V)"
	],
	"generated_text": "{\"iconclass-codes\": [\"49L171\", \"49M413\", \"49M3\", \"49L12(V)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 0.8
	},
	{
	"index": 9,
	"gt_codes": [
	"31A231",
	"31A2421",
	"61B(+53)"
	],
	"predicted_codes": [
	"31A2211",
	"61B(+2)",
	"61B(+53)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2211\", \"61B(+2)\", \"61B(+53)\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.8915422885572138,
	"code_precision": 0.3333333333333333,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 10,
	"gt_codes": [
	"41C5",
	"44B113",
	"44B1511"
	],
	"predicted_codes": [
	"82AA",
	"44B1",
	"44B113"
	],
	"generated_text": "{\"iconclass-codes\": [\"82AA\", \"44B1\", \"44B113\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.6182078853046595,
	"code_precision": 0.3333333333333333,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 11,
	"gt_codes": [
	"25F711"
	],
	"predicted_codes": [
	"25F711"
	],
	"generated_text": "{\"iconclass-codes\": [\"25F711\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": true,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 1.0
	},
	{
	"index": 12,
	"gt_codes": [
	"11Q712",
	"25I22",
	"34B11"
	],
	"predicted_codes": [
	"25I2"
	],
	"generated_text": "{\"iconclass-codes\": [\"25I2\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.38,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 13,
	"gt_codes": [
	"54A44(+4)",
	"54D3(+4)",
	"51MM11(+4)",
	"46C24111",
	"25F44(+5246)"
	],
	"predicted_codes": [
	"25F(+3)",
	"25F(+34)",
	"86(QUOD NEMO VIDEAT)",
	"86(QUE NULLE VUENT)",
	"86(QUE NULLE VUENT)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25F(+3)\", \"25F(+34)\", \"86(QUOD NEMO VIDEAT)\", \"86(QUE NULLE VUENT)\", \"86(QUE NULLE VUENT)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.15199999999999997,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 14,
	"gt_codes": [
	"61B:31A2212(+1)",
	"61B:31D14",
	"31A2518",
	"41D223(JERKIN)",
	"31A231"
	],
	"predicted_codes": [
	"31A2422",
	"61B(+2)",
	"61B(+53)",
	"61B:31A2212(+1)",
	"61B:31D14"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2422\", \"61B(+2)\", \"61B(+53)\", \"61B:31A2212(+1)\", \"61B:31D14\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.7480746666666667,
	"code_precision": 0.4,
	"code_recall": 0.4
	},
	{
	"index": 15,
	"gt_codes": [
	"94C133"
	],
	"predicted_codes": [
	"83(OVID, Metamorphoses)"
	],
	"generated_text": "{\"iconclass-codes\": [\"83(OVID, Metamorphoses)\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 16,
	"gt_codes": [
	"31A2422",
	"61B(+52)",
	"61B:31D14"
	],
	"predicted_codes": [
	"11H",
	"11H(JOHN)"
	],
	"generated_text": "{\"iconclass-codes\": [\"11H\", \"11H(JOHN)\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 17,
	"gt_codes": [
	"31A2422",
	"45B",
	"61B(+52)",
	"61B:31D14"
	],
	"predicted_codes": [
	"31A2211",
	"31A24(+1)",
	"61B(+2)",
	"61B(+52)",
	"61B:31A2212(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2211\", \"31A24(+1)\", \"61B(+2)\", \"61B(+52)\", \"61B:31A2212(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.78502732880456,
	"code_precision": 0.2,
	"code_recall": 0.25
	},
	{
	"index": 18,
	"gt_codes": [
	"46C13141(+763)"
	],
	"predicted_codes": [
	"46C1314"
	],
	"generated_text": "{\"iconclass-codes\": [\"46C1314\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.9500000000000001,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 19,
	"gt_codes": [
	"31D15",
	"49M32"
	],
	"predicted_codes": [
	"31D15",
	"31D13",
	"41D221(CAP)",
	"61B(+2)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D15\", \"31D13\", \"41D221(CAP)\", \"61B(+2)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.5418993135011442,
	"code_precision": 0.25,
	"code_recall": 0.5
	},
	{
	"index": 20,
	"gt_codes": [
	"48C1612",
	"48C2",
	"11Q712",
	"25H155",
	"25I13"
	],
	"predicted_codes": [
	"25G3",
	"25G41(ROSE)",
	"25G41(TULIP)",
	"25G41(ANEMONE)",
	"25G41(DAFFODIL)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25G3\", \"25G41(ROSE)\", \"25G41(TULIP)\", \"25G41(ANEMONE)\", \"25G41(DAFFODIL)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.22382608695652173,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 21,
	"gt_codes": [
	"31D15",
	"41A1",
	"31D112",
	"25H15",
	"25G3"
	],
	"predicted_codes": [
	"25G3",
	"25H",
	"25H15",
	"41A1",
	"46C11"
	],
	"generated_text": "{\"iconclass-codes\": [\"25G3\", \"25H\", \"25H15\", \"41A1\", \"46C11\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.6674556213017752,
	"code_precision": 0.6,
	"code_recall": 0.6
	},
	{
	"index": 22,
	"gt_codes": [
	"41D221"
	],
	"predicted_codes": [
	"44B1"
	],
	"generated_text": "{\"iconclass-codes\": [\"44B1\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.19,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 23,
	"gt_codes": [
	"73A6"
	],
	"predicted_codes": [
	"73",
	"25H11",
	"41A1",
	"73B641",
	"25G3"
	],
	"generated_text": "{\"iconclass-codes\": [\"73\", \"25H11\", \"41A1\", \"73B641\", \"25G3\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.21714285714285714,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 24,
	"gt_codes": [
	"25F24(STAG)",
	"25B3",
	"48A9876",
	"25F43",
	"12Q21"
	],
	"predicted_codes": [
	"25F23(LION)",
	"48A9876",
	"48A9875",
	"48A9871",
	"48A9831"
	],
	"generated_text": "{\"iconclass-codes\": [\"25F23(LION)\", \"48A9876\", \"48A9875\", \"48A9871\", \"48A9831\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.6931165311653117,
	"code_precision": 0.2,
	"code_recall": 0.2
	},
	{
	"index": 25,
	"gt_codes": [
	"73C8212",
	"73C8214",
	"73C8215"
	],
	"predicted_codes": [
	"73D312",
	"73D3121"
	],
	"generated_text": "{\"iconclass-codes\": [\"73D312\", \"73D3121\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.39493670886075954,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 26,
	"gt_codes": [
	"73A11"
	],
	"predicted_codes": [
	"73C721",
	"73C7211"
	],
	"generated_text": "{\"iconclass-codes\": [\"73C721\", \"73C7211\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.38,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 27,
	"gt_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(S)"
	],
	"predicted_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(S)"
	],
	"generated_text": "{\"iconclass-codes\": [\"49L171\", \"49M413\", \"49M3\", \"49L12(S)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": true,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 1.0
	},
	{
	"index": 28,
	"gt_codes": [
	"41C741",
	"43B31"
	],
	"predicted_codes": [
	"41A2",
	"41A775",
	"41C12",
	"41C74(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"41A2\", \"41A775\", \"41C12\", \"41C74(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.5391135573580533,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 29,
	"gt_codes": [
	"46C21",
	"46C223"
	],
	"predicted_codes": [
	"25H213",
	"25I",
	"46C24"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H213\", \"25I\", \"46C24\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.397452229299363,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 30,
	"gt_codes": [
	"49E391"
	],
	"predicted_codes": [
	"41C12",
	"41C74(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"41C12\", \"41C74(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.19,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 31,
	"gt_codes": [
	"31A231",
	"31D15",
	"41D2321",
	"61B(+55)"
	],
	"predicted_codes": [
	"31D15",
	"25G3",
	"31A231",
	"25G3(+22)",
	"31D14"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D15\", \"25G3\", \"31A231\", \"25G3(+22)\", \"31D14\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.5283018867924527,
	"code_precision": 0.4,
	"code_recall": 0.5
	},
	{
	"index": 32,
	"gt_codes": [
	"73D58",
	"73D641"
	],
	"predicted_codes": [
	"73D6",
	"73D641"
	],
	"generated_text": "{\"iconclass-codes\": [\"73D6\", \"73D641\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.8297897897897898,
	"code_precision": 0.5,
	"code_recall": 0.5
	},
	{
	"index": 33,
	"gt_codes": [
	"33B3",
	"41A221",
	"41A722",
	"56DD1"
	],
	"predicted_codes": [
	"31D112",
	"31D14",
	"31D15",
	"41A2",
	"41A721"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D112\", \"31D14\", \"31D15\", \"41A2\", \"41A721\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.45410016510731976,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 34,
	"gt_codes": [
	"25A3",
	"45C41",
	"45K21"
	],
	"predicted_codes": [
	"25A"
	],
	"generated_text": "{\"iconclass-codes\": [\"25A\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.285,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 35,
	"gt_codes": [
	"73C8172",
	"73C8171"
	],
	"predicted_codes": [
	"73C721",
	"73C7212"
	],
	"generated_text": "{\"iconclass-codes\": [\"73C721\", \"73C7212\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.5924050632911392,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 36,
	"gt_codes": [
	"25F23(PANTHER)",
	"31A2323",
	"31A2518",
	"42A1931",
	"46C161"
	],
	"predicted_codes": [
	"31D15",
	"46C14",
	"46A14",
	"31A221"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D15\", \"46C14\", \"46A14\", \"31A221\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.5292857142857142,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 37,
	"gt_codes": [
	"12B61",
	"25I",
	"41A12"
	],
	"predicted_codes": [
	"25G3",
	"25I",
	"25I1",
	"41A12"
	],
	"generated_text": "{\"iconclass-codes\": [\"25G3\", \"25I\", \"25I1\", \"41A12\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7002967359050445,
	"code_precision": 0.5,
	"code_recall": 0.6666666666666666
	},
	{
	"index": 38,
	"gt_codes": [
	"23I41",
	"41C32",
	"41C12",
	"31D14",
	"47I4221"
	],
	"predicted_codes": [
	"23D42",
	"23O",
	"23D43",
	"23D41",
	"23D421"
	],
	"generated_text": "{\"iconclass-codes\": [\"23D42\", \"23O\", \"23D43\", \"23D41\", \"23D421\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.13288135593220338,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 39,
	"gt_codes": [
	"31D13",
	"61B:31A2212(+1)",
	"31A2421",
	"41D222(RUFF)",
	"31AA533(EMPIREKRULLEN)"
	],
	"predicted_codes": [
	"31A2211",
	"31A24(+1)",
	"31D15:61BB",
	"61B(+2)",
	"61B(+52)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2211\", \"31A24(+1)\", \"31D15:61BB\", \"61B(+2)\", \"61B(+52)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.6246482649842271,
	"code_precision": 0.0,
	"code_recall": 0.0
	}
	]
	}

Xet Storage Details

Size:: 21.3 kB
Xet hash:: cc844a0338df2ea3a93028aa0ceab2a4b5abdcddd2a347a5e951fcae2d32e0b3

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.