Buckets:

davanstrien
/

iconclass-smoke-evals

Files

xet

davanstrien/iconclass-smoke-evals / v5full.json

davanstrien

4 days ago

download

raw

21.2 kB

	{
	"summary": {
	"model": "davanstrien/qwen35-4b-iconclass-grpo-v5full",
	"dataset": "davanstrien/iconclass-vlm-sft",
	"split": "test",
	"num_evaluated": 40,
	"num_skipped": 0,
	"elapsed_seconds": 258.5,
	"exact_match_accuracy": 0.075,
	"hierarchical_f1": 0.539,
	"code_precision": 0.2762,
	"code_recall": 0.25,
	"code_f1": 0.2625,
	"valid_json_pct": 1.0,
	"valid_code_pct": 1.0
	},
	"results": [
	{
	"index": 0,
	"gt_codes": [
	"92B361",
	"12B13",
	"12EE13",
	"25F33(EAGLE)",
	"31E3"
	],
	"predicted_codes": [
	"73",
	"73D721"
	],
	"generated_text": "{\"iconclass-codes\": [\"73\", \"73D721\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 1,
	"gt_codes": [
	"48C27(+321)",
	"92F2"
	],
	"predicted_codes": [
	"48C24"
	],
	"generated_text": "{\"iconclass-codes\": [\"48C24\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.5333333333333333,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 2,
	"gt_codes": [
	"86(MONSTRATUR IN UNDIS)",
	"22C5:24A6",
	"46C24",
	"25H214",
	"25I1"
	],
	"predicted_codes": [
	"25H23",
	"25H2321",
	"25H2322",
	"25I12",
	"86(AMORIS INFINITUS IMPULSUS)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H23\", \"25H2321\", \"25H2322\", \"25I12\", \"86(AMORIS INFINITUS IMPULSUS)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.6487940740740741,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 3,
	"gt_codes": [
	"11I613",
	"11I1"
	],
	"predicted_codes": [
	"44B113",
	"44B114(PRECEPTOR)"
	],
	"generated_text": "{\"iconclass-codes\": [\"44B113\", \"44B114(PRECEPTOR)\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 4,
	"gt_codes": [
	"25H11",
	"25H213",
	"25I",
	"46C112"
	],
	"predicted_codes": [
	"25H11",
	"25H213",
	"46C112",
	"46C513"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H11\", \"25H213\", \"46C112\", \"46C513\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.8716332378223495,
	"code_precision": 0.75,
	"code_recall": 0.75
	},
	{
	"index": 5,
	"gt_codes": [
	"25G31",
	"31D15",
	"25H11",
	"46C13142",
	"25H"
	],
	"predicted_codes": [
	"25H11",
	"25H213",
	"46C131",
	"46C13141"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H11\", \"25H213\", \"46C131\", \"46C13141\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7053654783277762,
	"code_precision": 0.25,
	"code_recall": 0.2
	},
	{
	"index": 6,
	"gt_codes": [
	"25G41(CARNATION)",
	"31A2421",
	"31D15:61BB",
	"41D221(SLUIERKAP)",
	"61B(+53)"
	],
	"predicted_codes": [
	"31A2421",
	"31D15:61BB",
	"61B(+53)",
	"61B:31A2212(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2421\", \"31D15:61BB\", \"61B(+53)\", \"61B:31A2212(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7175879396984924,
	"code_precision": 0.75,
	"code_recall": 0.6
	},
	{
	"index": 7,
	"gt_codes": [
	"61B(+51)",
	"41D221(TURBAN)",
	"73",
	"32B321",
	"61B:31D14"
	],
	"predicted_codes": [
	"31A2211",
	"31A2421",
	"31D14",
	"61B(+52)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2211\", \"31A2421\", \"31D14\", \"61B(+52)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.37114842175957025,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 8,
	"gt_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(V)",
	"49L12(E)"
	],
	"predicted_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(V)"
	],
	"generated_text": "{\"iconclass-codes\": [\"49L171\", \"49M413\", \"49M3\", \"49L12(V)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 0.8
	},
	{
	"index": 9,
	"gt_codes": [
	"31A231",
	"31A2421",
	"61B(+53)"
	],
	"predicted_codes": [
	"31A231",
	"31A2422",
	"61B(+54)",
	"61B:31A2212(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A231\", \"31A2422\", \"61B(+54)\", \"61B:31A2212(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.9288014311270126,
	"code_precision": 0.25,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 10,
	"gt_codes": [
	"41C5",
	"44B113",
	"44B1511"
	],
	"predicted_codes": [
	"83(Anonymus, Herzog Ernst)",
	"44B113",
	"44B191"
	],
	"generated_text": "{\"iconclass-codes\": [\"83(Anonymus, Herzog Ernst)\", \"44B113\", \"44B191\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.624,
	"code_precision": 0.3333333333333333,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 11,
	"gt_codes": [
	"25F711"
	],
	"predicted_codes": [
	"25F711"
	],
	"generated_text": "{\"iconclass-codes\": [\"25F711\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": true,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 1.0
	},
	{
	"index": 12,
	"gt_codes": [
	"11Q712",
	"25I22",
	"34B11"
	],
	"predicted_codes": [
	"11Q712",
	"25I2"
	],
	"generated_text": "{\"iconclass-codes\": [\"11Q712\", \"25I2\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.704,
	"code_precision": 0.5,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 13,
	"gt_codes": [
	"54A44(+4)",
	"54D3(+4)",
	"51MM11(+4)",
	"46C24111",
	"25F44(+5246)"
	],
	"predicted_codes": [
	"25F44(+512)",
	"25F44(+5262)",
	"54A7(+4)",
	"86(QUOD NEMO VIDEAT)",
	"86(QUE NEMO VIDEAT)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25F44(+512)\", \"25F44(+5262)\", \"54A7(+4)\", \"86(QUOD NEMO VIDEAT)\", \"86(QUE NEMO VIDEAT)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.469446088794926,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 14,
	"gt_codes": [
	"61B:31A2212(+1)",
	"61B:31D14",
	"31A2518",
	"41D223(JERKIN)",
	"31A231"
	],
	"predicted_codes": [
	"31A231",
	"31A2422",
	"61B(+53)",
	"61B:31A2212(+1)",
	"61B:31D14"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A231\", \"31A2422\", \"61B(+53)\", \"61B:31A2212(+1)\", \"61B:31D14\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.8130232558139536,
	"code_precision": 0.6,
	"code_recall": 0.6
	},
	{
	"index": 15,
	"gt_codes": [
	"94C133"
	],
	"predicted_codes": [
	"45H3(+22)"
	],
	"generated_text": "{\"iconclass-codes\": [\"45H3(+22)\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 16,
	"gt_codes": [
	"31A2422",
	"61B(+52)",
	"61B:31D14"
	],
	"predicted_codes": [
	"11H(JOHN)"
	],
	"generated_text": "{\"iconclass-codes\": [\"11H(JOHN)\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 17,
	"gt_codes": [
	"31A2422",
	"45B",
	"61B(+52)",
	"61B:31D14"
	],
	"predicted_codes": [
	"31A2422",
	"45B",
	"61B(+52)",
	"61B:31D14"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2422\", \"45B\", \"61B(+52)\", \"61B:31D14\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": true,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 1.0
	},
	{
	"index": 18,
	"gt_codes": [
	"46C13141(+763)"
	],
	"predicted_codes": [
	"46C13141"
	],
	"generated_text": "{\"iconclass-codes\": [\"46C13141\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 1.0,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 19,
	"gt_codes": [
	"31D15",
	"49M32"
	],
	"predicted_codes": [
	"31D15:61BB",
	"41D221(CAP)",
	"41D222(RUFF)",
	"41D222(JERKIN)",
	"41D26611"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D15:61BB\", \"41D221(CAP)\", \"41D222(RUFF)\", \"41D222(JERKIN)\", \"41D26611\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.39921348314606736,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 20,
	"gt_codes": [
	"48C1612",
	"48C2",
	"11Q712",
	"25H155",
	"25I13"
	],
	"predicted_codes": [
	"25G41(ROSE)",
	"25G411(COLUMBINE)",
	"25G411(FLAG)",
	"25G411(FLAG)",
	"25G411(FLAG)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25G41(ROSE)\", \"25G411(COLUMBINE)\", \"25G411(FLAG)\", \"25G411(FLAG)\", \"25G411(FLAG)\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.22857142857142856,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 21,
	"gt_codes": [
	"31D15",
	"41A1",
	"31D112",
	"25H15",
	"25G3"
	],
	"predicted_codes": [
	"25H15",
	"25H155",
	"41A17"
	],
	"generated_text": "{\"iconclass-codes\": [\"25H15\", \"25H155\", \"41A17\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.5741061810720698,
	"code_precision": 0.3333333333333333,
	"code_recall": 0.2
	},
	{
	"index": 22,
	"gt_codes": [
	"41D221"
	],
	"predicted_codes": [
	"44B112"
	],
	"generated_text": "{\"iconclass-codes\": [\"44B112\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.20000000000000004,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 23,
	"gt_codes": [
	"73A6"
	],
	"predicted_codes": [
	"73C7213"
	],
	"generated_text": "{\"iconclass-codes\": [\"73C7213\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.38,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 24,
	"gt_codes": [
	"25F24(STAG)",
	"25B3",
	"48A9876",
	"25F43",
	"12Q21"
	],
	"predicted_codes": [
	"48A9876",
	"48A9875",
	"48A98311",
	"48A98312",
	"48A98781"
	],
	"generated_text": "{\"iconclass-codes\": [\"48A9876\", \"48A9875\", \"48A98311\", \"48A98312\", \"48A98781\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.32808988764044944,
	"code_precision": 0.2,
	"code_recall": 0.2
	},
	{
	"index": 25,
	"gt_codes": [
	"73C8212",
	"73C8214",
	"73C8215"
	],
	"predicted_codes": [
	"73G326",
	"73G3261"
	],
	"generated_text": "{\"iconclass-codes\": [\"73G326\", \"73G3261\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.39493670886075954,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 26,
	"gt_codes": [
	"73A11"
	],
	"predicted_codes": [
	"73C721"
	],
	"generated_text": "{\"iconclass-codes\": [\"73C721\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.38,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 27,
	"gt_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(S)"
	],
	"predicted_codes": [
	"49L171",
	"49M413",
	"49M3",
	"49L12(S)"
	],
	"generated_text": "{\"iconclass-codes\": [\"49L171\", \"49M413\", \"49M3\", \"49L12(S)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": true,
	"hierarchical_f1": 1.0,
	"code_precision": 1.0,
	"code_recall": 1.0
	},
	{
	"index": 28,
	"gt_codes": [
	"41C741",
	"43B31"
	],
	"predicted_codes": [
	"41C12",
	"41C741",
	"41C741(+1)",
	"41C741(+1):46A14"
	],
	"generated_text": "{\"iconclass-codes\": [\"41C12\", \"41C741\", \"41C741(+1)\", \"41C741(+1):46A14\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7175879396984924,
	"code_precision": 0.25,
	"code_recall": 0.5
	},
	{
	"index": 29,
	"gt_codes": [
	"46C21",
	"46C223"
	],
	"predicted_codes": [
	"25I12",
	"46C24"
	],
	"generated_text": "{\"iconclass-codes\": [\"25I12\", \"46C24\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.5288135593220339,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 30,
	"gt_codes": [
	"49E391"
	],
	"predicted_codes": [
	"41C326",
	"41C741",
	"41C7411"
	],
	"generated_text": "{\"iconclass-codes\": [\"41C326\", \"41C741\", \"41C7411\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.19831932773109243,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 31,
	"gt_codes": [
	"31A231",
	"31D15",
	"41D2321",
	"61B(+55)"
	],
	"predicted_codes": [
	"25G3",
	"31D15",
	"41D221(HAT)"
	],
	"generated_text": "{\"iconclass-codes\": [\"25G3\", \"31D15\", \"41D221(HAT)\"]}",
	"valid_json": true,
	"valid_code_count": 3,
	"total_predicted": 3,
	"exact_match": false,
	"hierarchical_f1": 0.5596433878157503,
	"code_precision": 0.3333333333333333,
	"code_recall": 0.25
	},
	{
	"index": 32,
	"gt_codes": [
	"73D58",
	"73D641"
	],
	"predicted_codes": [
	"73D641"
	],
	"generated_text": "{\"iconclass-codes\": [\"73D641\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.8795518207282913,
	"code_precision": 1.0,
	"code_recall": 0.5
	},
	{
	"index": 33,
	"gt_codes": [
	"33B3",
	"41A221",
	"41A722",
	"56DD1"
	],
	"predicted_codes": [
	"31D112",
	"31D11221",
	"31D11222",
	"31D11223",
	"31D11224"
	],
	"generated_text": "{\"iconclass-codes\": [\"31D112\", \"31D11221\", \"31D11222\", \"31D11223\", \"31D11224\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.076,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 34,
	"gt_codes": [
	"25A3",
	"45C41",
	"45K21"
	],
	"predicted_codes": [
	"25A3",
	"45K"
	],
	"generated_text": "{\"iconclass-codes\": [\"25A3\", \"45K\"]}",
	"valid_json": true,
	"valid_code_count": 2,
	"total_predicted": 2,
	"exact_match": false,
	"hierarchical_f1": 0.711149825783972,
	"code_precision": 0.5,
	"code_recall": 0.3333333333333333
	},
	{
	"index": 35,
	"gt_codes": [
	"73C8172",
	"73C8171"
	],
	"predicted_codes": [
	"73C721"
	],
	"generated_text": "{\"iconclass-codes\": [\"73C721\"]}",
	"valid_json": true,
	"valid_code_count": 1,
	"total_predicted": 1,
	"exact_match": false,
	"hierarchical_f1": 0.57,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 36,
	"gt_codes": [
	"25F23(PANTHER)",
	"31A2323",
	"31A2518",
	"42A1931",
	"46C161"
	],
	"predicted_codes": [
	"46A14",
	"46A151",
	"46A152",
	"46A155"
	],
	"generated_text": "{\"iconclass-codes\": [\"46A14\", \"46A151\", \"46A152\", \"46A155\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.1817153996101365,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 37,
	"gt_codes": [
	"12B61",
	"25I",
	"41A12"
	],
	"predicted_codes": [
	"25I",
	"25I1",
	"25I1461",
	"41A12"
	],
	"generated_text": "{\"iconclass-codes\": [\"25I\", \"25I1\", \"25I1461\", \"41A12\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7210103329506313,
	"code_precision": 0.5,
	"code_recall": 0.6666666666666666
	},
	{
	"index": 38,
	"gt_codes": [
	"23I41",
	"41C32",
	"41C12",
	"31D14",
	"47I4221"
	],
	"predicted_codes": [
	"23K41",
	"41C7111",
	"23O42",
	"23K42",
	"41C7112"
	],
	"generated_text": "{\"iconclass-codes\": [\"23K41\", \"41C7111\", \"23O42\", \"23K42\", \"41C7112\"]}",
	"valid_json": true,
	"valid_code_count": 5,
	"total_predicted": 5,
	"exact_match": false,
	"hierarchical_f1": 0.3991764705882353,
	"code_precision": 0.0,
	"code_recall": 0.0
	},
	{
	"index": 39,
	"gt_codes": [
	"31D13",
	"61B:31A2212(+1)",
	"31A2421",
	"41D222(RUFF)",
	"31AA533(EMPIREKRULLEN)"
	],
	"predicted_codes": [
	"31A2421",
	"31D15:61BB",
	"61B(+52)",
	"61B:31A2212(+1)"
	],
	"generated_text": "{\"iconclass-codes\": [\"31A2421\", \"31D15:61BB\", \"61B(+52)\", \"61B:31A2212(+1)\"]}",
	"valid_json": true,
	"valid_code_count": 4,
	"total_predicted": 4,
	"exact_match": false,
	"hierarchical_f1": 0.7436889332003987,
	"code_precision": 0.5,
	"code_recall": 0.4
	}
	]
	}

Xet Storage Details

Size:: 21.2 kB
Xet hash:: b8b1b2205731bc77103ebac4c66ce9e86d63e090f401b9974425cb1f07109576

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.