Download evaluation.json from thelamapi/Next-Match: direct link, hf CLI and curl.
- Browser
- Download file 16.9 kB
-
https://huggingface.co/thelamapi/Next-Match/resolve/main/evaluation.json
- Command line
-
hf download hf://thelamapi/Next-Match/evaluation.json
-
curl -L -o evaluation.json https://huggingface.co/thelamapi/Next-Match/resolve/main/evaluation.json
16.9 kB
{
"project": "Next Match",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"type": "synthetic catalog rendered by render.py",
"combinations": 192,
"heldout_combinations": [
[
"bag",
"green",
"dotted"
],
[
"bag",
"green",
"plain"
],
[
"bag",
"green",
"striped"
],
[
"bag",
"orange",
"dotted"
],
[
"bag",
"white",
"striped"
],
[
"bottle",
"yellow",
"dotted"
],
[
"chair",
"black",
"plain"
],
[
"chair",
"green",
"dotted"
],
[
"chair",
"green",
"plain"
],
[
"hat",
"green",
"plain"
],
[
"lamp",
"black",
"plain"
],
[
"lamp",
"purple",
"dotted"
],
[
"lamp",
"red",
"plain"
],
[
"lamp",
"red",
"striped"
],
[
"lamp",
"white",
"striped"
],
[
"mug",
"green",
"dotted"
],
[
"mug",
"orange",
"plain"
],
[
"mug",
"yellow",
"plain"
],
[
"sneaker",
"black",
"plain"
],
[
"sneaker",
"red",
"plain"
],
[
"sneaker",
"white",
"striped"
],
[
"tshirt",
"black",
"dotted"
],
[
"tshirt",
"orange",
"plain"
],
[
"tshirt",
"red",
"striped"
]
],
"train_instances_per_combo": 10,
"gallery": "1 unseen rendering per combination (seed 2)",
"references": "unseen renderings (seed 3)",
"templates": {
"train": {
"color": {
"tr": [
"bunun {c} olanı",
"{c} renklisi",
"aynısının {c} olanı",
"{c} olsun",
"rengi {c} olsun"
],
"en": [
"the {c} one",
"make it {c}",
"same but {c}",
"in {c}"
]
},
"pattern": {
"tr": [
"{p} olanı",
"{p} olsun",
"aynısı ama {p}"
],
"en": [
"the {p} one",
"make it {p}",
"same but {p}"
]
},
"category": {
"tr": [
"aynısı ama {k}",
"bunun {k} olanı",
"aynı desende {k}"
],
"en": [
"same but a {k}",
"a {k} like this",
"same style {k}"
]
},
"color+pattern": {
"tr": [
"{c} ve {p} olanı",
"{p} ve {c} olsun"
],
"en": [
"{c} and {p}",
"make it {p} and {c}"
]
}
},
"heldout": {
"color": {
"tr": [
"{c} olanını göster",
"bundan {c} var mı"
],
"en": [
"show me the {c} one",
"do you have it in {c}"
]
},
"pattern": {
"tr": [
"{p} modelini göster"
],
"en": [
"show a {p} version"
]
},
"category": {
"tr": [
"bunun gibi bir {k} istiyorum"
],
"en": [
"I want a {k} like this"
]
},
"color+pattern": {
"tr": [
"{c} renkli ve {p} göster"
],
"en": [
"show it {p} in {c}"
]
}
}
},
"vocabulary_size": 61
},
"training": {
"combined": {
"seconds": 161.6,
"final_loss": 0.5543771022558213,
"steps": 6000,
"batch": 128
},
"image": {
"seconds": 158.2,
"final_loss": 3.290284743309021,
"steps": 6000,
"batch": 128
},
"text": {
"seconds": 138.7,
"final_loss": 2.8225105476379393,
"steps": 6000,
"batch": 128
}
},
"budget": "same encoder architecture, batch, steps, optimiser and schedule for all three models",
"results": {
"all": {
"n": 1600,
"combined": {
"R@1": 0.9325,
"R@5": 0.96625,
"R@10": 0.979375
},
"image": {
"R@1": 0.0,
"R@5": 0.315625,
"R@10": 0.578125
},
"text": {
"R@1": 0.054375,
"R@5": 0.263125,
"R@10": 0.47125
}
},
"edit=color": {
"n": 648,
"combined": {
"R@1": 0.9938271604938271,
"R@5": 1.0,
"R@10": 1.0
},
"image": {
"R@1": 0.0,
"R@5": 0.35802469135802467,
"R@10": 0.9583333333333334
},
"text": {
"R@1": 0.05092592592592592,
"R@5": 0.2175925925925926,
"R@10": 0.39197530864197533
}
},
"edit=pattern": {
"n": 311,
"combined": {
"R@1": 0.8360128617363344,
"R@5": 0.8971061093247589,
"R@10": 0.9356913183279743
},
"image": {
"R@1": 0.0,
"R@5": 0.8778135048231511,
"R@10": 0.9678456591639871
},
"text": {
"R@1": 0.01607717041800643,
"R@5": 0.07395498392282958,
"R@10": 0.19614147909967847
}
},
"edit=category": {
"n": 322,
"combined": {
"R@1": 1.0,
"R@5": 1.0,
"R@10": 1.0
},
"image": {
"R@1": 0.0,
"R@5": 0.0,
"R@10": 0.0
},
"text": {
"R@1": 0.031055900621118012,
"R@5": 0.18322981366459629,
"R@10": 0.37267080745341613
}
},
"edit=color+pattern": {
"n": 319,
"combined": {
"R@1": 0.8338557993730408,
"R@5": 0.9310344827586207,
"R@10": 0.9592476489028213
},
"image": {
"R@1": 0.0,
"R@5": 0.0,
"R@10": 0.009404388714733543
},
"text": {
"R@1": 0.12225705329153605,
"R@5": 0.6206896551724138,
"R@10": 1.0
}
},
"lang=tr": {
"n": 747,
"combined": {
"R@1": 0.9946452476572959,
"R@5": 1.0,
"R@10": 1.0
},
"image": {
"R@1": 0.0,
"R@5": 0.3172690763052209,
"R@10": 0.570281124497992
},
"text": {
"R@1": 0.050870147255689425,
"R@5": 0.2583668005354752,
"R@10": 0.47523427041499333
}
},
"lang=en": {
"n": 853,
"combined": {
"R@1": 0.8780773739742087,
"R@5": 0.936694021101993,
"R@10": 0.9613130128956624
},
"image": {
"R@1": 0.0,
"R@5": 0.3141852286049238,
"R@10": 0.5849941383352872
},
"text": {
"R@1": 0.05744431418522861,
"R@5": 0.2672919109026964,
"R@10": 0.46776084407971863
}
},
"template=seen": {
"n": 1104,
"combined": {
"R@1": 1.0,
"R@5": 1.0,
"R@10": 1.0
},
"image": {
"R@1": 0.0,
"R@5": 0.3233695652173913,
"R@10": 0.5769927536231884
},
"text": {
"R@1": 0.057065217391304345,
"R@5": 0.2509057971014493,
"R@10": 0.45742753623188404
}
},
"template=held-out": {
"n": 496,
"combined": {
"R@1": 0.782258064516129,
"R@5": 0.8911290322580645,
"R@10": 0.9334677419354839
},
"image": {
"R@1": 0.0,
"R@5": 0.29838709677419356,
"R@10": 0.5806451612903226
},
"text": {
"R@1": 0.04838709677419355,
"R@5": 0.2903225806451613,
"R@10": 0.5020161290322581
}
},
"combo=seen": {
"n": 1192,
"combined": {
"R@1": 0.9328859060402684,
"R@5": 0.9664429530201343,
"R@10": 0.9773489932885906
},
"image": {
"R@1": 0.0,
"R@5": 0.34815436241610737,
"R@10": 0.5956375838926175
},
"text": {
"R@1": 0.05704697986577181,
"R@5": 0.2961409395973154,
"R@10": 0.5016778523489933
}
},
"combo=held-out": {
"n": 408,
"combined": {
"R@1": 0.9313725490196079,
"R@5": 0.9656862745098039,
"R@10": 0.9852941176470589
},
"image": {
"R@1": 0.0,
"R@5": 0.22058823529411764,
"R@10": 0.5269607843137255
},
"text": {
"R@1": 0.04656862745098039,
"R@5": 0.16666666666666666,
"R@10": 0.38235294117647056
}
}
},
"failure_examples_combined": [
{
"ref": [
"lamp",
"black",
"striped"
],
"target": [
"lamp",
"black",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 15
},
{
"ref": [
"mug",
"blue",
"dotted"
],
"target": [
"mug",
"blue",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 53
},
{
"ref": [
"hat",
"yellow",
"dotted"
],
"target": [
"hat",
"black",
"striped"
],
"text": "show it striped in black",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 16
},
{
"ref": [
"mug",
"blue",
"dotted"
],
"target": [
"mug",
"blue",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 53
},
{
"ref": [
"bag",
"black",
"dotted"
],
"target": [
"bag",
"orange",
"striped"
],
"text": "show it striped in orange",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 8
},
{
"ref": [
"bottle",
"green",
"plain"
],
"target": [
"bottle",
"white",
"striped"
],
"text": "show it striped in white",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 50
},
{
"ref": [
"bag",
"red",
"plain"
],
"target": [
"bag",
"red",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 64
},
{
"ref": [
"hat",
"purple",
"striped"
],
"target": [
"hat",
"purple",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 25
},
{
"ref": [
"bottle",
"green",
"plain"
],
"target": [
"bottle",
"white",
"striped"
],
"text": "show it striped in white",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 50
},
{
"ref": [
"bag",
"purple",
"dotted"
],
"target": [
"bag",
"green",
"striped"
],
"text": "show it striped in green",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 6
},
{
"ref": [
"mug",
"red",
"dotted"
],
"target": [
"mug",
"red",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 28
},
{
"ref": [
"sneaker",
"white",
"plain"
],
"target": [
"sneaker",
"white",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 60
},
{
"ref": [
"hat",
"red",
"dotted"
],
"target": [
"hat",
"black",
"striped"
],
"text": "show it striped in black",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 5
},
{
"ref": [
"bottle",
"purple",
"dotted"
],
"target": [
"bottle",
"purple",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 8
},
{
"ref": [
"bag",
"blue",
"dotted"
],
"target": [
"bag",
"blue",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 22
},
{
"ref": [
"sneaker",
"red",
"striped"
],
"target": [
"sneaker",
"red",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 6
},
{
"ref": [
"mug",
"red",
"striped"
],
"target": [
"mug",
"red",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 73
},
{
"ref": [
"mug",
"purple",
"dotted"
],
"target": [
"mug",
"purple",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 125
},
{
"ref": [
"mug",
"red",
"dotted"
],
"target": [
"mug",
"red",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 89
},
{
"ref": [
"sneaker",
"green",
"striped"
],
"target": [
"sneaker",
"green",
"dotted"
],
"text": "show a dotted version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 8
},
{
"ref": [
"lamp",
"red",
"dotted"
],
"target": [
"lamp",
"black",
"striped"
],
"text": "show it striped in black",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 5
},
{
"ref": [
"tshirt",
"white",
"plain"
],
"target": [
"tshirt",
"white",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 15
},
{
"ref": [
"sneaker",
"black",
"dotted"
],
"target": [
"sneaker",
"black",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 80
},
{
"ref": [
"bottle",
"black",
"plain"
],
"target": [
"bottle",
"blue",
"striped"
],
"text": "show it striped in blue",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 23
},
{
"ref": [
"chair",
"black",
"striped"
],
"target": [
"chair",
"black",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 6
},
{
"ref": [
"sneaker",
"blue",
"plain"
],
"target": [
"sneaker",
"blue",
"striped"
],
"text": "show a striped version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 57
},
{
"ref": [
"chair",
"orange",
"striped"
],
"target": [
"chair",
"orange",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 10
},
{
"ref": [
"bag",
"green",
"dotted"
],
"target": [
"bag",
"green",
"plain"
],
"text": "show a plain version",
"kind": "pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": true,
"rank_combined": 11
},
{
"ref": [
"sneaker",
"yellow",
"dotted"
],
"target": [
"sneaker",
"orange",
"striped"
],
"text": "show it striped in orange",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 24
},
{
"ref": [
"hat",
"orange",
"dotted"
],
"target": [
"hat",
"white",
"striped"
],
"text": "show it striped in white",
"kind": "color+pattern",
"lang": "en",
"heldout_template": true,
"unseen_combo": false,
"rank_combined": 12
}
],
"tests": {
"reload": {
"returncode": 0,
"seconds": 0.24,
"process": "fresh python -E subprocess, cwd = temporary copy of the model folder",
"passed": true,
"cases": 6,
"mismatches": []
},
"node_parity": {
"cases": 6,
"failed": 0,
"maxDiff": 1.6653345369377348e-16,
"results": [
{
"name": "28:bunun kırmızı olanı",
"maxDiff": 1.3877787807814457e-17
},
{
"name": "17:çizgili olanı",
"maxDiff": 5.551115123125783e-17
},
{
"name": "102:same but a lamp",
"maxDiff": 2.7755575615628914e-17
},
{
"name": "84:make it dotted and yellow",
"maxDiff": 1.249000902703301e-16
},
{
"name": "190:İSTİYORUM, mor!",
"maxDiff": 2.949029909160572e-17
},
{
"name": "122:",
"maxDiff": 1.6653345369377348e-16
}
],
"ms": 749,
"runtime": "node v22.23.4",
"status": "passed"
},
"browser": {
"status": "skipped",
"reason": "playwright not installed"
}
}
}
{
"project": "Next Match",
"profile": "full",
"seed": 73,
"device": {
"python": "3.13.16",
"platform": "Linux-6.6.159+-x86_64-with-glibc2.39",
"cpu_count": 12,
"numpy": "2.4.6",
"torch": "2.14.1+cu130",
"cuda": true,
"gpu": "NVIDIA L4",
"colab": true
},
"data": {
"type": "synthetic catalog rendered by render.py",
"combinations": 192,
"heldout_combinations": [
[
"bag",
"green",
"dotted"
],
[
"bag",
"green",
"plain"
],
[
"bag",
"green",
"striped"
],
[
"bag",
"orange",
"dotted"
],
[
"bag",
"white",
"striped"
],
[
"bottle",
"yellow",
"dotted"
],
[
"chair",
"black",
"plain"
],
[
"chair",
"green",
"dotted"
],
[
"chair",
"green",
"plain"
],
[
"hat",
"green",
"plain"
],
[
"lamp",
"black",
"plain"
],
[
"lamp",
"purple",
"dotted"
],
[
"lamp",
"red",
"plain"
],
[
"lamp",
"red",
"striped"
],
[
"lamp",
"white",
"striped"
],
[
"mug",
"green",
"dotted"
],
[
"mug",