diff --git a/eval/bin_v1/00_seed1001.png b/eval/bin_v1/00_seed1001.png new file mode 100644 index 0000000000000000000000000000000000000000..cdd04e204539fd5406037a6c8a015c94b848ef17 --- /dev/null +++ b/eval/bin_v1/00_seed1001.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24985d95b666e752c21aed666db1d385a47038ac6d32074392ad7a508235e1f2 +size 1214004 diff --git a/eval/bin_v1/01_seed1002.png b/eval/bin_v1/01_seed1002.png new file mode 100644 index 0000000000000000000000000000000000000000..556dee5bd4b60755adff19d1559ed2d3b4b7fac6 --- /dev/null +++ b/eval/bin_v1/01_seed1002.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c6087a89e78e914d0b7d9ec8413c33a2edb437654e7706f8fdd541a22fdfe83a +size 742358 diff --git a/eval/bin_v1/02_seed1003.png b/eval/bin_v1/02_seed1003.png new file mode 100644 index 0000000000000000000000000000000000000000..f654a30bdaaab5bce0513ff9ed3d933c187f7682 --- /dev/null +++ b/eval/bin_v1/02_seed1003.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ae6f6dbba102b49a092047e079b277f2dd22831e867b0726961a75c809a8fb3a +size 1441596 diff --git a/eval/category_winners.md b/eval/category_winners.md new file mode 100644 index 0000000000000000000000000000000000000000..7d7a5e69cb42a95b0dfc4fe654a47d9ab0ef5c03 --- /dev/null +++ b/eval/category_winners.md @@ -0,0 +1,33 @@ +# Category winners + +Generated by scripts/category_winners.py from eval/*/metrics.json, eval/hps3_summary.json and the judge reports. 24 prompts, 768 px, one seed each; the judge is a Claude model +instance (+-0.2 noise per average); HPSv3 paired SE vs the teacher is ~0.2-0.3 (differences below ~0.5 mu are not distinguishable). + +| tag | GB | judge accept A | judge text T | text exact /6 | HPSv3 mu (diff to teacher +- SE) | DINOv2 | SSIM | latent cos | CLIP | +|---|---|---|---|---|---|---|---|---|---| +| mixH2_xm_f8 | 1.72 | 4.15 | 4.00 | 4 | 11.44 (-0.94 +- 0.20) | 0.851 | 0.594 | 0.656 | 36.61 | +| mixI_x | 1.74 | - | - | - | 9.66 (-2.73 +- 0.29) | 0.745 | 0.541 | 0.572 | 35.60 | +| mixI_tuned | 1.74 | 3.15 | 1.14 | 0 | 10.65 (-1.73 +- 0.31) | 0.788 | 0.554 | 0.607 | 35.93 | +| mixH2_xm | 1.83 | - | - | - | 11.49 (-0.90 +- 0.22) | 0.856 | 0.594 | 0.659 | 36.68 | +| mixH2_x_f8 | 1.85 | 4.21 (=mixH2_x) | 4.00 | 3 | 11.99 (-0.40 +- 0.18) | 0.877 | 0.641 | 0.720 | 36.39 | +| mixH_x | 1.95 | - | - | - | 11.88 (-0.50 +- 0.22) | 0.867 | 0.627 | 0.700 | 36.64 | +| mixH_tuned | 1.95 | 4.10 | 4.00 | 3 | 11.70 (-0.69 +- 0.30) | 0.860 | 0.615 | 0.680 | 36.11 | +| mixH2_x | 1.95 | 4.21 | 4.00 | 3 | 11.92 (-0.47 +- 0.19) | 0.877 | 0.640 | 0.720 | 36.33 | +| mixH2_tuned | 1.95 | 4.17 | 3.93 | 2 | 11.90 (-0.49 +- 0.23) | 0.864 | 0.622 | 0.691 | 36.26 | +| mixG_x | 2.14 | 4.29 | 4.43 | 4 | 12.09 (-0.30 +- 0.19) | 0.884 | 0.656 | 0.733 | 36.31 | +| mixG_tuned | 2.14 | 4.21 | 4.79 | 6 | 12.14 (-0.24 +- 0.29) | 0.901 | 0.666 | 0.750 | 36.35 | +| mixF_ptq | 2.42 | 4.02 | 4.36 | - | 10.86 (-1.53 +- 0.23) | 0.802 | 0.438 | 0.572 | 36.07 | + +## Winners + +| category | winner | evidence | statistically tied with | +|---|---|---|---| +| best overall (judge 'would accept') | mixG_x (A 4.29) | judge reports | mixG_tuned, mixH_tuned, mixH2_x, mixH2_tuned, mixH2_xm_f8 | +| best text rendering | mixG_tuned (T 4.79, 6/6 letter-exact) | judge reports | none (next best T 4.43) | +| best aesthetics (HPSv3) | mixG_tuned (mu 12.14, teacher 12.38) | eval/hps3_summary.md | mixG_x, mixH_x, mixH_tuned, mixH2_x, mixH2_x_f8, mixH2_tuned | +| closest to the teacher | mixG_tuned (DINOv2 0.901, SSIM 0.666, latent cos 0.750) | eval/*/metrics.json | none by DINOv2 (next 0.884) | +| smallest acceptable (judge A >= 4.0) | mixH2_xm_f8 (1.72 GB, A 4.15, HPSv3 -0.94 vs teacher) | judge + size | - | +| best size/quality value (A >= 4.0 and HPSv3 within 0.5 of the teacher) | mixH2_x_f8 (1.85 GB, HPSv3 -0.40) | judge + HPSv3 + size | - | + +Notes: mixH2_x_f8 was not judged separately (metrics identical to mixH2_x; same model within fp8 rounding of the LoRA factors). HPSv3 sees the int4 modulation layers (mixH2_xm*) as a real +aesthetic cost (-0.94 vs -0.40 for fp8-LoRA only), which agrees with the judge's observation of a small cool/low-contrast drift. 1-bit-attention (mixI) and ternary/binary (tern_v1) are not acceptable. diff --git a/eval/crops_vae_full_vs_small.png b/eval/crops_vae_full_vs_small.png new file mode 100644 index 0000000000000000000000000000000000000000..eab35ab9c84214d0cacb4472a6da97abc6e31b14 --- /dev/null +++ b/eval/crops_vae_full_vs_small.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4978e5a0f9c4ef8a04aa591bb732564e1240c3a9eb3b75cec43b106474b263a4 +size 2220439 diff --git a/eval/eval_prompts.json b/eval/eval_prompts.json new file mode 100644 index 0000000000000000000000000000000000000000..f97cb49bc663eef694934218edf938ca52d3276b --- /dev/null +++ b/eval/eval_prompts.json @@ -0,0 +1,146 @@ +[ + { + "id": 0, + "seed": 1001, + "category": "text", + "prompt": "Photograph of a small cheese shop storefront at dusk, a cream-painted facade with a navy hand-lettered sign reading \"LE PETIT FROMAGE\" above the door, a smaller line on the window glass \"Affineur depuis 1974\", warm lamplight falling on wheels of cheese, wooden crates and a ginger cat asleep on the step, a slate board on the pavement saying \"Tasting - Saturday 11 AM\", wet cobblestones reflecting the glow, shallow depth of field. The gold lettering is crisp and clearly readable, the slate writing is neat white chalk, and the rain-slick street shows glowing reflections of the shop windows." + }, + { + "id": 1, + "seed": 1002, + "category": "text", + "prompt": "Clean flat-design infographic poster on an off-white background, the title \"WATER CYCLE\" in bold blue sans-serif capitals at the top, a simple illustration of a sun, a cloud, a mountain with a river and a sea, with neat arrows and four labels \"Evaporation\", \"Condensation\", \"Precipitation\", \"Collection\" placed beside each stage, rounded icons, tidy margins, soft pastel blues and greens. Each label sits in a clean rounded box beside its illustration, the lines and arrows are neatly aligned, all text is crisp and perfectly readable, and the layout leaves generous margins around the diagram." + }, + { + "id": 2, + "seed": 1003, + "category": "text", + "prompt": "Vintage-style movie poster illustration of a lighthouse keeper's daughter rowing a small boat through storm waves toward a glowing lighthouse, the title \"BEYOND THE BREAKERS\" in large cream serif letters across the bottom, a tagline \"Some lights never go out\" in small italic text above it, painted look with textured brushwork, deep teal sea and warm yellow beam, slight print grain. A swirl of white spray crowns each wave, the girl wears a bright yellow raincoat, the beam of the lighthouse cuts through the storm clouds, and all lettering is sharp, evenly spaced and fully legible." + }, + { + "id": 3, + "seed": 1004, + "category": "text", + "prompt": "Mobile app UI mockup shown on a modern phone screen: a music player with a square album cover of a purple sunset, the song title \"Neon Horizon\" in white bold text, the artist line \"Luna Vale\" in grey below, a progress bar at \"1:24 / 3:48\", round play, previous and next buttons, a dark gradient background, polished glassy interface, phone resting on a desk with soft light. Small rounded icons sit beneath the controls, the interface is neatly aligned with crisp white and grey typography, soft shadows give the cards depth, and the phone screen glows gently in the dim room." + }, + { + "id": 4, + "seed": 1005, + "category": "text", + "prompt": "Chinese restaurant street scene at night: a large red illuminated signboard reading \"老王面馆\" in bold golden characters, a smaller line \"手工拉面 · 营业至凌晨两点\" underneath, paper lanterns hanging from the eaves, steam escaping from the kitchen door, a few customers seated at metal tables on the sidewalk, wet ground with colorful reflections, cinematic night photography. The golden characters are sharp and evenly spaced on the red board, the second line is smaller but clearly readable, and warm lantern light mixes with cool shadows across the wet pavement and the diners' faces." + }, + { + "id": 5, + "seed": 1006, + "category": "text", + "prompt": "Photograph of a roadside fruit stand under a striped awning in the countryside, wooden crates of fruit with hand-painted signs reading \"PEACHES $3\", \"CHERRIES $5\" and \"HONEY $8\", white brush letters on dark green boards, a straw hat hanging on a post, wildflowers in the foreground, rolling hills and a bright blue sky behind, soft morning sun, crisp detail on the painted wood. A few daisies lean against a crate, the painted numbers are perfectly legible, and sunbeams scatter warm highlights across the grain of the weathered wood and the glossy fruit skins." + }, + { + "id": 6, + "seed": 2001, + "category": "portrait", + "prompt": "Studio portrait of an elderly Himalayan monk with a shaved head and deeply lined face, wearing a maroon robe with a golden edge, a gentle half-smile and clear dark eyes looking just past the camera, soft window light from the left, a plain warm grey backdrop, fine skin pores, wrinkles and eyebrow hairs resolved, 85mm lens, shallow depth of field. His heavy-lidded eyes are warm and clear, tiny prayer beads loop around his wrist, the robe's wool weave is clearly visible, and the light gently models the bones of his skull and the silver stubble on his chin." + }, + { + "id": 7, + "seed": 2002, + "category": "portrait", + "prompt": "Cinematic close-up portrait of a young woman with curly auburn hair and green eyes, wearing a navy turtleneck, standing in a rain-streaked train window at night, colored city lights as soft bokeh circles behind her, a faint reflection of the glass over her cheek, droplets on her eyelashes, teal and amber color grading, subtle film grain. A single red lamp glow on the glass mixes with the cool blue of the night, tiny droplets scatter across her skin, her lips are slightly parted, and the soft grain gives the image a quiet intimate mood." + }, + { + "id": 8, + "seed": 2003, + "category": "portrait", + "prompt": "Anime-style portrait of a boy with messy black hair and golden eyes wearing a loose white shirt and a dark green scarf, standing in a field of blue flowers under a wide sky with drifting clouds, a gentle wind blowing his hair and scarf, soft cel shading, delicate detailed highlights in the eyes, pastel painterly background. Her fine hair strands catch the light, drifting petals cross the foreground, the white shirt billows with the breeze, the line art is clean and the eyes are rendered with glossy layered highlights." + }, + { + "id": 9, + "seed": 2004, + "category": "portrait", + "prompt": "Environmental portrait of a middle-aged female ceramicist in a sunlit workshop, standing with her arms crossed and clay-smeared linen apron, short grey hair and round glasses, shelves of glazed bowls and vases behind her, warm morning light from a large window, a confident calm smile, natural skin texture, shallow depth of field. Clay crusts her forearms and her apron, her wrinkles around the eyes deepen with the smile, a kiln glows faintly in the background, and the light falling across the shelves is warm, honest and unretouched." + }, + { + "id": 10, + "seed": 3001, + "category": "detail", + "prompt": "Extreme close-up photograph of a vintage mechanical typewriter, round chrome-rimmed glass keys with black letters, tiny typebars curled behind them, a sheet of cream paper with a few lines of typed text rolled in, worn enamel, scratches and dust in the crevices, soft side light, razor-thin depth of field, rich texture. Tiny serif letters on each key are readable, the carriage return lever is slightly bent, dust and ink smudges collect around the platen, and the warm light creates soft reflections along the metal rim." + }, + { + "id": 11, + "seed": 3002, + "category": "detail", + "prompt": "Macro photograph of a peacock feather's eye, iridescent barbs in turquoise, emerald, gold and violet, each fine strand visible with a metallic sheen, tiny water droplets resting on the surface, dark soft background, studio lighting that shifts the colors, ultra sharp detail, shallow focus at the edges. The barbs overlap like fine silk threads, tiny water drops magnify the underlying texture, soft bokeh in the dark background has warm gold tints, and the photograph shows extreme sharpness at the center of the eye." + }, + { + "id": 12, + "seed": 3003, + "category": "detail", + "prompt": "Photograph of an ancient oak tree trunk covered in thick moss, lichen, ferns and tiny mushrooms, deep bark furrows, snails on the surface, rain droplets on every leaf, soft green forest light with mist, fine texture visible all over, natural colors, shallow depth of field in the background. Droplets of water bead on tiny leaves of moss, bracket fungi in cream and orange cling to the bark, a ladybug crawls along a fern, and green light filters softly from above with delicate mist in the air." + }, + { + "id": 13, + "seed": 3004, + "category": "detail", + "prompt": "Highly detailed photograph of a chainmail and leather medieval gauntlet resting on a wooden table, individual interlocking metal rings, rivets, stitching and scratches, a worn leather strap, a candle flame casting warm uneven light and long shadows, dust on the metal, dark background, macro lens detail. A single bright highlight runs along each steel ring, stitched seams show tiny dust and wax stains on the leather, the table's wood has deep scratches, and the candle's warm glow fades into complete darkness behind." + }, + { + "id": 14, + "seed": 4001, + "category": "counting", + "prompt": "Photograph of exactly five colorful hot air balloons floating over a green valley at sunrise: a red one, an orange one, a yellow one, a blue one and a striped purple one, evenly spread across the sky with distance haze, misty hills, soft golden light, a tiny village below, wide shot. Each balloon has a distinct pattern and a small wicker basket with a tiny pilot, long morning shadows stretch across the fields, wisps of mist hang between hills, and the colors glow warmly in the low sun." + }, + { + "id": 15, + "seed": 4002, + "category": "counting", + "prompt": "Studio photograph of exactly three ripe red apples, two yellow pears and one green pear arranged in a wooden bowl on a white linen cloth, each fruit clearly separated and recognizable, soft window light from the left, subtle shadows, a muted grey wall behind, fresh and clean still-life composition. The fruits have small natural blemishes and glossy highlights, the leaves of the pears are fresh, the linen fabric shows fine folds, and the soft shadow of the bowl falls toward the right edge of the table." + }, + { + "id": 16, + "seed": 5001, + "category": "varied", + "prompt": "Epic cinematic landscape of a colossal ancient tree growing from the center of a mirror-like lake at twilight, its branches glowing with soft golden lanterns, roots reaching down into clear water, a small wooden boat with one traveler approaching, distant mountains under a deep violet sky with the first stars, painterly fantasy realism. Flowers glow in the water, the bridge between giant roots is lit by a handful of floating candles, a pale mist drifts across the lake, and the stars above are reflected in sharp points in the dark surface." + }, + { + "id": 17, + "seed": 5002, + "category": "varied", + "prompt": "Photograph of a rainy neon-lit Seoul street at night, a woman in a clear umbrella and yellow trench coat walking past a row of glowing shop signs, steam from a food cart, reflections on the wet asphalt, taxis blurred in the distance, teal and magenta color palette, 50mm lens, cinematic atmosphere. Her coat is bright and slightly wet, signs in pink, blue and orange glow behind her in blurred bokeh, a scooter passes in the distance, and the street is full of mirrored lights and thin rain streaks." + }, + { + "id": 18, + "seed": 5003, + "category": "varied", + "prompt": "Isometric 3D render of a cozy miniature bakery on a floating cube of ground, a pastel pink shop with a striped awning, tiny bread loaves in the window, a chimney puffing white smoke, a little tree, a cobblestone path and a tiny delivery bicycle, soft global illumination, clay-like materials, pastel blue backdrop. A tiny chimney, a delivery scooter, small round bread rolls and a hanging sign with a croissant icon create charming details, the soft shadows are smooth, and the pastel colors are clean and cheerful." + }, + { + "id": 19, + "seed": 5004, + "category": "varied", + "prompt": "Wildlife photograph of a brown bear catching a salmon in a rushing river, the fish leaping mid-air with water droplets frozen around it, the bear's wet fur, open mouth and focused eyes, green forest and mist behind, bright overcast light, fast shutter speed, telephoto compression. The bear's fur is soaked and shining, water streams from the salmon's tail, rocks and logs are wet and mossy, the forest in the background is soft green, and the splashes sparkle in the light." + }, + { + "id": 20, + "seed": 5005, + "category": "varied", + "prompt": "Watercolor and ink illustration of an old Venetian canal at sunrise, gondolas tied to striped poles, pastel-colored palazzos with arched windows reflected in the green water, a few pigeons on a stone bridge, loose washes with bleeding pigments, fine pen line details, visible paper texture. A small red flower pot hangs on one balcony, laundry lines cross above the canal, the reflections ripple in soft colors, a gondolier in a striped shirt rests his oar, and the paper shows a faint pencil underdrawing." + }, + { + "id": 21, + "seed": 5006, + "category": "varied", + "prompt": "Dark fantasy digital painting of a lone knight in battered black armor kneeling before a giant glowing crystal in a ruined cathedral, shafts of cold blue light through broken stained-glass windows, floating dust and embers, torn banners on the pillars, heavy shadows, dramatic scale, richly detailed metal and stone. Ancient runes glow faintly in the stone floor, floating embers drift upward, his helmet lies at his side, a broken sword is planted in the floor, and the light of the crystal casts strong cold edge highlights on the armor." + }, + { + "id": 22, + "seed": 5007, + "category": "varied", + "prompt": "High-key bright photograph of a minimalist Scandinavian kitchen with white cabinets, a light oak table set with a ceramic jug of tulips and two cups of coffee, morning sun streaming through a large window to create soft shadows, pale grey floor, a plant on the shelf, clean airy mood. Crumbs and a steaming coffee cup sit on the table, tulip petals cast delicate shadows, a pale wooden chair is pulled back, light bounces off the white cabinets, and the entire scene is bright and calm." + }, + { + "id": 23, + "seed": 5008, + "category": "varied", + "prompt": "Low-light photograph of a street food stall in a rainy alley at night, a vendor in a hooded jacket frying dumplings in a large pan, warm yellow bulbs above, rising steam and smoke, a few customers' backs in the foreground, wet pavement with orange reflections, deep shadows, gritty documentary style. Orange oil sizzles in the pan, a pile of dumpling wrappers sits beside a bowl of filling, a customer's hand reaches out with a coin, drops of rain fall under the awning, and the atmosphere is gritty and warm." + } +] diff --git a/eval/ghost_rate_part1.md b/eval/ghost_rate_part1.md new file mode 100644 index 0000000000000000000000000000000000000000..c5a9a28bfcc1e330b8bceb8601eafd2a5dda5cae --- /dev/null +++ b/eval/ghost_rate_part1.md @@ -0,0 +1,70 @@ +# Ghost-face rate, part 1 (gh_x_* folders, 4 text-encoder variants) + +Judge: a Claude model instance doing visual inspection with its own image viewing. Not a human, not a trained detector. +Counts carry judgement noise of a few images. Each variant has only 36 images, so differences of 1-3 images are not meaningful. + +## Method + +- Data: `eval\gh_x_bf16`, `gh_x_int8`, `gh_x_gptq`, `gh_x_i4e8`; 36 images each (prompt ids 06, 07, 08, 09, 17, 22; six seeds each). 144 images total. +- Blinding: one fixed random permutation of the four folders (`random.Random(20261030).shuffle`), mapping written to a temp file and not opened until all 144 were classified. Labels in composites: A, B, C, D. +- For each (prompt id, seed) one 2x2 composite (640 px per variant, 1280x1280) was built and viewed at full size: 36 composites. Faces were then re-checked at 1:1 / zoomed crops (all 07 seeds, 06/08/09 face contact sheets, 17 face crops, 08 seeds 2203/2303, kitchen leg regions for 22). +- Classes: clean / ghost (double or double-exposed face) / mushy (face smeared, no clear double) / other (heavy non-face artifact such as translucent or duplicated furniture). +- Mapping revealed after judging: A = gh_x_bf16, B = gh_x_i4e8, C = gh_x_gptq, D = gh_x_int8. +- Judging was blind to the mapping, but the images of one variant (later revealed as i4e8) often differed visibly in rendering style or composition from the other three, so I may have had an impression of "same variant" across composites. I did not know which encoder it was. +- Tooling note: Pillow/numpy came from an existing venv (`[private metadata removed] no GPU, no process other than short CPU Python runs. Composites were written to the OS temp folder and deleted afterwards. + +## Total counts per variant (36 images each) + +| variant | clean | ghost | mushy | other | total | +|---|---|---|---|---|---| +| bf16 | 35 | 0 | 0 | 1 | 36 | +| int8 | 35 | 0 | 0 | 1 | 36 | +| gptq | 35 | 0 | 0 | 1 | 36 | +| i4e8 | 29 | 2 | 0 | 5 | 36 | + +Of i4e8's 5 "other", 1 (08_seed2203) is mild (semi-transparent scarf end over the chest). Without it i4e8 would be 30 clean / 2 ghost / 4 other. +The single "other" for bf16/int8/gptq is the same image in each (22_seed5207, translucent table legs in all four variants). + +## Per prompt id (clean / ghost / mushy / other; 6 images per cell) + +| prompt id | bf16 | int8 | gptq | i4e8 | +|---|---|---|---|---| +| 06 monk | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 07 curly hair, rainy window | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | 4/2/0/0 | +| 08 anime boy | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | 5/0/0/1 (mild) | +| 09 pottery studio | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 17 umbrella street | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 22 white kitchen | 5/0/0/1 | 5/0/0/1 | 5/0/0/1 | 2/0/0/4 | +| total | 35/0/0/1 | 35/0/0/1 | 35/0/0/1 | 29/2/0/5 | + +Ghost count per prompt id (face ghosts only): 07 has 0 / 0 / 0 / 2 for bf16 / int8 / gptq / i4e8; all other prompt ids have 0 ghost faces in every variant. + +## Flagged images + +| image | class | note | +|---|---|---| +| gh_x_i4e8/07_seed2002.png | ghost | second 3/4-view face smeared over the right cheek, second mouth, doubled eyes (double exposure) | +| gh_x_i4e8/07_seed2202.png | ghost | two overlapping faces: doubled mouth and doubled eyes, mushy cheeks | +| gh_x_i4e8/08_seed2203.png | other (mild) | scarf end semi-transparent over the open collar/chest; face itself clean | +| gh_x_i4e8/22_seed5007.png | other | extra translucent table leg and pale ghost legs beside/under the chair | +| gh_x_i4e8/22_seed5107.png | other | translucent ghost chair floating under the table | +| gh_x_i4e8/22_seed5207.png | other | pale, doubled, see-through table legs on the left (strongest of the four for this seed) | +| gh_x_i4e8/22_seed5407.png | other | translucent ghost chair at left under the table | +| gh_x_bf16/22_seed5207.png | other | table legs washed out / see-through, almost white | +| gh_x_gptq/22_seed5207.png | other | table legs washed out / see-through, almost white | +| gh_x_int8/22_seed5207.png | other (milder) | one see-through table leg, the rest opaque | + +No image was classed as mushy. + +Borderline cases counted as clean (not in the tables above): +- gh_x_bf16, gh_x_gptq, gh_x_int8 / 22_seed5507: faint pale far table legs against the white floor. +- gh_x_int8 / 22_seed5307: a hairline-thin pale leg behind the table. +- gh_x_i4e8 and gh_x_int8 / 07_seed2102: faint window-haze veil on the face, no second face. +- gh_x_i4e8 / 08_seed2303: translucent scarf wisp beside the shoulder. + +## Observations + +- Face ghosting appears only in i4e8: 2 of 36 images, both in prompt 07 (seeds 2002 and 2202). bf16, int8 and gptq had no ghost faces and no mushy faces in any of the 108 images; the faces in 06, 08, 09 and 17 were clean in all four variants. +- The ghost faces are seed-specific: 2 of the 6 prompt-07 seeds in i4e8, while the same seeds were clean in the other three variants, and the other four 07 seeds were clean in all four variants. +- The "other" class is concentrated in the kitchen prompt 22: i4e8 has translucent/ghost furniture in 4 of 6 kitchen images (two clear ghost chairs on seeds 5107 and 5407), the other variants have it in 1 of 6. Seed 5207 shows see-through table legs in all four variants (same noise), so it does not separate the variants; seeds 5007, 5107 and 5407 flag i4e8 only. +- Overall i4e8 has 7 flagged images against 1 for each of the other three (6 against 0 if the shared 5207 case is set aside). That gap is larger than the 1-3 image band, but with 36 images per variant, a single judge, and some borderline calls, it is a rough signal, not a measured rate. bf16, int8 and gptq cannot be told apart from each other on this data. diff --git a/eval/ghost_rate_part2.md b/eval/ghost_rate_part2.md new file mode 100644 index 0000000000000000000000000000000000000000..0b1fadd82b96f15f0fdc12f70cf8c007312a0c2e --- /dev/null +++ b/eval/ghost_rate_part2.md @@ -0,0 +1,85 @@ +# Ghost-face rate, part 2 (gh_turbo_bf16, gh_x_bf16, gh_tuned_bf16) + +Judge: a Claude model instance doing visual inspection with its own image viewing. Not a human, not a trained detector. +Counts carry judgement noise of a few images. Each variant has only 36 images, so differences of 1-3 images are not meaningful. + +## Method + +- Data: `eval\gh_turbo_bf16`, `eval\gh_x_bf16`, `eval\gh_tuned_bf16`; 36 images each (prompt ids 06, 07, 08, 09, 17, 22; six seeds each). 108 images total. +- Blinding: one fixed random permutation of the three folders (PowerShell `Get-Random -SetSeed 918273`), mapping written to a temp file and not opened until all 108 were classified. Labels in composites: A, B, C. +- For each (prompt id, seed) one side-by-side composite (3 x 640 px = 1920x640) was built and viewed: 36 composites. Where something looked doubled or translucent I re-checked 1:1 / zoomed crops from the original images (crops were built in the same blinded A/B/C order, recomputed in the script, mapping file still unread). Zoomed crops: 07 seeds 2002, 2102, 2202, 2402, 2502; 08 seeds 2103, 2203; 17 seeds 5202, 5302, 5402; 22 seeds 5007, 5107, 5307, 5507. All other images were judged from the composites only (06 and 09 faces looked sharp and single at that scale). +- Classes: clean / ghost (double or double-exposed face) / mushy (face smeared, no clear double) / other (heavy non-face artifact such as translucent or duplicated furniture, see-through objects). +- Mapping revealed after judging: A = gh_turbo_bf16, B = gh_x_bf16, C = gh_tuned_bf16. +- Tooling note: Pillow/numpy were not installed in the default Python installs, so composites and crops were made with .NET System.Drawing from PowerShell (CPU only, no GPU, no other process). Composites, crops and helper scripts were written to the OS temp folder and deleted afterwards. No existing file was modified. +- Rule used for "other": flagged when a major object (chair, table leg, umbrella) was visibly translucent or doubled at composite scale; "mild" marks cases that were visible at composite scale but small. Things only visible at zoom, or plausible stylistic effects (wind-blown translucent scarf in the anime prompt, kitchen towel, rain-window haze), were counted clean and listed under borderline. + +## Total counts per variant (36 images each) + +| variant | clean | ghost | mushy | other | total | +|---|---|---|---|---|---| +| turbo (gh_turbo_bf16) | 31 | 3 | 0 | 2 | 36 | +| x (gh_x_bf16) | 34 | 0 | 0 | 2 | 36 | +| tuned (gh_tuned_bf16) | 28 | 1 | 0 | 7 | 36 | + +If the "other (mild)" cases were counted clean instead (to show how much the borderline calls matter; the faint ghost 07_seed2102 in turbo is kept as ghost): +turbo 33 clean / 3 ghost / 0 mushy / 0 other; x 35 / 0 / 0 / 1; tuned 29 / 1 / 0 / 6. + +## Per prompt id (clean / ghost / mushy / other; 6 images per cell) + +| prompt id | turbo | x | tuned | +|---|---|---|---| +| 06 monk | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 07 curly hair, rainy window | 2/3/0/1 | 6/0/0/0 | 4/1/0/1 | +| 08 anime boy | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 09 pottery studio | 6/0/0/0 | 6/0/0/0 | 6/0/0/0 | +| 17 umbrella street | 5/0/0/1 | 6/0/0/0 | 6/0/0/0 | +| 22 white kitchen | 6/0/0/0 | 4/0/0/2 | 0/0/0/6 | +| total | 31/3/0/2 | 34/0/0/2 | 28/1/0/7 | + +Ghost-face count per prompt id (face ghosts only): 07 has 3 / 0 / 1 for turbo / x / tuned; every other prompt id has 0 in all three variants. + +## Flagged images + +| image | class | note | +|---|---|---| +| gh_turbo_bf16/07_seed2102.png | ghost (faint) | second faint mouth/lips beside the face, behind the hair on the left | +| gh_turbo_bf16/07_seed2202.png | ghost | faint second face (eye, cheek) to the left of the real face, behind the glass | +| gh_turbo_bf16/07_seed2402.png | ghost | second faint face (eye and mouth) in the window area left of the real face | +| gh_turbo_bf16/07_seed2502.png | other (mild) | cyan translucent smear across cheek/hair at left of the face | +| gh_turbo_bf16/17_seed5202.png | other (mild) | second translucent umbrella overlapping behind the real umbrella | +| gh_x_bf16/22_seed5207.png | other | table legs washed out / see-through, almost white | +| gh_x_bf16/22_seed5507.png | other (mild) | faint pale vertical ghost legs behind the table | +| gh_tuned_bf16/07_seed2002.png | ghost | second blurred face (nose, eye, mouth) double-exposed over the real face | +| gh_tuned_bf16/07_seed2102.png | other (mild, borderline) | hard-edged translucent teal patch over the left eye/brow; no second face | +| gh_tuned_bf16/22_seed5007.png | other | translucent extra chair and pale ghost legs under/left of the table | +| gh_tuned_bf16/22_seed5107.png | other | translucent ghost chair floating under the table | +| gh_tuned_bf16/22_seed5207.png | other | see-through doubled table legs plus a ghost chair | +| gh_tuned_bf16/22_seed5307.png | other | chair back/seat see-through where it overlaps the table | +| gh_tuned_bf16/22_seed5407.png | other | heavily doubled translucent chairs and legs | +| gh_tuned_bf16/22_seed5507.png | other | several translucent table legs and chair, strong | + +No image was classed as mushy. + +Borderline cases counted as clean (not in the tables above): +- turbo / 22_seed5107: translucent plaid towel on the chair back; turbo / 22_seed5307: table edge slightly see-through over the chair seat (zoom only). +- x / 22_seed5107: small semi-transparent patches under the table edge (zoom only). +- x and tuned / 07_seed2202: faint beige smudge above the left eyebrow; faces soft but features defined. +- x / 07_seed2102: faint window-haze veil on the face, no second face. +- x and tuned / 08 seeds 2003, 2103, 2203, 2303, 2403: semi-transparent scarf tails and layered scarf wisps; turbo had opaque scarves in those seeds. Faces clean in all three variants. + +## Observations + +- Face ghosting appears in prompt 07 only (woman behind a rainy window): turbo 3 of 6, tuned 1 of 6, x 0 of 6. The faces in 06, 08, 09 and 17 were clean in all 108 images, and no image was mushy. The turbo ghosts are faint second faces/mouths sitting in the window area beside the real face; the tuned ghost (seed 2002) is a stronger double exposure on the face itself. +- The ghosts are seed-specific: turbo is ghosted on seeds 2102, 2202, 2402 (and has a mild cyan smear on 2502), tuned on seed 2002, which is clean in the other two variants. Only seed 2302 was clean for all three variants. Seed 2202 is ghosted in turbo and in part 1's i4e8, and 2002 is ghosted in tuned and in part 1's i4e8. +- The kitchen prompt 22 separates the variants on furniture, not faces: tuned has translucent/ghost furniture in 6 of 6 seeds, x in 2 of 6 (5207 clear, 5507 mild), turbo in 0 of 6. Seed 5207 (see-through table legs) is flagged in x and tuned, but not in turbo. +- Overall flagged images: tuned 8 (1 ghost + 7 other), turbo 5, x 2. With 36 images per variant, a single judge and several mild calls, the turbo-vs-x-vs-tuned difference in face ghosts (3 / 0 / 1) is within noise; the tuned kitchen result (6 of 6) is larger than the 1-3 image band. + +## Comparison with ghost_rate_part1.md + +Part 1 judged four variants that all use the quantized generator tmixG_x (the same as `gh_x_bf16` here): bf16 35/0/0/1, int8 35/0/0/1, gptq 35/0/0/1, i4e8 29/2/0/5 (clean/ghost/mushy/other). + +- `gh_x_bf16` is the shared reference. Part 1 gave it 35/0/0/1 (22_seed5207 only); I give 34/0/0/2: the same 22_seed5207 plus 22_seed5507, which part 1 listed as a borderline clean ("faint pale far table legs"). Both judgements agree on no face ghosts and no mushy in this folder. +- Face ghosts: part 1 found them only in i4e8 (2, both prompt 07). Here turbo has 3 and tuned has 1, also only in prompt 07. So the x-based bf16/int8/gptq variants (0 ghost) and this x_bf16 sit at 0, while turbo (3), i4e8 (2) and tuned (1) show a few; at 36 images each, 0-3 face ghosts is a rough signal at best. +- Furniture "other" in the kitchen: part 1 had 1 of 6 for bf16/int8/gptq and 4 of 6 for i4e8; here turbo has 0 of 6, x 2 of 6 (1 clear), tuned 6 of 6. Seed 5207 is see-through in all four part-1 variants and in tuned here, but not in turbo. +- Same-seed overlap in prompt 07 ghosts: seed 2202 (turbo here, i4e8 in part 1) and seed 2002 (tuned here, i4e8 in part 1); seed 2102 is flagged in turbo (faint ghost), tuned (mild patch) and in part 1 only as a haze veil. +- Caveats for the comparison: different judging passes (part 1 used 2x2 composites of four variants, this part 3-up composites), borderline calls were not identical (e.g. 22_seed5507, 07_seed2102), and both are single-judge counts with noise of a few images. diff --git a/eval/grid_enc_a3.png b/eval/grid_enc_a3.png new file mode 100644 index 0000000000000000000000000000000000000000..c5dfce1165ec0c71019dc37a6f666ba691f2613f --- /dev/null +++ b/eval/grid_enc_a3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d6717cefd046deda03881360e36298b77d11e8d7ec3290ef3863b004cc27406f +size 1530096 diff --git a/eval/grid_enc_gptq_pruned.png b/eval/grid_enc_gptq_pruned.png new file mode 100644 index 0000000000000000000000000000000000000000..1870cad01ba3dc63f27e5c646d96fb56a8ca56d9 --- /dev/null +++ b/eval/grid_enc_gptq_pruned.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a178a1fca8e2c722e3c565d473b4af321edf310a5e687470ee99900cfd2d442 +size 1547115 diff --git a/eval/grid_enc_i3.png b/eval/grid_enc_i3.png new file mode 100644 index 0000000000000000000000000000000000000000..9ae29cf5e1d359c6fe1aef9949c70ca6169ce7af --- /dev/null +++ b/eval/grid_enc_i3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fbdbb2a6a5be31f05786f532fa2ca8521ab4540b99606a60831d34270929393 +size 1844070 diff --git a/eval/grid_enc_i4.png b/eval/grid_enc_i4.png new file mode 100644 index 0000000000000000000000000000000000000000..9b9a9ea36b9f8c39e7eed1547d0214f501ffcfa1 --- /dev/null +++ b/eval/grid_enc_i4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebc65d0632635c4e97e2a21a5227cd5cb639a4a9f08b0f39cbdce5fad6dd5b79 +size 1569199 diff --git a/eval/grid_enc_i4e8.png b/eval/grid_enc_i4e8.png new file mode 100644 index 0000000000000000000000000000000000000000..68bc81a8e8f83c2d5b22b133703573e3dea28689 --- /dev/null +++ b/eval/grid_enc_i4e8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5fad735567b70c1969ace7aa59dae3a4044ccba2114cd2c66ad40a488db7dde1 +size 1552215 diff --git a/eval/grid_enc_int8_convrot.png b/eval/grid_enc_int8_convrot.png new file mode 100644 index 0000000000000000000000000000000000000000..9bca0ee8a5e31edb15cebee7b6bfc5220f6bfa3a --- /dev/null +++ b/eval/grid_enc_int8_convrot.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:06b825d5803f6cd5e09b08fe6c9845857e165cbbd09bd752364c4c05edf19e77 +size 1541095 diff --git a/eval/grid_enc_l3.png b/eval/grid_enc_l3.png new file mode 100644 index 0000000000000000000000000000000000000000..d1147518edccfac5206ef39c7500700435f52746 --- /dev/null +++ b/eval/grid_enc_l3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d46ec741f2162eb2ada32fc70c11a6cfccf815f99f7764e2c1032b22832d0ce3 +size 1478005 diff --git a/eval/grid_enc_m3.png b/eval/grid_enc_m3.png new file mode 100644 index 0000000000000000000000000000000000000000..4b07cb85284807f8711eb4dbd8c77c93f4c483a4 --- /dev/null +++ b/eval/grid_enc_m3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7388dec7c893a18ca2fc501554a1c236481bee024821752a4d84e9b20e00416d +size 1560123 diff --git a/eval/grid_enc_w4a8_gptq.png b/eval/grid_enc_w4a8_gptq.png new file mode 100644 index 0000000000000000000000000000000000000000..62f7fa1f1121f5b95a636a16fa5d3a66b6167de0 --- /dev/null +++ b/eval/grid_enc_w4a8_gptq.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:81cd537d988f63e62b8e2272c65a808df04977bf4f9bd48358f820c83418a732 +size 1554174 diff --git a/eval/grid_nd_mixG_tuned_gptq.png b/eval/grid_nd_mixG_tuned_gptq.png new file mode 100644 index 0000000000000000000000000000000000000000..45321579b06e8cc5916bb14a9c77708ce48ea462 --- /dev/null +++ b/eval/grid_nd_mixG_tuned_gptq.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:99bfe91e1614c635329001cd813b50cf3e52a6f26ad6686de3e27f4e926dfd76 +size 1766295 diff --git a/eval/grid_nd_mixH2_x_f8_gptq.png b/eval/grid_nd_mixH2_x_f8_gptq.png new file mode 100644 index 0000000000000000000000000000000000000000..c9c255788f53d68f14c2a9c7f3f77b3a3766305f --- /dev/null +++ b/eval/grid_nd_mixH2_x_f8_gptq.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:42f096ad75fd5673b58d2d8df14151b9d1aed39a1d06781fa43d459cbb6286cb +size 1739569 diff --git a/eval/grid_tmixG_traj_1024.png b/eval/grid_tmixG_traj_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..7fd9665c6462075ac539d2d3bbf11a46b7c61af4 --- /dev/null +++ b/eval/grid_tmixG_traj_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3cf091fcd22eddaa1c858da7a86f61893e446b3cf042ba155c0a7a57258cbec0 +size 1216201 diff --git a/eval/grid_tmixG_traj_768.png b/eval/grid_tmixG_traj_768.png new file mode 100644 index 0000000000000000000000000000000000000000..a3c7ef30beaa833a45029a0d6cd7696abc6e0c53 --- /dev/null +++ b/eval/grid_tmixG_traj_768.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:f2055efccbab7119088c3a1f4f5e97d66bcec469c9d99b6ff2bd9e4c704511aa +size 1252485 diff --git a/eval/grid_tmixG_tuned_1024.png b/eval/grid_tmixG_tuned_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..1968a651318c841830188082da9c7c35d704929d --- /dev/null +++ b/eval/grid_tmixG_tuned_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a5e795179db76d04d4cf8ded2e6d4105893590d2236ad770187b7f13d3d64e6 +size 1580520 diff --git a/eval/grid_tmixG_tuned_768.png b/eval/grid_tmixG_tuned_768.png new file mode 100644 index 0000000000000000000000000000000000000000..07fd02a46e14b3ebda0f403aabb2f6aad04450fa --- /dev/null +++ b/eval/grid_tmixG_tuned_768.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f6ab88ed0f101b3b9fefc44b544914a2386094f1e2047b2690b5582772cdffb +size 1529890 diff --git a/eval/grid_tmixG_x_1024.png b/eval/grid_tmixG_x_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..6fed26de5943826bf11453be21a55a87768a9a84 --- /dev/null +++ b/eval/grid_tmixG_x_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e74bfc1db66520d7794de11e61d9384bb5fd0e8e33b2d4dbdec20c05030d3db +size 1557706 diff --git a/eval/grid_tmixG_x_768.png b/eval/grid_tmixG_x_768.png new file mode 100644 index 0000000000000000000000000000000000000000..0c512abf0f3068ed1224cdfe9ecd043086919ff1 --- /dev/null +++ b/eval/grid_tmixG_x_768.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:74ef97c6d982ab81fb9b9912ff66a044650406b86b44e33df6ceafacba86048c +size 1495734 diff --git a/eval/grid_tmixH_x_f8_1024.png b/eval/grid_tmixH_x_f8_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..249a6789dfd273d9b87041fb7e99a63f04dffa5e --- /dev/null +++ b/eval/grid_tmixH_x_f8_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:60d7785f184976e18bb0f28361d0ce6887ed250e028d9cfab29801704c69f573 +size 1489108 diff --git a/eval/grid_tmixH_x_f8_768.png b/eval/grid_tmixH_x_f8_768.png new file mode 100644 index 0000000000000000000000000000000000000000..a5d1f4fe81f47bcc260442d0d5a54db2c7a33cbd --- /dev/null +++ b/eval/grid_tmixH_x_f8_768.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:77a0c4cd17c842b471b2f9d21accef25a1a5c52c9cea4b38c2723d6952ea39bc +size 1448066 diff --git a/eval/grid_tmixH_xm_f8_1024.png b/eval/grid_tmixH_xm_f8_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..c2e055c34496b8d6a0261d413c793df5ea27c051 --- /dev/null +++ b/eval/grid_tmixH_xm_f8_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3155d9df685af4e582dc27070dfc30d3ccb2760bb91c35c55bd41540e94b316e +size 1388827 diff --git a/eval/grid_tmixH_xm_f8_768.png b/eval/grid_tmixH_xm_f8_768.png new file mode 100644 index 0000000000000000000000000000000000000000..a41dfb462e9615e5337d3ae0a286493fa9de5458 --- /dev/null +++ b/eval/grid_tmixH_xm_f8_768.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3db9c0550db6538add036fa8b0c9887bd9795aa1442c3046656abddbed9727d9 +size 1372270 diff --git a/eval/grid_turbo4_1024.png b/eval/grid_turbo4_1024.png new file mode 100644 index 0000000000000000000000000000000000000000..fb05d9f67f1dcdb0a9f5d983ac3cec85ca16a41f --- /dev/null +++ b/eval/grid_turbo4_1024.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0ab06b3381a003c49e00642cc7257eaaba5357cc37cf12cefccc2324112eff81 +size 1666699 diff --git a/eval/grid_turbo6_on_mixG_tuned.png b/eval/grid_turbo6_on_mixG_tuned.png new file mode 100644 index 0000000000000000000000000000000000000000..c4d39aa2933b1449e9fb292bbb8da8e675e39703 --- /dev/null +++ b/eval/grid_turbo6_on_mixG_tuned.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2b671f64a05f64133667c80df9295d3ff7bfd2ef298c335397272bafd27664a3 +size 1828136 diff --git a/eval/grid_turbo6_on_mixH2_x_f8.png b/eval/grid_turbo6_on_mixH2_x_f8.png new file mode 100644 index 0000000000000000000000000000000000000000..7d17469db3bc9b904afbfa392b25d5dc7c9890fa --- /dev/null +++ b/eval/grid_turbo6_on_mixH2_x_f8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4b0d70975655aa803b555579109c728215d37f00bb45d444973e644cfe516cdd +size 1568718 diff --git a/eval/grid_turboF_mixG_tuned.png b/eval/grid_turboF_mixG_tuned.png new file mode 100644 index 0000000000000000000000000000000000000000..fc61c98c6ab4380c384d31cda57870529f7e2b47 --- /dev/null +++ b/eval/grid_turboF_mixG_tuned.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:11e82efcc3e92902af7bf3e6cab8538fdf382284ac766db6053da53a0a7eaef1 +size 1674207 diff --git a/eval/grid_turboF_mixH2_x_f8.png b/eval/grid_turboF_mixH2_x_f8.png new file mode 100644 index 0000000000000000000000000000000000000000..db6da76487f7a91caab79e09ac2a2b4b6bac0963 --- /dev/null +++ b/eval/grid_turboF_mixH2_x_f8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c318d0e993997f347535a02582ac7d9f1bee2513636f818c256d0deeaf2957f4 +size 1588663 diff --git a/eval/grid_turbo_on_mixG_tuned.png b/eval/grid_turbo_on_mixG_tuned.png new file mode 100644 index 0000000000000000000000000000000000000000..aa9ea2ad001b055dc184e12c2feaffc87fe880d7 --- /dev/null +++ b/eval/grid_turbo_on_mixG_tuned.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e065eaab8b10e09fbc32c8498b02f1387864d985f0c488c65df3cacac798a833 +size 1635705 diff --git a/eval/grid_turbo_on_mixH2_x_f8.png b/eval/grid_turbo_on_mixH2_x_f8.png new file mode 100644 index 0000000000000000000000000000000000000000..6a15ceab2dbc53559ff3d00608c0c77c844d749e --- /dev/null +++ b/eval/grid_turbo_on_mixH2_x_f8.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5dca90bfce851d97996f68c2d4f68f67cddb3d9fb300dac42add6ba66574ec6b +size 1506950 diff --git a/eval/hps3_enc_baseline.json b/eval/hps3_enc_baseline.json new file mode 100644 index 0000000000000000000000000000000000000000..7381fa7b0bc6b6a274638c1fc01c14ec2d86dcc7 --- /dev/null +++ b/eval/hps3_enc_baseline.json @@ -0,0 +1,45 @@ +{ + "enc_int8_convrot": { + "n": 24, + "mean_mu": 10.945838987827301, + "sd_mu": 1.6878424869155904, + "ci95": [ + 10.261139473319053, + 11.598907498021921 + ], + "category_mean_mu": { + "text": 12.098817825317383, + "portrait": 11.843674421310425, + "detail": 9.474143862724304, + "counting": 10.993349552154541, + "varied": 10.356157064437866 + }, + "mu_by_prompt": { + "0": 11.561762809753418, + "1": 10.07781982421875, + "2": 13.57741928100586, + "3": 12.749553680419922, + "4": 11.740704536437988, + "5": 12.88564682006836, + "6": 11.89348030090332, + "7": 11.907999038696289, + "8": 11.463834762573242, + "9": 12.109383583068848, + "10": 8.401968002319336, + "11": 12.902873039245605, + "12": 9.782557487487793, + "13": 6.809176921844482, + "14": 10.684173583984375, + "15": 11.302525520324707, + "16": 10.40369701385498, + "17": 13.471558570861816, + "18": 9.411077499389648, + "19": 8.697176933288574, + "20": 10.233926773071289, + "21": 10.562956809997559, + "22": 10.744577407836914, + "23": 9.324285507202148 + }, + "size_gb": null + } +} diff --git a/eval/hps3_enc_ptq.json b/eval/hps3_enc_ptq.json new file mode 100644 index 0000000000000000000000000000000000000000..b8eb5ce815275fe3e04c33a77c638d0694c7541b --- /dev/null +++ b/eval/hps3_enc_ptq.json @@ -0,0 +1,131 @@ +{ + "enc_i4": { + "n": 24, + "mean_mu": 11.034659087657928, + "sd_mu": 1.8064966564013927, + "ci95": [ + 10.30820912073056, + 11.735686944921811 + ], + "category_mean_mu": { + "text": 12.530771573384603, + "portrait": 11.501980543136597, + "detail": 9.691685795783997, + "counting": 11.013746738433838, + "varied": 10.355628728866577 + }, + "mu_by_prompt": { + "0": 12.212133407592773, + "1": 10.685227394104004, + "2": 14.403738021850586, + "3": 12.342391014099121, + "4": 11.99782657623291, + "5": 13.543313026428223, + "6": 11.655359268188477, + "7": 11.481369972229004, + "8": 10.31234073638916, + "9": 12.558852195739746, + "10": 8.624855041503906, + "11": 13.177989959716797, + "12": 10.35336685180664, + "13": 6.610531330108643, + "14": 10.301910400390625, + "15": 11.72558307647705, + "16": 10.535961151123047, + "17": 12.761947631835938, + "18": 9.778426170349121, + "19": 8.138744354248047, + "20": 10.932673454284668, + "21": 10.173169136047363, + "22": 11.53548812866211, + "23": 8.988619804382324 + }, + "size_gb": 5.61 + }, + "enc_m3": { + "n": 24, + "mean_mu": 10.635515789190928, + "sd_mu": 1.8093861358357015, + "ci95": [ + 9.91959085514148, + 11.339807620147864 + ], + "category_mean_mu": { + "text": 11.88214635848999, + "portrait": 11.379156112670898, + "detail": 9.323559641838074, + "counting": 10.494130611419678, + "varied": 10.020047068595886 + }, + "mu_by_prompt": { + "0": 13.073763847351074, + "1": 8.502704620361328, + "2": 13.960399627685547, + "3": 11.80362319946289, + "4": 11.661916732788086, + "5": 12.290470123291016, + "6": 11.188579559326172, + "7": 11.254002571105957, + "8": 10.09174919128418, + "9": 12.982293128967285, + "10": 7.603389263153076, + "11": 13.012289047241211, + "12": 10.081485748291016, + "13": 6.597074508666992, + "14": 9.626181602478027, + "15": 11.362079620361328, + "16": 10.498995780944824, + "17": 11.699702262878418, + "18": 10.247276306152344, + "19": 8.36837100982666, + "20": 9.855698585510254, + "21": 9.963851928710938, + "22": 9.955434799194336, + "23": 9.571045875549316 + }, + "size_gb": 5.01 + }, + "enc_i3": { + "n": 24, + "mean_mu": 4.856175818790992, + "sd_mu": 3.3988304751516716, + "ci95": [ + 3.5031733943149446, + 6.1502091315885385 + ], + "category_mean_mu": { + "text": 5.54188072681427, + "portrait": 7.1700427532196045, + "detail": 2.088006906211376, + "counting": 5.593181848526001, + "varied": 4.384796619415283 + }, + "mu_by_prompt": { + "0": -1.0470497608184814, + "1": 7.336244583129883, + "2": 7.800258159637451, + "3": 5.490494728088379, + "4": 7.72861385345459, + "5": 5.942722797393799, + "6": 6.834472179412842, + "7": 7.286423683166504, + "8": 10.505233764648438, + "9": 4.054041385650635, + "10": -3.5405421257019043, + "11": 9.378467559814453, + "12": -0.1173417866230011, + "13": 2.631443977355957, + "14": 4.578416347503662, + "15": 6.60794734954834, + "16": 8.415070533752441, + "17": 2.7991318702697754, + "18": 4.980869770050049, + "19": 2.071200132369995, + "20": 6.5670952796936035, + "21": 5.090610027313232, + "22": 1.5781588554382324, + "23": 3.5762364864349365 + }, + "size_gb": 4.75 + } +} diff --git a/eval/hps3_enc_ptq.md b/eval/hps3_enc_ptq.md new file mode 100644 index 0000000000000000000000000000000000000000..fcab08f229d1b08b8de6a5472939378bdcfd30dd --- /dev/null +++ b/eval/hps3_enc_ptq.md @@ -0,0 +1,12 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| enc_i4 | 5.61 | 11.03 [10.31, 11.74] | 12.53 | 11.50 | 9.69 | 11.01 | 10.36 | - | +| enc_m3 | 5.01 | 10.64 [9.92, 11.34] | 11.88 | 11.38 | 9.32 | 10.49 | 10.02 | - | +| enc_i3 | 4.75 | 4.86 [3.50, 6.15] | 5.54 | 7.17 | 2.09 | 5.59 | 4.38 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.81-3.40; bootstrap 95% half-width of a mean is 0.71-1.32. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.14-0.67 (median 0.65); a difference below about 2 x that (1.30 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_enc_ptq2.json b/eval/hps3_enc_ptq2.json new file mode 100644 index 0000000000000000000000000000000000000000..7d68f12da0d0d1262e18bef658f78d9789f92027 --- /dev/null +++ b/eval/hps3_enc_ptq2.json @@ -0,0 +1,131 @@ +{ + "enc_i4e8": { + "n": 24, + "mean_mu": 11.091991861661276, + "sd_mu": 1.7556452830536504, + "ci95": [ + 10.388231111566226, + 11.772854788104693 + ], + "category_mean_mu": { + "text": 12.595286051432291, + "portrait": 11.365147113800049, + "detail": 9.674987316131592, + "counting": 11.102847576141357, + "varied": 10.533731937408447 + }, + "mu_by_prompt": { + "0": 12.308874130249023, + "1": 10.906404495239258, + "2": 14.414641380310059, + "3": 12.643335342407227, + "4": 11.775763511657715, + "5": 13.522697448730469, + "6": 11.209823608398438, + "7": 10.562433242797852, + "8": 11.092175483703613, + "9": 12.596156120300293, + "10": 8.60431957244873, + "11": 12.998363494873047, + "12": 10.34162425994873, + "13": 6.755641937255859, + "14": 10.533465385437012, + "15": 11.672229766845703, + "16": 10.680486679077148, + "17": 12.985466957092285, + "18": 10.035730361938477, + "19": 8.486477851867676, + "20": 10.956826210021973, + "21": 10.306949615478516, + "22": 11.811293601989746, + "23": 9.006624221801758 + }, + "size_gb": 4.6 + }, + "enc_l3": { + "n": 24, + "mean_mu": 9.768357813358307, + "sd_mu": 1.3258938818064758, + "ci95": [ + 9.239143864810467, + 10.278334188461303 + ], + "category_mean_mu": { + "text": 10.216746012369791, + "portrait": 10.868505001068115, + "detail": 8.685726881027222, + "counting": 10.24935531616211, + "varied": 9.303059160709381 + }, + "mu_by_prompt": { + "0": 10.018077850341797, + "1": 9.047893524169922, + "2": 10.448965072631836, + "3": 11.477543830871582, + "4": 10.774165153503418, + "5": 9.533830642700195, + "6": 10.359161376953125, + "7": 11.376570701599121, + "8": 10.234749794006348, + "9": 11.503538131713867, + "10": 7.847609519958496, + "11": 11.483872413635254, + "12": 8.430160522460938, + "13": 6.981265068054199, + "14": 9.427104949951172, + "15": 11.071605682373047, + "16": 10.225643157958984, + "17": 10.968631744384766, + "18": 8.514876365661621, + "19": 7.952888488769531, + "20": 9.436090469360352, + "21": 9.479061126708984, + "22": 10.167335510253906, + "23": 7.679946422576904 + }, + "size_gb": 4.4 + }, + "enc_a3": { + "n": 24, + "mean_mu": 11.020188371340433, + "sd_mu": 1.8679766509915257, + "ci95": [ + 10.28298435707887, + 11.752896255254745 + ], + "category_mean_mu": { + "text": 12.489479700724283, + "portrait": 11.711003541946411, + "detail": 9.484590530395508, + "counting": 10.592373847961426, + "varied": 10.447564840316772 + }, + "mu_by_prompt": { + "0": 12.082544326782227, + "1": 9.998886108398438, + "2": 14.684210777282715, + "3": 12.428792953491211, + "4": 12.565485000610352, + "5": 13.176959037780762, + "6": 11.885496139526367, + "7": 11.376730918884277, + "8": 11.167491912841797, + "9": 12.414295196533203, + "10": 8.093016624450684, + "11": 13.321942329406738, + "12": 10.130970001220703, + "13": 6.392433166503906, + "14": 10.154117584228516, + "15": 11.030630111694336, + "16": 9.916529655456543, + "17": 13.19386100769043, + "18": 9.735214233398438, + "19": 8.763481140136719, + "20": 11.289271354675293, + "21": 10.299274444580078, + "22": 11.002974510192871, + "23": 9.379912376403809 + }, + "size_gb": 4.54 + } +} diff --git a/eval/hps3_enc_ptq2.md b/eval/hps3_enc_ptq2.md new file mode 100644 index 0000000000000000000000000000000000000000..6d838e4ae115ed3b07f7bfe9f57b7c2b591d6d1c --- /dev/null +++ b/eval/hps3_enc_ptq2.md @@ -0,0 +1,12 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| enc_i4e8 | 4.6 | 11.09 [10.39, 11.77] | 12.60 | 11.37 | 9.67 | 11.10 | 10.53 | - | +| enc_l3 | 4.4 | 9.77 [9.24, 10.28] | 10.22 | 10.87 | 8.69 | 10.25 | 9.30 | - | +| enc_a3 | 4.54 | 11.02 [10.28, 11.75] | 12.49 | 11.71 | 9.48 | 10.59 | 10.45 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.33-1.87; bootstrap 95% half-width of a mean is 0.52-0.73. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.10-0.23 (median 0.22); a difference below about 2 x that (0.44 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_enc_w4a8_gptq.json b/eval/hps3_enc_w4a8_gptq.json new file mode 100644 index 0000000000000000000000000000000000000000..70a04d49bc1d5a7337a74be51f3c2a1d03d25a79 --- /dev/null +++ b/eval/hps3_enc_w4a8_gptq.json @@ -0,0 +1,45 @@ +{ + "enc_w4a8_gptq": { + "n": 24, + "mean_mu": 10.98950481414795, + "sd_mu": 1.8582404056006003, + "ci95": [ + 10.239382770657539, + 11.696771595875422 + ], + "category_mean_mu": { + "text": 12.021718660990397, + "portrait": 11.957259893417358, + "detail": 9.272223472595215, + "counting": 10.894657135009766, + "varied": 10.613819479942322 + }, + "mu_by_prompt": { + "0": 10.913015365600586, + "1": 9.044050216674805, + "2": 13.881145477294922, + "3": 13.00432300567627, + "4": 12.235599517822266, + "5": 13.052178382873535, + "6": 11.801921844482422, + "7": 12.187568664550781, + "8": 11.914678573608398, + "9": 11.924870491027832, + "10": 8.108549118041992, + "11": 12.995185852050781, + "12": 9.44250774383545, + "13": 6.542651176452637, + "14": 10.373090744018555, + "15": 11.416223526000977, + "16": 10.093405723571777, + "17": 13.795523643493652, + "18": 9.823501586914062, + "19": 8.755511283874512, + "20": 11.18978214263916, + "21": 10.255514144897461, + "22": 11.602023124694824, + "23": 9.395294189453125 + }, + "size_gb": null + } +} diff --git a/eval/hps3_nd_gptq.json b/eval/hps3_nd_gptq.json new file mode 100644 index 0000000000000000000000000000000000000000..253fe41df713aa55dbc58f8d8772e566cd1f1aca --- /dev/null +++ b/eval/hps3_nd_gptq.json @@ -0,0 +1,174 @@ +{ + "mixG_tuned": { + "n": 24, + "mean_mu": 12.139749030272165, + "sd_mu": 1.9395526891947201, + "ci95": [ + 11.361157366633414, + 12.872107101480166 + ], + "category_mean_mu": { + "text": 13.390686511993408, + "portrait": 12.87514042854309, + "detail": 10.46267569065094, + "counting": 11.710683822631836, + "varied": 11.779653191566467 + }, + "mu_by_prompt": { + "0": 13.417725563049316, + "1": 12.467721939086914, + "2": 13.885729789733887, + "3": 11.765070915222168, + "4": 14.611907958984375, + "5": 14.195962905883789, + "6": 12.4324951171875, + "7": 12.14983081817627, + "8": 11.897542953491211, + "9": 15.020692825317383, + "10": 12.184309005737305, + "11": 14.310564041137695, + "12": 7.555306434631348, + "13": 7.800523281097412, + "14": 11.379426002502441, + "15": 12.04194164276123, + "16": 10.702643394470215, + "17": 13.159246444702148, + "18": 10.658286094665527, + "19": 10.344663619995117, + "20": 11.928624153137207, + "21": 11.974713325500488, + "22": 10.689987182617188, + "23": 14.779061317443848 + }, + "size_gb": 2.14 + }, + "nd_mixG_tuned_gptq": { + "n": 24, + "mean_mu": 12.163939317067465, + "sd_mu": 1.8067306763097963, + "ci95": [ + 11.441241880257925, + 12.849267401297887 + ], + "category_mean_mu": { + "text": 13.353586037953695, + "portrait": 13.092275142669678, + "detail": 10.582758665084839, + "counting": 11.811241149902344, + "varied": 11.686301231384277 + }, + "mu_by_prompt": { + "0": 13.440675735473633, + "1": 11.957268714904785, + "2": 13.895618438720703, + "3": 12.55988883972168, + "4": 14.599224090576172, + "5": 13.668840408325195, + "6": 12.092365264892578, + "7": 13.142611503601074, + "8": 12.048057556152344, + "9": 15.086066246032715, + "10": 12.212583541870117, + "11": 14.199559211730957, + "12": 8.30803394317627, + "13": 7.610857963562012, + "14": 11.580402374267578, + "15": 12.04207992553711, + "16": 10.661609649658203, + "17": 12.946452140808105, + "18": 11.319280624389648, + "19": 10.453644752502441, + "20": 11.457819938659668, + "21": 12.063944816589355, + "22": 10.652288436889648, + "23": 13.935369491577148 + }, + "size_gb": null + }, + "mixH2_x_f8": { + "n": 24, + "mean_mu": 11.989253520965576, + "sd_mu": 1.5468661955186895, + "ci95": [ + 11.359737481673559, + 12.58515132466952 + ], + "category_mean_mu": { + "text": 12.905888080596924, + "portrait": 12.12691354751587, + "detail": 11.099243640899658, + "counting": 12.004260063171387, + "varied": 11.674200892448425 + }, + "mu_by_prompt": { + "0": 13.653209686279297, + "1": 11.297388076782227, + "2": 12.705009460449219, + "3": 11.617015838623047, + "4": 13.824302673339844, + "5": 14.33840274810791, + "6": 11.460399627685547, + "7": 12.014626502990723, + "8": 10.534360885620117, + "9": 14.49826717376709, + "10": 11.64794921875, + "11": 13.963139533996582, + "12": 11.265219688415527, + "13": 7.520666122436523, + "14": 11.920021057128906, + "15": 12.088499069213867, + "16": 11.032934188842773, + "17": 13.643148422241211, + "18": 11.172962188720703, + "19": 10.405658721923828, + "20": 11.057677268981934, + "21": 12.540094375610352, + "22": 11.082413673400879, + "23": 12.458718299865723 + }, + "size_gb": 1.85 + }, + "nd_mixH2_x_f8_gptq": { + "n": 24, + "mean_mu": 12.035317579905191, + "sd_mu": 1.5277030513915637, + "ci95": [ + 11.423713621497154, + 12.625486142436662 + ], + "category_mean_mu": { + "text": 12.712045351664225, + "portrait": 12.027806520462036, + "detail": 11.226006746292114, + "counting": 12.206240177154541, + "varied": 11.893452048301697 + }, + "mu_by_prompt": { + "0": 13.797195434570312, + "1": 11.58507251739502, + "2": 12.74012565612793, + "3": 11.010663986206055, + "4": 13.852313995361328, + "5": 13.286900520324707, + "6": 11.62837028503418, + "7": 11.955574035644531, + "8": 9.93774127960205, + "9": 14.589540481567383, + "10": 11.754337310791016, + "11": 14.261083602905273, + "12": 10.913103103637695, + "13": 7.975502967834473, + "14": 12.154131889343262, + "15": 12.25834846496582, + "16": 10.834696769714355, + "17": 13.356339454650879, + "18": 11.414605140686035, + "19": 10.90980339050293, + "20": 11.242483139038086, + "21": 12.390066146850586, + "22": 11.085796356201172, + "23": 13.913825988769531 + }, + "size_gb": null + } +} diff --git a/eval/hps3_nd_gptq.md b/eval/hps3_nd_gptq.md new file mode 100644 index 0000000000000000000000000000000000000000..8daf3127a158943ff2312ed35cea6227bcd75993 --- /dev/null +++ b/eval/hps3_nd_gptq.md @@ -0,0 +1,13 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| mixG_tuned | 2.14 | 12.14 [11.36, 12.87] | 13.39 | 12.88 | 10.46 | 11.71 | 11.78 | - | +| nd_mixG_tuned_gptq | - | 12.16 [11.44, 12.85] | 13.35 | 13.09 | 10.58 | 11.81 | 11.69 | - | +| mixH2_x_f8 | 1.85 | 11.99 [11.36, 12.59] | 12.91 | 12.13 | 11.10 | 12.00 | 11.67 | - | +| nd_mixH2_x_f8_gptq | - | 12.04 [11.42, 12.63] | 12.71 | 12.03 | 11.23 | 12.21 | 11.89 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.53-1.94; bootstrap 95% half-width of a mean is 0.60-0.76. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.09-0.22 (median 0.19); a difference below about 2 x that (0.38 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_summary.json b/eval/hps3_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..8d9f8a641c5b750119997d5f8debcfb81a23fb90 --- /dev/null +++ b/eval/hps3_summary.json @@ -0,0 +1,731 @@ +{ + "teacher_bf16": { + "n": 24, + "mean_mu": 12.384581685066223, + "sd_mu": 1.6123343047925531, + "ci95": [ + 11.755278043945632, + 13.006242449084917 + ], + "category_mean_mu": { + "text": 13.332724253336588, + "portrait": 13.072895526885986, + "detail": 12.40628981590271, + "counting": 10.933318138122559, + "varied": 11.68127965927124 + }, + "mu_by_prompt": { + "0": 13.639022827148438, + "1": 11.825362205505371, + "2": 12.995671272277832, + "3": 12.526433944702148, + "4": 14.530313491821289, + "5": 14.479541778564453, + "6": 12.07623291015625, + "7": 13.430919647216797, + "8": 12.091690063476562, + "9": 14.692739486694336, + "10": 13.62648868560791, + "11": 14.686071395874023, + "12": 12.613874435424805, + "13": 8.698724746704102, + "14": 9.724932670593262, + "15": 12.141703605651855, + "16": 10.176751136779785, + "17": 13.937211036682129, + "18": 10.837376594543457, + "19": 11.01986312866211, + "20": 11.772106170654297, + "21": 12.926685333251953, + "22": 11.148569107055664, + "23": 11.631674766540527 + }, + "size_gb": 10.16 + }, + "mixA_x_tuned": { + "n": 24, + "mean_mu": 11.610886931419373, + "sd_mu": 1.7820989888716794, + "ci95": [ + 10.906349510947862, + 12.311402688423792 + ], + "category_mean_mu": { + "text": 11.760952949523926, + "portrait": 13.03148365020752, + "detail": 10.76393747329712, + "counting": 11.648460388183594, + "varied": 11.202120423316956 + }, + "mu_by_prompt": { + "0": 14.09498405456543, + "1": 11.100715637207031, + "2": 11.812602996826172, + "3": 10.70802116394043, + "4": 12.798552513122559, + "5": 10.050841331481934, + "6": 12.894301414489746, + "7": 13.858095169067383, + "8": 10.505816459655762, + "9": 14.867721557617188, + "10": 11.039589881896973, + "11": 14.058591842651367, + "12": 10.705621719360352, + "13": 7.251946449279785, + "14": 11.508694648742676, + "15": 11.788226127624512, + "16": 10.323859214782715, + "17": 13.585147857666016, + "18": 10.774235725402832, + "19": 9.016969680786133, + "20": 11.208295822143555, + "21": 11.186261177062988, + "22": 10.375581741333008, + "23": 13.146612167358398 + }, + "diff_to_teacher": { + "mean": -0.7736947536468506, + "se": 0.28107680515481503, + "sd": 1.3769895023219707, + "wins": 7 + }, + "size_gb": 1.67 + }, + "mixF_ptq": { + "n": 24, + "mean_mu": 10.857280095418295, + "sd_mu": 1.816638553281807, + "ci95": [ + 10.164406632383663, + 11.594348962108294 + ], + "category_mean_mu": { + "text": 12.170512040456137, + "portrait": 11.56724238395691, + "detail": 10.674527406692505, + "counting": 10.113127708435059, + "varied": 9.794789433479309 + }, + "mu_by_prompt": { + "0": 12.231085777282715, + "1": 12.160717964172363, + "2": 13.37264347076416, + "3": 11.19559097290039, + "4": 11.993592262268066, + "5": 12.069441795349121, + "6": 10.759611129760742, + "7": 10.656207084655762, + "8": 9.303426742553711, + "9": 15.549724578857422, + "10": 12.295470237731934, + "11": 13.153759002685547, + "12": 10.473901748657227, + "13": 6.7749786376953125, + "14": 10.282621383666992, + "15": 9.943634033203125, + "16": 8.922804832458496, + "17": 10.780318260192871, + "18": 9.721498489379883, + "19": 8.549283027648926, + "20": 10.071290016174316, + "21": 10.076916694641113, + "22": 10.67863655090332, + "23": 9.557567596435547 + }, + "diff_to_teacher": { + "mean": -1.527301589647929, + "se": 0.23380791654539382, + "sd": 1.145420186718895, + "wins": 4 + }, + "size_gb": 2.42 + }, + "mixG_x": { + "n": 24, + "mean_mu": 12.08710265159607, + "sd_mu": 1.7183072395722299, + "ci95": [ + 11.39186682899793, + 12.736877583463986 + ], + "category_mean_mu": { + "text": 13.42167854309082, + "portrait": 12.39479398727417, + "detail": 10.94544506072998, + "counting": 11.734143733978271, + "varied": 11.59139358997345 + }, + "mu_by_prompt": { + "0": 13.307634353637695, + "1": 11.922536849975586, + "2": 13.955156326293945, + "3": 12.162612915039062, + "4": 15.23203182220459, + "5": 13.950098991394043, + "6": 11.93759536743164, + "7": 12.26282787322998, + "8": 11.281140327453613, + "9": 14.097612380981445, + "10": 11.678287506103516, + "11": 14.476302146911621, + "12": 10.909077644348145, + "13": 6.718112945556641, + "14": 11.539464950561523, + "15": 11.92882251739502, + "16": 11.382481575012207, + "17": 13.015539169311523, + "18": 11.165454864501953, + "19": 10.521598815917969, + "20": 11.441516876220703, + "21": 12.8370361328125, + "22": 10.439844131469727, + "23": 11.927677154541016 + }, + "diff_to_teacher": { + "mean": -0.2974790334701538, + "se": 0.18837994685499737, + "sd": 0.9228694951347124, + "wins": 7 + }, + "size_gb": 2.14 + }, + "mixG_tuned": { + "n": 24, + "mean_mu": 12.139749030272165, + "sd_mu": 1.9395526891947201, + "ci95": [ + 11.361157366633414, + 12.872107101480166 + ], + "category_mean_mu": { + "text": 13.390686511993408, + "portrait": 12.87514042854309, + "detail": 10.46267569065094, + "counting": 11.710683822631836, + "varied": 11.779653191566467 + }, + "mu_by_prompt": { + "0": 13.417725563049316, + "1": 12.467721939086914, + "2": 13.885729789733887, + "3": 11.765070915222168, + "4": 14.611907958984375, + "5": 14.195962905883789, + "6": 12.4324951171875, + "7": 12.14983081817627, + "8": 11.897542953491211, + "9": 15.020692825317383, + "10": 12.184309005737305, + "11": 14.310564041137695, + "12": 7.555306434631348, + "13": 7.800523281097412, + "14": 11.379426002502441, + "15": 12.04194164276123, + "16": 10.702643394470215, + "17": 13.159246444702148, + "18": 10.658286094665527, + "19": 10.344663619995117, + "20": 11.928624153137207, + "21": 11.974713325500488, + "22": 10.689987182617188, + "23": 14.779061317443848 + }, + "diff_to_teacher": { + "mean": -0.24483265479405722, + "se": 0.28922744505696935, + "sd": 1.416919319996863, + "wins": 9 + }, + "size_gb": 2.14 + }, + "mixH_x": { + "n": 24, + "mean_mu": 11.880460023880005, + "sd_mu": 1.6990035552645357, + "ci95": [ + 11.192399097482364, + 12.525516930222512 + ], + "category_mean_mu": { + "text": 13.041773160298666, + "portrait": 12.490415334701538, + "detail": 10.68679404258728, + "counting": 11.49379014968872, + "varied": 11.397997975349426 + }, + "mu_by_prompt": { + "0": 13.99026870727539, + "1": 10.050540924072266, + "2": 13.606485366821289, + "3": 11.855009078979492, + "4": 14.160813331604004, + "5": 14.58752155303955, + "6": 11.589570999145508, + "7": 13.778044700622559, + "8": 10.534186363220215, + "9": 14.059859275817871, + "10": 11.494105339050293, + "11": 13.14444351196289, + "12": 10.829146385192871, + "13": 7.279480934143066, + "14": 11.513114929199219, + "15": 11.474465370178223, + "16": 10.609831809997559, + "17": 12.428728103637695, + "18": 10.733562469482422, + "19": 10.845394134521484, + "20": 10.504751205444336, + "21": 11.515116691589355, + "22": 11.304014205932617, + "23": 13.242585182189941 + }, + "diff_to_teacher": { + "mean": -0.5041216611862183, + "se": 0.2162678089390539, + "sd": 1.0594915593808092, + "wins": 8 + }, + "size_gb": 1.95 + }, + "mixH_tuned": { + "n": 24, + "mean_mu": 11.699441293875376, + "sd_mu": 2.029649702715248, + "ci95": [ + 10.869073146084943, + 12.448687643309436 + ], + "category_mean_mu": { + "text": 12.943698406219482, + "portrait": 12.36544394493103, + "detail": 10.056230425834656, + "counting": 11.240527629852295, + "varied": 11.3695809841156 + }, + "mu_by_prompt": { + "0": 14.136148452758789, + "1": 11.969319343566895, + "2": 12.627728462219238, + "3": 11.427092552185059, + "4": 14.123497009277344, + "5": 13.37840461730957, + "6": 12.329015731811523, + "7": 11.961216926574707, + "8": 10.635523796081543, + "9": 14.536019325256348, + "10": 12.294113159179688, + "11": 14.582603454589844, + "12": 6.157026290893555, + "13": 7.191178798675537, + "14": 10.864974975585938, + "15": 11.616080284118652, + "16": 10.46957778930664, + "17": 12.961014747619629, + "18": 10.876848220825195, + "19": 10.018270492553711, + "20": 11.60196304321289, + "21": 11.650176048278809, + "22": 10.586633682250977, + "23": 12.792163848876953 + }, + "diff_to_teacher": { + "mean": -0.6851403911908468, + "se": 0.2972779784018067, + "sd": 1.4563587177010893, + "wins": 7 + }, + "size_gb": 1.95 + }, + "mixH2_x": { + "n": 24, + "mean_mu": 11.917078256607056, + "sd_mu": 1.4976344977986062, + "ci95": [ + 11.3171836266915, + 12.492527563373248 + ], + "category_mean_mu": { + "text": 12.793494701385498, + "portrait": 12.09325361251831, + "detail": 10.962145805358887, + "counting": 11.970977306365967, + "varied": 11.635669708251953 + }, + "mu_by_prompt": { + "0": 13.581269264221191, + "1": 11.102155685424805, + "2": 12.775008201599121, + "3": 11.850469589233398, + "4": 13.954010963439941, + "5": 13.498054504394531, + "6": 11.450055122375488, + "7": 12.061519622802734, + "8": 10.337630271911621, + "9": 14.523809432983398, + "10": 11.705960273742676, + "11": 13.739737510681152, + "12": 10.581864356994629, + "13": 7.82102108001709, + "14": 11.763824462890625, + "15": 12.178130149841309, + "16": 11.282197952270508, + "17": 13.614313125610352, + "18": 11.304281234741211, + "19": 10.406351089477539, + "20": 10.849934577941895, + "21": 12.486143112182617, + "22": 10.684165000915527, + "23": 12.457971572875977 + }, + "diff_to_teacher": { + "mean": -0.4675034284591675, + "se": 0.19015638558118367, + "sd": 0.9315722320116647, + "wins": 5 + }, + "size_gb": 1.95 + }, + "mixH2_tuned": { + "n": 24, + "mean_mu": 11.895683626333872, + "sd_mu": 1.7010001135124344, + "ci95": [ + 11.21190027644237, + 12.53292619784673 + ], + "category_mean_mu": { + "text": 12.790374755859375, + "portrait": 12.12794017791748, + "detail": 10.82534396648407, + "counting": 11.896456718444824, + "varied": 11.643513560295105 + }, + "mu_by_prompt": { + "0": 13.311539649963379, + "1": 11.351049423217773, + "2": 12.961719512939453, + "3": 11.900381088256836, + "4": 13.324409484863281, + "5": 13.893149375915527, + "6": 12.187474250793457, + "7": 11.224508285522461, + "8": 10.200164794921875, + "9": 14.899613380432129, + "10": 11.416027069091797, + "11": 14.799453735351562, + "12": 10.560506820678711, + "13": 6.525388240814209, + "14": 11.732919692993164, + "15": 12.059993743896484, + "16": 10.965230941772461, + "17": 12.68959903717041, + "18": 11.342528343200684, + "19": 10.494524955749512, + "20": 11.686250686645508, + "21": 11.768655776977539, + "22": 11.264636039733887, + "23": 12.93668270111084 + }, + "diff_to_teacher": { + "mean": -0.48889805873235065, + "se": 0.22897005628476946, + "sd": 1.1217196085480594, + "wins": 8 + }, + "size_gb": 1.95 + }, + "mixH2_xm": { + "n": 24, + "mean_mu": 11.487676699956259, + "sd_mu": 1.6185009544025424, + "ci95": [ + 10.831634158889452, + 12.11318797369798 + ], + "category_mean_mu": { + "text": 12.731455167134603, + "portrait": 12.26175832748413, + "detail": 9.958102703094482, + "counting": 11.286506175994873, + "varied": 10.982881665229797 + }, + "mu_by_prompt": { + "0": 12.68850326538086, + "1": 11.272262573242188, + "2": 13.997017860412598, + "3": 11.66956901550293, + "4": 12.465785026550293, + "5": 14.29559326171875, + "6": 11.829322814941406, + "7": 12.416139602661133, + "8": 11.078559875488281, + "9": 13.723011016845703, + "10": 10.847691535949707, + "11": 12.856647491455078, + "12": 8.917652130126953, + "13": 7.210419654846191, + "14": 10.262415885925293, + "15": 12.310596466064453, + "16": 9.655908584594727, + "17": 12.390963554382324, + "18": 11.141194343566895, + "19": 10.1093111038208, + "20": 10.608089447021484, + "21": 11.77521800994873, + "22": 11.69253158569336, + "23": 10.489836692810059 + }, + "diff_to_teacher": { + "mean": -0.896904985109965, + "se": 0.2175312633346519, + "sd": 1.0656811965457924, + "wins": 5 + }, + "size_gb": 1.83 + }, + "mixH2_xm_f8": { + "n": 24, + "mean_mu": 11.442908763885498, + "sd_mu": 1.68771424350057, + "ci95": [ + 10.76411321957906, + 12.09367621441682 + ], + "category_mean_mu": { + "text": 12.668961842854818, + "portrait": 12.236000776290894, + "detail": 10.113464832305908, + "counting": 10.981910705566406, + "varied": 10.906794428825378 + }, + "mu_by_prompt": { + "0": 12.688547134399414, + "1": 10.991280555725098, + "2": 13.278096199035645, + "3": 11.797897338867188, + "4": 12.758256912231445, + "5": 14.499692916870117, + "6": 11.637022972106934, + "7": 12.572162628173828, + "8": 10.822105407714844, + "9": 13.912712097167969, + "10": 11.10090446472168, + "11": 13.388298034667969, + "12": 8.770405769348145, + "13": 7.19425106048584, + "14": 9.96194076538086, + "15": 12.001880645751953, + "16": 9.986132621765137, + "17": 12.722164154052734, + "18": 11.125619888305664, + "19": 10.118558883666992, + "20": 9.693978309631348, + "21": 11.55894660949707, + "22": 11.63875675201416, + "23": 10.410198211669922 + }, + "diff_to_teacher": { + "mean": -0.9416729211807251, + "se": 0.2028905801931563, + "sd": 0.9939567901809284, + "wins": 5 + }, + "size_gb": 1.72 + }, + "mixH2_x_f8": { + "n": 24, + "mean_mu": 11.989253520965576, + "sd_mu": 1.5468661955186895, + "ci95": [ + 11.359737481673559, + 12.58515132466952 + ], + "category_mean_mu": { + "text": 12.905888080596924, + "portrait": 12.12691354751587, + "detail": 11.099243640899658, + "counting": 12.004260063171387, + "varied": 11.674200892448425 + }, + "mu_by_prompt": { + "0": 13.653209686279297, + "1": 11.297388076782227, + "2": 12.705009460449219, + "3": 11.617015838623047, + "4": 13.824302673339844, + "5": 14.33840274810791, + "6": 11.460399627685547, + "7": 12.014626502990723, + "8": 10.534360885620117, + "9": 14.49826717376709, + "10": 11.64794921875, + "11": 13.963139533996582, + "12": 11.265219688415527, + "13": 7.520666122436523, + "14": 11.920021057128906, + "15": 12.088499069213867, + "16": 11.032934188842773, + "17": 13.643148422241211, + "18": 11.172962188720703, + "19": 10.405658721923828, + "20": 11.057677268981934, + "21": 12.540094375610352, + "22": 11.082413673400879, + "23": 12.458718299865723 + }, + "diff_to_teacher": { + "mean": -0.395328164100647, + "se": 0.1802226438840783, + "sd": 0.8829070352226305, + "wins": 5 + }, + "size_gb": 1.85 + }, + "mixI_x": { + "n": 24, + "mean_mu": 9.655051132043203, + "sd_mu": 1.679698239139186, + "ci95": [ + 8.966416811943056, + 10.272336814800898 + ], + "category_mean_mu": { + "text": 10.08133602142334, + "portrait": 10.162810564041138, + "detail": 8.78571367263794, + "counting": 10.202799320220947, + "varied": 9.379189431667328 + }, + "mu_by_prompt": { + "0": 10.26925277709961, + "1": 6.766984939575195, + "2": 10.589472770690918, + "3": 9.530107498168945, + "4": 12.589543342590332, + "5": 10.742654800415039, + "6": 10.981014251708984, + "7": 9.425098419189453, + "8": 9.351439476013184, + "9": 10.89369010925293, + "10": 11.534799575805664, + "11": 10.136062622070312, + "12": 8.623100280761719, + "13": 4.8488922119140625, + "14": 10.231926918029785, + "15": 10.17367172241211, + "16": 9.301717758178711, + "17": 11.269822120666504, + "18": 7.489152431488037, + "19": 8.805434226989746, + "20": 8.242559432983398, + "21": 8.819071769714355, + "22": 9.821023941040039, + "23": 11.284733772277832 + }, + "diff_to_teacher": { + "mean": -2.7295305530230203, + "se": 0.28547024864832266, + "sd": 1.3985128918676595, + "wins": 1 + }, + "size_gb": 1.74 + }, + "mixI_tuned": { + "n": 24, + "mean_mu": 10.650842626889547, + "sd_mu": 1.779264514185643, + "ci95": [ + 9.94127772450447, + 11.3256166746219 + ], + "category_mean_mu": { + "text": 11.08804210027059, + "portrait": 11.977301120758057, + "detail": 9.071760654449463, + "counting": 10.985382080078125, + "varied": 10.365619897842407 + }, + "mu_by_prompt": { + "0": 12.68350601196289, + "1": 8.426652908325195, + "2": 11.992571830749512, + "3": 9.905455589294434, + "4": 11.383698463439941, + "5": 12.136367797851562, + "6": 11.28021240234375, + "7": 12.629135131835938, + "8": 10.298370361328125, + "9": 13.701486587524414, + "10": 9.960355758666992, + "11": 12.58846664428711, + "12": 5.817440032958984, + "13": 7.920780181884766, + "14": 9.724404335021973, + "15": 12.246359825134277, + "16": 9.888011932373047, + "17": 11.552902221679688, + "18": 10.344181060791016, + "19": 8.561613082885742, + "20": 10.274347305297852, + "21": 10.694239616394043, + "22": 10.358518600463867, + "23": 11.251145362854004 + }, + "diff_to_teacher": { + "mean": -1.7337390581766765, + "se": 0.31209043163446204, + "sd": 1.5289246222187787, + "wins": 1 + }, + "size_gb": 1.74 + }, + "tern_v1": { + "n": 24, + "mean_mu": -8.961636026700338, + "sd_mu": 1.341661786296067, + "ci95": [ + -9.442805964748064, + -8.39396297633648 + ], + "category_mean_mu": { + "text": -8.419553756713867, + "portrait": -9.220449924468994, + "detail": -9.765491724014282, + "counting": -9.634195804595947, + "varied": -8.668722987174988 + }, + "mu_by_prompt": { + "0": -9.186239242553711, + "1": -5.7475481033325195, + "2": -7.011780738830566, + "3": -8.10580825805664, + "4": -10.226703643798828, + "5": -10.239242553710938, + "6": -9.475618362426758, + "7": -8.87272834777832, + "8": -8.483269691467285, + "9": -10.050183296203613, + "10": -10.196704864501953, + "11": -8.95408821105957, + "12": -10.226978302001953, + "13": -9.684195518493652, + "14": -9.198389053344727, + "15": -10.070002555847168, + "16": -9.747382164001465, + "17": -9.68046760559082, + "18": -5.4149274826049805, + "19": -9.003195762634277, + "20": -9.51219654083252, + "21": -7.509844779968262, + "22": -9.660802841186523, + "23": -8.820966720581055 + }, + "diff_to_teacher": { + "mean": -21.34621771176656, + "se": 0.46752777685313823, + "sd": 2.2904089877359692, + "wins": 0 + }, + "size_gb": 1.92 + } +} diff --git a/eval/hps3_summary.md b/eval/hps3_summary.md new file mode 100644 index 0000000000000000000000000000000000000000..db25e073d22340f6cea534ae3da497a3b3c229c5 --- /dev/null +++ b/eval/hps3_summary.md @@ -0,0 +1,24 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| teacher_bf16 | 10.16 | 12.38 [11.76, 13.01] | 13.33 | 13.07 | 12.41 | 10.93 | 11.68 | - | +| mixA_x_tuned | 1.67 | 11.61 [10.91, 12.31] | 11.76 | 13.03 | 10.76 | 11.65 | 11.20 | -0.77 +- 0.28 (7/24) | +| mixF_ptq | 2.42 | 10.86 [10.16, 11.59] | 12.17 | 11.57 | 10.67 | 10.11 | 9.79 | -1.53 +- 0.23 (4/24) | +| mixG_x | 2.14 | 12.09 [11.39, 12.74] | 13.42 | 12.39 | 10.95 | 11.73 | 11.59 | -0.30 +- 0.19 (7/24) | +| mixG_tuned | 2.14 | 12.14 [11.36, 12.87] | 13.39 | 12.88 | 10.46 | 11.71 | 11.78 | -0.24 +- 0.29 (9/24) | +| mixH_x | 1.95 | 11.88 [11.19, 12.53] | 13.04 | 12.49 | 10.69 | 11.49 | 11.40 | -0.50 +- 0.22 (8/24) | +| mixH_tuned | 1.95 | 11.70 [10.87, 12.45] | 12.94 | 12.37 | 10.06 | 11.24 | 11.37 | -0.69 +- 0.30 (7/24) | +| mixH2_x | 1.95 | 11.92 [11.32, 12.49] | 12.79 | 12.09 | 10.96 | 11.97 | 11.64 | -0.47 +- 0.19 (5/24) | +| mixH2_tuned | 1.95 | 11.90 [11.21, 12.53] | 12.79 | 12.13 | 10.83 | 11.90 | 11.64 | -0.49 +- 0.23 (8/24) | +| mixH2_xm | 1.83 | 11.49 [10.83, 12.11] | 12.73 | 12.26 | 9.96 | 11.29 | 10.98 | -0.90 +- 0.22 (5/24) | +| mixH2_xm_f8 | 1.72 | 11.44 [10.76, 12.09] | 12.67 | 12.24 | 10.11 | 10.98 | 10.91 | -0.94 +- 0.20 (5/24) | +| mixH2_x_f8 | 1.85 | 11.99 [11.36, 12.59] | 12.91 | 12.13 | 11.10 | 12.00 | 11.67 | -0.40 +- 0.18 (5/24) | +| mixI_x | 1.74 | 9.66 [8.97, 10.27] | 10.08 | 10.16 | 8.79 | 10.20 | 9.38 | -2.73 +- 0.29 (1/24) | +| mixI_tuned | 1.74 | 10.65 [9.94, 11.33] | 11.09 | 11.98 | 9.07 | 10.99 | 10.37 | -1.73 +- 0.31 (1/24) | +| tern_v1 | 1.92 | -8.96 [-9.44, -8.39] | -8.42 | -9.22 | -9.77 | -9.63 | -8.67 | -21.35 +- 0.47 (0/24) | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.34-2.03; bootstrap 95% half-width of a mean is 0.52-0.79. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.06-0.34 (median 0.23); a difference below about 2 x that (0.46 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_tmixG.json b/eval/hps3_tmixG.json new file mode 100644 index 0000000000000000000000000000000000000000..7e2557ce78278240187198eb3455d1dfa0d0d8d0 --- /dev/null +++ b/eval/hps3_tmixG.json @@ -0,0 +1,88 @@ +{ + "tmixG_x_1024": { + "n": 24, + "mean_mu": 11.093161900838217, + "sd_mu": 1.7534528688621807, + "ci95": [ + 10.383457724253336, + 11.765245208144187 + ], + "category_mean_mu": { + "text": 12.275560855865479, + "portrait": 12.011467218399048, + "detail": 9.47965931892395, + "counting": 10.9927339553833, + "varied": 10.579068303108215 + }, + "mu_by_prompt": { + "0": 12.029065132141113, + "1": 10.247049331665039, + "2": 13.635560035705566, + "3": 12.875860214233398, + "4": 11.864503860473633, + "5": 13.001326560974121, + "6": 12.741265296936035, + "7": 11.759806632995605, + "8": 11.334644317626953, + "9": 12.210152626037598, + "10": 8.151248931884766, + "11": 13.089639663696289, + "12": 9.873390197753906, + "13": 6.80435848236084, + "14": 10.72061538696289, + "15": 11.264852523803711, + "16": 10.157102584838867, + "17": 13.925820350646973, + "18": 9.778844833374023, + "19": 9.371624946594238, + "20": 10.581709861755371, + "21": 10.420363426208496, + "22": 11.24316120147705, + "23": 9.153919219970703 + }, + "size_gb": null + }, + "tmixG_x_768": { + "n": 24, + "mean_mu": 11.060355603694916, + "sd_mu": 1.8899038827082177, + "ci95": [ + 10.284124094247817, + 11.7712279672424 + ], + "category_mean_mu": { + "text": 12.452021280924479, + "portrait": 12.091742038726807, + "detail": 9.59649384021759, + "counting": 10.613999366760254, + "varied": 10.344433069229126 + }, + "mu_by_prompt": { + "0": 12.194013595581055, + "1": 11.119743347167969, + "2": 14.199522972106934, + "3": 12.216362953186035, + "4": 12.408049583435059, + "5": 12.574435234069824, + "6": 12.863862991333008, + "7": 13.082260131835938, + "8": 10.617880821228027, + "9": 11.802964210510254, + "10": 10.203123092651367, + "11": 13.07459545135498, + "12": 9.950275421142578, + "13": 5.1579813957214355, + "14": 9.728994369506836, + "15": 11.499004364013672, + "16": 10.289899826049805, + "17": 12.423584938049316, + "18": 10.51021957397461, + "19": 8.268671035766602, + "20": 10.916498184204102, + "21": 11.067235946655273, + "22": 9.782889366149902, + "23": 9.496465682983398 + }, + "size_gb": null + } +} diff --git a/eval/hps3_tmixG.md b/eval/hps3_tmixG.md new file mode 100644 index 0000000000000000000000000000000000000000..ee0b2b20e0414e770b8bf5d5e77f908e167b2877 --- /dev/null +++ b/eval/hps3_tmixG.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| tmixG_x_1024 | - | 11.09 [10.38, 11.77] | 12.28 | 12.01 | 9.48 | 10.99 | 10.58 | - | +| tmixG_x_768 | - | 11.06 [10.28, 11.77] | 12.45 | 12.09 | 9.60 | 10.61 | 10.34 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.75-1.89; bootstrap 95% half-width of a mean is 0.69-0.74. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.19-0.19 (median 0.19); a difference below about 2 x that (0.37 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_tmixG_traj.json b/eval/hps3_tmixG_traj.json new file mode 100644 index 0000000000000000000000000000000000000000..e1ad1dcf41061ae5ef01ed4dde389b74616d6c4e --- /dev/null +++ b/eval/hps3_tmixG_traj.json @@ -0,0 +1,88 @@ +{ + "tmixG_traj_1024": { + "n": 24, + "mean_mu": 4.709925166020791, + "sd_mu": 3.983988371755315, + "ci95": [ + 3.075492799654603, + 6.241217858375361 + ], + "category_mean_mu": { + "text": 4.4555767228206, + "portrait": 6.16073477268219, + "detail": 2.9998376071453094, + "counting": 6.154835224151611, + "varied": 4.66909795999527 + }, + "mu_by_prompt": { + "0": 0.9237642288208008, + "1": 1.3860772848129272, + "2": 11.350065231323242, + "3": 7.726625919342041, + "4": 0.4157429039478302, + "5": 4.931184768676758, + "6": 6.6820149421691895, + "7": 5.373495578765869, + "8": 6.486237049102783, + "9": 6.101191520690918, + "10": -5.721309185028076, + "11": 10.541007995605469, + "12": 6.0127854347229, + "13": 1.1668661832809448, + "14": 6.27398681640625, + "15": 6.035683631896973, + "16": 8.108414649963379, + "17": 9.131568908691406, + "18": -1.9423153400421143, + "19": 6.569377899169922, + "20": 6.460616588592529, + "21": 5.1261210441589355, + "22": 2.049818277359009, + "23": 1.8491816520690918 + }, + "size_gb": null + }, + "tmixG_traj_768": { + "n": 24, + "mean_mu": 4.852243738869826, + "sd_mu": 3.477296949103987, + "ci95": [ + 3.448883457357685, + 6.230218659092982 + ], + "category_mean_mu": { + "text": 4.018895824750264, + "portrait": 6.62620997428894, + "detail": 4.375384375452995, + "counting": 5.518620014190674, + "varied": 4.662107169628143 + }, + "mu_by_prompt": { + "0": 4.189118385314941, + "1": 1.9079973697662354, + "2": 9.796252250671387, + "3": 6.590702056884766, + "4": -1.5507299900054932, + "5": 3.180034875869751, + "6": 6.0802693367004395, + "7": 8.471721649169922, + "8": 5.010310173034668, + "9": 6.942538738250732, + "10": 0.11058086156845093, + "11": 10.911245346069336, + "12": 5.121137619018555, + "13": 1.3585736751556396, + "14": 4.785759925842285, + "15": 6.2514801025390625, + "16": 8.555427551269531, + "17": 8.209768295288086, + "18": 0.994794487953186, + "19": 6.622662544250488, + "20": 6.6200151443481445, + "21": 6.726078033447266, + "22": 1.2486697435379028, + "23": -1.680558443069458 + }, + "size_gb": null + } +} diff --git a/eval/hps3_tmixG_traj.md b/eval/hps3_tmixG_traj.md new file mode 100644 index 0000000000000000000000000000000000000000..0acc7022bfbb387c5ef5deb6344b270cb387fffe --- /dev/null +++ b/eval/hps3_tmixG_traj.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| tmixG_traj_1024 | - | 4.71 [3.08, 6.24] | 4.46 | 6.16 | 3.00 | 6.15 | 4.67 | - | +| tmixG_traj_768 | - | 4.85 [3.45, 6.23] | 4.02 | 6.63 | 4.38 | 5.52 | 4.66 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 3.48-3.98; bootstrap 95% half-width of a mean is 1.39-1.58. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.42-0.42 (median 0.42); a difference below about 2 x that (0.84 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_tmixG_tuned.json b/eval/hps3_tmixG_tuned.json new file mode 100644 index 0000000000000000000000000000000000000000..e7c9961e9a8f6e846774b526adb98b5e7f2e33fc --- /dev/null +++ b/eval/hps3_tmixG_tuned.json @@ -0,0 +1,88 @@ +{ + "tmixG_tuned_1024": { + "n": 24, + "mean_mu": 10.652918815612793, + "sd_mu": 2.184634115247501, + "ci95": [ + 9.773516083757084, + 11.474333416422208 + ], + "category_mean_mu": { + "text": 12.478143215179443, + "portrait": 11.159794330596924, + "detail": 8.479046821594238, + "counting": 10.683469772338867, + "varied": 10.109861016273499 + }, + "mu_by_prompt": { + "0": 12.857460021972656, + "1": 9.87504768371582, + "2": 13.805025100708008, + "3": 12.697933197021484, + "4": 12.118104934692383, + "5": 13.515288352966309, + "6": 10.928951263427734, + "7": 9.354809761047363, + "8": 11.5258150100708, + "9": 12.829601287841797, + "10": 6.571925163269043, + "11": 12.82712173461914, + "12": 9.568309783935547, + "13": 4.948830604553223, + "14": 9.851425170898438, + "15": 11.515514373779297, + "16": 10.392324447631836, + "17": 12.931724548339844, + "18": 9.329937934875488, + "19": 8.547661781311035, + "20": 11.10020637512207, + "21": 9.360323905944824, + "22": 10.160114288330078, + "23": 9.056594848632812 + }, + "size_gb": null + }, + "tmixG_tuned_768": { + "n": 24, + "mean_mu": 11.227271378040314, + "sd_mu": 1.773840908714257, + "ci95": [ + 10.51942848712206, + 11.91141223659118 + ], + "category_mean_mu": { + "text": 12.540873368581137, + "portrait": 12.05748462677002, + "detail": 10.594995856285095, + "counting": 10.761388301849365, + "varied": 10.25957179069519 + }, + "mu_by_prompt": { + "0": 12.522515296936035, + "1": 11.514143943786621, + "2": 14.11611270904541, + "3": 11.90596866607666, + "4": 12.564529418945312, + "5": 12.621970176696777, + "6": 10.930744171142578, + "7": 13.707110404968262, + "8": 11.02267074584961, + "9": 12.569413185119629, + "10": 10.090007781982422, + "11": 13.923852920532227, + "12": 11.461185455322266, + "13": 6.904937267303467, + "14": 10.090411186218262, + "15": 11.432365417480469, + "16": 10.16760540008545, + "17": 12.410406112670898, + "18": 10.793304443359375, + "19": 8.491320610046387, + "20": 11.738554000854492, + "21": 10.25912857055664, + "22": 8.409496307373047, + "23": 9.806758880615234 + }, + "size_gb": null + } +} diff --git a/eval/hps3_tmixG_tuned.md b/eval/hps3_tmixG_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..4f7345a47bdc731a4c4aeb92e64f7c64d09a7cfb --- /dev/null +++ b/eval/hps3_tmixG_tuned.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| tmixG_tuned_1024 | - | 10.65 [9.77, 11.47] | 12.48 | 11.16 | 8.48 | 10.68 | 10.11 | - | +| tmixG_tuned_768 | - | 11.23 [10.52, 11.91] | 12.54 | 12.06 | 10.59 | 10.76 | 10.26 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.77-2.18; bootstrap 95% half-width of a mean is 0.70-0.85. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.28-0.28 (median 0.28); a difference below about 2 x that (0.57 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_tmixH.json b/eval/hps3_tmixH.json new file mode 100644 index 0000000000000000000000000000000000000000..2e2841bfa153b54128684337849ccdcb9a95d25e --- /dev/null +++ b/eval/hps3_tmixH.json @@ -0,0 +1,174 @@ +{ + "tmixH_x_f8_1024": { + "n": 24, + "mean_mu": 10.131939927736918, + "sd_mu": 2.0625337601098073, + "ci95": [ + 9.292234958211582, + 10.923493075370788 + ], + "category_mean_mu": { + "text": 11.367913405100504, + "portrait": 11.457662343978882, + "detail": 7.8921799659729, + "counting": 10.364721298217773, + "varied": 9.603783249855042 + }, + "mu_by_prompt": { + "0": 10.626686096191406, + "1": 8.482626914978027, + "2": 14.132658958435059, + "3": 11.576177597045898, + "4": 10.897163391113281, + "5": 12.492167472839355, + "6": 10.915449142456055, + "7": 11.73303508758545, + "8": 11.375720024108887, + "9": 11.806445121765137, + "10": 5.906720161437988, + "11": 12.130553245544434, + "12": 8.532204627990723, + "13": 4.999241828918457, + "14": 9.31460952758789, + "15": 11.414833068847656, + "16": 9.578956604003906, + "17": 11.979984283447266, + "18": 9.499679565429688, + "19": 9.205005645751953, + "20": 9.6007719039917, + "21": 9.092829704284668, + "22": 9.583581924438477, + "23": 8.289456367492676 + }, + "size_gb": null + }, + "tmixH_x_f8_768": { + "n": 24, + "mean_mu": 10.135649581750235, + "sd_mu": 2.005867692361933, + "ci95": [ + 9.334704489509265, + 10.896111269295215 + ], + "category_mean_mu": { + "text": 11.428105354309082, + "portrait": 11.535491943359375, + "detail": 8.237787008285522, + "counting": 9.67943811416626, + "varied": 9.529370725154877 + }, + "mu_by_prompt": { + "0": 10.048906326293945, + "1": 10.509834289550781, + "2": 13.993431091308594, + "3": 10.891467094421387, + "4": 11.197443008422852, + "5": 11.927550315856934, + "6": 11.30863094329834, + "7": 11.40882682800293, + "8": 11.020305633544922, + "9": 12.404204368591309, + "10": 8.035388946533203, + "11": 11.674660682678223, + "12": 9.01162338256836, + "13": 4.229475021362305, + "14": 8.26537036895752, + "15": 11.093505859375, + "16": 9.869827270507812, + "17": 11.857256889343262, + "18": 8.995159149169922, + "19": 7.6623969078063965, + "20": 10.883493423461914, + "21": 9.832741737365723, + "22": 8.173702239990234, + "23": 8.96038818359375 + }, + "size_gb": null + }, + "tmixH_xm_f8_1024": { + "n": 24, + "mean_mu": 9.8137975136439, + "sd_mu": 1.870414495837523, + "ci95": [ + 9.072772541642188, + 10.562509364386399 + ], + "category_mean_mu": { + "text": 11.446102460225424, + "portrait": 10.730872869491577, + "detail": 7.68708336353302, + "counting": 9.740857124328613, + "varied": 9.212623298168182 + }, + "mu_by_prompt": { + "0": 10.68917465209961, + "1": 9.783720016479492, + "2": 14.103422164916992, + "3": 11.092169761657715, + "4": 10.242474555969238, + "5": 12.765653610229492, + "6": 10.40516471862793, + "7": 10.711195945739746, + "8": 10.342735290527344, + "9": 11.464395523071289, + "10": 7.277338027954102, + "11": 11.156170845031738, + "12": 6.727646350860596, + "13": 5.5871782302856445, + "14": 9.190208435058594, + "15": 10.291505813598633, + "16": 9.026238441467285, + "17": 10.811176300048828, + "18": 8.906768798828125, + "19": 7.4107136726379395, + "20": 8.796531677246094, + "21": 9.19455623626709, + "22": 10.38198184967041, + "23": 9.173019409179688 + }, + "size_gb": null + }, + "tmixH_xm_f8_768": { + "n": 24, + "mean_mu": 9.748612900575003, + "sd_mu": 2.1359685267381363, + "ci95": [ + 8.882039352258046, + 10.570001577834288 + ], + "category_mean_mu": { + "text": 11.439074516296387, + "portrait": 11.07733678817749, + "detail": 6.97668719291687, + "counting": 9.637810707092285, + "varied": 9.230068147182465 + }, + "mu_by_prompt": { + "0": 11.136442184448242, + "1": 10.389738082885742, + "2": 13.709424018859863, + "3": 10.463499069213867, + "4": 9.689776420593262, + "5": 13.245567321777344, + "6": 11.690040588378906, + "7": 11.433954238891602, + "8": 9.839062690734863, + "9": 11.34628963470459, + "10": 7.6889119148254395, + "11": 10.638336181640625, + "12": 5.348882675170898, + "13": 4.230618000030518, + "14": 8.386436462402344, + "15": 10.889184951782227, + "16": 8.703730583190918, + "17": 10.428938865661621, + "18": 9.208730697631836, + "19": 7.690370082855225, + "20": 9.681374549865723, + "21": 8.804404258728027, + "22": 9.608067512512207, + "23": 9.71492862701416 + }, + "size_gb": null + } +} diff --git a/eval/hps3_tmixH.md b/eval/hps3_tmixH.md new file mode 100644 index 0000000000000000000000000000000000000000..08e4c8c3676ff8c40bc6ba4e930347a7fdd8e91e --- /dev/null +++ b/eval/hps3_tmixH.md @@ -0,0 +1,13 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| tmixH_x_f8_1024 | - | 10.13 [9.29, 10.92] | 11.37 | 11.46 | 7.89 | 10.36 | 9.60 | - | +| tmixH_x_f8_768 | - | 10.14 [9.33, 10.90] | 11.43 | 11.54 | 8.24 | 9.68 | 9.53 | - | +| tmixH_xm_f8_1024 | - | 9.81 [9.07, 10.56] | 11.45 | 10.73 | 7.69 | 9.74 | 9.21 | - | +| tmixH_xm_f8_768 | - | 9.75 [8.88, 10.57] | 11.44 | 11.08 | 6.98 | 9.64 | 9.23 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.87-2.14; bootstrap 95% half-width of a mean is 0.74-0.84. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.14-0.24 (median 0.20); a difference below about 2 x that (0.40 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_turbo.json b/eval/hps3_turbo.json new file mode 100644 index 0000000000000000000000000000000000000000..aad6e81ec0d85c3083b4c49f9bb884643361b691 --- /dev/null +++ b/eval/hps3_turbo.json @@ -0,0 +1,88 @@ +{ + "turbo4_768": { + "n": 24, + "mean_mu": 11.934448599815369, + "sd_mu": 1.7504683247281883, + "ci95": [ + 11.23799334168434, + 12.610023081302643 + ], + "category_mean_mu": { + "text": 13.463109175364176, + "portrait": 12.022218227386475, + "detail": 10.613495826721191, + "counting": 11.166980743408203, + "varied": 11.59641170501709 + }, + "mu_by_prompt": { + "0": 12.864891052246094, + "1": 12.010098457336426, + "2": 14.524601936340332, + "3": 12.253418922424316, + "4": 14.202826499938965, + "5": 14.922818183898926, + "6": 11.319442749023438, + "7": 12.523279190063477, + "8": 10.392143249511719, + "9": 13.854007720947266, + "10": 10.229938507080078, + "11": 13.917720794677734, + "12": 11.214136123657227, + "13": 7.092187881469727, + "14": 10.747130393981934, + "15": 11.586831092834473, + "16": 12.07064437866211, + "17": 13.272892951965332, + "18": 11.954444885253906, + "19": 9.88763427734375, + "20": 12.504342079162598, + "21": 11.750227928161621, + "22": 10.750712394714355, + "23": 10.580394744873047 + }, + "size_gb": null + }, + "turbo4_1024": { + "n": 24, + "mean_mu": 11.926482478777567, + "sd_mu": 1.687956290993407, + "ci95": [ + 11.244729457298915, + 12.581498128175735 + ], + "category_mean_mu": { + "text": 13.278272787729898, + "portrait": 12.592132091522217, + "detail": 10.429693698883057, + "counting": 11.500927925109863, + "varied": 11.434597969055176 + }, + "mu_by_prompt": { + "0": 13.271171569824219, + "1": 11.38896369934082, + "2": 14.23133373260498, + "3": 12.569449424743652, + "4": 13.366289138793945, + "5": 14.842429161071777, + "6": 11.500049591064453, + "7": 13.33083724975586, + "8": 12.062450408935547, + "9": 13.475191116333008, + "10": 10.438334465026855, + "11": 13.073284149169922, + "12": 10.891487121582031, + "13": 7.315669059753418, + "14": 11.185186386108398, + "15": 11.816669464111328, + "16": 11.457283973693848, + "17": 13.996313095092773, + "18": 10.656420707702637, + "19": 9.06871509552002, + "20": 11.556375503540039, + "21": 12.088006019592285, + "22": 12.024413108825684, + "23": 10.629256248474121 + }, + "size_gb": null + } +} diff --git a/eval/hps3_turbo.md b/eval/hps3_turbo.md new file mode 100644 index 0000000000000000000000000000000000000000..6d7001e00c1d201097879da949cac90bc52663df --- /dev/null +++ b/eval/hps3_turbo.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| turbo4_768 | - | 11.93 [11.24, 12.61] | 13.46 | 12.02 | 10.61 | 11.17 | 11.60 | - | +| turbo4_1024 | - | 11.93 [11.24, 12.58] | 13.28 | 12.59 | 10.43 | 11.50 | 11.43 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.69-1.75; bootstrap 95% half-width of a mean is 0.67-0.69. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.15-0.15 (median 0.15); a difference below about 2 x that (0.30 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_turbo_fold.json b/eval/hps3_turbo_fold.json new file mode 100644 index 0000000000000000000000000000000000000000..3dca5244c69e11bd24d2c2ede845c29e89dfdd96 --- /dev/null +++ b/eval/hps3_turbo_fold.json @@ -0,0 +1,88 @@ +{ + "turboF_mixG_tuned": { + "n": 24, + "mean_mu": 12.054207305113474, + "sd_mu": 1.8837308279838634, + "ci95": [ + 11.288835290074347, + 12.768072972695032 + ], + "category_mean_mu": { + "text": 13.483850161234537, + "portrait": 13.5546133518219, + "detail": 10.24861991405487, + "counting": 11.129275798797607, + "varied": 11.365798711776733 + }, + "mu_by_prompt": { + "0": 13.425480842590332, + "1": 11.913114547729492, + "2": 13.817068099975586, + "3": 13.241583824157715, + "4": 13.62537956237793, + "5": 14.880474090576172, + "6": 13.737812042236328, + "7": 14.302459716796875, + "8": 12.1873140335083, + "9": 13.990867614746094, + "10": 10.120655059814453, + "11": 12.881285667419434, + "12": 10.899666786193848, + "13": 7.092872142791748, + "14": 10.200028419494629, + "15": 12.058523178100586, + "16": 10.911789894104004, + "17": 14.066827774047852, + "18": 11.410252571105957, + "19": 9.03049087524414, + "20": 11.292652130126953, + "21": 10.377947807312012, + "22": 12.601527214050293, + "23": 11.234901428222656 + }, + "size_gb": null + }, + "turboF_mixH2_x_f8": { + "n": 24, + "mean_mu": 11.57393209139506, + "sd_mu": 1.7908379583449878, + "ci95": [ + 10.833748201529184, + 12.231900214155514 + ], + "category_mean_mu": { + "text": 12.760419686635336, + "portrait": 12.790683269500732, + "detail": 9.96740460395813, + "counting": 10.70246934890747, + "varied": 11.09682023525238 + }, + "mu_by_prompt": { + "0": 12.584124565124512, + "1": 11.796751022338867, + "2": 14.096269607543945, + "3": 11.595819473266602, + "4": 12.833593368530273, + "5": 13.655960083007812, + "6": 11.765027046203613, + "7": 13.449390411376953, + "8": 12.012411117553711, + "9": 13.935904502868652, + "10": 10.283557891845703, + "11": 13.074844360351562, + "12": 10.601030349731445, + "13": 5.910185813903809, + "14": 10.230630874633789, + "15": 11.174307823181152, + "16": 10.800506591796875, + "17": 12.757822036743164, + "18": 10.795364379882812, + "19": 8.53349494934082, + "20": 11.309234619140625, + "21": 11.119672775268555, + "22": 11.693855285644531, + "23": 11.76461124420166 + }, + "size_gb": null + } +} diff --git a/eval/hps3_turbo_fold.md b/eval/hps3_turbo_fold.md new file mode 100644 index 0000000000000000000000000000000000000000..f2c87603fc855d00caad32e477e5916b8079e103 --- /dev/null +++ b/eval/hps3_turbo_fold.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| turboF_mixG_tuned | - | 12.05 [11.29, 12.77] | 13.48 | 13.55 | 10.25 | 11.13 | 11.37 | - | +| turboF_mixH2_x_f8 | - | 11.57 [10.83, 12.23] | 12.76 | 12.79 | 9.97 | 10.70 | 11.10 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 1.79-1.88; bootstrap 95% half-width of a mean is 0.70-0.74. +- paired SE between two candidate models (excluding teacher and tern_v1) is 0.14-0.14 (median 0.14); a difference below about 2 x that (0.28 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_turbo_quant.json b/eval/hps3_turbo_quant.json new file mode 100644 index 0000000000000000000000000000000000000000..5920a6f012a844a95d854491e33fb1b8cca3ae90 --- /dev/null +++ b/eval/hps3_turbo_quant.json @@ -0,0 +1,88 @@ +{ + "turbo_on_mixG_tuned": { + "n": 24, + "mean_mu": 0.6785016457239786, + "sd_mu": 11.03595436855944, + "ci95": [ + -3.7482088724772136, + 5.1595202038685475 + ], + "category_mean_mu": { + "text": -2.4730655352274575, + "portrait": 6.126515626907349, + "detail": -4.580440521240234, + "counting": 11.082052230834961, + "varied": 0.34675347805023193 + }, + "mu_by_prompt": { + "0": -10.2711181640625, + "1": -10.214679718017578, + "2": 13.671292304992676, + "3": 12.482805252075195, + "4": -10.241397857666016, + "5": -10.265295028686523, + "6": 11.479754447937012, + "7": 12.140230178833008, + "8": 11.125768661499023, + "9": -10.239690780639648, + "10": -10.271653175354004, + "11": 12.425623893737793, + "12": -10.228824615478516, + "13": -10.246908187866211, + "14": 10.417734146118164, + "15": 11.746370315551758, + "16": -10.166426658630371, + "17": -9.324441909790039, + "18": 11.024897575378418, + "19": 9.750812530517578, + "20": -10.206877708435059, + "21": -9.391433715820312, + "22": 11.950613021850586, + "23": 9.136884689331055 + }, + "size_gb": null + }, + "turbo_on_mixH2_x_f8": { + "n": 24, + "mean_mu": -0.6272683143615723, + "sd_mu": 10.71604236269574, + "ci95": [ + -4.7650328497091925, + 3.597271713614463 + ], + "category_mean_mu": { + "text": -2.403357982635498, + "portrait": 0.5396475791931152, + "detail": -1.4173829555511475, + "counting": -10.278879165649414, + "varied": 2.9293010234832764 + }, + "mu_by_prompt": { + "0": -10.26313591003418, + "1": -10.048715591430664, + "2": 14.382139205932617, + "3": 12.05323600769043, + "4": -10.281204223632812, + "5": -10.262467384338379, + "6": 10.741776466369629, + "7": 11.967440605163574, + "8": -10.27446174621582, + "9": -10.276165008544922, + "10": 8.19097900390625, + "11": -9.42135238647461, + "12": -10.258614540100098, + "13": 5.819456100463867, + "14": -10.278194427490234, + "15": -10.279563903808594, + "16": 10.89321517944336, + "17": 11.099827766418457, + "18": 10.622716903686523, + "19": -10.273361206054688, + "20": -10.268458366394043, + "21": 10.475646018981934, + "22": 11.14976692199707, + "23": -10.264945030212402 + }, + "size_gb": null + } +} diff --git a/eval/hps3_turbo_quant.md b/eval/hps3_turbo_quant.md new file mode 100644 index 0000000000000000000000000000000000000000..89a1f96c2dc8f3ad3f7e7ad58a78260de293f3ef --- /dev/null +++ b/eval/hps3_turbo_quant.md @@ -0,0 +1,11 @@ +| tag | size GB | mean mu [bootstrap 95%] | text | portrait | detail | counting | varied | diff to teacher +- SE (wins/24) | +|---|---|---|---|---|---|---|---|---| +| turbo_on_mixG_tuned | - | 0.68 [-3.75, 5.16] | -2.47 | 6.13 | -4.58 | 11.08 | 0.35 | - | +| turbo_on_mixH2_x_f8 | - | -0.63 [-4.77, 3.60] | -2.40 | 0.54 | -1.42 | -10.28 | 2.93 | - | + +mu = HPSv3 raw score (higher = preferred), mean over 24 prompts, one seed per prompt, 768px images scored at ~448px (HPSv3 input). +diff to teacher = mean of (student mu - teacher mu) per prompt, SE = sd of the 24 paired differences / sqrt(24); wins = prompts where student mu > teacher mu. + +Noise: only 24 prompts and ONE seed per prompt, so seed-to-seed variation of a model is not measured at all and the intervals only reflect which 24 prompts were drawn. +- sd of mu across prompts is 10.72-11.04; bootstrap 95% half-width of a mean is 4.18-4.45. +- paired SE between two candidate models (excluding teacher and tern_v1) is 2.84-2.84 (median 2.84); a difference below about 2 x that (5.67 mu) cannot be told apart from prompt-sampling noise. diff --git a/eval/hps3_turbo_quant_steps.json b/eval/hps3_turbo_quant_steps.json new file mode 100644 index 0000000000000000000000000000000000000000..5990e9faea8adc7dac902b96cfc485fef4c0139c --- /dev/null +++ b/eval/hps3_turbo_quant_steps.json @@ -0,0 +1,45 @@ +{ + "turbo6_on_mixG_tuned": { + "n": 24, + "mean_mu": -0.6928242444992065, + "sd_mu": 10.821398370797311, + "ci95": [ + -4.866530988613764, + 3.6030108163754138 + ], + "category_mean_mu": { + "text": -1.8798831303914387, + "portrait": 1.453338623046875, + "detail": 0.5357625484466553, + "counting": -10.246141910552979, + "varied": 0.898424506187439 + }, + "mu_by_prompt": { + "0": -10.282100677490234, + "1": -10.157089233398438, + "2": 14.185490608215332, + "3": -9.268797874450684, + "4": -10.180964469909668, + "5": 14.424162864685059, + "6": -10.258037567138672, + "7": -9.60051155090332, + "8": 11.324554443359375, + "9": 14.347349166870117, + "10": 9.369885444641113, + "11": -3.812710762023926, + "12": -10.178215980529785, + "13": 6.764091491699219, + "14": -10.208133697509766, + "15": -10.284150123596191, + "16": 10.37563419342041, + "17": 14.226563453674316, + "18": 10.835710525512695, + "19": -10.139874458312988, + "20": -9.82712459564209, + "21": 10.206365585327148, + "22": -10.25687026977539, + "23": -8.23300838470459 + }, + "size_gb": null + } +} diff --git a/eval/hps3_vae_final.json b/eval/hps3_vae_final.json new file mode 100644 index 0000000000000000000000000000000000000000..d052099975eb357702cddecf842429b8522ae80a --- /dev/null +++ b/eval/hps3_vae_final.json @@ -0,0 +1,45 @@ +{ + "enc_gptq_pruned_flux2-small-decoder-vae": { + "n": 24, + "mean_mu": 11.05746477842331, + "sd_mu": 1.9355683246858872, + "ci95": [ + 10.286498762170474, + 11.80013428032398 + ], + "category_mean_mu": { + "text": 12.212622165679932, + "portrait": 12.439714431762695, + "detail": 9.339466214179993, + "counting": 11.171565055847168, + "varied": 10.330446124076843 + }, + "mu_by_prompt": { + "0": 11.371161460876465, + "1": 9.201536178588867, + "2": 13.873811721801758, + "3": 12.936747550964355, + "4": 12.383365631103516, + "5": 13.509110450744629, + "6": 13.020609855651855, + "7": 11.980722427368164, + "8": 11.782705307006836, + "9": 12.974820137023926, + "10": 8.424834251403809, + "11": 12.533751487731934, + "12": 9.609766006469727, + "13": 6.789513111114502, + "14": 10.499263763427734, + "15": 11.843866348266602, + "16": 10.027688980102539, + "17": 13.780057907104492, + "18": 9.438458442687988, + "19": 8.03177261352539, + "20": 10.843424797058105, + "21": 10.101381301879883, + "22": 11.073204040527344, + "23": 9.347580909729004 + }, + "size_gb": null + } +} diff --git a/eval/hps3_vae_full_ref.json b/eval/hps3_vae_full_ref.json new file mode 100644 index 0000000000000000000000000000000000000000..0d65fdd9f29cbe845063dda8b367a389dd26e444 --- /dev/null +++ b/eval/hps3_vae_full_ref.json @@ -0,0 +1,45 @@ +{ + "enc_gptq_pruned": { + "n": 24, + "mean_mu": 11.02418977022171, + "sd_mu": 1.8188911126730065, + "ci95": [ + 10.301669026911258, + 11.714808073143164 + ], + "category_mean_mu": { + "text": 12.023759841918945, + "portrait": 11.918961763381958, + "detail": 9.534263491630554, + "counting": 11.000334739685059, + "varied": 10.5780531167984 + }, + "mu_by_prompt": { + "0": 10.913015365600586, + "1": 9.044050216674805, + "2": 13.893392562866211, + "3": 13.00432300567627, + "4": 12.235599517822266, + "5": 13.052178382873535, + "6": 11.615826606750488, + "7": 12.187568664550781, + "8": 11.94758129119873, + "9": 11.924870491027832, + "10": 8.12902545928955, + "11": 13.062519073486328, + "12": 10.2658052444458, + "13": 6.679704189300537, + "14": 10.553024291992188, + "15": 11.44764518737793, + "16": 10.093405723571777, + "17": 13.655694961547852, + "18": 9.823501586914062, + "19": 8.609209060668945, + "20": 11.18978214263916, + "21": 10.255514144897461, + "22": 11.602023124694824, + "23": 9.395294189453125 + }, + "size_gb": null + } +} diff --git a/eval/judge_enc_i4e8.md b/eval/judge_enc_i4e8.md new file mode 100644 index 0000000000000000000000000000000000000000..083124c3a97036b03ffe4031d9495154cc37de28 --- /dev/null +++ b/eval/judge_enc_i4e8.md @@ -0,0 +1,114 @@ +# Visual judge: enc_i4e8 (tmixG_x generator with the 4.6 GB int4-layer + int8-embedding text encoder, 4 steps, CFG 1.0) vs the same generator with the 19.3 GB bf16 text encoder + +Question: WOULD A USER ACCEPT THE IMAGES MADE WITH THE SHRUNKEN TEXT ENCODER INSTEAD OF THE IMAGES MADE WITH THE bf16 ENCODER FOR THE SAME PROMPTS? + +Scales 0-5, same rubric as judge_turbo4.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept the enc_i4e8 image instead of the bf16-encoder image (tmixG_x_1024). "-" = not applicable. Half points used. Scored images: eval\enc_i4e8 (24 x 1024x1024) against eval\tmixG_x_1024. + +Method and caveats: +- All 24 candidate PNGs were opened alone at native 1024x1024. For all 24 prompts a reference | candidate side-by-side (2056x1024, reference on the left, viewer showed it at about 97%) was also opened. The six text prompts (00-05) and the typewriter (10) were additionally opened for the reference at native size; every quoted string was read letter by letter from native views plus 2-4x crops for both sets (00 sign, window line and slate; 01 labels; 02 tagline; 03 progress bar; 04 title and 11-character sub-line; 05 all boards). Scratch files were written to the OS temp folder and deleted afterwards; this report is the only file written in the project. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the six quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- The reference set was scored by me on the same scale before reading judge_tmixG_x.md (that report judges the same reference images against turbo4); its numbers are compared at the end. +- Because the conditioning differs, a different layout was not penalised by itself; missing prompt elements, wrong text, artifacts and loss of detail were. In 20 of 24 pairs the layout is the same as the reference, so most verdicts are direct detail comparisons; the layout differs clearly in 00 (framing), 01, 07 (head pose) and 10 (framing). +- This judge is a model instance; differences of +-0.2 on any average are within noise. One seed per prompt. + +## Per-image table (enc_i4e8, with the reference scored on the same scale) + +| id | category | F | T | C | Q | A | vs bf16 enc | ref F/T/C/Q | Note | +|---|---|---|---|---|---|---|---|---|---| +| 00 | text | 4 | 3 | 4 | 4 | 4 | better | 4/2.5/4/4.5 | Full storefront, cat asleep on the step, cheese wheels, wet cobbles, lamp, blue-hour street, slate on the pavement. Sign "LE PETIT FROMAGE" exact (hand-lettered, the I of PETIT carries a dot). Window line wrong: "Affineur / duus dopuis 1974". Slate "Tasting - / Tasting / Saturday / 11 AM" (duplicated word, otherwise exact) with a garbled scribble header. Unrequested garbled chalk-like text on the left wall ("Affineur Coparina Cfe"). Reference sign is "L LE PETTT FROMAGE", window and slate also wrong. | +| 01 | text | 4 | 4 | 3.5 | 3.5 | 3.5 | same | 3.5/2.5/3/4 | "WATER CYCLE" exact, "Condensation", "Precipitation", "Collection" exact, "Evaportion" (minor typo for Evaporation). All four labels in rounded boxes beside sun / cloud / mountain with waterfall / sea. But the picture is pale, there is no rain, the arrows are weak, a stray capsule icon sits beside "Collection" and the sea is a flat slab. Reference is bolder with raindrops but has "Evapoation", "Condenation", "Conipitation" and no Collection. | +| 02 | text | 4.5 | 4 | - | 4.5 | 4 | worse | 4.5/5/-/4.5 | Title "BEYOND THE BREAKERS" exact. Tagline reads "Some lights nover go out" (minor typo, visible at full size). Girl in yellow raincoat rowing in profile, curled wave, lighthouse beam, print grain, strong. Same scene as the reference, whose tagline is exact. | +| 03 | text | 4.5 | 4.5 | 4.5 | 4.5 | 4.5 | same | 4.5/5/4.5/4.5 | "Neon Horizon", "Luna Vale", "1:24 /" and "3:48" exact (a slash after 1:24, a few dotted specks near 3:48), purple-sunset square cover, round previous / play / next, three icons below, desk with blurred lamp and pen pot. Extra status-bar "1:48". Practically the same image as the reference. | +| 04 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | 4.5/5/-/4.5 | "老王面馆" exact; sub-line "手工拉面 · 营业至凌晨两点" all 11 characters plus the dot in order (凌 slightly ambiguous at native size, identical to the reference). Lanterns, steam, diners at metal tables, wet street. Same layout as the reference; one more diner on the right. | +| 05 | text | 4 | 3.5 | 4 | 4.5 | 4 | same | 4/4/4/4.5 | "PEACHES $3" (top left), "CHERRIES $5" and "HONEY $8" (right) exact on dark green boards. The biggest board in the foreground reads "CHEFRRIES $3" (unrequested, wrong price pairing) and a blurred middle board is garbled. Straw hat hangs on the left post (clean), daisies, hills, awning, crates of fruit. Reference is exact on the three boards with a cut-off unrequested "HONEY $2" board and a hat plus a rope ball on the post. | +| 06 | portrait | 4 | - | - | 4.5 | 4 | same | 3.5/-/-/4.5 | Shaved head, silver stubble, maroon robe with golden edge and visible wool weave, window light from the left, plain grey wall, heavy-lidded warm eyes, prayer beads looped on the wrist as asked (reference has the beads on the chest). Face is less deeply lined than the reference, gaze goes to the camera. Excellent sharpness. | +| 07 | portrait | 3 | - | - | 2.5 | 2.5 | worse | 4/-/-/4.5 | Curly auburn hair, navy turtleneck, red lamp glow, teal/amber bokeh, droplets on the glass. HEAVY LOCAL ARTIFACT: the face is smeared as a double exposure - a ghost second jaw / mouth on the right cheek, mushy eyes, lashes lost, face much softer than the hair. The reference has a sharp face with cheek droplets. Readable but a user would reject it. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Messy black hair, golden eyes, loose white shirt, green scarf, blue flower field, petals, clouds. Clean line art, no stray arcs. Hair is truer black than the reference; eye highlights a little flatter than the reference's glossy ones. | +| 09 | portrait | 4 | - | - | 4.5 | 4 | same | 4.5/-/-/4.5 | Grey short hair, round glasses, arms crossed, smile, shelves of glazed bowls, large window, clay speckles on apron, forearms and table. The glowing kiln of the reference is missing (only faint warm blur). Sharp and warm. | +| 10 | detail | 3 | 3 | - | 3.5 | 3 | worse | 4/3.5/-/4 | Worn chrome-rimmed keys with dust, shallow focus, cream paper. The mechanics are poorer: no platen / typebar structure, a torn perforated sheet fragment in the middle, soft keys. Only about 6-8 legends readable (Y, IX, AB, 00, F, L), paper text pseudo-letters ("NO. S", "BLX"). Reference has sharp typebars, rusty platen and about 10 readable legends (H, X, T, LO, S, G, E). | +| 11 | detail | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Peacock eye, near-identical to the reference; slightly more saturated, droplets and barbs sharp. | +| 12 | detail | 4.5 | - | - | 4 | 4 | same | 4.5/-/-/4.5 | Mossy oak, ferns, two snails, ladybugs, many mushrooms, mist. More crowded: mushrooms and snails look a little pasted on, bark furrows less deep than the reference. | +| 13 | detail | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Rivet-studded cuff, individually resolved mail rings, candle with wax drips, scratched wood, dark background. Practically identical to the reference. | +| 14 | counting | 4 | - | 5 | 4.5 | 4 | same | 4/-/5/4.5 | Exactly five balloons in a row: red, red-orange check (reads red more than orange), yellow, blue-top/yellow with a small bird-like emblem glitch, purple/white stripes; tiny pilots, mist, village, sunrise. Reference also has five, with a pale teal-yellow fourth. | +| 15 | counting | 4 | - | 4.5 | 4.5 | 4.5 | better | 3.5/-/3/4.5 | Three red apples, a yellow pear, a second yellow fruit half hidden behind it, and a green pear = the requested 3 / 2 / 1 (the hidden yellow pear is ambiguous). Reference shows four red apples (one hidden) and only one yellow pear. Window light, linen folds, grey wall. | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Colossal lantern tree, candle-lit path along the roots, one boat, mountains, violet sky with stars and crescent, long golden reflections, pink flowers. Near-identical to the reference. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Yellow trench coat, clear umbrella, steaming cart with Hangul signs, wet reflections, scooter, teal/magenta palette. Near-identical to the reference. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Pastel pink clay bakery, striped awning, bread in the window, chimney smoke, tree, cobbles, one delivery scooter, croissant sign (clearer than the reference's). No separate bicycle in either. | +| 19 | varied | 4 | - | - | 4.5 | 4 | same | 4/-/-/4.5 | Brown bear in a rushing river, open mouth, wet fur, mossy rocks, misty forest, spray. TWO salmon (prompt: one), one right at the jaw. Reference has one fish beside the head and an odd raised paw. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | 4.5/-/-/4.5 | Watercolor canal, striped poles, gondolas, striped-shirt gondolier, palazzi and reflections, pigeons on the bridge, pennant lines. Same as the reference. | +| 21 | varied | 4 | - | - | 4.5 | 4 | same | 4/-/-/4.5 | Black-armoured knight kneeling before a faceted blue crystal, stained-glass light shafts, embers, planted sword, small flames. Helmet is on his head (an object at his side may be a second helmet or shield). Dark, rich. Same as the reference. | +| 22 | varied | 4.5 | - | 4.5 | 4 | 4.5 | same | 4.5/-/4.5/4 | Bright Scandinavian kitchen, white cabinets, oak table, jug of pink tulips, two cups with saucers and steam, crumbs, petals, pulled-back pale chair, plant, big window. A pale translucent table leg (also in the reference). | +| 23 | varied | 4 | - | - | 4 | 4 | same | 4/-/-/4 | Hooded vendor frying dumplings, warm bulbs, steam, rain, bowl of filling, tray of dumplings, customers' backs. No visible coin (also missing in the reference). Dark, as the reference. | + +## Averages (enc_i4e8 vs bf16-encoder reference, both scored by me on this scale) + +| | F | T (7 imgs) | T (6 quoted-text) | C (7 imgs) | Q | A | +|---|---|---|---|---|---|---| +| enc_i4e8 | 4.17 | 3.86 | 4.00 | 4.29 | 4.25 | 4.10 | +| bf16 encoder (tmixG_x_1024) | 4.21 | 3.93 | 4.00 | 4.00 | 4.42 | - | +| delta | -0.04 | -0.07 | 0.00 | +0.29 | -0.17 | | + +All deltas are inside or at the edge of the +-0.2 noise band (Q -0.17 borderline, C +0.29 on n=7 comes from 15 and 01). + +| category | n | F (cand/ref) | T (cand/ref) | C (cand/ref) | Q (cand/ref) | A (cand) | +|---|---|---|---|---|---|---| +| text | 6 | 4.25 / 4.17 | 4.00 / 4.00 | 4.00 / 3.88 (4 imgs) | 4.25 / 4.42 | 4.08 | +| portrait | 4 | 3.88 / 4.13 | - | - | 4.00 / 4.50 | 3.75 | +| detail | 4 | 4.13 / 4.38 | 3.0 / 3.5 (typewriter) | - | 4.13 / 4.38 | 4.00 | +| counting | 2 | 4.00 / 3.75 | - | 4.75 / 4.00 | 4.50 / 4.50 | 4.25 | +| varied | 8 | 4.31 / 4.31 | - | 4.5 / 4.5 (1 img) | 4.38 / 4.38 | 4.31 | + +Without 07 (the one heavy-artifact image) the candidate averages are F 4.22, Q 4.33, A 4.17. + +## Verdict versus the bf16-encoder image (all 24) + +Better 2: 00, 15. +Same 19: 01, 03, 04, 05, 06, 08, 09, 11, 12, 13, 14, 16, 17, 18, 19, 20, 21, 22, 23. +Worse 3: 02, 07, 10. + +Reasons in brief: +- better: 00 sign exact where the reference has "L LE PETTT FROMAGE"; 15 the 3 / 2 / 1 fruit mix is nearly right where the reference shows four apples. +- worse: 02 tagline "nover" (rest identical); 07 ghosted double-face smear on the face; 10 degraded typewriter mechanics, fewer legible keys. +- same (small leanings): 06 wrist beads but less deeply lined; 09 no kiln glow; 12 more crowded and slightly pasted-on mushrooms; 05 stray CHEFRRIES $3 board (reference: stray HONEY $2); 01 all four labels present but pale and no rain (reference: bolder, three labels, mostly misspelled); 19 second salmon. + +## Text-prompt acceptance and letter-by-letter reading + +- Six quoted-text prompts accepted (A >= 4): 5/6 for the candidate (00, 02, 03, 04, 05; 01 at A 3.5). +- Letter-exact (all required strings of a prompt exact): candidate 3/6 (03, 04, 05), reference 4/6 (02, 03, 04, 05). By individual string (19 strings in total): candidate 15 exact / 3 minor typo / 1 wrong; reference 12 exact / 3 minor typo / 4 wrong (one of them missing). +- 00 candidate: sign "LE PETIT FROMAGE" exact (dotted I); window "Affineur duus dopuis 1974" WRONG; slate "Tasting - / Tasting / Saturday / 11 AM" minor typo (duplicated "Tasting"). Reference: sign "L LE PETTT FROMAGE" WRONG; window "Affineur" + garbled glyphs + "1974" WRONG; slate "Tasting / Satting / Saturday / 11 AM" minor typo. +- 01 candidate: "WATER CYCLE" exact, "Evaportion" minor typo, "Condensation", "Precipitation", "Collection" exact. Reference: "WATER CYCLE" exact, "Evapoation" minor typo, "Condenation" minor typo, "Conipitation" WRONG, "Collection" missing. +- 02 candidate: "BEYOND THE BREAKERS" exact, "Some lights nover go out" minor typo. Reference: both exact. +- 03 both: "Neon Horizon", "Luna Vale", "1:24", "3:48" exact (candidate adds a slash after 1:24 and an extra "1:48" status clock). +- 04 both: "老王面馆" and "手工拉面 · 营业至凌晨两点" (11 characters plus dot, in order) exact; 凌 equally ambiguous at native size in both. +- 05 both: "PEACHES $3", "CHERRIES $5", "HONEY $8" exact. Extra unrequested wrong boards: candidate "CHEFRRIES $3" (foreground, large) plus a garbled middle board; reference a cut-off "HONEY $2". +- Typewriter keys (10): candidate about 6-8 legible legends, paper text pseudo-letters; reference about 10 legible legends, paper text pseudo-letters. + +## Broken images and artifacts + +- Broken images: 0. +- Heavy local artifact: 1 (07, smeared double-exposure face). Lighter defects: 10 (loss of mechanical structure, torn-sheet fragment), 14 (small emblem glitch on the blue balloon), 22 (translucent leg, also in the reference), 00 and 05 (hallucinated extra text on the wall / extra board). +- No colour shift, banding, blocking, wash-out or over-sharpening. + +## Comparison with judge_tmixG_x.md (same reference images judged against turbo4) + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| tmixG_x_1024 as scored in judge_tmixG_x.md (vs turbo4) | 4.08 | 3.79 | 3.86 | 4.23 | 4.06 | +| tmixG_x_1024 as scored here (this session) | 4.21 | 3.93 | 4.00 | 4.42 | - | +| enc_i4e8 (this report) | 4.17 | 3.86 | 4.29 | 4.25 | 4.10 | + +- My reference scores are +0.13 to +0.19 above that report on F, T, C and Q (a different baseline and a different judge instance; the gaps are at the edge of the noise band). The readings agree where it matters: the reference fails text on 00 and 01 with the same strings ("L LE PETTT FROMAGE", "Evapoation", "Condenation", "Conipitation", no Collection), 02, 03, 04, 05 are exact, and 15 shows four apples plus two pears. +- Against that report's own numbers the candidate is F +0.09, T +0.07, C +0.43, Q +0.02, A +0.04, i.e. equal within noise on every axis except C, which comes from 15 and 01. + +## Summary (10 lines) + +1. enc_i4e8 overall: F 4.17, T 3.86 (4.00 on the six quoted-text prompts), C 4.29, Q 4.25, A 4.10; the bf16-encoder reference scored on the same scale: F 4.21, T 3.93 (4.00), C 4.00, Q 4.42. +2. Side-by-side versus the bf16-encoder image: 2 better, 19 same, 3 worse out of 24. +3. Letter-exact text: candidate 3/6 prompts (15 of 19 strings exact), reference 4/6 prompts (12 of 19 strings exact). +4. Broken images: 0; heavy local artifact: 1 (07, ghosted double face). +5. 20 of 24 pairs share the reference layout; many (03, 04, 11, 13, 16, 17, 18, 20) are practically identical. +6. Worse images: 07 (face smear), 10 (typewriter mechanics and legends), 02 (tagline "nover" only). +7. Better images: 00 (sign exact) and 15 (fruit mix closer to 3 / 2 / 1). +8. A >= 4 in 21 of 24 images (A < 4: 01, 07, 10), and for 5 of 6 text prompts. +9. Prompt details dropped silently are minor and few: 09 kiln glow, 06 deep wrinkles and gaze, 07 lashes and window; 06 gained the wrist beads. +10. Caveat: this judge is a model instance; +-0.2 differences are within noise; one seed per prompt. diff --git a/eval/judge_mixA_lora32.md b/eval/judge_mixA_lora32.md new file mode 100644 index 0000000000000000000000000000000000000000..e96399a06f6bff9302ef05532ad1c02e6353d979 --- /dev/null +++ b/eval/judge_mixA_lora32.md @@ -0,0 +1,76 @@ +# Visual judge: mixA_lora32 (student) vs teacher_bf16 + +Scales 0-5. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable (no quoted text / no count or spatial requirement). Half points used where borderline. +Judged by opening all 24 student PNGs at full resolution, teacher PNGs for 00, 03, 04, 10, 15 at full resolution, and the other teachers on the contact sheets. Teacher reference: all six quoted-text prompts are rendered fully correctly by the teacher (including 老王面馆); the teacher has exactly 5 balloons; the teacher has 3 apples + 1 yellow pear + 1 green pear (it also misses one yellow pear). + +## Per-image table + +| id | category | F | T | C | Q | A | Note (most visible difference from teacher) | +|---|---|---|---|---|---|---|---| +| 00 | text | 3 | 2 | 4 | 4 | 2 | Sign "LE PETIT FROMAGE" correct, but window line renders "Afineer dedio 1977" (should be "Affineur depuis 1974") and slate renders "Tastina / Seatial / 11 AM" (should be "Tasting - Saturday 11 AM"); teacher has all three correct. Cat and wet cobbles fine. | +| 01 | text | 2 | 1 | 2 | 3 | 1 | Title "WATER CCLE" (Y missing); labels "Eviapation", "Condeation", "Podluiton" (=Precipitation, unrecognisable), "Colecton": 0 of 4 labels correct. No cycle arrows, one big blob of sea, stray raindrop. Teacher is a clean correct cycle. | +| 02 | text | 4 | 2 | - | 4 | 2 | Title "BEYOID TIE BREAKERS" (BEYOND THE), tagline "Sone ligtt nener go out" (Some lights never go out). Painting itself (girl in yellow coat, lighthouse beam, spray) is good. | +| 03 | text | 2 | 2 | 2 | 4 | 2 | Title "Neon Horinn" (Horizon); "Luna Vale" ok; time reads "1.24 / 3.48" (dots not colons). Device is a squat boxy gadget, not a modern phone; album art is landscape, not square; buttons are play / play / previous (no "next"); four empty circles with no icons. | +| 04 | text | 3 | 1 | - | 4 | 1 | Sign reads "北子面鍋" instead of "老王面馆" (only 面 correct); second line is pseudo-hanzi, nothing from "手工拉面 · 营业至凌晨两点" is readable. Scene (lanterns, steam, diners, wet reflections) is fine. Teacher is perfect. | +| 05 | text | 3 | 1 | 3 | 4 | 2 | Signs read "PECHES SS", "CHERIES SS", "HONY SS": all three words misspelled and every price lost ($3/$5/$8 became "SS"). Two straw hats instead of one. Fruit, awning, hills, daisies good. | +| 06 | portrait | 4 | - | - | 4 | 3.5 | Strong portrait, pores/wrinkles resolved. Bead string hangs at the neck and a loose cluster of beads floats beside the ear (prompt asked for beads around the wrist); robe weave looks terry-towel-like; gaze straight into camera, not past it. | +| 07 | portrait | 3.5 | - | - | 3.5 | 3.5 | Face skin shows a cracked/speckled craquelure texture (droplets rendered as crackle), hair a bit mushy; no glass reflection over the cheek, lips closed not parted, hair is a short bob not long curls. Teacher is more cinematic with the layered reflection. | +| 08 | portrait | 4.5 | - | - | 4 | 4 | Faithful: black messy hair, golden eyes, white shirt, green scarf, blue flower field, petals. Shirt and scarf shading flatter and more simplified than teacher; no real defects. | +| 09 | portrait | 4 | - | - | 3.5 | 4 | Round glasses, grey hair, crossed arms, clay apron, kiln glow all present. Hair is longer/wispier than "short"; crossed hands slightly blobby; clay smear looks like uniform mud texture. | +| 10 | detail | 2.5 | 1 | - | 3.5 | 2 | Every key carries the same glyph ("I8"/"T8"/"16"), teacher has distinct Q W E R... keys; typed paper text is gibberish at a smaller size. Typebars not curled behind keys. Metal/dust texture itself is good. | +| 11 | detail | 3 | - | - | 4 | 3 | Sharp barbs and droplets, but no recognisable peacock "eye": a green/gold lobe with no turquoise/violet centre. Contrast and saturation more crunchy than teacher. | +| 12 | detail | 4 | - | - | 4 | 4 | Moss, ferns, bracket fungi, droplets, mist present. "Snails" are black glossy slug/beetle blobs, no ladybug, less lichen than teacher; fine detail slightly softer. | +| 13 | detail | 3 | - | - | 3.5 | 3 | Mail reads as a regular grid of punched discs/rivets rather than interlocking rings, and the object is a dome/hood shape with no fingers (not a gauntlet). Candle, scratched wood, darkness good. Repetitive mosaic-like texture. | +| 14 | counting | 4 | - | 1 | 4.5 | 2 | SIX balloons (red, orange, yellow, an extra yellow patterned, blue, striped purple) instead of exactly five; clustered, not evenly spread. Teacher has exactly five. Landscape quality excellent. | +| 15 | counting | 2.5 | - | 1 | 3 | 2 | 4 red apples + 1 yellow pear + 1 green pear (asked 3 apples + 2 yellow pears + 1 green pear); teacher gets apples right. Apples are dull matte with brown pits (pomegranate-like), no glossy highlights, bowl and linen fine. | +| 16 | varied | 4.5 | - | - | 4.5 | 4 | Tree, lanterns, boat with traveler, mountains, stars, candle-lit root path all there. Extra figure standing on the water at right; tree reflection missing (only lights reflect). | +| 17 | varied | 4.5 | - | - | 3.5 | 3.5 | Yellow coat, clear umbrella, food cart steam, neon reflections good. Face is smeared/muddy, legs and boots awkward; teacher face and anatomy sharper. | +| 18 | varied | 3.5 | - | - | 3.5 | 3 | Bakery, bread, chimney smoke, tree, bike, croissant sign present, but the base is a beige suitcase-like box with a latch (no green ground, not a floating cube); awning roof geometry is odd; slightly gritty. | +| 19 | varied | 3.5 | - | - | 4 | 3.5 | Wet fur and splashes excellent. Salmon is beside the head, not caught; bear looks at camera and reads cub-like. | +| 20 | varied | 4.5 | - | - | 4.5 | 4 | Gondolas, striped poles, palazzi, bridge, pigeons, red flower pot, laundry line, striped-shirt gondolier, paper texture all present. Very close to the teacher in quality. | +| 21 | varied | 4 | - | - | 4 | 4 | Crystal, stained glass, embers, helmet, runes, sword present; knight seen from behind half-melted into the wet floor, sword leaning not planted. Dark areas a bit murky. | +| 22 | varied | 3 | - | 2 | 3 | 2.5 | One coffee cup instead of two; table top is an irregular polygon with ~6 legs, chair is a bar-stool; washed-out, flatter lighting than teacher's coherent table and chair. | +| 23 | varied | 4.5 | - | - | 4 | 4 | All elements present (hooded vendor, dumplings, pan, bulbs, rain, steam, hand with coin). Dark foreground figures are featureless blocks; faces slightly smeared. | + +## Averages + +Overall (24 images): F 3.50, T 1.4 (7 images with legible-text requirement: 6 quoted-text prompts plus typewriter key legends; 1.5 on the six quoted-text prompts alone), C 2.14 (7 images), Q 3.81, A 2.90. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 2.83 | 1.50 | 2.75 (4 imgs) | 3.83 | 1.67 | +| portrait | 4 | 4.00 | - | - | 3.75 | 3.75 | +| detail | 4 | 3.13 | 1.0 (typewriter only) | - | 3.75 | 3.00 | +| counting | 2 | 3.25 | - | 1.00 | 3.75 | 2.00 | +| varied | 8 | 4.00 | - | 2.0 (1 img) | 3.88 | 3.56 | + +## Five worst images + +1. 01 water cycle (A 1): 0 of 5 text strings correct (title missing the Y, all four labels garbled, "Precipitation" unreadable), and the diagram does not show a cycle (no arrows, giant sea blob); an infographic whose labels are wrong is useless. +2. 04 Chinese noodle shop (A 1): the signboard says 北子面鍋 not 老王面馆 and the second line is pseudo-characters; the teacher renders both lines perfectly. Chinese text fidelity is essentially lost (1 of ~4 glyphs on the main sign). +3. 05 fruit stand (A 2): all three painted signs misspelled (PECHES, CHERIES, HONY) and all prices replaced by "SS"; the prompt's whole point (legible hand-painted prices) fails. +4. 03 music player (A 2): "Neon Horinn", wrong device (boxy gadget instead of phone), landscape art instead of square, button row is play/play/previous, empty icon circles; time stamp uses dots. +5. 15 fruit bowl (A 2): counts wrong (4 apples, 1 yellow pear, 1 green pear vs 3/2/1; teacher has apples right), and the apples lost gloss and look like pitted pomegranates. +Close behind: 10 typewriter (every key has the identical glyph "I8"), 14 balloons (six, not exactly five), 02 poster ("BEYOID TIE BREAKERS"), 00 cheese shop (window and slate text garbled). + +## Systematic patterns + +- Text is the dominant failure. Teacher gets 6/6 quoted-text prompts right; the student gets none fully right. Errors are glyph-level: dropped/swapped letters (CYCLE to CCLE, THE to TIE, BEYOND to BEYOID, PEACHES to PECHES, HONEY to HONY, Horizon to Horinn), "$3" collapsing to "SS", colons in the time becoming dots. Large title lettering survives with 1-2 errors; small/secondary text (window line, slate, taglines, labels, hanzi sub-line) is largely illegible. Chinese is the worst script (main sign 1/4 characters, sub-line pseudo-glyphs). Repeated-glyph collapse on the typewriter keys (all keys show the same "I8"). +- Counting is unreliable: both explicit-count prompts fail (6 balloons; wrong apple/pear mix). Small-set compositional constraints from late in the long prompt ("two cups", "one hat", "beads around wrist", "ladybug") are frequently dropped. +- Object-level geometry/coherence drifts: gauntlet becomes a dome, phone becomes a boxy device, cube becomes a suitcase, table with six legs, a person standing on water, floating beads next to the monk's ear. +- What is preserved: photographic texture, fur, moss, water, painterly and anime styles, bokeh, and lighting. No banding, no mosaic/blocking, no global colour shift or contrast loss was seen; palette and exposure match the teacher. Backgrounds and landscapes are essentially teacher-level (14, 16, 20, 23). +- Mild local defects: crackled/speckled skin texture (07), muddy faces in small or distant subjects (17, 23), dull materials with lost specular gloss (15 apples), repetitive punched-disc texture instead of true fine structure (13), slightly crunchy contrast on macro (11), murky near-black regions (21, 23). +- Composition changes versus the teacher are large everywhere (as expected), so judgements are against the prompts, not teacher layouts. + +## Summary (10 lines) + +1. Overall acceptance 2.9/5; faithfulness 3.5, quality 3.8, counts 2.1, text 1.4. +2. For non-text, non-counting content (portraits, landscapes, painting, anime, nature, macro) the student is near-teacher: A 3.5-4. +3. Every text prompt fails: 0 of 6 are correct; text-category acceptance averages 1.7/5. +4. Failures are letter/glyph errors, worst on small text, digits and "$", and on Chinese (老王面馆 became 北子面鍋). +5. Both counting prompts fail (six balloons instead of five; apple/pear mix wrong and worse than the teacher). +6. Typewriter keys all show an identical glyph; repeated-token collapse on legends. +7. Geometry/object identity errors: gauntlet, phone, floating cube base, six-legged table, person on water. +8. Fine detail (fur, moss, water drops, feather barbs, pores) is largely preserved; no banding, mosaic, or colour shift observed. +9. Minor degradation: crackled skin (07), muddy small faces (17, 23), matte materials (15), murky darks (21, 23). +10. Verdict: usable as a general image model, not as a text-rendering or exact-count model; text prompts would be rejected by a user in 6/6 cases. diff --git a/eval/judge_mixF_ptq.md b/eval/judge_mixF_ptq.md new file mode 100644 index 0000000000000000000000000000000000000000..6f82ade06509dc13d5ccc317c4633ac22e8518e7 --- /dev/null +++ b/eval/judge_mixF_ptq.md @@ -0,0 +1,104 @@ +# Visual judge: mixF_ptq (student, 2.42 GB) vs teacher_bf16 + +Scales 0-5, same rubric as judge_mixA_lora32.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used where borderline. +Judged by opening all 24 mixF_ptq PNGs at full resolution; teacher PNGs 00, 03, 04, 05, 10, 14, 15, 19 at full resolution (other teachers not re-opened; I rely on the previous report that the teacher renders all six quoted-text prompts correctly and gets exactly 5 balloons); mixA_x_tuned PNGs 00, 01, 02, 03, 04, 05, 10, 14, 15 for the side-by-side. Caveat: I could not zoom beyond 768x768, so single-glyph claims on small cursive/hanzi text ("Affineur", "凌晨") carry some uncertainty. +C is scored for the same 7 images as the previous report (00, 01, 03, 05, 14, 15, 22). T is scored for the 6 quoted-text prompts plus the typewriter key legends (10). + +## Per-image table + +| id | category | F | T | C | Q | A | Note (most visible difference from teacher) | +|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 4 | 4.5 | 4.5 | 4 | Sign "LE PETIT FROMAGE" exact; slate "Tasting / -Saturday / 11 AM" exact (teacher has a dash line instead of a leading hyphen); window cursive reads "Affineur depuis 1974" but the first word's letters are soft/smeared (teacher crisp). Ginger cat on step, cheese wheels and crates, wet cobbles all present. Window is flatter/more uniformly yellow than teacher. | +| 01 | text | 3.5 | 5 | 2.5 | 3.5 | 3.5 | Title "WATER CYCLE" and all four labels (Evaporation, Condensation, Precipitation, Collection) spelled exactly right, in rounded boxes. But it is not a cycle: labels sit in a right-hand column with arrows that point at nothing, sun/cloud/mountain/sea are separate, stray raindrops; image slightly soft/haloed. Teacher is a coherent diagram. | +| 02 | text | 4.5 | 5 | - | 4 | 4.5 | "BEYOND THE BREAKERS" and "Some lights never go out" both exact (THE set smaller, tagline above title as asked). Girl in yellow raincoat rowing, spray-crowned wave, beam through clouds. Small figure, heavy print grain/halftone texture. | +| 03 | text | 4 | 4.5 | 3 | 4.5 | 4 | "Neon Horizon", "Luna Vale", "1:24", "3:48" all exact and on a modern phone (teacher near-identical layout). Differences: album art is a landscape rounded rectangle, not square; controls are rewind / pause / fast-forward (no play, no skip glyphs) and the three small bottom icons are odd glyphs; no desk lamp. | +| 04 | text | 4.5 | 4.5 | - | 4 | 4.5 | Main sign "老王面馆" exact and golden on red; second line "手工拉面 · 营业至凌晨两点" has all 12 characters in order, with 晨 slightly malformed (teacher perfect). Lanterns, steam, diners, wet reflections all present. Pavement reflections look like coloured tile-blocks, a few diner faces smeared. Huge jump from the previous student. | +| 05 | text | 4.5 | 3.5 | 3.5 | 4.5 | 4 | Signs read "PEACES $3" (H dropped), "CHERRIES $5", "HONEY $8": 2 of 3 words exact, all three prices exact. One straw hat, daisies, awning, hills, sky fine. Crates are less richly filled than the teacher's and the wood grain flatter. | +| 06 | portrait | 4 | - | - | 4.5 | 4 | Strong portrait: lined face, pores, silver stubble, maroon robe with golden edge, half-smile. Bead string hangs at the chest (prompt: around the wrist), robe weave is smooth (no visible wool weave), gaze goes to camera not past it; skin slightly airbrushed. | +| 07 | portrait | 4.5 | - | - | 4 | 4 | Good: curly auburn hair (short), green eyes, navy turtleneck, rain-streaked window with a real reflection of the face over the cheek, droplets, red lamp glow, parted lips, teal/amber grade. Reflection edge is a hard vertical seam; much better than previous student's cracked skin. | +| 08 | portrait | 5 | - | - | 4.5 | 4.5 | All elements present (black messy hair, golden eyes, white shirt, green scarf blowing, blue flower field, petals, clouds). One stray thin white curved line at the right edge; otherwise clean cel shading. | +| 09 | portrait | 4.5 | - | - | 4.5 | 4.5 | Grey hair pulled back, round glasses, arms crossed, clay-flecked apron, shelves of glazed bowls, kiln glow, warm window light, convincing skin. Hands tucked under arms (not visible). | +| 10 | detail | 4 | 4 | - | 4.5 | 4 | Keys carry distinct legends (N, 5, E, 6, C, F, 8, 2, A, D, O...) - the identical-glyph collapse is gone - but they are a random mix of letters and digits with repeats, not QWERTY like the teacher. Paper text is pseudo-English (as is the teacher's). Typebars read as straight rods, no bent carriage lever. Metal/rust texture excellent. | +| 11 | detail | 4.5 | - | - | 4 | 4 | Now a real peacock eye: turquoise ring, dark blue centre, gold/orange arcs, barbs and water droplets on a dark bokeh background. Barbs show a beaded/dotted aliasing pattern at the left strands and colours are over-saturated. | +| 12 | detail | 4.5 | - | - | 4 | 4 | Moss, lichen, fern with droplets, three cream mushrooms (not bracket fungi), a ladybug on the fern (previous student had none), one small dark snail, misty green bokeh. Greens over-saturated and background flat; bark slightly soft. | +| 13 | detail | 3.5 | - | - | 4.5 | 4 | Real interlocking rings now (previous student had punched discs), rivet, stitched leather strap, scratched wood, candle, darkness. But the object is a rolled tube/sleeve with no fingers - not a gauntlet. | +| 14 | counting | 3.5 | - | 1 | 4.5 | 2.5 | FOUR balloons (red; orange/yellow patterned; teal/yellow patterned; striped purple) instead of exactly five; no plain orange, yellow or blue balloon as asked. Teacher has exactly 5 plain-colour balloons. Landscape, haze and tiny village are excellent. | +| 15 | counting | 3 | - | 1 | 4.5 | 2 | 2 red apples + 2 pears (one yellow, one lime) = 4 fruits instead of 3 apples + 2 yellow pears + 1 green pear = 6; teacher has 3 apples + 2 pears (5). Rendering is excellent (glossy skin, blemishes, linen, soft shadow) but the explicit "exactly" counts fail, and the grey wall is dark slate. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | Colossal tree on an island, lantern-lit branches, candle-lit steps on the roots, boat with traveler, mountains, violet sky with stars, glowing lotus flowers, clean tree reflection. Minor: lanterns small. | +| 17 | varied | 5 | - | - | 4 | 4 | Yellow trench coat, clear umbrella, steaming food cart with vendor, Hangul neon signs, wet reflections, blurred scooter/taxi, teal-magenta grade. Woman seen from behind; legs/shoes mismatched and slightly awkward. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | Pink isometric bakery on a floating grass/brick slab, striped awning, bread in windows, smoking chimney, tree, cobble path, scooter, sign with a pastry/sunburst icon, pastel blue backdrop (previous student made a suitcase). Odd faceted dormer on the roof; scooter not bicycle. | +| 19 | varied | 3.5 | - | - | 3.5 | 3.5 | Bear with open mouth and a salmon beside its head, wet fur, river, forest. Over-saturated/high-contrast "crunchy" look, fur less detailed and face softer than teacher; fish is dark/blurry and not clearly leaping with frozen droplets. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | Gondolas at striped poles, palazzi with arched windows, bridge, red flower pot on balcony, laundry line, striped-shirt gondolier, pigeons (flying and one on the water, not on the bridge), washes and paper texture. Very close to expectation. | +| 21 | varied | 4.5 | - | - | 4.5 | 4.5 | Kneeling armoured knight, large glowing blue crystal, stained-glass shafts, embers, glowing floor runes, sword leaning at the left (not planted), visor/helm at his knee. Murky near-black regions but good detail. | +| 22 | varied | 4.5 | - | 3.5 | 4.5 | 4.5 | Bright Scandinavian kitchen, oak table, jug of tulips, crumbs, steaming cup, pale chair pulled back, plant shelf, window light. THREE cups on the table (prompt: two, plus a steaming one - ambiguous); table geometry plausible (previous student: one cup, six-legged table). | +| 23 | varied | 5 | - | - | 4.5 | 4.5 | Hooded vendor, dumplings frying in a large pan, steam, warm bulbs, dumplings and filling bowls, hand with coin, rain under awning, wet orange reflections. Foreground customers are near-black silhouettes. | + +## Averages + +Overall (24 images): F 4.29, T 4.36 (7 images with legible-text requirement: 6 quoted-text prompts + typewriter legends; 4.42 on the six quoted-text prompts alone), C 2.71 (7 images), Q 4.29, A 4.02. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.25 | 4.42 | 3.38 (4 imgs) | 4.17 | 4.08 | +| portrait | 4 | 4.50 | - | - | 4.38 | 4.25 | +| detail | 4 | 4.13 | 4.0 (typewriter only) | - | 4.25 | 4.00 | +| counting | 2 | 3.25 | - | 1.00 | 4.50 | 2.25 | +| varied | 8 | 4.56 | - | 3.5 (1 img) | 4.31 | 4.31 | + +## Comparison with the previous report (mixA_lora32) + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixA_lora32 (previous report) | 3.50 | 1.4 | 2.14 | 3.81 | 2.90 | +| mixF_ptq (this report) | 4.29 | 4.36 | 2.71 | 4.29 | 4.02 | + +Per category A: text 1.67 -> 4.08, portrait 3.75 -> 4.25, detail 3.00 -> 4.00, counting 2.00 -> 2.25, varied 3.56 -> 4.31. The gain is largest on text; counting is the only category that did not recover. + +## Side-by-side vs previous student (eval\mixA_x_tuned), 9 prompts + +Note: the mixA_x_tuned images I opened are not pixel-identical to what the previous report describes (e.g. its 14 has exactly 5 balloons and its 15 has the right 3/2/1 mix, whereas the report scored six balloons and four apples), so x_tuned is a slightly better student than the one scored in the report, especially on counting. + +| id | verdict | reason | +|---|---|---| +| 00 | better | mixF: sign, slate exact, window line legible. x_tuned: window "Affinas dogaun 1975", slate "Tasting / Satlad / 11 AM". x_tuned composition is a bit richer (flower pot, lanterns). | +| 01 | better | mixF: title + 4 labels all exact. x_tuned: "Eviation", "Condestion", "Podliation", "Collecion" (3 of 4 wrong plus 1 close). x_tuned's illustration looks more like a water system, mixF's diagram is a list with arrows to nowhere. | +| 02 | better | mixF: title and tagline exact. x_tuned: "BEYOD THE BRAKERS" plus garbled extra text in the top-left corner. x_tuned painting is more dramatic (bigger boat, textured waves), mixF has a small figure. | +| 03 | better | mixF: all four strings exact on a modern phone. x_tuned: "Neon Horion", "3:143", boxy gadget, wrong controls. | +| 04 | better | mixF: 老王面馆 exact + sub-line mostly correct. x_tuned: pseudo-hanzi on both lines (first glyph 北, last 鍋). | +| 05 | better | mixF: PEACES / CHERRIES / HONEY with $3 / $5 / $8. x_tuned: "PEACHES S8", "CUERNIS S8", "HONY S8" (prices lost). x_tuned crates richer. | +| 10 | better | mixF: distinct key legends. x_tuned: nearly every key shows "18"/"15"/"88" (repeated-glyph collapse). x_tuned has more mechanical-detail richness (bent lever, curly typebars). | +| 14 | worse | x_tuned has exactly five balloons (red, orange, yellow, blue, striped purple); mixF has four. | +| 15 | worse | x_tuned has exactly 3 apples + 2 yellow pears + 1 green pear (correct); mixF has 2 apples + 2 pears. | + +Net: 7 better, 0 same, 2 worse. Both "worse" cases are explicit counting prompts. + +## Five worst images + +1. 15 fruit bowl (A 2): four fruits (2 apples, 2 pears) where six were asked ("exactly 3 apples, 2 yellow pears, 1 green pear"); teacher is closer (5 fruits, 3 apples right). Quality is fine, the count is the point of the prompt. +2. 14 hot air balloons (A 2.5): four balloons, not "exactly five"; the requested plain orange, yellow and blue balloons are merged into two patterned ones. The previous student's x_tuned output had this right. +3. 01 water cycle (A 3.5): every string is spelled correctly, but the diagram is not a cycle (labels in a column, arrows point at nothing, no evaporation/condensation flow), so the infographic fails its purpose despite correct text. +4. 19 bear and salmon (A 3.5): over-saturated crunchy contrast, softer face and fur than the teacher, salmon dark/blurry and not caught or clearly leaping. +5. 05 fruit stand (A 4, T 3.5): "PEACES" (H dropped) is the only remaining misspelled sign word in the whole set, and a visible one on a large painted board. +Close behind: 13 chainmail (a sleeve/tube, not a gauntlet), 03 phone (landscape art, odd button glyphs), 06 monk (beads at the neck, not wrist), 17 (awkward legs/shoes). + +## Systematic failure patterns + +- Counting is the only systematic failure left: both explicit-count prompts fail (four balloons for five; four fruits for six). The previous student's tuned variant got both right, so this looks like a regression from the PTQ model, not a fundamental limit. No count mistakes elsewhere except 22's three cups (ambiguous prompt). +- Residual text errors are rare and single-glyph: "PEACES" (05), a soft first word of the cursive window line (00), a malformed 晨 (04). No repeated-glyph collapse, no dropped dollar signs, no pseudo-hanzi; Chinese now works for the 4-character sign and the 12-character sub-line. +- Compositional/semantic structure for infographics is weak: 01 has correct labels but no real cycle; 03 has a landscape album cover and incomplete control glyphs. +- Object identity drifts mildly remain: chainmail "gauntlet" becomes a sleeve (13), beads at the neck instead of wrist (06), keys use random letters/digits (10). +- Over-saturation / crunchy contrast on natural subjects (11 peacock, 12 moss, 19 bear); teacher is more neutral and finer in fur. +- Preserved/strong: photographic skin (06, 07, 09), anime (08), painterly and watercolor (16, 20, 21), isometric render (18), night street scenes (04, 17, 23), still-life gloss (15 apples), macro metal (10, 13). No banding, blocking, global colour shift, or the cracked skin / muddy faces seen in the previous student. Remaining local issues: wet-pavement reflection tiling (04), hard reflection seam (07), stray white line (08), awkward feet (17). + +## Summary (10 lines) + +1. Overall acceptance 4.0/5 (previous 2.9); faithfulness 4.3 (3.5), quality 4.3 (3.8), text 4.4 (1.4), counts 2.7 (2.1). +2. Text prompts: 5/6 are acceptable to a user (A >= 4); 3/6 are letter-for-letter exact (01, 02, 03), 2/6 have all strings readable with one soft or malformed glyph (00, 04), 1/6 has one dropped letter (05 "PEACES"). Previous student: 0/6. +3. Chinese 老王面馆 and its 12-character sub-line are correctly rendered (previous: 北子面鍋 and pseudo-glyphs); the worst previous failure is fixed. +4. Typewriter keys have distinct legends; the identical-"I8" collapse is gone. +5. Both counting prompts fail (4 balloons instead of 5; 4 fruits instead of 6): counting acceptance 2.25/5, and the previous x_tuned images got these two right. +6. Portraits, landscapes, painting, anime, watercolor and isometric images are at or near teacher level (A 4-4.5); crackled skin and muddy faces from the previous student are gone. +7. Geometry fixes versus previous: real chain rings, modern phone, bakery on a floating slab, two-to-three-cup table with proper legs, ladybug present; the gauntlet is still a sleeve. +8. Remaining quality issues are mild: over-saturation on macro/nature (11, 12, 19), faint aliasing in peacock barbs, reflection tiling on wet pavement (04). +9. Worst five: 15, 14, 01 (diagram logic), 19, 05 (PEACES). +10. Verdict: 2.42 GB mixF_ptq is a clear step up from mixA_lora32 on every category (text +2.4 A, overall +1.1 A) and acceptable as a general image model including text; it should not be trusted for exact-count prompts. diff --git a/eval/judge_mixG_tuned.md b/eval/judge_mixG_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..b243726d05ef8816afcdacf4feb5c1860fc410b6 --- /dev/null +++ b/eval/judge_mixG_tuned.md @@ -0,0 +1,115 @@ +# Visual judge: mixG_tuned (student, 2.14 GB, low-bit attention + 3-bit FFN, after the extra end-to-end tuning round) vs teacher_bf16 and vs mixG_x + +Scales 0-5, same rubric as judge_mixG_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixG_tuned PNGs opened individually at full 768x768 resolution. +- For all 24 prompts the teacher and the mixG_x image were also opened, as one teacher | mixG_x side-by-side file per prompt (1536x768, so each half at native resolution; scratch composites were written to the OS temp folder, not into the project, and deleted afterwards). No comparison was skipped. +- No zoom beyond native 768x768; the small time stamps in 03 and the 12-character Chinese sub-line in 04 were read at native size only. +- Same scoring subsets as the previous report: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness: no 5 on Q or A (fine micro-detail is still below bf16). Because this time I compared every image directly against mixG_x, I docked Q by 0.5 where mixG_x was visibly crisper (06, 07, 08, 09). Part of the Q drop versus the old report average is therefore calibration, not only the model. +- Reading difference on 03: in the mixG_x file I read "3:48" with its colon (the previous report read "3 48"); the stray dotted glyph beside "Luna Vale" in mixG_x is real. Roughly +0.2 of the T gain below comes from that reading difference, not from the tuning. +- This judge is a model instance; differences of +-0.2 on any average are within noise. The per-image side-by-side verdicts are the firmer evidence. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixG_x | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 5 | 4.5 | 4.5 | 4.5 | same | "LE PETIT FROMAGE" (top edge of sign slightly cropped), window "Affineur depuis 1974" and slate "Tasting / - Saturday / 11 AM" all exact; cat on step, cheese wheels on a crate, wet cobbles, lamp glow. Window line thin white script, not gold; teacher has a wider street scene. | +| 01 | text | 3 | 5 | 2 | 3.5 | 3 | worse | Title and four labels spelled exactly, but still a column of boxes at the right with arrows, not a cycle (teacher: labels beside each stage on a loop). Arrows are now a tangle of thin light-blue strokes, plus a stray blue triangle under the cloud and spiky blue shapes at the shoreline; mixG_x had cleaner arrows and a cleaner mountain/river. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "Some lights never go out" and "BEYOND THE BREAKERS" exact (THE smaller). Girl in yellow raincoat rowing toward the lit lighthouse, dramatic curled wave, beam through clouds. Girl seen from behind, a partial stray glyph at bottom-right, title sits right on the bottom edge. | +| 03 | text | 4 | 5 | 3 | 4 | 4 | better | "Neon Horizon", "Luna Vale" and "1:24 / 3:48" all exact; the stray dotted glyph next to "Luna Vale" that mixG_x had is gone. Square sunset album art, tilted phone on desk with a blurred cup. Controls are play / pause / a malformed bow-tie "next" (no previous); bottom icons are three junk glyphs; no lamp; screen darker and lower contrast than the teacher. | +| 04 | text | 4.5 | 4.5 | - | 4 | 4.5 | same | "老王面馆" exact in golden glowing characters on the red board; sub-line "手工拉面 · 营业至凌晨两点" reads all 12 characters in order, cream-white and small. Lanterns, steam, cook, diners present. Wet pavement is a flat patchwork of coloured blobs (looks like tiles) and faces are smeared; sign glow is weaker than mixG_x. | +| 05 | text | 4 | 4.5 | 3.5 | 4 | 4 | better | "PEACHES $3", "CHERRIES $5", "HONEY $8" all exact and clean; one straw hat (as the teacher). No more garbled "PERRCIES" sign. But "HONEY $8" appears twice (large board plus a crate board) and the H of the big board is slightly mangled; palette is hot/saturated. | +| 06 | portrait | 4 | - | - | 4 | 4 | same | Lined face, maroon robe with golden edge, silver stubble, warm grey backdrop, window light left. Beads hang at the neck (none on the wrist), gaze into the camera, skin slightly waxy and eyebrows not silver; a touch softer than mixG_x. | +| 07 | portrait | 4 | - | - | 3.5 | 3.5 | worse | Curly auburn hair, green left eye, navy turtleneck, red lamp glow, teal/amber bokeh all present, but the whole face is soft and smeared (no readable lashes, freckles or hair strands), few droplets, and a hard vertical glass seam runs straight through forehead and brow (also in teacher/mixG_x but far more visible here). mixG_x was clearly sharper. | +| 08 | portrait | 5 | - | - | 4 | 4 | same | Messy black hair, golden eyes with highlights, loose white shirt, dark green scarf, blue flower field, petals, clouds. Two thin white arcs (left edge across the scarf, right edge) instead of mixG_x's single one, and a ghost white drape behind the left shoulder; scarf has less detail and no fringe. | +| 09 | portrait | 5 | - | - | 4 | 4 | same | Short grey hair, round glasses, arms crossed, clay-smeared apron and forearms, shelves of bowls and vases, glowing kiln, big window. Skin and shelves are slightly smoother/mushier than in mixG_x, hands tucked away (mixG_x showed clay-crusted hands). | +| 10 | detail | 4.5 | 4.5 | - | 4.5 | 4.5 | same | Distinct serif letters on chrome-rimmed keys (Z E F / A S E P G H L / B T Y W N M plus 5, 6, 6), dust, worn enamel, cream paper with pseudo-English lines (like the teacher). Carriage lever not clearly bent (a chrome loop at upper left). | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | Proper eye (turquoise ring, dark violet centre, gold/green arcs), fine barbs, droplets left of the eye, gold bokeh. Very close to mixG_x; still more saturated than the teacher. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | better | Mossy oak with lichen, ferns, droplets, misty bokeh, cream/orange mushrooms and now a real cluster of bracket fungi, several snails, 2 ladybugs (prompt: one; mixG_x had four). Upper trunk slightly soft. | +| 13 | detail | 4.5 | - | - | 4 | 4.5 | better | Leather cuff with rivets and red stitching over interlocking mail that now ends in dark leather finger-like segments, so it reads as a gauntlet rather than a bracer; candle with melted wax, scratched wood, black background. Rings have fewer bright highlights than the teacher; mail slightly mushy. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | same | Exactly five balloons: red, orange striped, yellow with red patches, blue with a colour band, purple striped; pilots in baskets, sunrise, mist, village, shadows. Balloons in a loose arc, candy-saturated; village slightly mushy. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 3.5 | better | Three red apples (correct), one yellow pear and one olive-green pear = 5 fruits instead of 3+2+1 = 6 (a yellow pear is missing); mixG_x had only 4 fruits. Glossy skin, leaves, linen folds, muted grey wall, shadow to the right all good. Exact-count wording still not met. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | same | Colossal lantern tree on an island, lit walkway through the roots, boat with one traveller, mountains, violet sky with stars, glowing flowers and floating lights, soft mist. Fewer, smaller lanterns and a hazier reflection than mixG_x. | +| 17 | varied | 4.5 | - | - | 4.5 | 4 | worse | Yellow trench coat, clear umbrella, steaming food cart, Hangul neon signs, blurred taxi and scooter, wet reflections. Woman now walks away from the camera (teacher and mixG_x face the camera), with an odd white under-hem below the coat; the rest is as good as mixG_x. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Pink clay bakery on a floating slab, striped awning, bread in window, smoking chimney, tree, cobble path, delivery scooter, croissant sign. Odd croissant lying on the flat roof (mixG_x: a loaf), sandy slab, fuzzy broccoli-like tree; bicycle is a scooter (prompt mentions both). | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | better | Wet brown bear with open mouth, salmon now touching the jaw, droplets frozen around it, mossy log, rushing river, misty forest. Sharper and more convincing "catch" than mixG_x; slightly crunchy contrast. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Gondolas at striped poles, arched palazzi reflected in green water, stone bridge with figures, red-flower balcony, laundry line, striped-shirt gondolier, pigeons in the air and on the water (none on the bridge), paper texture. Pencil underdrawing barely visible. | +| 21 | varied | 4.5 | - | - | 4 | 4 | same | Black-armoured knight before a giant glowing crystal, stained-glass shafts, embers, glowing floor runes, torn banners, sword planted with a broken blade, helmet on the floor. The knight wears a horned helmet AND a second helmet lies beside him; the pose is a crouching lunge rather than a kneel; very crushed blacks. | +| 22 | varied | 5 | - | 4.5 | 4.5 | 4.5 | same | Bright Scandinavian kitchen: two cups on the oak table with steam between them, jug of pink tulips, crumbs, pulled-back pale chair, plant on shelf, big window, grey floor, petals. Table perspective slightly odd; otherwise clean. | +| 23 | varied | 5 | - | - | 4 | 4.5 | better | Hooded vendor frying dumplings, warm bulbs, steam, tray of dumplings, bowl of filling, customer backs in the foreground and the customer's hand with a clearly visible coin (mixG_x: coin not visible). Foreground customers are near-black crushed silhouettes; no burner flame. | + +## Averages + +Overall (24 images): F 4.48, T 4.79 (7 text-bearing images; 4.83 on the six quoted-text prompts alone), C 3.64 (7 images), Q 4.23, A 4.21. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.17 | 4.83 | 3.25 (4 imgs) | 4.08 | 4.08 | +| portrait | 4 | 4.50 | - | - | 3.88 | 3.88 | +| detail | 4 | 4.63 | 4.5 (typewriter only) | - | 4.38 | 4.50 | +| counting | 2 | 4.25 | - | 4.00 (5 and 3) | 4.50 | 4.00 | +| varied | 8 | 4.69 | - | 4.5 (1 img) | 4.38 | 4.38 | + +## Comparison with the mixG_x report + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixF_ptq (2.42 GB) | 4.29 | 4.36 | 2.71 | 4.29 | 4.02 | +| mixG_x (2.14 GB, previous report) | 4.50 | 4.43 | 3.36 | 4.44 | 4.29 | +| mixG_tuned (2.14 GB, this report) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| delta tuned - G_x | -0.02 | +0.36 | +0.28 | -0.21 | -0.08 | + +Per category A (mixG_x -> mixG_tuned): text 4.17 -> 4.08, portrait 4.38 -> 3.88, detail 4.38 -> 4.50, counting 3.50 -> 4.00, varied 4.50 -> 4.38. + +Reading the deltas: F and A are unchanged within noise. T (+0.36) and C (+0.28) gains are real in direction (03 stray glyph gone, 05 garbled sign gone, 15 five fruits instead of four) but part of T is the 03 reading difference described above and part of C is one image (15). The Q drop (-0.21) is partly my stricter side-by-side docking of portraits, but the portrait softness in 07 is a real regression I would see in any judge pass. Net: no clear overall win from the tuning round; it traded some fine-detail sharpness on faces and diagram cleanliness for fewer text and count errors. + +## Side-by-side verdict versus mixG_x (all 24) + +Better 7: 03, 05, 12, 13, 15, 19, 23. +Same 14: 00, 02, 04, 06, 08, 09, 10, 11, 14, 16, 18, 20, 21, 22. +Worse 3: 01, 07, 17. + +Reasons in brief: +- better: 03 no stray glyph by "Luna Vale", all four strings exact; 05 no garbled "PERRCIES" sign, one hat (but a duplicated HONEY sign); 12 bracket fungi cluster and 2 ladybugs instead of 4; 13 mail ends in leather finger segments so it reads as a gauntlet; 15 five fruits (three apples) instead of four; 19 sharper bear, salmon touches the mouth; 23 coin clearly visible in the reaching hand. +- worse: 01 arrows turned into a tangle with stray triangle/spikes; 07 face soft and smeared, seam through the forehead; 17 woman now faces away with an odd white under-hem. +- same (with small leanings): 06, 08, 09 are a touch softer or have extra stray white arcs (08 has two) but within what I would call unchanged; 04 equal text, but pavement looks like coloured tiles and sign glow is weaker; 21 crouching knight pose and double helmet (same flaw class as before); 00, 02, 10, 11, 14, 16, 18, 20, 22 near-identical in quality. + +## Text-prompt acceptance + +- Six quoted-text prompts: 5/6 accepted (A >= 4: 00, 02, 03, 04, 05); 01 at A 3 because the diagram is neither a cycle nor clean. +- All requested strings are letter-for-letter exact in 6/6 images: 00 (three strings), 01 (title and four labels), 02 (title and tagline), 03 (title, artist, "1:24 / 3:48"), 04 (老王面馆 and all 12 characters of the sub-line), 05 (three signs). No garbled requested string anywhere; the only text defects are a duplicated "HONEY $8" sign (05) and one stray partial glyph at the bottom-right of 02. +- Typewriter keys (10) are legible, distinct serif letters. + +## Five worst images + +1. 01 water cycle (A 3): text exact, but a list of boxes with a tangle of arrows and stray shapes; regression versus mixG_x. +2. 07 train-window portrait (A 3.5, Q 3.5): soft smeared face, visible seam through the forehead; regression versus mixG_x. +3. 15 fruit bowl (A 3.5, C 3): five fruits instead of six, one yellow pear missing; improved from four, still fails "exactly". +4. 03 music player (A 4): all strings exact now, but a malformed next button, junk bottom icons, no previous button, dark low-contrast screen. +5. 08 anime boy (A 4): two stray thin white arcs plus a ghost white drape; otherwise a good image. +Close behind (A 4): 05 (duplicated HONEY sign), 06 (beads at neck), 09 (smoothed skin), 17 (back view), 21 (crouching knight, double helmet). + +## Systematic failure patterns + +- Exact-count prompts: balloons exact (5/5), fruit bowl moved from 4 to 5 of 6 fruits with correct three apples; the composite 3/2/1 mix still fails (the second yellow pear is dropped). Acceptance on counting: 4.0/5 average, one pass and one near-miss. +- Infographic logic: 01 still produces labels in a right-hand column rather than beside each stage, and after tuning the arrows and icons are noisier. +- Close-up portrait sharpness: skin and hair in 06, 07, 09 are a notch softer than mixG_x; 07 is the clear case (no readable lashes or freckles). Not a colour or blocking problem, a detail-loss problem. +- Duplicated or unrequested objects: 05 repeats the HONEY sign, 21 shows a helmet on his head and another on the floor, 18 has a croissant on the roof. +- Stray linework in the anime image (08): thin white arcs at the edges (two now). +- Text: no systematic garbling; residual small items are the duplicated sign (05), a stray partial glyph (02), junk icon glyphs on the phone (03). +- Crushed near-black regions remain in 21 and 23; no banding, blocking, or global colour/saturation shift seen. Slightly hot colour in 05, 14 and a crunchy contrast in 19. + +## Summary (10 lines) + +1. Overall: F 4.48, T 4.79, C 3.64, Q 4.23, A 4.21 (mixG_x report: 4.50 / 4.43 / 3.36 / 4.44 / 4.29); F and A unchanged within noise, T and C up, Q down. +2. Side-by-side versus mixG_x: 7 better, 14 same, 3 worse out of 24. +3. Text prompts: 5/6 acceptable (A >= 4); every requested string is exact in 6/6 images, the weak one is 01 (diagram logic and arrow noise). +4. Chinese 老王面馆 and its 12-character sub-line still render correctly. +5. Counting: balloons exactly five; fruit bowl improved from four to five of six fruits but still fails the exact-count requirement. +6. Gains on specific defects of mixG_x: no stray glyph in 03, no garbled sign in 05, visible coin in 23, bracket fungi and fewer ladybugs in 12, gauntlet fingers in 13, sharper bear and salmon in 19. +7. Regressions: 07 face softness and seam, 01 arrow tangle, 17 back view with odd hem; mild extra softness in 06 and 09, extra white arcs in 08. +8. No global colour shift, banding or blocking; defects are local. +9. Worst five: 01, 07, 15, 03, 08. +10. Verdict: the tuning round fixed several text and count defects but did not raise overall acceptance (A 4.21 vs 4.29, within noise) and cost some portrait/diagram quality; the old caveats (exact counts with several object types, infographic layout logic) remain. Caveat: this judge is a model instance and +-0.2 differences are within noise. diff --git a/eval/judge_mixG_tuned_t.md b/eval/judge_mixG_tuned_t.md new file mode 100644 index 0000000000000000000000000000000000000000..ed0ae8746b2cfa5fd7ba264d8f267388ab134b58 --- /dev/null +++ b/eval/judge_mixG_tuned_t.md @@ -0,0 +1,140 @@ +# Visual judge: mixG_tuned_t (2.93 GB quantized mixG_tuned with the 4-step turbo LoRA folded in as extra low-rank factors, 4 steps, CFG 1.0, euler) vs turbo4 (its unquantized distilled teacher: bf16 model + the same LoRA, same steps) + +Question: HOW GOOD IS THE QUANTIZED, TURBO-FOLDED MODEL COMPARED WITH ITS OWN UNQUANTIZED DISTILLED TEACHER, AND WOULD A USER ACCEPT THE QUANTIZED IMAGE INSTEAD OF THE TEACHER IMAGE FOR THE SAME PROMPT? + +Scales 0-5, same rubric as judge_turbo4.md / judge_tmixG_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image (turbo4_1024). "-" = not applicable. Half points used. Scored images: eval\turboF_mixG_tuned (24 PNGs, 1024x1024, primary). Reference: eval\turbo4_1024. + +Method and caveats: +- All 24 candidate PNGs were opened alone at native 1024x1024, and for all 24 prompts a candidate | turbo4_1024 side-by-side (2056x1024; the viewer showed it at about 97%, so each half was slightly under native) was opened. Text was read from the native single views plus 1.8-2x crops (01 labels, 04 Chinese sub-line); ghost checks used 1.5-1.8x crops (07 face, 22 chair and table, 21 knight, 05 crates). Scratch files were written to the OS temp folder and deleted afterwards; the only file written in the project is this report. +- Scoring was done on all 24 images BEFORE judge_tmixG_x.md was read; judge_tmixG_tuned.md and judge_tmixH_x_f8.md were not read. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the six quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- Compositions differ from the reference in most pairs (different sampling trajectory); a different layout alone was not penalised. Missing prompt elements, wrong or garbled text, artifacts, colour/contrast drift and loss of detail were. "vs teacher" = would I rather have this image or the turbo4 image for the same prompt. +- Pixel statistics (System.Drawing, CPU, all 24 images resized to 768, mean over images) are an objective cross-check. +- This judge is a model instance; differences of +-0.2 on any average are within noise. One seed per prompt. + +## Per-image table (turboF_mixG_tuned, 1024) + +| id | category | F | T | C | Q | A | vs teacher | Note | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 5 | 4 | 4.5 | 4.5 | same | Full storefront, sign not cropped. Sign "LE PETIT FROMAGE" exact (navy serif, no swash), window line "Affineur depuis 1974" exact, slate "Tasting - Saturday 11 AM" exact (an extra short dash above "Tasting" and a small diamond at the bottom). One slate only (reference has two, one truncated). Two ginger cats asleep on the step (prompt: one). Warm window full of cheese wheels, wet cobbles, blue-hour street. | +| 01 | text | 3.5 | 3 | 3 | 3.5 | 3 | worse | "WATER CYCLE" exact; labels "Evaporation" (exact, but the box appears TWICE: beside the sun and again at the left of the cloud), "Condenssion" (typo: "Condensation" with "at" lost), "Precipitation" exact, "Collection" exact. Bold, saturated and clean (no ghost panel, no glass sphere), but the logic is poor: arrows point sun -> right label and cloud -> left label, an upward arrow stands in the rain, drops fall from no cloud, the sea is a flat tank. Misspelled word plus duplicated label; the reference has all four labels exact (but is pale with a ghost panel). | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "Some lights never go out" and "BEYOND THE BREAKERS" (THE in small caps) exact, cream serif, sharp. Girl in yellow raincoat rowing, big curled wave with spray, lighthouse beam through clouds, teal sea, print grain. Left oar tip leaves the frame; boat rim dissolves into stipple (same family as the reference). | +| 03 | text | 4.5 | 5 | 4.5 | 4.5 | 4.5 | same | "Neon Horizon", "Luna Vale", "1:24" and "3:48" at the two ends of the progress bar, all exact. Square purple-sunset cover, previous / play / next as skip icons and a round play button, three dim icons below, upright phone with lamp and book on the desk. Bluer and cleaner than the reference; desk background simpler (no laptop and pens), no status-bar clock. | +| 04 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "老王面馆" exact in glowing golden characters on the red board; sub-line "手工拉面 · 营业至凌晨两点" reads all 11 characters and the dot in order (checked on a 1.8x crop). Lanterns clean (no pseudo-glyphs), cooks in a lit kitchen, steam, diners at metal tables, wet reflections. A green side sign shows garbled Latin letters (harmless). Frontal, symmetrical; the reference has a scooter and richer lantern detail. | +| 05 | text | 4 | 5 | 4.5 | 4.5 | 4 | worse | "PEACHES $3", "CHERRIES $5", "HONEY $8" exact in white brush letters on dark green boards (smaller than the reference's huge signs). One straw hat on the post, daisies, hills, striped awning. The crates hold a generic mix of oranges, apples and pears; the CHERRIES crate contains red apples, no distinct peaches or cherries (reference has real peaches and cherries). | +| 06 | portrait | 4.5 | - | - | 4.5 | 4.5 | better | Deeply lined elderly monk, shaved head with silver stubble, maroon robe with golden edge and visible wool weave, beads on the wrist, window light from the left, plain grey wall (reference: a younger, jollier monk with a sofa behind him). Still looks into the camera instead of past it. Skin pores and wrinkles resolved. | +| 07 | portrait | 4.5 | - | - | 4 | 4.5 | better | One face, no ghosting (see ghost check). Curly auburn hair, grey-green eyes, navy turtleneck, parted lips, droplets on forehead and cheeks, rain-streaked window with a dark frame at the right edge, red lamp glow, strong teal/amber bokeh. Missing: droplets on the lashes. The reference has no window at all and softer hair. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | same | Messy black hair, golden eyes, loose white shirt billowing at the right, dark green fringed scarf flying, field of blue flowers, petals, clouds, mountains. Clean line art, neutral pose (reference: hand-to-cheek smile). Eye highlights simple. No stray arcs. | +| 09 | portrait | 4 | - | - | 4 | 4 | worse | Grey short hair, round glasses, arms crossed, smile, shelves of glazed bowls, glowing kiln, big window. Clay crust on apron and forearms only as faint speckles (reference: clearer clay stains); whole picture hazy and lower in contrast, kiln and shelves simpler. Face natural. | +| 10 | detail | 3.5 | 3.5 | - | 4 | 4 | same | Close macro of a dark-blue worn enamel typewriter with dust, chrome-rimmed glass keys, sharp where in focus. About 18 legends readable (G F 2 3 L F G T 8 H F E G C X C N X) but the sequence is nonsense (repeated F, G, X, C). Paper text is pseudo-English ("The se neet de na / lad goue iny dev / the wint the ee tca") with no strike-through artifact. No visible typebar basket or platen detail, carriage lever not visible. The reference shows the typebars and has a strike-through artifact in the typed line. | +| 11 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Excellent peacock eye: blue ring, deep blue centre, gold-brown halo, green barbs resolved individually, many well-formed droplets, dark background with warm gold bokeh. Inner dark pupil has a slightly offset darker lobe (the reference shows a similar split). More saturated than the reference. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Mossy oak with ferns, lichen, droplets, several ladybugs (prompt: one), 3-4 snail-like shapes, stalked mushrooms and a cream/orange cluster, misty green bokeh. Moss is very dense; bark furrows less visible than in the reference. | +| 13 | detail | 4 | - | - | 4.5 | 4 | same | Rivet-studded leather cuff, stitching, wax drops, uniform interlocking mail rings sharply resolved, candle on a holder with wax, dark background. Mail forms a regular mitt without finger segments and with no single-ring highlights; table scratches and leather scratches are fewer than in the reference. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | same | Exactly five balloons in a row: red with yellow band, orange, yellow, blue, purple/white striped; baskets with tiny pilots, layered mist, hills, village, sunrise at the right edge, long shadows. Plain stripes/bands, no blocky patches. As good as the reference. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 4 | same | Two red apples, two yellow pears and one green pear = five fruits (prompt: three apples, two yellow pears, one green pear = six). Pears right, apples one short; the reference has exactly the same count and the same miss. Glossy skin, leaves, linen folds, grey wall, window at upper left. | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Colossal lantern tree on an island, roots into the lake, candle-lit path along the roots to an arched bridge, one traveller in a boat, mountains, violet sky with stars and a moon (unrequested), mist, long golden reflections. The glowing flowers in the water are replaced by pink flowers on the shore. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart with Hangul signs, neon signs, wet reflections, blurred cars and a scooter, teal/magenta palette. Woman faces the camera (reference: profile). Very saturated, no artifacts. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Pink clay bakery with striped awning, bread in the window, chimney smoke, tree, cobbles, croissant sign, both a scooter and a delivery bicycle, fence with bread. Pastel blue backdrop, soft shadows. No clouds (reference has two); equally charming. | +| 19 | varied | 4 | - | - | 4.5 | 4 | same | Brown bear in a rushing river, open mouth, wet fur, salmon leaping beside the head with frozen droplets, mossy rocks and logs, misty forest. Sharp fur and spray. Fish is next to, not at, the jaws; bear's raised paw is a bit odd. Same as the reference in this respect. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Watercolor-and-ink canal: red/white striped poles, three gondolas and a gondolier in a striped shirt with an oar, arched palazzi and their reflections, pigeons on the bridge, laundry lines, red flowers on balconies, faint paper texture. No torn edge, no visible pencil underdrawing; cleaner and more digital than the reference's looser washes. | +| 21 | varied | 4 | - | - | 4 | 4 | worse | Black-armoured knight kneeling (seen from behind) before a giant glowing blue faceted crystal in a cathedral, stained glass, light shafts, embers, torn banners, many candles. Helmet on his head AND one on the floor (as in the reference), no visible floor runes, three swords (a planted one at right, a small one at left, and a jagged blade held up in his raised hand), pseudo-text on the plinth. Very blue and saturated; armour is dark with less detail than the reference. | +| 22 | varied | 5 | - | 4.5 | 4.5 | 4.5 | same | Bright Scandinavian kitchen: white cabinets, oak table, ceramic jug of pink tulips, two cups with saucers (one steaming), crumbs, petals, pulled-back pale chair, plant on the shelf, big window, pale grey floor with sun streaks. Chair and table are solid (no ghosting). Table top is an odd rounded wedge. Cleaner and simpler than the reference (no pendant lamp). | +| 23 | varied | 4 | - | - | 4 | 4 | same | Hooded vendor frying dumplings in a large pan, warm bulbs, thick steam, rain under the awning, customers' backs in the foreground, wet orange reflections, bowl of filling, tray of dumplings. No visible coin in the reaching hand, no wrapper pile; large near-black foreground areas. Reference is warmer, with a coin hand. | + +## Averages (turboF_mixG_tuned, 1024) + +Overall (24 images): F 4.29, T 4.50 (7 text-bearing images; 4.67 on the six quoted-text prompts alone), C 4.07 (7 images), Q 4.35, A 4.27. 23 of 24 images have A >= 4 (only 01 is below, A 3). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.25 | 4.67 | 4.00 (4 imgs) | 4.33 | 4.17 | +| portrait | 4 | 4.38 | - | - | 4.25 | 4.38 | +| detail | 4 | 4.13 | 3.5 (typewriter only) | - | 4.38 | 4.25 | +| counting | 2 | 4.25 | - | 4.00 (5 and 3) | 4.50 | 4.25 | +| varied | 8 | 4.38 | - | 4.5 (1 img) | 4.38 | 4.31 | + +## Comparison with the reference and with the other distilled quantized student (tmixG_x, 2.14 GB; judge_tmixG_x.md read only after scoring) + +| | F | T | C | Q | A | A >= 4 | quoted-text prompts letter-exact | better / same / worse | +|---|---|---|---|---|---|---|---|---| +| turbo4_1024 vs the 25-step teacher (judge_turbo4.md) | 4.23 | 4.71 | 4.14 | 4.31 | 4.21 | - | 6/6 | 8 / 11 / 5 (vs 25-step) | +| tmixG_x_1024 vs turbo4 (2.14 GB) | 4.08 | 3.79 | 3.86 | 4.23 | 4.06 | 18/24 | 4/6 | 1 / 14 / 9 | +| mixG_tuned_t (turboF_mixG_tuned) vs turbo4 (2.93 GB, this report) | 4.29 | 4.50 | 4.07 | 4.35 | 4.27 | 23/24 | 5/6 (18/19 strings) | 2 / 18 / 4 | +| delta (this report - tmixG_x) | +0.21 | +0.71 | +0.21 | +0.12 | +0.21 | +5 | +1 | | + +Reading the deltas: F, C, Q and A differences are at the edge of the +-0.2 noise band; T is the one clearly different number (+0.71), caused by 00 (exact here, wrong in tmixG_x) and 01 (one typo here versus 3 wrong labels plus a missing one). Both reports were written by model instances, so a leniency offset between judges cannot be excluded; the tmixG_x report's own 05 / 13 / 18 / 23 / 21 critiques have partial counterparts here (generic fruit in 05, fewer scratches in 13, simpler detail in 21) but the images here score higher on 18 and 23. + +## Verdict versus the turbo4 image (all 24) + +Better 2: 06, 07. +Same 18: 00, 02, 03, 04, 08, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 22, 23. +Worse 4: 01, 05, 09, 21. + +Reasons in brief: +- better: 06 plain grey wall and a deeply lined face where the reference shows a younger monk with a sofa; 07 a real rain-streaked window and teal/amber grade where the reference has none, single clean face. +- worse: 01 misspelled "Condenssion" and a duplicated "Evaporation" box with confusing arrows (reference text is exact, although the reference is pale with a ghost panel); 05 generic fruit (no cherries or peaches) and smaller signs; 09 clay crust mostly lost, hazy; 21 extra blades, a raised arm holding a jagged blade, darker armour with less detail. +- same: all others; leanings are small (00 and 03 cleaner but simpler; 10 sharper keys but no typebars; 13 fewer scratches; 20 cleaner, less watercolour feel; 14, 15, 19 equal to the reference including the same count/fish misses). + +## Text-prompt acceptance and letter-by-letter reading + +- Six quoted-text prompts: 5/6 accepted (A >= 4: 00, 02, 03, 04, 05); 01 at A 3. +- Letter-exact prompts: 5 of 6 (00, 02, 03, 04, 05). By string: 18 of 19 exact (00: 3, 01: 5, 02: 2, 03: 4, 04: 2, 05: 3). Turbo4 had 6 of 6; tmixG_x 4 of 6. +- 00 exact: sign "LE PETIT FROMAGE", window "Affineur depuis 1974", slate "Tasting - Saturday 11 AM" (extra short dash above "Tasting", harmless). +- 01: "WATER CYCLE" exact; "Evaporation" exact (shown twice); "Condenssion" = typo (Condensation, 12 letters, written with 11: "at" lost, extra "s"); "Precipitation" exact; "Collection" exact. +- 02 exact: "Some lights never go out", "BEYOND THE BREAKERS". +- 03 exact: "Neon Horizon", "Luna Vale", "1:24", "3:48" (at the two ends of the progress bar). +- 04 exact: "老王面馆" and "手工拉面 · 营业至凌晨两点" (all 11 characters and the dot, in order). +- 05 exact: "PEACHES $3", "CHERRIES $5", "HONEY $8"; no extra boards. +- Typewriter keys (10): about 18 sharp serif legends in a nonsense sequence, paper text pseudo-English, no strike-through artifact. + +## Broken images, local artifacts, ghosting + +- No image looks broken: no colour fill, mush, banding, blocking or burnt/over-sharpened look. No image has a heavy local artifact. +- Moderate local defects: 01 (duplicated "Evaporation" box, upward arrow in the rain, drops from no cloud, misspelled label), 21 (arm holding a jagged blade, extra swords, helmet on head plus helmet on floor, pseudo-text "KAHOCHI" on the plinth), 10 (garbled paper text, repeated key letters), 02 (stipple along the boat rim, same as the reference). +- Ghost / double faces (prompt 07): none. Single face, no second contour, no seam, no doubled eyes or mouth; hair and bokeh are clean. The only translucent element is a few faint bokeh discs over the forehead near the hairline, consistent with the requested glass reflection (checked on a 1.8x crop). +- Ghost / translucent furniture (prompt 22): none. Table, four-legged spindle chair, jug and cups are opaque with consistent shadows (checked on a 1.5x crop); the only oddity is the rounded-wedge table top, also present in tmixG_x. +- Doubled structures anywhere: 00 two ginger cats (prompt: one; both clearly separate, opaque); 01 the duplicated "Evaporation" label; 21 two helmets (head plus floor, as in the reference) and extra swords; 12 several ladybugs (prompt: one); 11 a mildly doubled pupil lobe (the reference has the same). No semi-transparent duplicate of any object or face anywhere. The ghost panel the reference has in 01 is absent here. + +## Five worst images + +1. 01 water cycle (A 3, T 3, F 3.5, C 3, Q 3.5): "Condenssion", duplicated "Evaporation" box, confusing arrows; the only image with A < 4. +2. 10 typewriter (A 4, F 3.5, T 3.5): nonsense key sequence, garbled paper text, no typebars; but crisp keys and no artifacts. +3. 15 fruit (A 4, F 3.5, C 3): five fruits instead of six (apples one short); identical miss to the reference. +4. 09 ceramicist (A 4, F 4, Q 4): clay crust mostly lost, hazy. +5. 21 knight (A 4, F 4, Q 4): jagged blade held up, extra swords, two helmets, darker armour detail. +Close behind: 05 (generic fruit), 23 (no coin, near-black foreground), 19 (fish not caught). + +## Systematic failure patterns + +- Small multi-label text: 01 has one misspelling and a duplicated label; the other five text prompts (including the Chinese sub-line) are exact. No loss of letters in 00, unlike tmixG_x. +- Exact multi-type counts (15): five of six fruits, apples one short; identical to the reference. Single-type count (14) and kitchen cups (22) exact. +- Prompt details dropped silently, as in the reference: droplets on lashes (07), clay crust (09), typebars and bent lever (10), glowing water flowers (16), fish in the jaws (19), pencil underdrawing (20), runes (21), coin (23), distinct peaches/cherries (05), gaze past the camera (06). +- Cleaner and more saturated rendering than the reference in several pairs (03, 07, 11, 21, 17); less rich detail in the dark knight (21) and in the fruit crates (05). +- No colour shift, banding or blocking; no over-sharpened look. + +## Style differences versus the teacher (turbo4_1024) + +Mean over all 24 images, both sets resized to 768 (System.Drawing, same method for both): + +| | luminance mean | luminance std (contrast) | mean saturation | fine-detail energy (Laplacian) | pixels < 0.04 | pixels > 0.96 | +|---|---|---|---|---|---|---| +| turbo4_1024 (reference) | 0.389 | 0.191 | 0.430 | 0.0290 | 7.8% | 3.6% | +| mixG_tuned_t (turboF_mixG_tuned) | 0.410 | 0.206 | 0.434 | 0.0286 | 9.4% | 4.2% | + +- Saturation is the same on average (0.434 vs 0.430): no global colour shift. Image-specific: more saturated in 21 (0.75 vs 0.40), 11 (0.70 vs 0.50), 07 (0.61 vs 0.52), 03, 17; less in 00 (0.36 vs 0.51), 05 (0.41 vs 0.55), 09, 10 (0.24 vs 0.39), 14. +- Contrast is about 8% higher than the reference (std 0.206 vs 0.191) and the image is 0.02 brighter on average, with slightly more crushed blacks and clipped whites (9.4% vs 7.8% near-black). The reference has about 7% lower contrast than the 25-step teacher (judge_turbo4.md), so this sits close to the 25-step level. +- Fine-detail energy is equal on average (0.0286 vs 0.0290, -1%) and not systematically lower: lower in 10 of 24 pairs (largest: 20 at 0.049 vs 0.086, 15, 21, 16, 05), higher in 14 (07, 08, 12, 11, 23). This differs from tmixG_x, whose report gives about 35% lower fine-detail energy than turbo4 (0.024 vs 0.031 on 1024-downscaled files). +- Visual impression matches the statistics: sharp barbs, mail rings, fur and key legends; skin and cloth natural; softest areas are the hazy 09 and the dark foregrounds in 23 and 21. + +## Summary (10 lines) + +1. mixG_tuned_t overall: F 4.29, T 4.50, C 4.07, Q 4.35, A 4.27 (turbo4 vs the 25-step teacher: 4.23 / 4.71 / 4.14 / 4.31 / 4.21; tmixG_x: 4.08 / 3.79 / 3.86 / 4.23 / 4.06). +2. Side-by-side versus the turbo4 image: 2 better, 18 same, 4 worse out of 24; A >= 4 in 23 of 24. +3. Text: 5/6 prompts and 18/19 strings letter-exact, including the Chinese sign and its 11-character sub-line; the one failure is 01 ("Condenssion", plus a duplicated "Evaporation" box). +4. No image looks broken; no heavy local artifacts; moderate defects in 01, 21 and 10. +5. Ghost faces (07): none. Ghost furniture (22): none. No semi-transparent duplicate of any object anywhere; only ordinary duplicates (two cats in 00, two helmets in 21, duplicate label in 01). +6. Style: no colour shift; contrast about 8% above turbo4; fine-detail energy equal to turbo4 (not lower). +7. Strengths: scene prompts (00, 02, 03, 04, 14, 16, 17, 18, 22) equal to the reference; 06 and 07 follow the prompt better than the reference. +8. Weaknesses: exact multi-type count (15, apples one short, same as the reference), generic fruit (05), hazy 09, jagged-blade knight (21), nonsense key legends and paper text (10). +9. Versus tmixG_x (2.14 GB): +0.21 F, +0.71 T, +0.21 C, +0.12 Q, +0.21 A; 23 vs 18 images with A >= 4; text exact 5/6 vs 4/6; 2/18/4 vs 1/14/9 better/same/worse. +10. ANSWER: yes, a user would accept the quantized image instead of the teacher image in 23 of 24 cases; it is prompt-faithful and artifact-free at the level of the unquantized distilled teacher, with its flaws (exact mixed counts, dropped second-level details) shared with the teacher. Caveat: this judge is a model instance; +-0.2 on any average is noise; one seed per prompt; scores of different judge instances may carry a leniency offset. diff --git a/eval/judge_mixG_x.md b/eval/judge_mixG_x.md new file mode 100644 index 0000000000000000000000000000000000000000..03d94b9543fe74b33b70ef9fe97d8907b46be0a6 --- /dev/null +++ b/eval/judge_mixG_x.md @@ -0,0 +1,103 @@ +# Visual judge: mixG_x (student, 2.14 GB, low-bit attention + 3-bit FFN, QAT-distilled) vs teacher_bf16 + +Scales 0-5, same rubric as judge_mixF_ptq.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used where borderline. + +Method and caveats: +- All 24 mixG_x PNGs opened at full 768x768 resolution. Teacher PNGs opened only for 01, 03, 05 (the rest judged against the prompt text and the earlier reports' statement that the teacher renders all six quoted-text prompts correctly and gets exactly 5 balloons). +- mixF_ptq PNGs opened for 00, 01, 02, 03, 04, 05, 07, 11, 12, 13, 15, 19, 22. For 06, 08, 09, 10, 14, 16, 17, 18, 20, 21, 23 the F side of the comparison relies on the text of judge_mixF_ptq.md (marked "(F per report)" below). +- Could not zoom beyond 768x768; the 12-character Chinese sub-line in 04 and the tiny "3 48" in 03 were read at native size only. +- C is scored on the same 7 images as before (00, 01, 03, 05, 14, 15, 22). T is scored for the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness note: I give no 5 on Q or A because fine micro-detail is still below a bf16 render; faithfulness 5 is given when every prompt element is present. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixF | Note (most visible difference from teacher) | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 5 | 4.5 | 4.5 | 4.5 | better | Sign "LE PETIT FROMAGE" exact (top edge of the sign is cropped by the frame); window line "Affineur depuis 1974" now crisp white chalk-style and exact (F's was a soft cursive smear); slate "Tasting / - Saturday / 11 AM" exact; ginger cat on step, cheese wheels, crates, wet cobbles, lamp glow all present. Composition differs from teacher but nothing missing. | +| 01 | text | 3.5 | 5 | 2.5 | 4 | 3.5 | same | "WATER CYCLE" and all four labels spelled exactly. Still a list, not a cycle: labels in a right-hand column of boxes, arrows point at boxes, only an up-arrow mountain-to-cloud and a curved arrow hint at flow; teacher has labels beside each stage with circulating arrows. Illustration (ice-capped mountain, river into sea) is somewhat better than F's; verdict same. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | better | "Some lights never go out" and "BEYOND THE BREAKERS" exact (THE set smaller). Girl in yellow raincoat rowing a boat toward the lit lighthouse, beam through storm clouds, spray-crowned wave, natural painterly colour. F was strongly yellow-green over-saturated with a tiny figure. Tiny stray glyph fragment in the bottom-right corner; girl seen from behind. | +| 03 | text | 4 | 3.5 | 3 | 4.5 | 4 | same | "Neon Horizon", "Luna Vale", "1:24" exact, but "3:48" renders as "3 48" (colon lost) and a stray dotted glyph sits next to "Luna Vale". Now a modern notched phone with a square album cover (purple sunset, sun disc) like the teacher; controls play / pause / next (no previous), bottom icons are two empty circles and an odd glyph; no desk lamp. F had all four strings exact but landscape art and rewind/ff controls. | +| 04 | text | 5 | 4.5 | - | 4.5 | 4.5 | better | Main sign "老王面馆" exact, large golden on red; second line "手工拉面 · 营业至凌晨两点" reads all 12 characters in order, larger and cleaner than F (F's 晨 was malformed; here it reads correctly at native size). Lanterns, steam, cook, diners, wet reflections natural (F's pavement looked like coloured tiles). Vertical menu board beside the door is pseudo-hanzi (background, not asked). | +| 05 | text | 4 | 4 | 3 | 4 | 4 | same | "PEACHES $3", "CHERRIES $5", "HONEY $8" all exact (F had "PEACES"), richer crates, natural colour. But a fourth sign on the left crate reads "PERRCIES $5" (garbled, not requested) and there are two straw hats instead of one. Teacher: three clean signs, one hat. | +| 06 | portrait | 4 | - | - | 4.5 | 4 | same (F per report) | Strong monk portrait, lined face, pores, silver brows and stubble, maroon robe with golden edge, faint wool texture. Beads hang at the neck/chest, not around the wrist; gaze goes to camera. | +| 07 | portrait | 4.5 | - | - | 4.5 | 4.5 | better | Curly short auburn hair, green eyes, freckles, navy turtleneck, parted lips, droplets, red lamp glow upper left, teal/amber grade, soft reflection stripe over the forehead. F had a hard vertical glass seam through the face and a cropped composition. | +| 08 | portrait | 5 | - | - | 4.5 | 4.5 | same (F per report) | All elements present (messy black hair, golden eyes, loose white shirt, dark green scarf, blue flower field, petals, clouds). One thin stray white arc at the right edge (same defect as F); shirt folds a bit faceted. | +| 09 | portrait | 5 | - | - | 4.5 | 4.5 | better (F per report) | Short grey hair, round glasses, arms crossed, linen apron and forearms crusted with clay, shelves of glazed bowls and vases, glowing kiln, warm window, smiling. Hands now visible (F tucked them away). | +| 10 | detail | 4.5 | 4 | - | 4.5 | 4.5 | better (F per report) | Keys carry distinct serif letters (I, E, E, F, S, U, R, Y, H, W, N...) readable, with repeats and not QWERTY; chrome-rimmed glass keys, typebars visible, a clearly bent carriage lever, cream paper with pseudo-English text (as in the teacher). F had random letters/digits and straight rods. | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | better | Proper peacock eye (turquoise ring, dark blue centre, gold/orange arcs), fine barbs, droplets, dark gold-tinted bokeh. F had beaded aliasing along the barbs and heavier saturation; this is cleaner. Still vivid. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | better | Mossy oak with lichen patches, ferns, droplets, misty forest bokeh, cream/orange mushrooms (not bracket fungi), 3 snails, ladybugs (four, prompt says one). F had a neon-green flat background with 3 mushrooms and one ladybug; this is far richer and more natural. | +| 13 | detail | 4 | - | - | 4.5 | 4 | better | Leather shell with rivets, stitching, wax stains over interlocking mail, candle, scratched wood, darkness. Reads as a bracer/vambrace; still no fingers, so not truly a gauntlet. F was a bare mail tube with a strap. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | better (F per report) | Exactly five balloons: red, orange striped, yellow, blue striped, striped purple, pilots in baskets, sunrise haze, misty hills, tiny village. Balloons sit in a loose row rather than depth-spread; colours slightly candy-like. F had four. | +| 15 | counting | 3 | - | 1 | 4.5 | 2.5 | better (quality) / same (count) | Two red apples + one yellow pear + one green pear = 4 fruits instead of 3 apples + 2 yellow pears + 1 green pear = 6; teacher gets the apples right. Render is excellent (glossy skin, blemishes, linen folds, muted grey wall as asked; F's wall was dark slate). The exact-count requirement still fails. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | same (F per report) | Colossal lantern-lit tree on an island, lit walkway among roots, boat with traveler, mountains, violet sky with stars, glowing flowers and floating lights, clean tree reflection. Lanterns small. | +| 17 | varied | 5 | - | - | 4.5 | 4.5 | better (F per report) | Yellow trench coat, clear umbrella, food cart with steaming food and a helmeted vendor, Hangul neon signs, scooter and taxi blurred, wet reflections. Woman now faces camera with natural legs and shoes (F: seen from behind, awkward feet). | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same (F per report) | Pink clay-like bakery on a floating slab, striped awnings, bread in window, smoking chimney, tree, cobble path, scooter with box, hanging croissant sign, pastel blue backdrop. Odd loaf lying on the flat roof; scooter instead of bicycle (prompt itself mentions scooter). | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | better | Brown bear with open mouth, soaked fur well resolved, salmon leaping beside its head with droplets, rushing river, mossy logs, misty forest. F had crunchy over-saturated look, soft face and dark blurry fish. Fish is next to, not in, the mouth. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same (F per report) | Gondolas at striped poles, arched palazzi reflected in green water, stone bridge, red flower pot, laundry line, striped-shirt gondolier, pigeons (in air and on water, none on bridge), paper texture and pencil underdrawing visible. | +| 21 | varied | 4.5 | - | - | 4.5 | 4.5 | better (F per report) | Kneeling black-armoured knight facing a giant glowing crystal, stained-glass shafts, embers, glowing runes in the floor, torn banners, helmet on the floor at his side, sword upright in the floor (blade looks chipped). Matches the prompt more fully than F; dark areas are crushed but fine. | +| 22 | varied | 5 | - | 4.5 | 4.5 | 4.5 | better | Bright Scandinavian kitchen: two cups on the oak table with steam between them, jug of tulips, crumbs and petals, pulled-back wooden chair, plant on shelf, big window, pale floor. F had three cups and a six-legged-looking table; this one is cleaner. | +| 23 | varied | 4.5 | - | - | 4 | 4.5 | same (F per report) | Hooded vendor frying dumplings in a large pan, steam, warm bulbs, bowl of dumplings and filling, hand reaching in the foreground (coin not clearly visible), wet alley with orange reflections. Foreground customers are near-black crushed silhouettes. | + +## Averages + +Overall (24 images): F 4.50, T 4.43 (7 text-bearing images; 4.50 on the six quoted-text prompts alone), C 3.36 (7 images), Q 4.44, A 4.29. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.33 | 4.50 | 3.25 (4 imgs) | 4.33 | 4.17 | +| portrait | 4 | 4.63 | - | - | 4.50 | 4.38 | +| detail | 4 | 4.50 | 4.0 (typewriter only) | - | 4.50 | 4.38 | +| counting | 2 | 4.00 | - | 3.00 (5 and 1) | 4.50 | 3.50 | +| varied | 8 | 4.69 | - | 4.5 (1 img) | 4.44 | 4.50 | + +## Comparison with the mixF_ptq report + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixA_lora32 | 3.50 | 1.4 | 2.14 | 3.81 | 2.90 | +| mixF_ptq (2.42 GB, previous report) | 4.29 | 4.36 | 2.71 | 4.29 | 4.02 | +| mixG_x (2.14 GB, this report) | 4.50 | 4.43 | 3.36 | 4.44 | 4.29 | + +Per category A (mixF report -> mixG_x): text 4.08 -> 4.17, portrait 4.25 -> 4.38, detail 4.00 -> 4.38, counting 2.25 -> 3.50, varied 4.31 -> 4.50. + +Caveat on comparability: scores are absolute judgements by a different judge instance, so +0.1 to +0.2 differences in F, T and Q are within judge noise. The side-by-side verdicts below are the firmer evidence. + +## Side-by-side verdict versus mixF_ptq (all 24) + +Better 15: 00, 02, 04, 07, 09, 10, 11, 12, 13, 14, 15, 17, 19, 21, 22. +Same 9: 01, 03, 05, 06, 08, 16, 18, 20, 23. +Worse 0. + +Reasons in brief: 00 crisp window text; 02 natural colour instead of yellow-green cast; 04 cleaner sub-line and natural reflections; 07 no glass seam; 09 hands and clay visible; 10 readable serif keys and bent lever; 11 no barb aliasing; 12 natural colour and richer detail; 13 leather shell with rivets and stitching; 14 five balloons instead of four; 15 muted grey wall and distinct yellow and green pears (count still wrong); 17 natural legs; 19 sharp fur and clear leaping fish; 21 helmet and planted sword; 22 two cups. Ties: 01 (same list-style diagram), 03 (square art and play button gained, but colon of "3:48" lost and a stray glyph), 05 (PEACHES fixed, but an extra garbled "PERRCIES $5" sign and two hats), the others near-identical in quality. Verdicts marked "(F per report)" in the table rely on the earlier report's description of the F image rather than a re-opened file. + +## Five worst images + +1. 15 fruit bowl (A 2.5): four fruits (2 apples, 1 yellow pear, 1 green pear) instead of six; the prompt says "exactly"; render itself is excellent. +2. 01 water cycle (A 3.5): all five strings exact, but the layout is a list of boxes with arrows, not a cycle; the teacher puts labels beside each stage. +3. 03 music player (A 4, T 3.5): "3:48" loses its colon, stray glyph beside "Luna Vale", no previous button, odd bottom icons. +4. 05 fruit stand (A 4, T 4): requested strings exact now, but an extra garbled sign "PERRCIES $5" and two straw hats. +5. 13 gauntlet (A 4): richly detailed leather and mail but a closed sleeve without fingers. +Close behind: 06 monk (beads at neck, not wrist). + +## Systematic failure patterns + +- Exact-count prompts: improved from 0/2 to 1/2. Balloons are now exactly five and distinct; the fruit bowl is still four fruits instead of six with a 3/2/1 mix. The failure is on the larger, more composite count; the model tends to draw 4 fruits in a bowl. +- Text: no systematic garbling any more. Residual single-item defects: a dropped colon (03 "3 48"), a garbled extra sign (05 "PERRCIES"), stray glyph fragments (03 next to "Luna Vale", 02 bottom-right corner). Quoted strings in 00, 01, 02, 04 and 05's three requested signs are exact; Chinese 老王面馆 and its 12-character sub-line are correct. +- Extra, unrequested text objects: where text is asked for the model sometimes adds an additional pseudo-text object (05 extra sign, 04 vertical pseudo-hanzi menu board). Mild. +- Infographic structure: 01 still has right-hand labels in a column rather than beside each stage; text correct, logic weak (as in F). +- Object identity drift: gauntlet becomes a bracer (13), beads at the neck (06), two straw hats (05), four ladybugs and no bracket fungi (12). +- Oversaturation/crunch seen in F (11, 12, 19, 02) is much reduced in G; I see no banding, blocking or global colour shift. Remaining local issues: crushed near-black regions (21, 23), stray white arc (08), small loaf on roof (18). +- Details at 3-bit FFN did not visibly degrade skin, fur, metal or water texture; the improvement over F is largest on natural-image sharpness and on colour naturalness. + +## Summary (10 lines) + +1. Overall: F 4.50, T 4.43, C 3.36, Q 4.44, A 4.29 (mixF_ptq report: 4.29 / 4.36 / 2.71 / 4.29 / 4.02); the smaller 2.14 GB model is not worse than the 2.42 GB one on any category. +2. Versus mixF_ptq side-by-side: 15 better, 9 same, 0 worse out of 24. +3. Text prompts: 5/6 acceptable (A >= 4: 00, 02, 03, 04, 05), 01 at 3.5 only because the diagram is not a cycle; 4/6 are letter-for-letter exact on every requested string (00, 01, 02, 04), 05's three requested signs are exact but with an extra garbled sign, 03 loses a colon. +4. Chinese 老王面馆 plus the 12-character sub-line render correctly with larger, cleaner glyphs than F. +5. Counting improved: balloons exactly five and distinct (F: four); fruit bowl still fails (four fruits for six), so counting acceptance is 3.5/5 and the fruit prompt remains the weakest image (A 2.5). +6. Natural-image colour and detail improved: no more yellow-green cast on the poster, no aliased peacock barbs, no neon-green moss, bear and salmon crisp, two-cup kitchen. +7. Portraits, anime, watercolor, isometric, dark fantasy and night street scenes stay at 4.5/5 acceptance with no regressions found. +8. Remaining defects are small and local (dropped colon, stray glyphs, extra garbled sign, crushed shadows, stray white arc). +9. Worst five: 15, 01, 03, 05, 13. +10. Verdict: mixG_x is a clear step up from mixF_ptq in absolute acceptance (+0.27 A) at 12 percent smaller size; still do not trust exact-count prompts with six objects, and infographic layout logic remains weak. diff --git a/eval/judge_mixH2_tuned.md b/eval/judge_mixH2_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..350a1adc4c6db270429bbf42d426eefb4990d30d --- /dev/null +++ b/eval/judge_mixH2_tuned.md @@ -0,0 +1,115 @@ +# Visual judge: mixH2_tuned (student, 1.95 GB, same model as mixH2_x plus one extra end-to-end tuning round that includes text-heavy training prompts) vs teacher_bf16, vs mixH2_x (untuned) and vs mixG_tuned + +Scales 0-5, same rubric as judge_mixH2_x.md / judge_mixH_tuned.md / judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixH2_tuned PNGs were opened at full 768x768 resolution, each inside a 1536x1536 2x2 composite (teacher | mixH2_x on top, mixH2_tuned | mixG_tuned below), so every tile was seen at native resolution. Composites and zoom crops were scratch files in the OS temp folder and were deleted afterwards; nothing else was modified. +- Zoom crops (2-3x, mixH2_x next to mixH2_tuned where useful) for the text prompts 00 (sign, window line, slate), 01 (all five labels), 02 (tagline and title, teacher next to tuned), 03 (title, artist, time stamps, controls, icons), 04 (both Chinese lines), 05 (both hanging boards and both crate boards) and 10 (key legends, paper text). +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. The mixH2_x scores from the previous report are used as the baseline; I re-looked at all its images in the composites and agree with them, so deltas are on the same scale. +- Calibration: as before, the teacher itself has only 5 fruits in 15 (3 apples + 2 pears), so 5 of 6 there is not a student regression; 22 (three cups instead of two) scored C 3.5 as for mixH2_x since it is the same error. +- Single seed per prompt: a changed text error in one image may be a trajectory draw rather than a systematic effect. This judge is a model instance; differences of +-0.2 on any average are within noise. The per-image findings (misspelled strings, extra sign, inverted keys) are firmer evidence than the averages. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixH2_x | vs mixG_tuned | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 3.5 | 4.5 | 4.5 | 4 | worse | worse | Sign "LE PETIT FROMAGE" exact (top clipped by the frame, as in the teacher). Slate "Tasting / Saturday. / 11 AM" letters exact, the dash is gone (stray period after Saturday). Window line reads "Afineur depuis 1974" (one f dropped, wrapped to two lines with an underline swish): in mixH2_x it was exact. Cat asleep on the step, crate with a cheese wheel, wheels in the window, wet cobbles, trees instead of the teacher's lamps. | +| 01 | text | 2.5 | 2 | 1.5 | 3 | 2.5 | worse | worse | Title "WATER CYCLE" exact; every label is wrong or duplicated: "Evaparton" (top), "Evapation" (second Evaporation box, lower left), "Condenstion", "Precipation"; only "Collection" is exact. Layout fell apart: floating boxes, stray double arrows, mountain/sea blob, no tidy column and no cycle. mixH2_x had a clean column with two misspellings; mixG_tuned had all four exact. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | same | "BEYOND THE BREAKERS" (THE smaller, as in the teacher) and "Some lights never go out" both exact; the tagline sits low and right instead of centred. Girl in yellow raincoat (white cap) rowing toward the lit lighthouse; the breaking wave is bigger and closer to the teacher than in mixH2_x. Tiny stray mark at the bottom edge (also in the teacher). | +| 03 | text | 4 | 4.5 | 3 | 4 | 4 | better | same | "Neon Horizon", "Luna Vale" and the time stamp "1:24 / 3:48" (with the slash) are all exact and legible (mixH2_x: "1:24 3:46"-like). Sun disc is back on the square purple cover. Controls are play / pause / one up-pointing triangle (no real previous and next); the bottom icons are three junk glyphs (basket, folder, lines). | +| 04 | text | 4.5 | 4.5 | - | 4 | 4.5 | same | same | "老王面馆" bold, golden, exact. Sub-line "手工拉面 · 营业至凌晨两点": all 12 characters in order, now bold white and larger and crisper than before; 凌 still renders as the variant 淩, so I do not count it as strictly exact (same ruling as for mixH2_x). Kitchen door with steam, diners at three tables, wet reflections. Lanterns are pale white instead of red, faces smeared, side sign junk (as in the teacher). | +| 05 | text | 3.5 | 3.5 | 3 | 4 | 3.5 | worse | same | Lower crate boards "PEACHES $3" and "HONEY $8" and hanging board "CHERRIES $5" are exact, but a second hanging board reads "PANCHES $3": a duplicated, misspelled PEACHES sign (four signs instead of three). One straw hat, daisies, striped awning, saturated mixed fruit with no dark cherries. mixH2_x had exactly three clean exact signs. | +| 06 | portrait | 4 | - | - | 4.5 | 4.5 | better | same | Bald head with pores, deep wrinkles by the eyes, grey stubble on the chin, maroon robe with golden edge, beads on the chest (none on the wrist, as in the teacher), window light from the left. Sharper and much less waxy than mixH2_x; eyebrows are dark (teacher: white). | +| 07 | portrait | 4 | - | - | 4 | 4 | better | same | Curly auburn hair, navy turtleneck, red lamp glow, teal/amber bokeh, rain drops. The face is sharper than in mixH2_x (eye colour, lashes, freckles readable); the glass seam through forehead and nose remains (the teacher has it too) and the frame is still dark and a bit murky. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | same | better | Messy black hair, 3/4 view, white billowing shirt, flowing dark-green scarf (no fringe), blue flowers, petals, clouds, no stray arcs. Eyes are duller olive-gold than the teacher's bright golden eyes (mixH2_x was closer). | +| 09 | portrait | 5 | - | - | 4.5 | 4.5 | same | same | Grey hair, round glasses, arms crossed, clay-crusted apron and forearms, shelves of bowls, big window, kiln glowing. Skin and hands a little smooth. | +| 10 | detail | 3.5 | 4.5 | - | 4.5 | 3.5 | worse | worse | Key legends are all real serif Latin letters (M E R S T B R J), no Cyrillic-like glyphs (mixH2_x had two). But the keys are black with cream letters: the prompt and the teacher want cream keys with black letters (the same inversion mixH_tuned had). Large bent lever, chrome rims, dust, razor focus; paper text is faint pseudo-English. | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | same | Proper peacock eye, turquoise/violet centre with droplets, fine barbs, gold bokeh; top feathers softly out of focus. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | better | Mossy oak, lichen, droplet ferns, cream/orange mushrooms, smaller bracket cluster at the right, 3 snails, exactly one ladybug on the fern. Misty background a little flat. | +| 13 | detail | 3.5 | - | - | 4 | 3.5 | same | worse | Crisp rings, leather strap, scratched table, candle now on a proper holder (mixH2_x fused it with leather discs). Still a fingerless mail mitt, with fewer rivets and almost no stitching compared with the teacher. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | better | same | Exactly five balloons: red, checkered orange, yellow with red panels, blue/white/purple striped, small striped purple; baskets, sunrise, mist, village, long shadows. Patterns are cleaner than mixH2_x's blotchy ornaments; sizes are uneven (one large foreground balloon) rather than "evenly spread". | +| 15 | counting | 3.5 | - | 3 | 4.5 | 4 | same | same | 3 red apples + 1 yellow pear + 1 green pear = 5 fruits instead of 6; the teacher has the same 5. Glossy skins, leaves, linen folds, grey wall; a pale vertical strip at the left edge. | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | better | same | Tree on the lake with a denser crown (closer to the teacher than mixH2_x's thin branches), fewer lanterns, lit walkway between roots, boat with one traveller, mountains, violet sky, a shooting star, glowing flowers. | +| 17 | varied | 5 | - | - | 4.5 | 4.5 | same | better | Yellow trench coat, clear umbrella, walking towards the camera, steaming cart, Korean neon, taxi and scooter, wet reflections. Subject smaller in frame; the vendor is not visible. | +| 18 | varied | 5 | - | - | 4.5 | 4.5 | better | same | Pink clay bakery with striped awning, bread in the window, chimney smoke, tree, cobbles, a real hanging sign with croissant, delivery scooter AND a small bicycle (the prompt mentions both; mixH2_x had no bicycle). Croissant on the flat roof (teacher: roof sign). | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Wet brown bear with open mouth, salmon leaping beside the jaw, droplets, mossy log, river, misty forest. Fur has a bit more texture than mixH2_x; fish and mouth do not quite touch. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Gondolas at striped poles, arched palazzi, flower balcony, bridge, laundry line, striped-shirt gondolier, pigeons on the water (not on the bridge), birds in the sky. Fills the frame instead of the teacher's vignette; strong ink lines. | +| 21 | varied | 4.5 | - | - | 4 | 4 | better | same | Knight kneeling before the crystal, a single planted sword (mixH2_x had two), helmet on the floor (and on his head, as in the teacher), stained glass, light shafts, embers, glowing runes. Very dark, crushed blacks. | +| 22 | varied | 4.5 | - | 3.5 | 4.5 | 4.5 | same | worse | Bright Scandinavian kitchen, oak table, jug of pink tulips, crumbs, steam, pulled-back pale chair, plant, window. THREE cups on the table (mug, steaming cup on a saucer, small cup on a saucer) instead of two; mixG_tuned shows two cups plus a coaster. | +| 23 | varied | 4.5 | - | - | 4 | 4 | better | same | Hooded vendor frying dumplings, bulbs, steam, bowl of dumplings, sauce bowls, rain under the awning, wet orange reflections. Several customers' backs (mixH2_x had one) and a small hand with a coin at the centre. Large near-black foreground masses, no wrapper pile. | + +## Averages + +Overall (24 images): F 4.29, T 3.93 (7 text-bearing images; 3.83 on the six quoted-text prompts alone), C 3.36 (7 images), Q 4.29, A 4.17. 20 of 24 images have A >= 4 (same count as mixH2_x). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 3.92 | 3.83 | 3.00 (4 imgs) | 4.00 | 3.83 | +| portrait | 4 | 4.38 | - | - | 4.38 | 4.38 | +| detail | 4 | 4.13 | 4.5 (typewriter only) | - | 4.38 | 4.00 | +| counting | 2 | 4.25 | - | 4.00 (5 balloons, 5 of 6 fruits) | 4.50 | 4.25 | +| varied | 8 | 4.63 | - | 3.5 (1 img) | 4.38 | 4.38 | + +## Comparison with mixH2_x and mixG_tuned + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixG_tuned (2.14 GB) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixH2_x (1.95 GB, no e2e tuning) | 4.33 | 4.00 | 3.43 | 4.27 | 4.21 | +| mixH2_tuned (1.95 GB, + e2e round with text-heavy prompts; this report) | 4.29 | 3.93 | 3.36 | 4.29 | 4.17 | +| delta H2_tuned - H2_x | -0.04 | -0.07 | -0.07 | +0.02 | -0.04 | +| delta H2_tuned - G_tuned | -0.19 | -0.86 | -0.28 | +0.06 | -0.04 | + +Per category A (mixH2_x -> mixH2_tuned): text 4.00 -> 3.83, portrait 4.13 -> 4.38, detail 4.25 -> 4.00, counting 4.25 -> 4.25, varied 4.38 -> 4.38. + +Every overall delta is inside the +-0.2 noise band. What moved is the distribution: the portraits gained about +0.25 on Q and A (06 and 07 clearly sharper), and several varied scenes gained a prompt attribute (18 bicycle, 21 single sword, 23 more customers, 16 denser crown, 14 cleaner balloons); the text category lost about 0.2 and the detail category 0.25 (10 inverted keys, 13 unchanged). The text-heavy tuning data did not buy better small-text spelling. + +## Side-by-side verdict (all 24) + +Versus mixH2_x: better 8 (03, 06, 07, 14, 16, 18, 21, 23), same 12 (02, 04, 08, 09, 11, 12, 13, 15, 17, 19, 20, 22), worse 4 (00, 01, 05, 10). +Versus mixG_tuned (secondary, lower confidence): better 3 (08, 12, 17), same 16, worse 5 (00, 01, 10, 13, 22). + +Reasons in brief: +- better: 03 time stamps "1:24 / 3:48" exact and the sun disc back; 06 pores, stubble, less waxy; 07 sharper face, lashes and freckles readable; 14 cleaner balloon patterns, still exactly five; 16 denser tree crown; 18 bicycle added to the scooter; 21 one planted sword and a real kneeling pose; 23 several customers and a visible coin hand. +- worse: 00 window text lost an f ("Afineur") and the slate dash is gone; 01 four of five labels misspelled plus a duplicate Evaporation box and a messy layout; 05 duplicated, misspelled "PANCHES $3" board; 10 keys inverted to black with cream letters. +- same: 02 title and tagline exact in both; 04 sub-line a little crisper but 淩 variant remains; 08 eye colour a bit duller, but cleaner composition; 09, 11, 12, 15, 17, 19, 20 unchanged in kind; 13 candle holder better, leather detail poorer; 22 three cups in both. + +## Text-prompt acceptance + +- Six quoted-text prompts: 4/6 accepted (A >= 4: 00 4, 02 4.5, 03 4, 04 4.5); 01 (A 2.5) and 05 (A 3.5) fail. mixH2_x: 4/6 (00, 02, 04, 05), mixG_tuned: 5/6. Same count as mixH2_x but a different set: 03 was fixed (time stamps), 05 broke. +- Letter-for-letter exact: 2/6 strictly (02, 03); 04 reads fully but with the 淩 variant, so 3/6 if it is counted. mixH2_x: 3/6 (00, 02, 05), mixG_tuned: 6/6. Failing: 00 ("Afineur" and no dash), 01 ("Evaparton", "Evapation", "Condenstion", "Precipation"), 05 ("PANCHES $3" duplicate board). +- Errors moved rather than disappeared: mixH2_x misspelled two labels in 01 and one numeral group in 03; mixH2_tuned fixed 03 and the keys' letters but broke 00's window line, 05's extra sign and made 01 worse. +- Typewriter keys (10): 8 of 8 visible key legends are real letters, an improvement in the glyphs, but the colour scheme is inverted. +- Large clean text (02 title and tagline, 04 title, 00 sign) stays intact; failures are small text in busy graphics (01) and secondary signs (00 window, 05 hanging board). + +## Five worst images + +1. 01 water cycle (A 2.5, T 2, C 1.5): four misspelled labels, a duplicate label, stray arrows and a broken layout; clearly worse than mixH2_x. +2. 05 fruit stand (A 3.5, T 3.5, C 3): extra hanging board "PANCHES $3". +3. 10 typewriter (A 3.5, F 3.5): cream-on-black keys instead of black letters on cream keys (legends themselves are clean). +4. 13 gauntlet (A 3.5): fingerless mail mitt with a leather band, fewer rivets/stitching than the teacher. +5. 00 cheese shop (A 4, T 3.5): window line "Afineur depuis 1974" and a missing slate dash; otherwise a very good image. +Close behind (A 4): 15 (5 of 6 fruits, same as the teacher), 03 (controls and icons), 07 (still murky), 21 (very dark), 23 (black foreground masses). + +## Systematic failure patterns + +- Small text in the infographic and on secondary signs remains the dominant weakness (01, 00 window, 05 hanging board). Large and medium text is intact. +- Layout collapse on dense graphic prompts: 01 lost its tidy box column; this looks like a trajectory outcome rather than a systematic loss, but it is one of six text prompts. +- Duplicated elements: 05 (second PEACHES board), 01 (second Evaporation label); 22 (third cup) and 15 (5 of 6 fruits, shared with the teacher) are the existing count errors. Balloon count stays exact (14). +- Colour/attribute inversion on technical objects (10 black keys); fingerless gauntlet (13) unchanged. +- UI semantics (03): previous/next buttons are still missing; icons are junk glyphs, but the time stamps are now right. +- Portraits: improved; 06 and 07 now show pores, stubble, lashes and freckles. Skin is still slightly smoother than the teacher in 09. +- No banding, blocking, global colour or saturation shift; defects are local. + +## Summary (10 lines) + +1. Overall: F 4.29, T 3.93, C 3.36, Q 4.29, A 4.17 (mixH2_x: 4.33 / 4.00 / 3.43 / 4.27 / 4.21; mixG_tuned: 4.48 / 4.79 / 3.64 / 4.23 / 4.21). +2. Versus mixH2_x: 8 better, 12 same, 4 worse (out of 24); versus mixG_tuned: 3 better, 16 same, 5 worse. +3. Text prompts: 4/6 with A >= 4 (00, 02, 03, 04), letter-exact 2/6 strictly (02, 03), 3/6 counting 04; mixH2_x had 4/6 and 3/6, mixG_tuned 5/6 and 6/6. +4. Concrete text outcomes: fixed 03 time stamps and the 10 key letters; new errors "Afineur" (00), "PANCHES $3" duplicate board (05) and a collapsed 01 with four wrong labels. +5. Counts: five balloons exact, fruit bowl 5 of 6 like the teacher, three cups in 22; 05 gained an extra sign and 01 an extra label. +6. Portraits improved (Q and A +0.25 as a category): sharper, textured faces in 06 and 07. +7. Several varied scenes gained attributes (bicycle in 18, one sword in 21, more customers in 23, denser crown in 16). +8. Regressions: 01 (layout and labels), 05 (extra misspelled sign), 00 (window f), 10 (inverted keys). +9. Worst five: 01, 05, 10, 13, 00. +10. Answer: the tuning round neither clearly helped nor hurt on the average (A 4.17 vs 4.21, all deltas inside the +-0.2 noise band). It helped portraits and some scene attributes, and it did NOT improve small-text spelling: letter-exact text went from 3/6 to 2/6 and T from 4.00 to 3.93, with one text prompt clearly worse (01) and one fixed (03); counts are unchanged; mixG_tuned (T 4.79, 6/6) remains the choice where exact lettering matters. Caveat: one seed per prompt and a model-instance judge, so the text swings of single images are weak evidence of a systematic effect. diff --git a/eval/judge_mixH2_x.md b/eval/judge_mixH2_x.md new file mode 100644 index 0000000000000000000000000000000000000000..48e04ed3d7a0364c891f00f29d0be6e8d3ffac01 --- /dev/null +++ b/eval/judge_mixH2_x.md @@ -0,0 +1,116 @@ +# Visual judge: mixH2_x (student, 1.95 GB, low-bit attention, 2-bit middle-block FFN with 3/4 bits at the ends, rank-64 low-rank correction, about twice the calibration data of mixH, NO end-to-end tuning) vs teacher_bf16, vs mixH_tuned and vs mixG_tuned + +Scales 0-5, same rubric as judge_mixH_tuned.md / judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixH2_x PNGs were opened at full 768x768 resolution, each inside a 1536x1536 2x2 composite (teacher | mixH2_x on top, mixH_tuned | mixG_tuned below), so every tile was shown at native resolution. Composites and crops were scratch files in the OS temp folder and were deleted afterwards; nothing else was modified. +- Native-size 1.2-2.5x zoom crops (scratch, deleted) for the text prompts: 00 (sign, window line, slate), 01 (four labels), 03 (title / artist / time stamps), 04 (both Chinese lines), 05 (three price signs), 10 (key letters); also 22 (cup count). +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness as before: no 5 on Q or A. The scores of mixH_tuned and mixG_tuned from the previous reports are used as baselines; I re-looked at all of their images in the composites and agree with those verdicts. +- One calibration point: in 15 (fruit bowl) the teacher itself has 3 apples + 2 pears (5 fruits), so the same shortfall in the student is not a regression against the teacher. I scored C 3 as before but A 4 (previous reports: 3.5); this is worth about +0.02 on the overall A average. +- This judge is a model instance; differences of +-0.2 on any average are within noise. The per-image side-by-side verdicts and the specific text/count findings are firmer evidence than the averages. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixH_tuned | vs mixG_tuned | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 4.5 | 4.5 | 4.5 | 4.5 | better | same | Navy "LE PETIT FROMAGE" exact (top edge of the letters touches the frame, as in the teacher), window "Affineur depuis 1974" exact in white script inside the glass (H_tuned put 1974 outside the glass in navy), slate "Tasting / Saturday / 11 AM" letters exact but the dash is replaced by an underline swish below. Cat on step, cheese wheels, crate, wet cobbles all there; wider street with trees instead of the teacher's lamps. | +| 01 | text | 3 | 2.5 | 2 | 3.5 | 3 | better | worse | Title "WATER CYCLE" exact. Labels: "Evaporation" ok, "Collection" ok, but "Condenstion" (missing a) and "Precipatton" (missing i, ti fused to tt). Clean column of rounded boxes at the right with 3 arrows (none for Collection), clean mountain/river/sea, no stray arrows (H_tuned had a mess); still no cycle (teacher: labels beside each stage on a loop). mixG_tuned had all four labels exact. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | same | "BEYOND THE BREAKERS" (THE smaller) and tagline "Some lights never go out" both exact with even spacing (H_tuned had a wide gap). Girl in yellow raincoat seen from behind rowing toward the lit lighthouse, beam through clouds. Wave smaller and less dramatic than the teacher; no stray glyph. | +| 03 | text | 3.5 | 3.5 | 2.5 | 4 | 3.5 | worse | worse | "Neon Horizon" and "Luna Vale" exact, square purple sunset cover (no sun disc), tilted phone on desk. Time stamps are tiny and malformed ("1:2A 3:4G"-like, no slash). Controls are play / pause / one plain left triangle: no real previous and next. Bottom icons are three junk glyphs (dash-like marks). Screen darker and lower contrast than the teacher. | +| 04 | text | 4.5 | 4 | - | 4 | 4.5 | better | same | "老王面馆" in bold golden glowing characters on the red board (H_tuned: pale cream); sub-line "手工拉面 · 营业至凌晨两点" all 12 characters in order, white, but 凌 is slightly malformed (reads like 淩) and 面 is soft. Cook at the door, steam, cream/orange lanterns, three tables of diners, wet reflections. Faces smeared; side sign is junk (as in the teacher); lanterns paler than the teacher's red ones. | +| 05 | text | 4 | 4.5 | 3.5 | 4 | 4 | better | better | "PEACHES $3", "CHERRIES $5", "HONEY $8" all exact and legible (the flat-topped 3 is fine), one straw hat, daisies in front (one touches the C of CHERRIES but it stays readable), no duplicated sign. PEACHES board is cut by the left frame edge; fruit is a mixed red/green/yellow mass in all crates (no dark cherries, no separate peach crate), saturated colours. | +| 06 | portrait | 4 | - | - | 4 | 4 | better | same | Smooth bald head, deep wrinkles by the eyes, faint silver stubble on chin and slightly grey eyebrows, maroon robe with golden edge, window light left, warm backdrop. Beads hang at the neck (none on the wrist), gaze into the camera, skin a bit smooth/waxy compared with the teacher's pores. Clearly better than H_tuned (no stubble, waxy). | +| 07 | portrait | 4 | - | - | 3.5 | 3.5 | worse | same | Curly auburn hair, navy turtleneck, red lamp glow, teal/amber bokeh, rain on glass, lips parted. But the frame is dark and murky, the face is soft (eye colour, lashes and freckles barely readable) and a hard vertical glass seam crosses forehead and nose. Not as sharp as H_tuned; comparable to G_tuned. | +| 08 | portrait | 5 | - | - | 4.5 | 4.5 | same | better | Messy black hair, golden eyes, 3/4 turn, white billowing shirt, flowing dark-green scarf (no fringe), blue flowers, petals, clouds. No stray arcs or ghost drape (G_tuned had two white arcs). Eyes a little less glossy than the teacher's. | +| 09 | portrait | 5 | - | - | 4.5 | 4.5 | same | same | Short grey hair, round glasses, arms crossed, clay-crusted hands and apron, shelves of bowls, big window, kiln glowing at right edge. Skin and hands slightly smooth/mushy; sleeves heavier than the teacher's rolled shirt. | +| 10 | detail | 4.5 | 4 | - | 4.5 | 4.5 | better | same | Cream keys with black serif letters and chrome-gold rims, as in the prompt (H_tuned had inverted black keys). Zoomed legends: I R P U P B R D Г P D V Б A R M R, i.e. mostly real letters but two Cyrillic-like glyphs and repeated letters. Bent metal lever at the top, dust and wear, razor-thin focus. Paper text is faint pseudo-English (as in the teacher). | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | same | Proper peacock eye (turquoise ring, dark violet centre), fine barbs, many droplets, warm gold bokeh. More orange/brown surround and denser blue-green barbs than the teacher. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | better | Mossy oak, lichen, ferns with droplets, two orange/cream mushrooms, bracket-fungi cluster at the right (smaller than the teacher's), 3 snails, exactly one ladybug on the fern (prompt: one; G_tuned had two and dark blobby snails). Misty bokeh a bit flat. | +| 13 | detail | 3.5 | - | - | 4 | 3.5 | same | worse | Crisp interlocking rings, leather band with rivets, scratched table, candle with melted wax, dark background. Still a rounded mail mitt/sleeve with no fingers (teacher: finger tips), reads less like a gauntlet; candle sits on a stack of leather discs that merge with the holder. G_tuned had finger-like segments and red stitching. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | better | same | Exactly five balloons: red, orange, yellow, blue, striped purple, each with a basket and a tiny pilot; sunrise, mist, village, long shadows. Patterns on red/orange/yellow/blue are odd blotchy ornaments rather than the teacher's clean stripes; staggered heights instead of a row. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 4 | same | same | Three red apples, one yellow pear and one green pear = 5 fruits instead of 6 (a second yellow pear is missing); the teacher has exactly the same 5 fruits. Glossy skins, leaves, linen folds, shadow to the right, grey wall all good. | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Tree on the lake, lit walkway through the roots, boat with one traveller, mountains, violet sky, mist, glowing flowers and floating lights. Crown is sparse with thin branches and big lanterns (teacher: dense foliage with small lanterns); fewer stars. | +| 17 | varied | 5 | - | - | 4.5 | 4.5 | better | better | Yellow trench coat, clear umbrella, steaming food cart with a vendor in a yellow helmet, Hangul neon, scooter and taxi, wet reflections. The woman walks towards the camera as in the teacher (H_tuned and G_tuned both turned her away). Face a bit soft; very close to the teacher. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | better | same | Pink clay bakery, striped awning, bread in the window, chimney smoke (cropped at the top), tree, cobbles, delivery scooter, and a real hanging sign with a croissant icon (H_tuned: free-standing board). Croissant lying on the flat roof (teacher: roof sign); the cream base blends into the pale backdrop. | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Wet brown bear, open mouth, salmon right next to the jaw, droplets, mossy log, rushing river, misty forest. Fur smoother and less soaked than the teacher's; fish and mouth do not quite touch. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Gondolas at striped poles, palazzi with arched windows reflected in the green water, bridge with figures, red-flower balcony, laundry line, striped-shirt gondolier, pigeons on the water, paper texture and faint pencil lines. Pigeons on the bridge not visible. | +| 21 | varied | 4 | - | - | 4 | 4 | same | same | Black-armoured knight crouching before the crystal, light shafts, stained glass, embers, glowing floor runes, torn banners. Helmet on his head AND one on the floor (the teacher has this too), two swords (one upright, one leaning at the left), crushed blacks, pose is a lunge not a kneel. | +| 22 | varied | 4.5 | - | 3.5 | 4.5 | 4.5 | same | same | Bright Scandinavian kitchen, oak table, jug of pink tulips, crumbs and petals, pulled-back pale chair, plant on a shelf, big window, grey floor. THREE cups on the table (two steaming coffee cups plus a third small cup) instead of two; table perspective steep; chair slightly odd at the back rail. | +| 23 | varied | 4 | - | - | 4 | 4 | same | same | Hooded vendor frying dumplings in a large pan, bulbs, steam, tray and bowl of dumplings, rain streaks under the awning, wet orange reflections, one customer's back at the right with an outstretched hand holding a small coin (visible but tiny). Only one customer back, no wrapper pile or filling bowl, large near-black masses, no flame. | + +## Averages + +Overall (24 images): F 4.33, T 4.00 (7 text-bearing images; also 4.00 on the six quoted-text prompts alone), C 3.43 (7 images), Q 4.27, A 4.21. 20 of 24 images have A >= 4. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.08 | 4.00 | 3.13 (4 imgs) | 4.08 | 4.00 | +| portrait | 4 | 4.50 | - | - | 4.13 | 4.13 | +| detail | 4 | 4.38 | 4.0 (typewriter only) | - | 4.38 | 4.25 | +| counting | 2 | 4.25 | - | 4.00 (5 balloons, 5 of 6 fruits) | 4.50 | 4.25 | +| varied | 8 | 4.44 | - | 3.5 (1 img) | 4.38 | 4.38 | + +## Comparison with the two earlier reports + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixG_tuned (2.14 GB) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixH_tuned (1.95 GB, less data, e2e tuned) | 4.31 | 4.00 | 3.29 | 4.29 | 4.10 | +| mixH2_x (1.95 GB, 2x data, no e2e tuning; this report) | 4.33 | 4.00 | 3.43 | 4.27 | 4.21 | +| delta H2_x - H_tuned | +0.02 | 0.00 | +0.14 | -0.02 | +0.11 | +| delta H2_x - G_tuned | -0.15 | -0.79 | -0.21 | +0.04 | 0.00 | + +Per category A (mixG_tuned -> mixH_tuned -> mixH2_x): text 4.08 -> 3.83 -> 4.00, portrait 3.88 -> 4.25 -> 4.13, detail 4.50 -> 4.13 -> 4.25, counting 4.00 -> 3.50 -> 4.25, varied 4.38 -> 4.38 -> 4.38. + +Reading the deltas: versus mixH_tuned everything is within noise (A +0.11, C +0.14); the only visible structural gain is counting (14 has five balloons again) and the fixed prompt attributes in 10 (black letters on cream keys), 17 (woman faces the camera), 18 (hanging sign), 06 (stubble back). Versus mixG_tuned, A is identical (4.21) and Q is level; the remaining gap is T (-0.79) and, to a lesser degree, F/C, concentrated in small-text and prompt-attribute details (01, 03, 13). So doubling the calibration data without e2e tuning recovered about what the e2e tuning round had given or lost elsewhere, at the 1.95 GB size: overall acceptance is level with the 2.14 GB model, but exact spelling of small text is still worse than mixG_tuned. Caveat: the F/Q/A deltas are all inside the +-0.2 noise band. + +## Side-by-side verdict (all 24) + +Versus mixH_tuned: better 9 (00, 01, 04, 05, 06, 10, 14, 17, 18), same 13 (02, 08, 09, 11, 12, 13, 15, 16, 19, 20, 21, 22, 23), worse 2 (03, 07). +Versus mixG_tuned: better 4 (05, 08, 12, 17), same 17 (00, 02, 04, 06, 07, 09, 10, 11, 14, 15, 16, 18, 19, 20, 21, 22, 23), worse 3 (01, 03, 13). + +Reasons in brief: +- better than H_tuned: 00 "1974" on the glass instead of outside it, no bent lamp; 01 clean label column, no stray arrows (but still two misspellings); 04 golden title instead of cream; 05 all three prices correct (H_tuned had a malformed digit on the PEACHES sign); 06 stubble and a less waxy face; 10 black letters on cream keys as asked; 14 five balloons instead of six; 17 subject faces the camera; 18 hanging sign. +- worse than H_tuned: 03 H_tuned had previous/play/next and exact "1:24 / 3:48"; 07 H_tuned's face was sharp with freckles and lashes, H2_x is murky and soft. +- better than G_tuned: 05 no duplicated HONEY sign and no mangled H; 08 no stray white arcs; 12 one ladybug, clean snails; 17 faces the camera, no odd hem. +- worse than G_tuned: 01 G had all four labels exact (H2_x: two misspelled); 03 G's time stamps and strings were exact; 13 G's mail ends in finger-like segments with red stitching. + +## Text-prompt acceptance + +- Six quoted-text prompts: 4/6 accepted (A >= 4: 00 4.5, 02 4.5, 04 4.5, 05 4). 01 (A 3) and 03 (A 3.5) fail. Previous reports: mixH_tuned 4/6, mixG_tuned 5/6. +- Letter-for-letter exact: 3/6 (00, 02, 05). 00 is exact in letters but the dash on the slate became an underline swish; 04 reads completely but one glyph of the sub-line (凌) is slightly malformed, so I do not count it as strictly exact; 01 has "Condenstion" and "Precipatton"; 03 has exact title and artist but garbled tiny time stamps. Previous reports: mixH_tuned 3/6, mixG_tuned 6/6. +- Large clean text (02 title and tagline, 04 title, 05 signs, 00 sign and window line) is intact; the failures are small text in busy graphics (01 labels) and tiny numerals (03 time stamps). +- Typewriter keys (10): legible black serif letters on cream keys, two Cyrillic-like glyphs among 17 keys. + +## Five worst images + +1. 01 water cycle (A 3, T 2.5): "Condenstion" and "Precipatton", no cycle, only 3 arrows; much cleaner than H_tuned but still the weakest. +2. 03 music player (A 3.5, T 3.5, C 2.5): garbled time stamps, no real previous/next buttons, junk bottom icons. +3. 07 train-window portrait (A 3.5, Q 3.5): dark, murky, soft face, seam through the forehead. +4. 13 gauntlet (A 3.5): mail mitt with leather band, no fingers, candle fused with leather discs. +5. 15 fruit bowl (A 4, C 3): 5 of 6 fruits (the teacher has the same 5). +Close behind (A 4): 05 (mixed fruit, board cut by the frame), 06 (waxy skin, beads at the neck), 21 (double helmet, two swords), 23 (one customer, tiny coin). + +## Systematic failure patterns + +- Small or dense text is the main loss: two dropped letters in the 01 labels, malformed tiny numerals in 03, one malformed glyph in the 04 sub-line. Large text is fine. +- UI semantics: the 03 controls are play/pause/triangle instead of previous/play/next; icons are junk glyphs. +- Prompt-attribute drift on technical objects: gauntlet without fingers (13), cream-key glyph oddities (10, two Cyrillic-like letters), croissant on the roof (18), third cup on the table (22), double helmet and two swords (21; helmet also in the teacher). +- Exact counts: balloons now exact (5/5); the fruit bowl still has 5 of 6 fruits like the teacher; the table has one cup too many (22). The 3/2/1 composite is not a student-specific failure. +- Low-light close-up portrait (07) is murky and soft; the other portraits (06, 08, 09) are fine. +- Infographic logic: 01 is a tidy column of labelled boxes but not a cycle (the teacher is a loop). +- No banding, blocking, global colour or saturation shift; defects are local. Skin and fur are slightly smoother than the teacher (06, 09, 19). + +## Summary (10 lines) + +1. Overall: F 4.33, T 4.00, C 3.43, Q 4.27, A 4.21 (mixH_tuned: 4.31 / 4.00 / 3.29 / 4.29 / 4.10; mixG_tuned: 4.48 / 4.79 / 3.64 / 4.23 / 4.21). +2. Versus mixH_tuned: 9 better, 13 same, 2 worse; versus mixG_tuned: 4 better, 17 same, 3 worse (out of 24). +3. Text prompts: 4/6 with A >= 4 (00, 02, 04, 05); letter-exact 3/6 (00, 02, 05); 01 and 03 fail. +4. Concrete text defects: "Condenstion" and "Precipatton" (01), garbled time stamps and wrong controls (03), slightly malformed 凌 (04), slate dash replaced by an underline (00). +5. Counting: five balloons (fixed versus mixH_tuned), fruit bowl 5 of 6 like the teacher, three cups instead of two in 22. +6. Gains over mixH_tuned come from prompt attributes and cleaner layout (10, 17, 18, 06, 00, 05) rather than from a general quality jump (Q is level). +7. Regressions: 07 (soft murky face), 03 (controls and time stamps); 13 still reads as a mail mitt. +8. No global colour shift, banding or blocking; defects are local. +9. Worst five: 01, 03, 07, 13, 15. +10. Verdict: doubling the calibration data without e2e tuning brings the 1.95 GB model level with mixG_tuned (2.14 GB) on overall acceptance (A 4.21 vs 4.21) and ahead of mixH_tuned in several prompt-attribute cases, but exact spelling of small text (T 4.00 vs 4.79) is still the gap. Caveat: this judge is a model instance and +-0.2 differences are within noise; the T gap and the per-image findings are the firm evidence. diff --git a/eval/judge_mixH2_xm_f8.md b/eval/judge_mixH2_xm_f8.md new file mode 100644 index 0000000000000000000000000000000000000000..aa248d9bcbcb255f3d45d63ab58ded8dda973a64 --- /dev/null +++ b/eval/judge_mixH2_xm_f8.md @@ -0,0 +1,110 @@ +# Visual judge: mixH2_xm_f8 (student, 1.72 GB: mixH2_x with shared_block_modulation, video_head_modulation and time_embedding.2 rounded to 4 bits and the low-rank branches stored as 8-bit floats, no retraining) vs teacher_bf16 and vs mixH2_x (1.95 GB) + +Scales 0-5, same rubric as judge_mixH2_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixH2_xm_f8 PNGs were opened at full 768x768 resolution, each in a 1536x1536 2x2 composite (teacher | mixH2_x on top, mixH2_xm_f8 | 2x centre crop of mixH2_xm_f8 below), so every tile was at native resolution. Additional 2-3x crops of mixH2_xm_f8 (and mixH2_x where noted) for the text prompts: 00 (sign, window line, slate), 01 (title and four labels), 02 (title and tagline), 03 (time stamps and controls, with mixH2_x for comparison), 04 (title and sub-line, with mixH2_x), 05 (three signs), 10 (key legends). All composites, crops and helper scripts were scratch files in the OS temp folder and were deleted afterwards; nothing else was modified. +- Same scoring subsets as before: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness as before: no 5 on Q or A. mixH2_x scores are taken from judge_mixH2_x.md; I re-looked at all its images in the composites and agree with them (one calibration point: 15 has 5 fruits in the teacher too, so not a regression). +- Besides eyeballing I computed per-image colour statistics (mean luminance, luminance std, 1st-99th percentile range, mean HSV saturation, mean R-B "warmth") on all 72 PNGs, only to look for a global shift. Compositions differ between models, so single-image values are confounded by content; only the consistent sign across the 24 images is used as evidence. +- This judge is a model instance; differences of +-0.2 on any average are within noise. The per-image side-by-side verdicts and the specific text findings are firmer than the averages. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixH2_x | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 4.5 | 4.5 | 4.5 | 4.5 | same | "LE PETIT FROMAGE" exact (top edge touches the frame, as in the teacher), window "Affineur depuis 1974" exact in white script, slate "Tasting / Saturday / 11 AM" exact (dash replaced by an underline swish, same as mixH2_x). Cat is smaller and sits at the far end of the step, fewer cheese wheels in the window, background more heavily blurred; wet cobbles, crate and pot all there. | +| 01 | text | 2.5 | 2 | 2 | 3.5 | 2.5 | worse | Title "WATER CYCLE" exact (wider gap, stray underline below it). Labels: "Evapoation" (missing r), "Condenstion" (missing a), "Precipittion" (missing a), "Collection" exact: 1 of 4 exact (mixH2_x: 2 of 4). Arrows do not match the labels (double-headed vertical arrow between cloud and mountain, "Precipittion" box sits inside the sea), no cycle; background is cream, not off-white. Clearly the weakest image. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "BEYOND THE BREAKERS" (THE smaller) and "Some lights never go out" both exact, even spacing. Girl in yellow raincoat rowing toward the lit lighthouse, beam through clouds, curling wave at left. Girl is small and hunched (hood a blob), waves smaller and less dramatic than the teacher. | +| 03 | text | 4 | 4 | 3 | 4 | 4 | better | "Neon Horizon" and "Luna Vale" exact; both time stamps now legible and correct ("1:24" at the left, "3:48" at the right of a violet progress bar; no slash). mixH2_x had tiny garbled stamps. Controls: play, play, skip-forward (no previous, duplicated play); three junk glyph icons below. Album cover is a more saturated violet/magenta than the teacher's dusty one, no sun disc, glow halo. | +| 04 | text | 4 | 4 | - | 4 | 4 | worse | Title "老王面馆" exact in golden characters (thinner and less glowing than mixH2_x); sub-line "手工拉面 · 营业至凌晨两点" all 12 characters in order and sharp, 凌 malformed (reads like 淩) exactly as in mixH2_x. Scene is darker and duller: lanterns pale cream (teacher red-orange), no cook at the door, diners small and murky; the wet pavement reflections are nice. | +| 05 | text | 4 | 5 | 4 | 4.5 | 4.5 | better | "PEACHES $3", "CHERRIES $5", "HONEY $8" all exact, large and whole (mixH2_x cut the PEACHES board with the frame). One straw hat on the post, daisies at the foot, striped awning, hills and sky. Fruit is a mixed red/orange mass in all crates (no dark cherries, no separate peaches), signs sit at the bottom of a low-angle crop. | +| 06 | portrait | 4 | - | - | 4 | 4 | same | Smooth bald head, deep wrinkles, silver stubble on chin, silver eyebrows, maroon robe with golden edge, window light left. Beads hang on the chest (prompt: wrist), gaze into the camera, skin smooth/waxy at a glance but pores and wrinkles resolve in the 2x crop. | +| 07 | portrait | 4 | - | - | 4 | 4 | better | Curly (more wavy) auburn hair, navy turtleneck, red lamp bokeh, teal window frame, droplets on the cheek, lips parted. Face is sharper than mixH2_x with freckles readable and no hard glass seam through the forehead. Whole frame is dark and murky, eye catchlights weak. | +| 08 | portrait | 4.5 | - | - | 4 | 4 | worse | Black messy hair, golden eyes, white shirt, dark-green scarf (reads like a stand-up collar, no fringe), blue flowers at the bottom, a few petals. Pale and washed out (mean luminance 0.70 vs 0.63 mixH2_x and 0.58 teacher), flat distant field, fewer flowers, eyes less glossy. | +| 09 | portrait | 5 | - | - | 4.5 | 4.5 | same | Short grey hair, round glasses, arms crossed, clay-smeared linen apron and forearm, shelves of bowls, big window, kiln glowing. Sleeves short instead of rolled, skin smooth, background soft; no artefacts. | +| 10 | detail | 4.5 | 3.5 | - | 4 | 4 | worse | Chrome-rimmed keys with black serif letters, bent lever, dust and wear, paper with pseudo-English. Key tops are cool grey-green (mixH2_x and the teacher: warm cream), so the keys clash with the warm paper (saturation 0.31 vs 0.46 mixH2_x). Legends read B F E E H M E U K M S U R plus 3-4 mirrored or Cyrillic-like glyphs among about 14 visible keys (mixH2_x: 2 of 17). | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | Proper peacock eye (turquoise ring, dark blue-violet centre), fine barbs, cluster of droplets on the eye, warm gold bokeh at the lower right, dark background. Barbs above the eye are strongly blue-violet instead of the teacher's green/gold, fewer droplets than mixH2_x, bokeh discs show contour steps (the teacher too). | +| 12 | detail | 3.5 | - | - | 4 | 3.5 | worse | Dark mossy oak with lichen, a moss strip, three mushrooms (two at left, one in the middle), fern at the bottom, one ladybug (on the bark, not the fern). No visible snails, no bracket-fungi cluster, droplets sparse; trunk much darker than mixH2_x, background mist shows large flat colour planes (posterisation, also in the teacher but larger here). | +| 13 | detail | 3.5 | - | - | 4 | 3.5 | same | Crisp interlocking rings, leather band with rivets, candle on stacked leather discs, dark background. Still a rounded mail mitt with no fingers (teacher: finger tips); table darker and less scratched than mixH2_x. | +| 14 | counting | 5 | - | 5 | 4 | 4.5 | same | Exactly five balloons: red, orange, yellow, blue, striped purple, each with a basket and tiny pilots; sunrise, village, mist. Balloons are cleaner than mixH2_x's blotchy ornaments but nearly plain (no distinct patterns except the blue ribs and purple stripes); the valley is hazy, soft and low-contrast. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 4 | same | Three red apples, one yellow pear, one green pear = 5 fruits instead of 6; the teacher has the same 5. Glossy skins, leaves, linen folds, shadow to the right, grey wall all good. | +| 16 | varied | 4 | - | - | 4.5 | 4.5 | same | Huge tree on the lake, lit walkway through the roots, boat with one traveller, mountains, violet sky, stars, mist, reflections excellent. Only about 5 hanging lanterns (teacher many, mixH2_x about 20), fewer glowing flowers and floating candles. | +| 17 | varied | 5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart with vendor in yellow helmet, Hangul neon, scooter and taxi, wet reflections; the woman faces the camera. Background is more strongly blurred and has a purple/blue cast on the awning and signs. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Pink clay bakery on a cobbled cube, striped awning, bread in the window and in crates, chimney smoke (cropped), tree, delivery scooter, hanging sign with a faint croissant icon. No loose croissant on the roof (mixH2_x had one), but the base is a pale beige slab instead of grass and the icon is barely visible. | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Wet brown bear, open mouth, salmon leaping right next to the jaw, droplets, mossy log, rushing river, misty forest. Fur smoother than the teacher's; fish and mouth do not quite touch. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Gondolas at striped poles, palazzi with arched windows reflected in green water, bridge with two figures, red-flower balcony, laundry line, striped-shirt gondolier, pigeons on the water, paper texture and pencil lines. Palette more beige/sepia than the teacher's pastels (same as mixH2_x). | +| 21 | varied | 4.5 | - | - | 4 | 4 | same | Black-armoured knight genuinely kneeling and leaning on a planted sword, glowing ice-like crystal, stained glass, embers, faint runes, helmet on the floor (and one on his head). Silhouette is very dark with little armour detail, blacks crushed, icy blue dominates; second small object at left. | +| 22 | varied | 4.5 | - | 5 | 4 | 4.5 | better | Bright Scandinavian kitchen, oak table, jug of pink tulips, crumbs, pulled-back pale chair, plant, big window. Exactly TWO cups on the table, one steaming (mixH2_x: three). Very high-key (mean luminance 0.79 vs 0.71 teacher), whites and background close to washed out, table top warped, chair back rail odd. | +| 23 | varied | 4 | - | - | 4 | 4 | same | Hooded vendor frying dumplings, bulbs, steam, bowl and tray of dumplings, orange oil, wet orange reflections, two customers' backs, one hand holding a small coin. Extremely dark (mean luminance 0.09, luminance std 0.11 vs 0.17 teacher), no wrapper pile, no flame. | + +## Averages + +Overall (24 images): F 4.25, T 4.00 (7 text-bearing images; 4.08 on the six quoted-text prompts alone), C 3.79 (7 images), Q 4.21, A 4.15. 21 of 24 images have A >= 4 (A < 4: 01, 12, 13). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.00 | 4.08 | 3.38 (4 imgs) | 4.17 | 4.00 | +| portrait | 4 | 4.38 | - | - | 4.13 | 4.13 | +| detail | 4 | 4.13 | 3.5 (typewriter only) | - | 4.13 | 3.88 | +| counting | 2 | 4.25 | - | 4.00 | 4.25 | 4.25 | +| varied | 8 | 4.44 | - | 5.0 (1 img) | 4.31 | 4.38 | + +## Comparison with mixH2_x + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixH2_x (1.95 GB; previous report) | 4.33 | 4.00 | 3.43 | 4.27 | 4.21 | +| mixH2_xm_f8 (1.72 GB; this report) | 4.25 | 4.00 | 3.79 | 4.21 | 4.15 | +| delta | -0.08 | 0.00 | +0.36 | -0.06 | -0.06 | + +Per category A (mixH2_x -> mixH2_xm_f8): text 4.00 -> 4.00, portrait 4.13 -> 4.13, detail 4.25 -> 3.88, counting 4.25 -> 4.25, varied 4.38 -> 4.38. + +Reading the deltas: F, T, Q and A are all inside the +-0.2 noise band (A -0.06). The C gain (+0.36) is not a general counting improvement: it comes from three image-specific changes (22 now has exactly two cups, 03 has a correct pair of time stamps and a skip button, 05 has whole signs), n=7, and 00/01/14/15 are unchanged or worse. The only category that moves is detail (A -0.37, n=4), driven by 10 (cool grey keys, mirrored glyphs) and 12 (dark trunk, missing snails and fungi cluster). + +## Side-by-side verdict (all 24, versus mixH2_x) + +Better 4 (03, 05, 07, 22); same 15 (00, 02, 06, 09, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23); worse 5 (01, 04, 08, 10, 12). + +Reasons in brief: +- better: 03 both time stamps legible and a real skip-forward glyph (mixH2_x garbled them); 05 three whole, exact signs (mixH2_x cut the PEACHES board by the frame); 07 sharper face with freckles and no glass seam through the forehead; 22 two cups (mixH2_x three). +- worse: 01 "Evapoation", "Condenstion", "Precipittion" (3 of 4 labels wrong versus 2 of 4), incoherent arrows, underline under the title; 04 darker and duller, lanterns cream, cook missing (text equal); 08 washed out and plainer, scarf like a collar; 10 key tops cool grey-green instead of cream and more mirrored glyphs; 12 darker trunk, snails and fungi cluster lost, posterised mist. +- same: 00, 02, 06, 09, 11, 13, 14, 15, 16, 17, 18, 19, 20, 21, 23 -- the differences are re-arrangements of the same content (cleaner balloons but hazier valley in 14, fewer lanterns in 16, no roof croissant but fainter sign icon in 18, kneeling knight with one planted sword in 21). + +## Text-prompt acceptance + +- Six quoted-text prompts: 5/6 accepted (A >= 4: 00 4.5, 02 4.5, 03 4, 04 4, 05 4.5); only 01 (A 2.5) fails. mixH2_x: 4/6. +- Letter-for-letter exact: 4/6 (00, 02, 03, 05). 00 is exact in letters but the slate dash became an underline swish; 03 has title, artist and both time stamps exact but no slash (the progress bar sits between them) and the wrong controls; 04 reads completely but 凌 is malformed (same as mixH2_x), so I do not count it; 01 has three misspelled labels. mixH2_x: 3/6. +- Large and clean text (02, 04 title, 05, 00) is intact and as sharp as in mixH2_x; fine detail was not lost to the modulation rounding. Failures remain dense small text in graphics (01). +- Typewriter keys (10): black serif letters on pale keys, but 3-4 of about 14 visible legends are mirrored or Cyrillic-like. + +## Five worst images + +1. 01 water cycle (A 2.5, T 2): three misspelled labels, arrows and labels unrelated, no cycle. +2. 12 oak trunk (A 3.5): dark, snails and bracket fungi missing, ladybug on bark, posterised mist. +3. 13 gauntlet (A 3.5): mail mitt without fingers (same as mixH2_x). +4. 10 typewriter (A 4, T 3.5): grey-green key tops (colour shift) and mirrored glyphs. +5. 08 anime boy (A 4, Q 4): washed-out, sparse flowers, collar-like scarf. +Close behind (A 4): 04 (dark and muted, cook missing), 15 (5 of 6 fruits, same as the teacher), 23 (extremely dark), 21 (very dark armour), 03 (wrong control glyphs, junk icons). + +## Systematic failure patterns + +- Global colour/contrast: no per-image tint is obvious at normal viewing, but a small consistent drift exists. Over the 24 images, mixH2_xm_f8 minus mixH2_x: R-B "warmth" -0.025 (79% of images cooler; mixH2_x versus the teacher is -0.001, i.e. none), luminance std -0.009 (75% lower; versus the teacher -0.017 vs -0.008 for mixH2_x), 1st-99th percentile luminance range -0.042 (versus the teacher -0.066 vs -0.025), mean saturation -0.014 (versus the teacher -0.023 vs -0.009, not significant). In plain words: slightly cooler white balance and about 8% less luminance contrast than the teacher, twice the loss of mixH2_x. Visible instances: 10 (cream keys became grey-green), 04 (lanterns cream, sign glow weaker), 17 (purple/blue cast on awning and signs), 11 (barbs shift to violet), 23, 12 and 04 (darker and less saturated), 08, 22 and 14 (paler and lower contrast, 08 and 22 about +0.05 to +0.065 brighter). The shift is scene dependent: warm/dark scenes lose saturation and warmth, bright scenes become paler. +- No banding, blocking or noise from the modulation layers. Contour steps in the bokeh and mist (11, 12) are also in the teacher. Foreground detail stays crisp (04 sub-line, 05 signs, 13 rings, 20 pen lines); softness is limited to backgrounds (00, 14, 17, 22), which can be a trajectory change rather than a defect. +- Dense small text in graphics is the main content loss (01), as in mixH2_x; large text is fine. +- Prompt-attribute drift: gauntlet without fingers (13, unchanged), mirrored key glyphs (10), missing snails and fungi cluster (12), sparse lanterns (16), wrong UI controls (03). +- Counts: balloons exact (5/5), two cups exact (22, improved), fruit bowl 5 of 6 like the teacher. +- Composition changed versus the teacher in almost every image (expected from the different sampling trajectory); the structural drift reported by DINOv2/SSIM is visible as re-arranged compositions, not as lower quality. + +## Summary (10 lines) + +1. Overall: F 4.25, T 4.00, C 3.79, Q 4.21, A 4.15 (mixH2_x: 4.33 / 4.00 / 3.43 / 4.27 / 4.21). +2. All averages except C are within the +-0.2 noise band; the C gain (+0.36, n=7) comes from three image-specific changes, not a general improvement. +3. Versus mixH2_x: 4 better (03, 05, 07, 22), 15 same, 5 worse (01, 04, 08, 10, 12). +4. Text prompts: 5/6 with A >= 4 (mixH2_x 4/6), letter-exact 4/6 (mixH2_x 3/6; 00, 02, 03, 05); 01 fails with three misspelled labels. +5. Global shift: yes, but small -- about 0.025 cooler (R-B) and about 8% less luminance contrast than the teacher on average, scene dependent (grey-green keys in 10, muted lanterns in 04, pale 08/22); no banding, blocking or noise. +6. The shift is invisible in most single images (portraits, 17, 19, 20 look like mixH2_x); it shows in warm dark scenes and in very bright high-key scenes. +7. Fine detail and large text are as sharp as in mixH2_x. +8. Worst five: 01, 12, 13, 10, 08. +9. 21 of 24 images have A >= 4 (mixH2_x 20 of 24). +10. Answer: YES, the visible quality of mixH2_xm_f8 is acceptable as a replacement for the teacher in 21 of 24 images and for 5 of 6 text prompts. It is at most marginally worse than mixH2_x on the overall level (A 4.15 vs 4.21, within noise): fewer exact labels in the infographic (01), duller warm/dark scenes (04, 10, 12) and washed-out 08, compensated by gains in 03, 05, 07, 22. The cost of the 12% smaller file is a slight cool/low-contrast drift and a handful of per-image regressions, not a general loss of quality. Caveat: this judge is a model instance and +-0.2 differences are within noise. diff --git a/eval/judge_mixH_tuned.md b/eval/judge_mixH_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..3ec83c12bc698dcf6674ea6543129d47e8aef56b --- /dev/null +++ b/eval/judge_mixH_tuned.md @@ -0,0 +1,115 @@ +# Visual judge: mixH_tuned (student, 1.95 GB, low-bit attention, 2-bit middle-block FFN with 3/4 bits at the ends, rank-64 low-rank correction, after end-to-end tuning) vs teacher_bf16 and vs mixG_tuned + +Scales 0-5, same rubric as judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixH_tuned PNGs opened at full 768x768 resolution (inside 1536x768 composites, so each half at native resolution). For every prompt two composites were opened: teacher | mixH_tuned and mixG_tuned | mixH_tuned. No comparison was skipped. The scratch composites were written to the OS temp folder and deleted afterwards. +- Two extra native-size 2-3x zoom crops (scratch, deleted): the four labels of 01 and the two price signs of 05, because both were decisive for the T score. +- Same scoring subsets as the previous report: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness kept as in the previous report: no 5 on Q or A. All images were compared directly against mixG_tuned, not only against the teacher. +- The previous report's scores for mixG_tuned are used as the baseline; I re-looked at every mixG_tuned image and agree with its verdicts (no re-scoring), apart from small wording differences. +- This judge is a model instance; differences of +-0.2 on any average are within noise. The per-image side-by-side verdicts and the specific text/count findings are firmer evidence than the averages. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixG_tuned | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 4.5 | 4.5 | 4.5 | 4.5 | same | "LE PETIT FROMAGE" fully visible in navy (G cropped it), slate "Tasting / - Saturday / 11 AM" exact, cat on the step, cheese wheels, crate, wet cobbles. Window line "Affineur depuis 1974": "Affineur depuis" in white script on the glass but "1974" in dark navy on the wall outside the glass (colour/placement break). Right lamp has a bent arm. | +| 01 | text | 2.5 | 2.5 | 2 | 3 | 2.5 | worse | Title "WATER CYCLE" exact (wide gap between the words). Labels now misspelled: "Evapioration" (extra i), "Condenation" (missing s); "Precipitation" and "Collection" correct (the e of Collection slightly malformed). Labels float around the scene with a few unrelated stray blue arrows/arrowheads, mountain/river/sea clutter and a cut-off sea shape; no cycle. mixG_tuned had all four labels exact in a clean column. | +| 02 | text | 4.5 | 4.5 | - | 4.5 | 4.5 | same | "BEYOND THE BREAKERS" exact (THE smaller), tagline "Some lights never go out" exact but with an irregular double gap between "lights" and "never". Girl in yellow raincoat seen from behind rowing toward the lit lighthouse, big curled wave, beam across clouds. Lighthouse/girl smaller than teacher; title not cropped (better than G). | +| 03 | text | 4 | 5 | 3.5 | 4 | 4 | better | Upright phone, "Neon Horizon", "Luna Vale" and "1:24 / 3:48" all exact, square purple sunset cover, desk and lamp. Controls are a thin "<", play and a fast-forward ">>" (previous/play/next as asked, G had pause plus a bow-tie next and no previous). Progress bar sits above the artist line instead of below; bottom icons are three junk glyphs (a "u" circle and two grey dots); screen dark and low contrast. | +| 04 | text | 4.5 | 4.5 | - | 4 | 4 | same | "老王面馆" exact; sub-line "手工拉面 · 营业至凌晨两点" reads all 12 characters in order. Characters are pale cream rather than bold gold and the board glows weaker than the teacher; lanterns are pale/cream instead of red; crowd of diners at right is smeared; wet ground is a patchwork of coloured blobs. Neighbouring side sign is junk (teacher too). | +| 05 | text | 4 | 3 | 3 | 4 | 3.5 | worse | "CHERRIES $5" and "HONEY $8" are exact, but the large board reads "PEACHES $?" with a malformed digit that looks like an 8 (zoomed: 3/8 hybrid) - wrong price on the most prominent sign. A daisy covers the C of CHERRIES. The duplicated HONEY sign of G is gone, one hat hangs on the post. Saturated, hot colours; fruit is a generic orange/red mass (teacher: separate crates of peaches, cherries, lemons). | +| 06 | portrait | 3.5 | - | - | 3.5 | 3.5 | worse | Smooth bald head, maroon robe with thin golden edge, window light left, grey backdrop, wrinkles by the eyes good. Skin is waxy, the silver stubble on the chin is gone (teacher and G have it), beads hang at the neck (none on the wrist), gaze into the camera. Softer pores than G. | +| 07 | portrait | 4.5 | - | - | 4.5 | 4.5 | better | Wavy auburn hair with bangs (not tightly curly), green eyes, freckles, readable lashes, parted lips, navy turtleneck, red lamp glow and teal/amber bokeh, sharp face behind the glass. Hard vertical glass seam through the forehead remains (also in teacher), few skin droplets. Clearly sharper than G (soft, smeared). | +| 08 | portrait | 5 | - | - | 4.5 | 4.5 | better | Messy black hair, golden eyes with highlights, billowing white shirt, flowing dark-green scarf (no fringe), blue flowers, petals, clouds. The stray white arcs and ghost drape of G are gone; eyes slightly less glossy than G. | +| 09 | portrait | 5 | - | - | 4.5 | 4.5 | better | Short grey hair, round glasses, arms crossed with clay-smeared hands visible, smeared apron, shelves of sage-green bowls, big window, kiln clearly glowing at right (G: hands tucked away, kiln faint). Skin a notch smoother than the teacher. | +| 10 | detail | 4 | 4 | - | 4.5 | 4 | worse | Keys are black glass with pale letters (prompt and G: black letters on cream keys); gold/chrome rims, very shallow depth of field, dust and wear, bent hook-shaped carriage part at the right. Letters readable (T E B U R E G Y B T D L) but a few keys carry odd glyphs; paper text is pseudo-English like the teacher. | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | Proper peacock eye (turquoise ring, violet centre, green/gold arcs), fine barbs, droplets on the eye, warm bokeh. Upper-left barbs soft by depth of field; more orange than the teacher. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | better | Mossy oak, lichen, ferns with droplets, mushrooms, bracket-fungi cluster at the right, 3-4 snails, a ladybug on the fern plus a second on the bark (prompt: one). Cleaner mushrooms and snails than G, which had dark blobby shapes beside the fungi. | +| 13 | detail | 3.5 | - | - | 4.5 | 3.5 | worse | Crisp interlocking rings, leather strap with stitching, scratched table, slender candle in a holder, dark background. But it is a mail tube/sleeve with one strap and no fingers or hand: reads as a sleeve, not a gauntlet; few rivets (G had finger-like segments, rivets and red stitching). | +| 14 | counting | 4 | - | 2 | 4.5 | 3.5 | worse | SIX balloons: red, orange, yellow with red patches, a second yellow with green/blue patches, blue, striped purple (prompt: exactly five, one yellow). Pilots, sunrise, mist, village, shadows all good. G had exactly five. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 3.5 | same | Three red apples (correct) but only two pears, both yellow-olive (no distinct green one): 5 fruits instead of 6. G also had 5 (yellow pear + green pear). Glossy skin, leaves, linen folds, shadow right, grey wall all good. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | same | Lantern tree on an island, lit walkway through the roots, boat with one traveller, mountains, violet sky with a shooting star, mist, glowing flowers and floating lights. Fewer lanterns than G, a few more lights in the water. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart, Hangul neon, scooter and blurred car, wet reflections. Woman again walks away from the camera (teacher faces it) but smaller and without G's odd white under-hem; cart has no vendor. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Pink clay bakery, striped awning, bread in the window, chimney smoke (cropped at top), tree, cobbles, delivery scooter with box, croissant lying on the flat roof (teacher: roof sign), a free-standing sign board rather than a hanging one; scooter instead of bicycle. | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Wet brown bear, open mouth, salmon touching the jaw on a diagonal, droplets, mossy log, river, misty forest. Small transparent water-bubble outline around the fish, fish body trout-like; otherwise as good as G. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Gondolas at striped poles, palazzi with arched windows reflected in green water, bridge with tiny figures, red flower balcony, laundry line, striped-shirt gondolier, pigeons in the air and on the water, paper texture, faint pencil lines in the margin. | +| 21 | varied | 4.5 | - | - | 4 | 4 | same | Black-armoured knight kneeling before the crystal, light shafts, stained glass, embers, glowing runes, torn banners, helmet on the floor, sword leaning (not clearly broken or planted). Knight still wears a helmet AND one lies on the floor; foreground cape/floor is a mushy dark mass; crushed blacks. | +| 22 | varied | 5 | - | 5 | 4.5 | 4.5 | same | Bright Scandinavian kitchen, two cups on the oak table (one on a saucer with steam), jug of pink-white tulips, crumbs, pulled-back pale chair, plant on a shelf, big window, grey floor, petal on the floor. Table perspective steep; chair has an extra rail. | +| 23 | varied | 4 | - | - | 4 | 4 | worse | Hooded vendor, large pan of orange oil with dumplings, bulbs, steam, trays, wrapper pile, sauce bowls, wet pavement. The customer's hand reaches in with a ladle/stick but no coin is visible (G: clear coin), only one customer back in the foreground, large near-black masses, no flame. | + +## Averages + +Overall (24 images): F 4.31, T 4.00 (7 text-bearing images; also 4.00 on the six quoted-text prompts alone), C 3.29 (7 images), Q 4.29, A 4.10. 18 of 24 images have A >= 4. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.08 | 4.00 | 3.25 (4 imgs) | 4.00 | 3.83 | +| portrait | 4 | 4.50 | - | - | 4.25 | 4.25 | +| detail | 4 | 4.25 | 4.0 (typewriter only) | - | 4.50 | 4.13 | +| counting | 2 | 3.75 | - | 2.50 (6 balloons, 5 of 6 fruits) | 4.50 | 3.50 | +| varied | 8 | 4.56 | - | 5.0 (1 img) | 4.38 | 4.38 | + +## Comparison with the mixG_tuned report + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixF_ptq (2.42 GB) | 4.29 | 4.36 | 2.71 | 4.29 | 4.02 | +| mixG_x (2.14 GB) | 4.50 | 4.43 | 3.36 | 4.44 | 4.29 | +| mixG_tuned (2.14 GB, previous report) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixH_tuned (1.95 GB, this report) | 4.31 | 4.00 | 3.29 | 4.29 | 4.10 | +| delta H - G_tuned | -0.17 | -0.79 | -0.35 | +0.06 | -0.11 | + +Per category A (mixG_tuned -> mixH_tuned): text 4.08 -> 3.83, portrait 3.88 -> 4.25, detail 4.50 -> 4.13, counting 4.00 -> 3.50, varied 4.38 -> 4.38. + +Reading the deltas: A (-0.11), Q (+0.06) and F (-0.17) are within noise. Two real regressions: T (-0.79: two misspelled labels in 01, a wrong price digit in 05, part of it the 1974 colour break in 00) and C (-0.35: six balloons in 14; the fruit bowl is unchanged). The portrait/anime category got better (07 sharp again, 08 no stray arcs, 09 hands and kiln). The 8.9% smaller model (1.95 vs 2.14 GB) costs about 0.1 A overall, but the loss is concentrated in text-heavy, exact-count and object-identity prompts rather than spread evenly; photographic/painterly scenes (varied) are unchanged. + +## Side-by-side verdict versus mixG_tuned (all 24) + +Better 5: 03, 07, 08, 09, 12. +Same 12: 00, 02, 04, 11, 15, 16, 17, 18, 19, 20, 21, 22. +Worse 7: 01, 05, 06, 10, 13, 14, 23. + +Reasons in brief: +- better: 03 proper previous/play/next controls, all strings exact, cleaner phone; 07 sharp face with freckles and lashes (G was soft); 08 no stray white arcs or ghost drape; 09 clay-smeared hands visible, kiln glowing; 12 cleaner mushrooms/snails. +- worse: 01 two labels misspelled, scattered arrows; 05 malformed price digit on the PEACHES sign; 06 waxy skin, stubble lost; 10 key colours inverted (pale letters on black keys) plus a few odd glyphs; 13 mail sleeve with a strap, not a gauntlet; 14 six balloons instead of five; 23 coin gone, fewer customer backs. +- same (with small leanings): 00 sign now complete but "1974" in the wrong colour; 04 paler sign and lanterns; 15 five fruits in both (colours of the pears less distinct than G); 17 no odd hem but smaller subject; 22 cleaner cups but odd chair rail; 21 same double helmet flaw. + +## Text-prompt acceptance + +- Six quoted-text prompts: 4/6 accepted (A >= 4: 00, 02, 03, 04). 01 (A 2.5) and 05 (A 3.5) fail. Previous report: 5/6. +- Strings letter-for-letter exact in only 3/6 images (02, 03, 04). 00 is exact in letters but "1974" is dark and off the glass; 01 has "Evapioration" and "Condenation"; 05 has a malformed price digit on the peaches sign. This is a regression versus mixG_tuned, where all requested strings were exact in 6/6. +- The Chinese 老王面馆 and its 12-character sub-line still render correctly; "Neon Horizon / Luna Vale / 1:24 / 3:48" is exact and the music-player controls are now correct. +- Typewriter keys (10) are legible, with a few odd glyphs. + +## Five worst images + +1. 01 water cycle (A 2.5, T 2.5): misspelled labels "Evapioration" and "Condenation", no cycle, stray arrows; clear regression versus mixG_tuned. +2. 05 fruit stand (A 3.5, T 3): the big "PEACHES" sign shows a malformed price digit (reads $8), daisy covers the C of CHERRIES. +3. 14 hot-air balloons (A 3.5, C 2): six balloons with two yellow ones; the "exactly five" prompt fails (G passed). +4. 15 fruit bowl (A 3.5, C 3): three apples but only two pears, 5 of 6 fruits; unchanged from G. +5. 13 gauntlet (A 3.5): reads as a mail sleeve with one strap, no fingers; the rings are crisp. +Close behind: 06 (A 3.5, waxy skin, stubble lost), then A 4: 03, 04, 10 (inverted key colours), 21 (double helmet), 23 (no coin). + +## Systematic failure patterns + +- Small text on busy graphics is the main loss: label spellings in 01 (one dropped and one inserted letter, the kind of error mixG_tuned did not show) and a digit shape in 05. Large clean text (02 title, 03, 04 Chinese) is intact. +- Exact-count prompts regressed: balloons went from 5/5 to 6, the fruit bowl is still 5 of 6 with the pear colours less distinct. The 3/2/1 composite still fails. +- Object-identity drift on technical objects: gauntlet to mail sleeve (13), keys with inverted colours (10), a hat and sign layout fine but a wrong price (05). Prompt attributes such as "black letters", "gauntlet", "silver stubble", "coin" are the first things lost. +- Infographic logic: 01 still produces no cycle, now with stray arrows. +- Portraits are the strongest improvement (07, 08, 09); the remaining soft spot is waxy skin in 06. +- Residual small items: croissant on the roof (18), double helmet (21), junk icon glyphs (03), crushed blacks in 21/23, hot saturation in 05, dim signage in 04. No banding, blocking or global colour/saturation shift; defects are local. + +## Summary (10 lines) + +1. Overall: F 4.31, T 4.00, C 3.29, Q 4.29, A 4.10 (mixG_tuned report: 4.48 / 4.79 / 3.64 / 4.23 / 4.21); F, Q, A within noise, T and C clearly lower. +2. Side-by-side versus mixG_tuned: 5 better, 12 same, 7 worse out of 24. +3. Text prompts: 4/6 acceptable (A >= 4: 00, 02, 03, 04); 01 (A 2.5) and 05 (A 3.5) fail; only 3/6 images have every requested string exact, versus 6/6 before. +4. Concrete text defects: "Evapioration" and "Condenation" in 01, malformed price digit on the PEACHES sign in 05, "1974" dark and off the window in 00. +5. Counting: six balloons in 14 (was five); fruit bowl still 5 of 6 fruits; the kitchen (22) is correct. +6. Gains: portraits (07 sharp face, 08 no stray arcs, 09 hands and kiln), 03 controls and strings, 12 cleaner fungi; varied scenes unchanged in quality. +7. Regressions: 01, 05, 14, 13 (gauntlet reads as sleeve), 10 (inverted key colours), 06 (waxy skin), 23 (coin lost). +8. No global colour shift, banding or blocking; defects are local and prompt-attribute specific. +9. Worst five: 01, 05, 14, 15, 13. +10. Verdict: mixH_tuned (1.95 GB) is about 0.1 A below mixG_tuned (2.14 GB), which is near noise overall, but the exact-text and exact-count behaviour that mixG_tuned had gained is partly lost, so mixG_tuned remains the safer candidate for text/count prompts while mixH_tuned is competitive on portraits and general scenes. Caveat: this judge is a model instance and +-0.2 differences are within noise. diff --git a/eval/judge_mixI_tuned.md b/eval/judge_mixI_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..c7a853cbda179c801672f7ee203225bc31c9cf84 --- /dev/null +++ b/eval/judge_mixI_tuned.md @@ -0,0 +1,114 @@ +# Visual judge: mixI_tuned (student, 1.74 GB, 1-bit attention in the middle blocks, 1-bit ffn_down in the most robust blocks, FFN otherwise 2-4 bits, rank-64 LoRA, after the end-to-end tuning round) vs teacher_bf16 and vs mixH_tuned + +Scales 0-5, same rubric as judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. + +Method and caveats: +- All 24 mixI_tuned PNGs were opened individually at full 768x768 resolution (text transcribed from these native files). +- For all 24 prompts a teacher | mixH_tuned | mixI_tuned side-by-side file (2312x768, displayed by the viewer at about 0.87x) was also opened. The mixH and teacher halves were therefore seen slightly downscaled, mixI at native size; fine-detail comparisons against mixH are a little less firm than the mixI scores. Scratch composites were written to the OS temp folder and deleted afterwards. +- Scoring subsets as in the previous report: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- Strictness matched to the mixG_tuned report: no 5 on Q or A. Differences in composition from the teacher were not penalised by themselves; F/C are scored against the prompt text. +- Letter-by-letter text reading: wrong or substituted letters, dropped words and invented glyphs count as errors even when the word is guessable. +- This judge is a model instance; differences of +-0.2 on any average are within noise. Here the deltas versus the previous reports are far larger than that, but individual per-image half points are still noisy. + +## Per-image table + +| id | category | F | T | C | Q | A | vs mixH_tuned | Note (most visible difference from the teacher) | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4 | 2 | 3 | 4 | 3 | worse | Sign reads "LE PETIT FPRONAGE" (R/O/M mangled), window line is a smeared "Affineur depus 1974" (hard to read), slate reads "Tasting / - / Sonnelang / 11 AM" (Saturday lost). Cheese wheels, wet cobbles, lamp glow and the sleeping cat are fine, but the cat lies on the pavement, not on the step; very different framing (side view, much of the facade cropped). mixH had all three strings exact. | +| 01 | text | 2.5 | 0.5 | 1 | 3 | 1.5 | worse | Title "WATER CCCLLE"; only three labels, all garbled ("Conobcion", "Coniioation", "Coniigtiion"), none of Evaporation/Precipitation/Collection readable. A pale blue mountain with a swooping stroke, small sun and cloud, no sea, no arrow cycle. Diagram intent lost. | +| 02 | text | 4 | 1 | - | 3.5 | 2.5 | worse | Title "BYOD TEN REAKERS" (should be BEYOND THE BREAKERS); tagline appears twice, both garbled ("Se ughleever go pun", "vthe nelluohlqr novever nhtulor"). Painting is good: yellow-raincoat rower in a cap, big curling wave, beam through clouds, but the lighthouse is thin and pale and the girl faces away. | +| 03 | text | 2.5 | 1 | 1 | 3 | 1.5 | worse | "Neon Hoiion", "Lurnal, Vale", progress line "1 :24 3:21 4:48" (nothing readable as 1:24 / 3:48). The album cover is merged into a sunset banner at the top of the screen, there are no round play/previous/next buttons (three abstract glyph blobs) and a row of four junk icons; the phone has a bulging trapezoid shape. | +| 04 | text | 3.5 | 1 | - | 3.5 | 2.5 | worse | Board reads "吉五面馆" (first two characters wrong, third is only a look-alike of 面), the 12-character sub-line is illegible pseudo-strokes. Red board, lanterns, steam, diners and wet colourful pavement are all there, but the board has lost its golden glow (flat yellow lettering) and faces are smeared. | +| 05 | text | 3.5 | 1 | 2.5 | 3.5 | 2.5 | worse | Signs read "PEURES $$8" (large board), "PEARNES $5" and "HERRIRES" (crate): none of PEACHES/CHERRIES/HONEY is correct, only "$5" survives, "$$8" is wrong. Awning, straw hat, daisies, hills and fruit crates are fine; the fruit is a mushy mix. | +| 06 | portrait | 3 | - | - | 3.5 | 3.5 | worse | A bald elderly man in a maroon robe with gold edge, window light from the left, beads at the neck (not the wrist); pores and wrinkles are resolved but he does not read as a Himalayan monk (light eyes, Western face) and the skin is waxy. mixH had a convincing monk face. | +| 07 | portrait | 3.5 | - | - | 3.5 | 3.5 | worse | Curly auburn hair, green eyes, navy turtleneck, red lamp bokeh and teal/amber grading are present, but the face is narrow, pasty and waxy with pursed (not parted) lips, few visible droplets on the lashes. No glass seam (an improvement over the teacher/mixG), but mixH had a more natural, sharper face with freckles. | +| 08 | portrait | 4.5 | - | - | 3.5 | 3.5 | worse | Boy with messy black hair, golden eyes, white shirt, wide green scarf, blue flower field and petals. Eyes are slightly unequal with brown cheek patches, scarf reads like a cape, an odd zipper-like seam runs down the shirt, sleeves are a mushy fold heap; mixH was cleaner. | +| 09 | portrait | 3.5 | - | - | 4 | 3.5 | worse (mild) | Grey short hair, round glasses, clay-smeared apron and hands, shelves of bowls, glowing kiln at the right. She is drawn clearly elderly (prompt: middle-aged) and the hands are clasped, not arms crossed; skin and clay texture are good. | +| 10 | detail | 3.5 | 1.5 | - | 3.5 | 3 | worse | Extreme close-up of brass-and-glass key tops, but the key legends are illegible smudged pseudo-glyphs (teacher and mixH: readable serif letters), paper text is gibberish, no carriage lever. Warm side light and tarnish are nice, depth of field is good. | +| 11 | detail | 4 | - | - | 4 | 4 | worse (mild) | Detailed barbs and water droplets, dark background with gold bokeh. The eye centre is a dark slit in a green-yellow ring with no violet pupil (teacher/mixH: clear turquoise ring and violet centre), and the palette is more saturated blue. | +| 12 | detail | 4 | - | - | 4 | 4 | same | Mossy trunk with lichen, ferns, droplets, mist, cream/orange mushrooms, a dark glossy bracket-fungus cluster and one ladybug. The snails read as grey pebbles and the bracket fungi are dark lumps; mixH had clearer snails and cream bracket fungi. Different strengths, equal acceptance. | +| 13 | detail | 3 | - | - | 3 | 3 | worse | The "chainmail" is a coarse square wire mesh with blurry blobs instead of small interlocking rings, folded like cloth next to a stitched leather panel; it does not read as a gauntlet. Candle, scratched wood and dark background are good. mixH's rings were convincing. | +| 14 | counting | 3 | - | 1.5 | 3.5 | 2.5 | worse | Only FOUR balloons (red, red-yellow striped, yellow with emblems, purple striped with an orange bottom); the blue and the orange one are missing. Balloon patterns are blobby with smeared logos, baskets are boxes. Valley, mist, village and sunrise are fine. mixH had exactly five in the right colours. | +| 15 | counting | 3 | - | 1.5 | 3.5 | 3 | worse | About four to five fruits: two red apples with pear-like necks, a red fruit behind, one large yellow-green pear, a pale pear peeking at the top; expected 3 apples, 2 yellow pears, 1 green pear. Skin texture, bowl, linen and soft light are good. | +| 16 | varied | 4 | - | - | 4 | 4 | same | Colossal tree on a root island, a few big glowing lanterns, mountains, violet sky with stars, glowing flowers, strong reflections. Fewer lanterns than the teacher; an empty boat on the left plus a standing figure on a skiff near the roots (instead of one boat with one traveller). Pretty and clean. | +| 17 | varied | 4 | - | - | 3.5 | 3.5 | worse | Yellow trench coat, clear umbrella, steaming cart, Hangul neon, wet reflections, blurred taxi. Woman is seen from behind (as in mixH), with a second umbrella shape stacked over her head and translucent ghostly legs. | +| 18 | varied | 3.5 | - | - | 3.5 | 3.5 | worse | Pink bakery, striped awning, bread in windows, chimney smoke, tree, scooter, hanging sign (scribbled text). The ground is a ragged sand-coloured slab rather than a clean floating cube, the cobble path is washed out, the scooter has an odd white box. mixH kept the neat cube. | +| 19 | varied | 3 | - | - | 3 | 2.5 | worse | Teddy-like bear with a distorted face and dry fur (not soaked), the salmon is a red/black mottled shape bent into a water stream at the mouth, not a leaping silver salmon. Forest, mist, river and log are fine. mixH was a convincing wildlife photo. | +| 20 | varied | 4 | - | - | 4 | 4 | worse (mild) | Gondolas at striped poles, arched palazzi, bridge, laundry lines, red flower pot, striped-shirt gondolier, pigeons on the water, paper texture and faint pencil lines. Sunrise glow is weak (pale, washed), details blurrier than mixH, and a black cap-like blob tops the left building. | +| 21 | varied | 3 | - | - | 3.5 | 3 | worse | The knight is sitting on the floor facing away from the viewer in an open-faced helmet, not kneeling before a crystal; the giant crystal has collapsed into a rubble heap. Stained glass, embers, planted broken sword and a helmet on the floor are there; runes barely visible; heavy blue haze and crushed blacks. mixH had the right scene. | +| 22 | varied | 4.5 | - | 4 | 3.5 | 4 | same | Bright high-key kitchen: white cabinets, oak table, jug of pink tulips, two cups (one steaming), crumbs, pulled-back chair, shelf plant, large window, grey floor. The table is a distorted wedge with floating grey legs; otherwise clean. mixH has cleaner furniture but three cups. | +| 23 | varied | 4 | - | - | 3.5 | 4 | same | Hooded vendor, warm bulbs, steam, dumplings in orange oil and in a bowl, two customer backs in the foreground, a reaching hand with a ladle, wet reflections. Vendor face grins oddly, no coin visible (not in mixH either), very crushed blacks. | + +## Averages + +Overall (24 images): F 3.54, T 1.14 (7 text-bearing images; 1.08 on the six quoted-text prompts alone), C 2.07 (7 images), Q 3.54, A 3.15. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 3.33 | 1.08 | 1.88 (4 imgs) | 3.42 | 2.25 | +| portrait | 4 | 3.63 | - | - | 3.63 | 3.50 | +| detail | 4 | 3.63 | 1.5 (typewriter only) | - | 3.63 | 3.50 | +| counting | 2 | 3.00 | - | 1.50 (4 balloons, ~4-5 fruits) | 3.50 | 2.75 | +| varied | 8 | 3.75 | - | 4.0 (1 img) | 3.56 | 3.56 | + +## Comparison with the previous reports + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixF_ptq (2.42 GB) | 4.29 | 4.36 | 2.71 | 4.29 | 4.02 | +| mixG_x (2.14 GB) | 4.50 | 4.43 | 3.36 | 4.44 | 4.29 | +| mixG_tuned (2.14 GB) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixI_tuned (1.74 GB, this report) | 3.54 | 1.14 | 2.07 | 3.54 | 3.15 | +| delta mixI_tuned - mixG_tuned | -0.94 | -3.65 | -1.57 | -0.69 | -1.06 | + +All deltas are far outside the +-0.2 noise band. Per category A (mixG_tuned -> mixI_tuned): text 4.08 -> 2.25, portrait 3.88 -> 3.50, detail 4.50 -> 3.50, counting 4.00 -> 2.75, varied 4.38 -> 3.56. Text is the collapse (T 4.79 -> 1.14, six of six garbled); the non-text categories lose roughly 0.4-1.0 on A; the least affected are landscape/still-life-type images (11, 12, 16, 20, 22, 23 at A 4). + +## Side-by-side verdict versus mixH_tuned (all 24) + +Better 0. +Same 4: 12, 16, 22, 23. +Worse 20: 00, 01, 02, 03, 04, 05, 06, 07, 08, 09 (mild), 10, 11 (mild), 13, 14, 15, 17, 18, 19, 20 (mild), 21. + +Reasons in brief: +- worse, text: 00, 01, 02, 03, 04, 05, 10 all had exact or legible text in mixH and have garbled or invented glyphs now. +- worse, counting/structure: 14 four balloons instead of five; 15 fruits mixed up; 13 chainmail turned into a coarse mesh; 18 base no longer a cube; 21 sitting knight and collapsed crystal; 19 teddy-like bear and odd salmon; 17 double umbrella and ghost legs. +- worse, portraits: 06 no longer reads as a Himalayan monk, 07 a waxy narrow face, 08 messier eyes and clothes, 09 older than prompted and clasped hands. +- worse, mild: 09, 11 (eye centre), 20 (pale sunrise, blurrier). +- same: 12 (snails vs bracket fungi trade off), 16 (fewer lanterns, boat detail), 22 (correct two cups but warped table), 23 (equal; both lack a visible coin). + +## Text-prompt acceptance + +- Six quoted-text prompts: 0/6 accepted (A >= 4): A values are 3, 1.5, 2.5, 1.5, 2.5, 2.5. +- Requested strings exact: 0 of 6 images fully exact (previous mixG_tuned report: 6 of 6). Nearly right: only the "$5" price (05), "11 AM" and "Tasting" (00), and "WATER" / "CYCLE" letters (01 title has an extra C: "CCCLLE"); every title, tagline, artist line, label and the Chinese board has at least one wrong character. +- Chinese board (04): "老王面馆" is "吉五面馆" with a look-alike third character; the sub-line has lost all legible characters. +- Typewriter keys (10): no readable letters at all. + +## Five worst images + +1. 01 water cycle (A 1.5): title and all labels garbled, no cycle diagram. +2. 03 music player (A 1.5): all three text elements wrong, no album cover or playback buttons. +3. 02 lighthouse poster (A 2.5): good painting but the title and tagline are garbled. +4. 04 noodle shop (A 2.5): the Chinese board and sub-line are wrong, scene is otherwise fine. +5. 05 fruit stand (A 2.5): none of the three sign words is correct. +Close behind (A 2.5-3): 14 (four balloons), 19 (teddy bear, odd salmon), 10 (illegible keys), 00 (garbled sign and slate), 13 (mesh chainmail), 21 (sitting knight, rubble crystal), 15 (fruit count). + +## Systematic failure patterns + +- Text rendering collapses: letters are dropped, doubled or replaced by invented glyphs (FPRONAGE, BYOD TEN REAKERS, Hoiion, Lurnal, CCCLLE, PEURES/PEARNES/HERRIRES), second-line text (taglines, sub-lines, small labels) is the worst, and the Chinese board keeps its shape but not its characters. Layout cues (a red board with a golden line, a poster title zone) survive, the glyphs do not. +- Counting and "set" constraints: balloons 4 of 5, fruit bowl mixed up; the model keeps the right general scene but drops one colour or one object class. +- Structured/geometric objects lose logic: the phone UI has no buttons, the chainmail becomes a mesh grid, the isometric base is not a cube, the table is a warped wedge, the knight sits instead of kneeling and the crystal collapses. These are structural errors rather than blur. +- Anatomy and identity drift in animals/people: teddy-like bear, wrong fish, double umbrella, translucent legs, a portrait that no longer matches the described ethnicity/age. Faces remain generally plausible and textured (06, 07, 09), which is better than expected. +- Natural scenes, landscapes, watercolor and still-life images survive best (11, 12, 16, 20, 22, 23: A 4); no global colour shift, banding or blocking; local issues are crushed blacks (21, 23), heavier saturation (11) and softness in portraits. + +## Summary (10 lines) + +1. Overall: F 3.54, T 1.14, C 2.07, Q 3.54, A 3.15 (mixG_tuned: 4.48 / 4.79 / 3.64 / 4.23 / 4.21); every average is well outside the +-0.2 noise band, so mixI_tuned is clearly worse, not just different. +2. Side-by-side versus mixH_tuned (1.95 GB): 0 better, 4 same, 20 worse out of 24. +3. Text prompts: 0/6 acceptable (A >= 4), 0/6 fully exact; the best, 00, still has "FPRONAGE", "Sonnelang" and a smeared window line. +4. The Chinese board and sub-line (04) are wrong; the typewriter key legends (10) are unreadable. +5. Counting fails: four balloons instead of five, fruit mix wrong; spatial and structural logic is also lost in 03, 13, 18, 21. +6. Natural scenes, landscapes, watercolor and still life still look good (A 4 in 11, 12, 16, 20, 22, 23); portraits are plausible but less faithful (06, 07, 09). +7. Regressions are mostly local and structural (garbled glyphs, odd anatomy, collapsed objects); there is no global colour shift, banding or blocking. +8. Worst five: 01, 03, 02, 04, 05, all text-bearing images. +9. Verdict: at 1.74 GB the 1-bit-attention mix gives up text rendering, counting and structured objects; it is acceptable only for text-free natural scenes, not as a general replacement for the teacher or for mixG_tuned/mixH_tuned. +10. Caveat: this judge is a model instance and +-0.2 differences are within noise; the mixH side-by-side halves were seen at about 0.87x scale, so the finest-detail verdicts (09, 11, 12, 20) are less firm than the text and count findings. diff --git a/eval/judge_nd_mixG_tuned_gptq.md b/eval/judge_nd_mixG_tuned_gptq.md new file mode 100644 index 0000000000000000000000000000000000000000..9cd9d26ccf5bff0a5b0f7173c42b47039b86cfb6 --- /dev/null +++ b/eval/judge_nd_mixG_tuned_gptq.md @@ -0,0 +1,149 @@ +# Visual judge: nd_mixG_tuned_gptq (mixG_tuned student, 2.14 GB, with the 4.9 GB 4-bit GPTQ text encoder instead of the 19 GB bf16 encoder) vs teacher_bf16 and vs mixG_tuned (bf16 encoder) + +Scales 0-5, same rubric as judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher. "-" = not applicable. Half points used. No 5 on Q or A. + +Method and caveats: +- All 24 candidate PNGs (eval\nd_mixG_tuned_gptq) opened individually at native 768x768, then side by side with the teacher (eval\teacher_bf16), one 1536x768 composite per prompt (teacher left, candidate right; scratch files in the OS temp folder, deleted afterwards). No comparison was skipped. +- Only AFTER all 24 candidates were scored did I open the earlier bf16-encoder images (eval\mixG_tuned) side by side with the candidate (earlier left, candidate right, again all 24) and read the numeric part of judge_mixG_tuned.md (averages, summary). +- Exposure disclosure: my first read of judge_mixG_tuned.md (to learn the method) used a 40-line window that also showed the per-image score table for ids 00-22 (not the averages, not id 23, not the summary). I could not un-see it, so partial anchoring on ids 00-22 cannot be excluded. Where my score differs from that table (05 text, 04 text, 10 text, 20/22/23 F) it is because of what I observed in the image, not the table. +- Small text read at native size; the 12-character sub-line in 04 and the two price signs in 05 were additionally cropped and enlarged 2.5-3x to read them letter by letter. +- Same scoring subsets as the previous report: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). +- This judge is a model instance; differences of +-0.2 on any average are within noise. The paired side-by-side verdicts versus the bf16-encoder images are firmer evidence than the difference of averages (different judging pass, different calibration). + +## Per-image table + +"vs teacher": same = no preference (A 4.5); worse = I would point out a visible loss in a side-by-side (worse (slight) = A 4, clear = A <= 3.5). "vs bf16-enc" = this image versus the earlier mixG_tuned image of the same prompt and seed. + +| id | category | F | T | C | Q | A | vs teacher | vs bf16-enc | Note | +|---|---|---|---|---|---|---|---|---|---| +| 00 | text | 5 | 5 | 4.5 | 4.5 | 4.5 | same | same | "LE PETIT FROMAGE" (top edge slightly cropped), window "Affineur / depuis 1974" (two lines, white script), slate "Tasting / - Saturday / 11 AM" all exact; cat on the step, cheese wheels, crate, wet cobbles, lamp glow. One small navy squiggle stray on the wall right of the window; narrower street view than the teacher. | +| 01 | text | 3 | 5 | 2 | 3.5 | 3 | worse (clear) | better | "WATER CYCLE" and the four labels exact. Still a column of four boxes at the right with three arrows, not a cycle beside each stage (teacher: labels around a loop). Noticeably cleaner than the bf16-encoder image (no stray blue triangle, no spiky shapes, fewer arrows); a short stray dash under the cloud. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | same | "Some lights never go out" and "BEYOND THE BREAKERS" exact. Girl in yellow raincoat rowing toward the lit lighthouse, curled wave, beam through clouds. Girl seen from behind; partial stray glyph at the bottom-right corner (the teacher has one too). | +| 03 | text | 4 | 5 | 3 | 4 | 3.5 | worse (clear) | same | "Neon Horizon", "Luna Vale", "1:24" (left of bar) and "3:48" (right) exact. Square purple-sunset album art, tilted phone on a desk with a blurred cup. Controls are play / pause / a malformed bow-tie "next" (no previous); the three bottom icons are junk glyphs; screen dark with a glowing album art, no lamp; teacher has clean prev/play/next and real icons. | +| 04 | text | 4.5 | 5 | - | 4 | 4.5 | same | same | "老王面馆" exact in glowing golden characters; sub-line "手工拉面 · 营业至凌晨两点" reads all 12 characters in order. Lanterns, steam, cook, diners present. Pavement is a coloured-tile patchwork and faces are smeared (the teacher has the same patchwork pavement). | +| 05 | text | 4 | 3 | 3 | 4 | 3.5 | worse (clear) | worse | Real text error: the large board reads "PEACHE$" / "$5" (S turned into a dollar sign, price 5 instead of 3). "CHERRIES $5" appears twice (large board plus a crate board, both exact) and "HONEY $8" is exact. One straw hat on the post (as the teacher). Awning, crates of fruit, daisies, hills, blue sky all good; palette hot. | +| 06 | portrait | 4 | - | - | 4 | 4 | worse (slight) | same | Lined face, silver stubble, maroon robe with orange collar, warm grey backdrop, window light left. Beads hang at the neck (none on the wrist), eyes toward the camera (prompt: just past it), eyebrows dark rather than silver, golden robe edge reduced to the shoulder patch; slightly warmer/higher contrast than the teacher. | +| 07 | portrait | 4 | - | - | 3.5 | 3.5 | worse (clear) | same | Curly auburn hair, green eye, navy turtleneck, red lamp glow, teal/amber bokeh all present. Face soft and smeared (no readable lashes, freckles or hair strands), few droplets; a hard vertical seam runs through forehead and brow (the teacher shows a seam too, but a fainter one over a much sharper face). | +| 08 | portrait | 4.5 | - | - | 4 | 4 | worse (slight) | same | Messy black hair, golden eyes with highlights, white shirt, green scarf, blue flower field, petals, clouds. Two thin stray white arcs (left across the scarf, right edge) and a translucent white drape behind the left shoulder (a petal shows through it); scarf has no fringe and less detail than the teacher. | +| 09 | portrait | 4.5 | - | - | 4 | 4 | worse (slight) | same | Short grey hair, round glasses, arms crossed, clay-smeared apron, shelves of bowls, glowing kiln, big window. Skin and shelves a notch smoother, forearm clay lighter, kiln smaller than in the teacher; warm golden light a bit stronger. | +| 10 | detail | 4.5 | 4 | - | 4.5 | 4.5 | same | same | Distinct serif letters on chrome-rimmed keys (Z E F 6 B / A S E P G H L / B T Y U N N; not QWERTY order, E and N repeated, teacher has near-QWERTY plus digits), dust, worn enamel, cream paper with pseudo-English lines (the teacher's paper text is pseudo-English too). Carriage lever is a chrome loop at the upper left, not clearly bent. | +| 11 | detail | 5 | - | - | 4.5 | 4.5 | same | same | Proper eye (turquoise ring, violet centre, gold/green arcs), fine barbs, droplets, gold bokeh. Sharper barbs and more saturated than the teacher (teacher is more muted). | +| 12 | detail | 4.5 | - | - | 4 | 4 | worse (slight) | same | Mossy oak, lichen, ferns, droplets on ferns, misty bokeh, cream/orange mushrooms, a cluster of bracket fungi, snails (read more like slugs/beetles, no clear spiral shells), 2 ladybugs (prompt: one; teacher: one). Fewer droplet-covered leaves and less fine texture than the teacher. | +| 13 | detail | 4 | - | - | 4 | 4 | worse (slight) | same | Leather cuff with rivets and red stitching over interlocking mail, candle with melted wax, scratched wood, black background. Mail ends as a rounded sleeve (no finger segments; the bf16-encoder image had them), no bright highlight on each ring, mail slightly mushy, little wax detail. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | same | same | Exactly five balloons: red striped, orange striped, yellow with red patches, blue with a colour band, purple striped; pilots in baskets, sunrise, mist, village, long shadows. Loose arc instead of an even row; candy-saturated. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 4.5 | same | same | Three red apples, one yellow and one green pear = 5 fruits instead of 3+2+1 = 6 (second yellow pear missing). The teacher has the identical 3+1+1 = 5, so relative to the teacher the count error is shared. Glossy skin, leaves, linen folds, grey wall, shadow to the right all good. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | same | same | Large lantern tree on an island, lit walkway through the roots, boat with one traveller, mountains, violet sky with stars, glowing flowers and floating lights, soft mist. Tree a little smaller, fewer lanterns, bluer water than the teacher. | +| 17 | varied | 4.5 | - | - | 4.5 | 4 | worse (slight) | same | Yellow trench coat, clear umbrella, steaming food cart, Hangul neon signs, blurred taxi and scooter, wet reflections. Woman walks away from the camera (teacher: face visible), with a semi-transparent white under-hem below the coat. | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Pink clay bakery on a floating slab, striped awning, bread in window, chimney smoke (cloud cut by the top edge), tree, cobble path, delivery scooter, croissant sign. Croissant lying on the flat roof, sandy base instead of the teacher's green grass, fuzzy broccoli-like tree; scooter rather than bicycle (prompt mentions both). | +| 19 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Wet brown bear, open mouth, salmon horizontal and touching the jaw, droplets frozen around it, mossy log, rushing river, misty forest. Fish less "leaping" than the teacher's vertical one; slightly crunchy contrast. | +| 20 | varied | 4 | - | - | 4.5 | 4 | worse (slight) | same | Gondolas at striped poles, arched palazzi reflected in green water, stone bridge with figures, red-flower balcony, laundry line, striped-shirt gondolier, pigeons in the air and on the water (none on the bridge). Fills the frame like a digital painting: no paper margin, no visible pencil underdrawing, fewer loose pigment bleeds than the teacher. | +| 21 | varied | 4 | - | - | 4 | 4 | worse (slight) | same | Black-armoured knight before a giant glowing crystal, stained-glass shafts, embers, glowing floor runes, torn banners, sword planted with a broken blade, helmet on the floor. Knight crouches on one hand (not kneeling) and wears a horned helmet while a second helmet lies beside him; lower sword blade fades into the blue glow; very crushed blacks. | +| 22 | varied | 4.5 | - | 4.5 | 4 | 4 | worse (slight) | same | Bright Scandinavian kitchen: two cups on an oak table with steam between them, jug of pink tulips, crumbs, pulled-back pale chair, plant on the shelf, big window, grey floor, petals. Table and chair geometry slightly off (chair legs and back posts lean), otherwise clean. | +| 23 | varied | 4.5 | - | - | 4.5 | 4.5 | same | same | Hooded vendor frying dumplings, two warm bulbs, steam, tray of dumplings, bowl of filling, two customers' backs in the foreground, a hand with a visible coin, rain streaks under the awning, wet pavement with orange reflections. Foreground customers near-black silhouettes (the teacher has the same). | + +## Averages + +Overall (24 images): F 4.33, T 4.57 (7 text-bearing images; 4.67 on the six quoted-text prompts alone), C 3.57 (7 images), Q 4.21, A 4.12. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.17 | 4.67 | 3.13 (4 imgs) | 4.08 | 3.92 | +| portrait | 4 | 4.25 | - | - | 3.88 | 3.88 | +| detail | 4 | 4.50 | 4.0 (typewriter only) | - | 4.25 | 4.25 | +| counting | 2 | 4.25 | - | 4.00 (5 and 3) | 4.50 | 4.50 | +| varied | 8 | 4.44 | - | 4.5 (1 img) | 4.38 | 4.25 | + +## Comparison with the bf16-encoder report (judge_mixG_tuned.md) + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixG_tuned, bf16 encoder (earlier report) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixG_tuned, 4-bit GPTQ encoder (this report) | 4.33 | 4.57 | 3.57 | 4.21 | 4.12 | +| delta | -0.15 | -0.22 | -0.07 | -0.02 | -0.09 | + +Reading the deltas: C, Q and A are inside the +-0.2 noise band. F (-0.15) is inside it too. T (-0.22) is just outside and is real in direction: it comes from one image (05, "PEACHES $3" now wrong) plus a stricter reading of the typewriter keys (10) in this pass. Because the two passes were done by different judge instances, part of every delta is calibration; the paired comparison below is the better evidence. Paired, 22 of 24 images are the same composition and the same quality as the bf16-encoder image (many are near pixel-identical: 02, 06, 07, 08, 09, 11, 14, 16, 17, 19, 21, 22, 23); one is clearly cleaner (01), one clearly worse (05). + +## Side-by-side verdict versus the bf16-encoder image (all 24) + +Better 1: 01 (diagram much cleaner: no stray blue triangle, no spiky shoreline shapes, fewer arrows; layout logic still a box column). +Worse 1: 05 (large board "PEACHES $3" became "PEACHE$ / $5"; the CHERRIES board is now shown twice instead of the duplicated HONEY). +Same 22: 00, 02, 03, 04, 06, 07, 08, 09, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23. +Small differences inside "same": 00 window line now on two lines plus a tiny navy stray on the wall; 03 time stamps split to the two ends of the bar (1:24 left, 3:48 right) instead of bunched at the right; 10 key set slightly different (U N N instead of W N M); 13 mail no longer ends in leather finger segments (reads slightly less like a gauntlet); 06 golden shoulder panel more prominent. + +## Side-by-side verdict versus the teacher (all 24) + +Better 0. +Same 11: 00, 02, 04, 10, 11, 14, 15, 16, 18, 19, 23. +Worse (slight, A 4) 9: 06, 08, 09, 12, 13, 17, 20, 21, 22. +Worse (clear, A <= 3.5) 4: 01, 03, 05, 07. + +## Text prompts: letter-by-letter + +Candidate (4-bit GPTQ encoder): + +| id | string | result | +|---|---|---| +| 00 | "LE PETIT FROMAGE" | exact (top edge of the letters slightly cropped by the frame) | +| 00 | "Affineur depuis 1974" | exact (set on two lines) | +| 00 | "Tasting - Saturday 11 AM" | exact ("Tasting / - Saturday / 11 AM") | +| 01 | "WATER CYCLE" | exact | +| 01 | "Evaporation", "Condensation", "Precipitation", "Collection" | all four exact | +| 02 | "BEYOND THE BREAKERS" | exact | +| 02 | "Some lights never go out" | exact | +| 03 | "Neon Horizon" | exact | +| 03 | "Luna Vale" | exact | +| 03 | "1:24 / 3:48" | exact digits "1:24" and "3:48"; no slash (placed at the two ends of the bar, as in the teacher) | +| 04 | "老王面馆" | exact (4/4) | +| 04 | "手工拉面 · 营业至凌晨两点" | exact (all 12 characters in order, with the middle dot) | +| 05 | "PEACHES $3" | WRONG: reads "PEACHE$" over "$5" (letter S replaced by a dollar sign, price wrong) | +| 05 | "CHERRIES $5" | exact (appears on two boards) | +| 05 | "HONEY $8" | exact | +| 10 | key legends | distinct, well-formed serif letters (Z E F 6 B / A S E P G H L / B T Y U N N), not QWERTY order; paper text pseudo-English like the teacher's | + +Letter-exact quoted strings per prompt: 00 3/3, 01 2/2 groups (title + 4 labels = 5/5 strings), 02 2/2, 03 3/3, 04 2/2, 05 2/3. +Count of the six quoted-text prompts with every string letter-exact: 5/6 (fails: 05). Counting individual strings: 17/18. +bf16-encoder version (earlier report, and confirmed in my side-by-side view of the earlier images, including a native-size read of 04 and 05): 6/6 prompts exact, 18/18 strings. +Teacher: all quoted strings exact in 6/6 (it also carries a stray glyph at the bottom-right of 02). +Acceptance on the six text prompts (A >= 4): 3/6 (00, 02, 04); 03 and 05 at 3.5, 01 at 3. (The earlier report had 5/6; for the same 03 image I give 3.5 where the earlier report gave 4, so ~1 of the difference is calibration, 05 is real.) + +## Broken, artifacts, ghosting, colour/contrast drift + +- Broken images: none. All 24 are coherent, full-frame, no tiling, banding, blocking, blank regions or global noise. +- Heavy local artifacts: none heavy. Moderate local defects: 01 (box-column layout, one stray dash), 03 (malformed "next" button, three junk icon glyphs, no previous button), 05 (PEACHE$ / $5 board), 07 (smeared face and forehead seam), 08 (two thin white arcs, translucent drape), 21 (double helmet, crouching pose, blade fading into the glow), 00 (one small navy squiggle on the wall). +- Ghosted or translucent objects: 08 (white drape behind the left shoulder, a petal visible through it); 17 (semi-transparent white under-hem below the coat); 21 (lower part of the sword blade dissolves into the crystal glow). None is severe; 08 and 17 are the most visible. +- Colour/contrast drift versus the teacher: no global shift. Local leanings: more saturated and contrasty in 05, 06, 09, 11 and 14, bluer water in 16, sandy instead of green base in 18, darker low-contrast screen in 03, crushed blacks in 21 and 23 (the teacher is also dark in 23). Versus the bf16-encoder images: no colour or contrast difference seen in any of the 24. + +## Acceptance + +A >= 4: 20 of 24. Below 4: 01 (3), 03 (3.5), 05 (3.5), 07 (3.5). +By category (mean A): text 3.92, portrait 3.88, detail 4.25, counting 4.50, varied 4.25. + +## Five worst images + +1. 01 water cycle (A 3): text exact but a right-hand column of boxes and arrows, not a cycle; the teacher is a real loop. +2. 05 fruit stand (A 3.5, T 3): "PEACHE$ / $5" instead of "PEACHES $3"; the only letter-inexact quoted string in the set (bf16-encoder image: exact). +3. 07 train-window portrait (A 3.5, Q 3.5): soft smeared face, no readable lashes or freckles, seam through the forehead (same softness as the bf16-encoder image). +4. 03 music player (A 3.5): all strings exact, but malformed next button, no previous, junk icons, dark low-contrast screen. +5. 08 anime boy (A 4): two stray thin white arcs plus a translucent white drape; otherwise a good image. +Close behind (A 4): 06, 09, 12, 13, 17, 20, 21, 22. + +## Systematic patterns + +- Infographic layout logic (01) and fine UI glyphs (03 icons, next button) remain the weak spots; the same weak spots as the bf16-encoder version. +- Exact-count prompts: balloons exact (5/5); fruit bowl 5 of 6 fruits (second yellow pear missing), identical to the teacher's own count. +- Close-up portrait sharpness: 07 clearly soft, 06 and 09 a notch softer than the teacher; unchanged versus the bf16-encoder images. +- Style: 20 loses the watercolour paper margin and pencil underdrawing (digital-painting look); unchanged versus the bf16-encoder image. +- Text: the single new text failure relative to the bf16-encoder version is the Peaches board in 05; the Chinese sign and sub-line in 04 are exact. + +## Summary (10 lines) + +1. Overall (this candidate): F 4.33, T 4.57, C 3.57, Q 4.21, A 4.12. Earlier bf16-encoder report: F 4.48, T 4.79, C 3.64, Q 4.23, A 4.21. F, C, Q, A inside noise; T -0.22 (one wrong string). +2. Paired against the bf16-encoder images: 1 better (01), 1 worse (05), 22 same; many are near pixel-identical. +3. Against the teacher: 0 better, 11 same, 13 worse (9 slight, 4 clear: 01, 03, 05, 07). +4. A >= 4 on 20 of 24 images; the four below are 01 (3), 03 (3.5), 05 (3.5), 07 (3.5). +5. Text: 17/18 quoted strings and 5/6 prompts letter-exact (miss: 05 "PEACHES $3" shown as "PEACHE$ / $5"); bf16-encoder version 18/18 and 6/6. +6. Chinese 老王面馆 and all 12 characters of the sub-line are exact; "Condensation", "BEYOND THE BREAKERS", "Neon Horizon", "LE PETIT FROMAGE" exact. +7. Counting: balloons exactly five; fruit bowl 5 of 6 fruits, same count as the teacher. +8. No broken image; no heavy artifact; mild ghosting in 08 (drape) and 17 (hem), 21 (blade tip). +9. No global colour shift versus the teacher or versus the bf16-encoder images; only local saturation/contrast leanings. +10. Verdict: a user comparing against the teacher would accept 20 of 24 and would reject-or-redo 01, 03, 05, 07; relative to the bf16-encoder version of the same student the images are nearly the same, with one text regression (05) and one cleaner diagram (01). Caveat: this judge is a model instance and +-0.2 differences are within noise. diff --git a/eval/judge_nd_mixH2_x_f8_gptq.md b/eval/judge_nd_mixH2_x_f8_gptq.md new file mode 100644 index 0000000000000000000000000000000000000000..06f155a555266fb130d473e93b7fbf4f9bd6e55e --- /dev/null +++ b/eval/judge_nd_mixH2_x_f8_gptq.md @@ -0,0 +1,133 @@ +# Visual judge: nd_mixH2_x_f8_gptq (student mixH2_x_f8, 1.85 GB, 25 steps, CFG 2.5, 768x768, text encoder replaced by the 4.9 GB 4-bit GPTQ encoder) vs teacher_bf16, and vs the same student with the bf16 encoder (mixH2_x_f8 / judge_mixH2_x.md) + +Scales 0-5, same rubric and subsets as judge_mixH2_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image. "-" = not applicable. Half points used, no 5 on Q or A. F, Q, A on all 24 images; T on the six quoted-text prompts plus the typewriter key legends (10); C on 00, 01, 03, 05, 14, 15, 22. + +Method and caveats: +- Order of work: (1) all 24 candidate PNGs were opened at native 768x768 inside a 1536x768 composite (teacher left | candidate right) and scored against the teacher alone, with 1.5-3.6x native crops for every text prompt (00 sign, window line and slate; 01 labels; 02 tagline and title; 03 title, artist, time stamp; 04 both Chinese lines; 05 three signs; 10 key legends) and for 06, 07, 09, 21, 22, 23. (2) Only after all 24 were scored, judge_mixH2_x.md was read in full and the 24 earlier images (eval\mixH2_x_f8, bf16 encoder, same weights) were opened side by side with the candidate (earlier left | candidate right, plus crops for 01, 03, 04). +- The per-image F/T/C/Q/A columns below are the blind scores from step 1 and were not changed after the comparison. Where the comparison changed my view it is written under "Post-comparison remarks". +- Reference note: judge_mixH2_x.md judges mixH2_x (before the fp8 low-rank factors, bf16 encoder). The direct encoder-swap comparison here uses the images of mixH2_x_f8 (fp8 factors, bf16 encoder); the two sets looked the same in every case I checked (same typos in 01 and 03, same composition elsewhere), but I did not re-score the f8 bf16-encoder images independently, so the numeric comparison with the earlier report is across two judge instances. +- Scratch composites and crops were written to the OS temp folder and deleted afterwards. This judge is a model instance; +-0.2 on any average is within noise. +- "worse than teacher" = a defect visible side by side that the teacher does not have. "vs bf16-enc" = this candidate against the earlier image of the same weights with the bf16 encoder. + +## Per-image table + +| id | category | F | T | C | Q | A | vs teacher | vs bf16-enc | Note (most visible difference) | +|---|---|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 4.5 | 4.5 | 4 | 4.5 | same | same | "LE PETIT FROMAGE" exact, window "Affineur depuis 1974" exact (two lines), slate "Tasting / Saturday / 11 AM" letters exact but the dash is an underline swish (the teacher has a dash between Tasting and Saturday). Cat asleep on the step, cheese wheels, crate with a wheel, wet cobbles, lamps. Wider street with trees instead of the teacher's close facade; window shelves a bit less defined. | +| 01 | text | 4 | 4 | 4 | 4 | 3.5 | worse | better | "WATER CYCLE" exact; labels Evaporation (i slightly fused but reads right), **Condenstion** (missing a), Precipitation exact, Collection exact. Tidy column of rounded boxes with three arrows (none for Collection), mountain/river/sea; no cycle loop (teacher has a loop with labels at each stage). The bf16-encoder image has two typos (Condenstion and Precipaton); this one has one. | +| 02 | text | 4 | 3.5 | - | 4 | 3.5 | worse | worse | Title "BEYOND THE BREAKERS" exact; tagline reads "Some /light never go out" (lights -> light, the first l is a stick, "never" has a malformed e). Girl in yellow raincoat rowing from behind towards the lit lighthouse, beam through clouds. Wave much smaller and the sea calmer than the teacher; bright yellow patch on the water under the boat. The bf16-encoder image has the tagline exact. | +| 03 | text | 3.5 | 2.5 | 3.5 | 4 | 3 | worse | worse | Title **"Neon Hoiizon"** (r became i), artist "Luna Vale" exact, only "3:48" shown, "1:24" missing (no slash). Controls are play / pause / play: no previous and next. Three junk glyphs under the controls (dash-like marks). Purple sunset cover with sun disc, tilted phone on a desk; screen darker and flatter than the teacher's. bf16-encoder image has exact title and both (malformed) time stamps. | +| 04 | text | 4.5 | 3.5 | - | 4 | 4 | worse | worse | "老王面馆": bold golden characters, 面 drawn with a 西-like top (identical in the bf16-encoder image, teacher is clean). Sub-line "手工拙面 · 营业至淩晨两点": the third character is **拙 instead of 拉**, 凌 drawn as 淩 (also in the bf16-encoder image, whose 拉 is closer to right). Scene, steam, lanterns, diners, wet reflections as the teacher. Sub-line white instead of gold. | +| 05 | text | 4 | 5 | 4.5 | 4 | 4 | same | same | "PEACHES $3", "CHERRIES $5", "HONEY $8" all exact, white brush on dark green boards, one straw hat, daisies, hills, blue sky. Fruit is a mixed red/green/orange mass in all crates (no dark cherries as in the teacher); PEACHES board partly cut by the left edge. | +| 06 | portrait | 4.5 | - | - | 4 | 4.5 | same | same | Shaved head, deep wrinkles, silver stubble and grey brows, maroon robe with gold edge, window light from the left. Beads at the neck (none on the wrist), looks into the camera; skin warmer/more orange and a little smoother than the teacher's. | +| 07 | portrait | 4.5 | - | - | 4 | 4.5 | worse | same | Curly auburn hair, navy turtleneck, red lamp glow, teal/amber bokeh, rain, lips parted, a translucent face reflection on the cheek (the teacher has the same reflection). Frame darker and lower contrast than the teacher, face a little softer, eyes less crisp. | +| 08 | portrait | 4.5 | - | - | 4 | 4 | same | same | Messy black hair, golden eyes, 3/4 turn, billowing white shirt, flowing dark-green scarf (no fringe, one leaf-like flap), blue flowers, petals. Eyes smaller and less glossy than the teacher's. | +| 09 | portrait | 4.5 | - | - | 4 | 4.5 | same | same | Short grey hair, round glasses, arms crossed, clay on forearms and apron, shelves of bowls, big window, kiln glowing at the right. Hands plausible but soft; heavier sleeves than the teacher's rolled ones. | +| 10 | detail | 4 | 3.5 | - | 4 | 4 | worse | same | Cream keys with black serif legends and chrome rims, bent metal lever, dust, razor-thin focus. Legends read roughly P E E P U Г B E R D P E D V A Б T M R: mostly real letters but 2-3 Cyrillic-like glyphs and repeated letters (teacher: a clean QWERTY block with digits). Typed paper text is faint pseudo-text (teacher: sharper). Same as the bf16-encoder image. | +| 11 | detail | 4.5 | - | - | 4 | 4.5 | same | same | Proper peacock eye, turquoise ring, violet centre, fine barbs, many droplets, gold bokeh. More orange surround and denser barbs than the teacher's. | +| 12 | detail | 4.5 | - | - | 4 | 4 | same | same | Mossy oak, lichen, ferns with droplets, mushrooms, bracket fungi at the right, exactly one ladybug. The snails read as dark blobs (teacher: clear spiral shells). | +| 13 | detail | 3.5 | - | - | 4 | 3.5 | worse | same | Crisp interlocking rings, leather band with rivets, scratched table, candle on a wooden/leather base. A mail tube/sleeve with no fingers (teacher: finger tips), a few metal balls hang at the open end. Reads less like a gauntlet. | +| 14 | counting | 4.5 | - | 5 | 4 | 4.5 | same | same | Exactly five balloons: red, orange, yellow, blue, striped purple, with baskets, sunrise, mist, village. Blotchy ornaments on red/orange/yellow/blue instead of clean stripes; staggered heights; smaller than the teacher's; pilots not visible. | +| 15 | counting | 4 | - | 3 | 4 | 4 | same | same | 3 apples + 1 yellow pear + 1 green pear = 5 fruits (prompt: 6); the teacher has the same 5. Glossy skins, leaves, linen folds, shadow, grey wall all good. | +| 16 | varied | 4 | - | - | 4 | 4 | worse | worse | Beautiful but recomposed: a huge tree seen close with the canopy cut off by the top edge, lantern branches, lit bridge between roots, boat with TWO travellers (prompt: one), mist, flowers, violet sky. The lake does not mirror the tree. The teacher and the bf16-encoder image show the whole tree and a strong mirror reflection. | +| 17 | varied | 4.5 | - | - | 4 | 4.5 | same | same | Yellow trench coat, clear umbrella, food cart, vendor in yellow helmet, Hangul neon, scooter and taxi, wet reflections. Face a bit soft; almost identical to the teacher in structure. | +| 18 | varied | 4.5 | - | - | 4 | 4.5 | same | same | Pink clay bakery, striped awning, bread in the window, chimney smoke, tree, cobbles, scooter, hanging croissant sign. Croissant lying on the flat roof, baguettes lying on the cobbles, cream/beige base with more muted palette than the teacher's green grass. | +| 19 | varied | 4.5 | - | - | 4 | 4.5 | same | same | Wet brown bear, open mouth, salmon beside the jaw, droplets, mossy log, rushing river, misty forest. Fur slightly smoother than the teacher's. | +| 20 | varied | 4 | - | - | 4 | 4 | same | same | Gondolas at striped poles, palazzi with arched windows reflected in green water, bridge, red flower pot, laundry line, striped-shirt gondolier, paper texture. Two pigeons stand on the water instead of on the bridge. | +| 21 | varied | 4 | - | - | 4 | 4 | worse | same | Black armour knight in a lunge/sprawl rather than kneeling, glowing crystal, light shafts, stained glass, embers, floor runes, helmet on the floor. Two swords (one upright, one leaning at the left; the teacher has one); armour detail good. | +| 22 | varied | 4 | - | 3 | 4 | 4 | worse | same | Bright Scandinavian kitchen, oak table, tulips in a jug, crumbs and petals, pulled-back chair, plant on a shelf, big window. THREE cups on the table (two with handles plus a small one) instead of two; the teacher has two. Table perspective steeper than the teacher's. | +| 23 | varied | 3.5 | - | - | 3.5 | 3.5 | worse | worse | Hooded vendor, bulbs, steam, tray and bowl of dumplings, rain streaks, wet orange reflections, customer backs in front. The pan holds orange sauce rather than visible dumplings. A pink disembodied arm stump pokes out beside the right customer's cap (the bf16-encoder image has an outstretched hand with a coin), no wrapper pile or filling bowl, large near-black masses. | + +## Averages + +Overall (24 images): F 4.19, T 3.79 (7 text-bearing images; 3.83 on the six quoted-text prompts alone), C 3.93 (7 images), Q 3.98, A 4.06. **19 of 24 images have A >= 4** (below 4: 01, 02, 03, 13, 23). The earlier non-distilled bar was 20 of 24, so the candidate is one image short of that bar, inside the noise of a single half-point. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.08 | 3.83 | 4.13 (4 imgs) | 4.00 | 3.75 | +| portrait | 4 | 4.50 | - | - | 4.00 | 4.38 | +| detail | 4 | 4.13 | 3.5 (typewriter only) | - | 4.00 | 4.00 | +| counting | 2 | 4.25 | - | 4.00 (5 balloons, 5 of 6 fruits) | 4.00 | 4.25 | +| varied | 8 | 4.13 | - | 3.0 (1 img) | 3.94 | 4.13 | + +## Versus the teacher (24) + +better 0, same 13 (00, 05, 06, 08, 09, 11, 12, 14, 15, 17, 18, 19, 20), worse 11 (01, 02, 03, 04, 07, 10, 13, 16, 21, 22, 23). +No image looks broken. Most "worse" calls are a single localised difference (a misspelled word, a third cup, a second sword, a recomposed scene), not a general loss of quality. + +## Versus the bf16-encoder version + +Side-by-side (earlier mixH2_x_f8 vs this candidate, same prompt and seed): 18 images are near-duplicates (same composition, same colours, same text; small shifts such as a smaller cat in 00 or different key legends in 10), 1 better (01), 5 worse (02, 03, 04, 16, 23). + +| | F | T (7) | C | Q | A | A >= 4 | +|---|---|---|---|---|---|---| +| bf16 encoder, judge_mixH2_x.md (other judge instance, mixH2_x before fp8 factors) | 4.33 | 4.00 | 3.43 | 4.27 | 4.21 | 20/24 | +| GPTQ encoder, this report (blind scores) | 4.19 | 3.79 | 3.93 | 3.98 | 4.06 | 19/24 | +| raw difference | -0.14 | -0.21 | +0.50 | -0.29 | -0.15 | -1 | + +The two judge instances are not on exactly the same scale (see below), so I also give the paired estimate, which applies only my own side-by-side deltas (GPTQ minus bf16-encoder image) to my scores: 01 T +1.0 / A +0.5; 02 T -1.5 / A -1.0; 03 F -0.5 / T -1.0 / A -0.5; 04 T -0.5 / A -0.5; 16 F -0.5 / A -0.5; 23 F -0.5 / Q -0.5 / A -0.5; all other images 0. Resulting change from the bf16 encoder to the GPTQ encoder: F -0.06, T -0.29, C 0.00, Q -0.02, A -0.10. On my scale that puts the bf16-encoder version at about F 4.25 / T 4.07 / C 3.93 / Q 4.00 / A 4.17. Only T (spelling) moves beyond the noise band; A moves by about the noise level. + +Scale offsets between the two judge instances, for the same-looking images: 07 earlier A 3.5 / Q 3.5 (murky) vs my blind A 4.5 / Q 4; 01 earlier C 2 / A 3 vs my C 4 / A 3.5 for an image with strictly fewer errors; 05 earlier C 3.5 vs my 4.5; 03 earlier C 2.5 vs my 3.5. This explains the +0.50 raw C difference and a part of the Q difference (the earlier judge gave 4.5 to several images that I gave 4, e.g. 11, 19, 20); the paired deltas above are the safer measure. + +## Text-prompt acceptance and spelling + +- Quoted-text prompts with A >= 4: 3/6 (00, 04, 05); fail: 01 (3.5), 02 (3.5), 03 (3). Earlier report: 4/6 (00, 02, 04, 05). +- Letter-exact prompts: **2/6 (00, 05)**. Earlier report: 3/6 (00, 02, 05). 00 counts as exact in letters (only the slate dash is an underline swish, as in the earlier image); 04 is not counted (one wrong character in the sub-line). +- Per string (18 quoted strings: 00 three, 01 five incl. the title, 02 two, 03 three, 04 two, 05 three): GPTQ 13/18 exact, earlier version 14/18. + +| prompt | string | GPTQ candidate | bf16-encoder image (earlier mixH2_x_f8) | +|---|---|---|---| +| 00 | LE PETIT FROMAGE | exact | exact | +| 00 | Affineur depuis 1974 | exact | exact | +| 00 | Tasting - Saturday 11 AM | letters exact, dash replaced by an underline swish | same | +| 01 | WATER CYCLE | exact | exact | +| 01 | Evaporation | exact (i slightly fused) | exact | +| 01 | Condensation | **minor typo: "Condenstion"** | same typo | +| 01 | Precipitation | exact | typo: "Precipaton" | +| 01 | Collection | exact | exact | +| 02 | BEYOND THE BREAKERS | exact | exact | +| 02 | Some lights never go out | **minor typo: "Some light never go out"** (first l stick-like, malformed e in "never") | exact | +| 03 | Neon Horizon | **minor typo: "Neon Hoiizon"** | exact | +| 03 | Luna Vale | exact | exact | +| 03 | 1:24 / 3:48 | **only "3:48"; "1:24" missing**, no slash | both present but malformed (like "1:2A 3:4B"), no slash | +| 04 | 老王面馆 | readable, 面 drawn 西-like (same as earlier image) | same | +| 04 | 手工拉面 · 营业至凌晨两点 | **拉 became 拙; 凌 drawn as 淩**; other characters in order | 拉 close to correct; 凌 drawn as 淩 | +| 05 | PEACHES $3 / CHERRIES $5 / HONEY $8 | all three exact | all three exact | +| 10 | typewriter legends | P E E P U Г B E R D P E D V A Б T M R: 2-3 Cyrillic-like glyphs, repeated letters | same set of glyph problems | + +- Large clean text (titles of 01, 02, 05 signs, 00 sign and window line, 04 title) is intact. The losses sit in small or dense text: tagline (02), small title and time stamps (03), second line of the Chinese sign (04). Misspelling moved to different strings than with the bf16 encoder (01 improved by one string, 02, 03, 04 each lost one). + +## Artifacts, ghosting and colour drift + +- Broken images: none. +- Heavy local artifacts: 23 (disembodied pink arm stump beside the right customer, pan without readable dumplings, near-black masses); 03 (junk icon glyphs, wrong control glyphs, garbled title and time stamp); 13 (mail tube with metal balls at its open end, no fingers). +- Smaller local defects: 02 (bright yellow patch on the water), 01 (arrows starting from nowhere), 20 (pigeons standing on the water), 21 (two swords, sprawled pose), 22 (third cup), 10 (Cyrillic-like key glyphs). +- Ghosted / translucent objects: only the face reflection in 07, which the prompt asks for and the teacher shows too. No translucent or double-exposed objects elsewhere. +- Colour and contrast drift: no global shift, no banding, no blocking. Local drift: 06 skin warmer and slightly smoother, 07 darker with lower contrast and a softer face, 03 darker screen, 18 more muted palette with a beige base, 05 fruit crates read as a mixed fruit mass instead of dark cherries and orange peaches. + +## Five worst images (by A) + +1. 03 music player (A 3, T 2.5): "Neon Hoiizon", "1:24" missing, play/pause/play controls, junk icons. +2. 01 water cycle (A 3.5): "Condenstion", no cycle, three arrows. +3. 02 lighthouse poster (A 3.5): tagline "Some light never go out", calmer sea than the teacher. +4. 13 gauntlet (A 3.5): fingerless mail tube with metal balls at the end. +5. 23 street food (A 3.5): arm stump artifact, dark, pan content unclear. +Next (A 4): 04 (wrong character 拙), 05 (no cherries), 16 (cropped tree, two travellers), 22 (three cups), 21, 20, 12, 10, 08, 15. + +## Post-comparison remarks (not applied to the table above) + +- After reading judge_mixH2_x.md I re-checked 07 with a 1.8x face crop against the teacher: the candidate face is clearly softer and darker with less crisp eyes. If re-scored it would be Q 3.5 / A 4 instead of Q 4 / A 4.5: overall A 4.04, Q 3.96, A >= 4 count unchanged (19/24). The earlier report scored the bf16-encoder 07 lower still (A 3.5), so this is a scale difference between judges, not a difference between the two encoders (the two images are near-identical). +- 01: under the earlier report's rubric (two misspelled labels = T 2.5) one misspelled label of four would map to about T 3.5, so my blind T 4 and C 4 for 01 are probably half a point to one point generous relative to that report. + +## Summary (10 lines) + +1. Overall (blind): F 4.19, T 3.79, C 3.93, Q 3.98, A 4.06. Earlier bf16-encoder report (mixH2_x): 4.33 / 4.00 / 3.43 / 4.27 / 4.21. +2. Paired side-by-side against the bf16-encoder images: 18 near-identical, 1 better (01), 5 worse (02, 03, 04, 16, 23); paired change F -0.06, T -0.29, C 0.00, Q -0.02, A -0.10. +3. Against the teacher: 0 better, 13 same, 11 worse; A >= 4 on 19 of 24 (earlier bar 20/24; earlier report 20/24); no image looks broken. +4. Text: letter-exact 2/6 (00, 05) vs 3/6 earlier (00, 02, 05); per string 13/18 vs 14/18; A >= 4 on 3/6 text prompts vs 4/6. +5. Text defects now: "Condenstion" (01, unchanged), "Some light never go out" (02, new), "Neon Hoiizon" and missing "1:24" (03, new), 拉 -> 拙 (04, new). Better than the bf16 encoder: "Precipitation" is correct in 01. +6. Two compositions changed visibly with the GPTQ encoder: 16 (tree cropped, no mirror reflection, two travellers) and 23 (coin hand lost, arm stump). +7. Defects shared with the bf16-encoder version: 07 soft/dark face, 13 fingerless mail tube, 15 five of six fruits (also in the teacher), 22 three cups, 21 two swords, 10 Cyrillic-like key glyphs. +8. Large clean text and all portraits, details, landscapes, anime, watercolor, kitchen and bear images are unchanged in quality; no global colour shift, banding or blocking; defects are local. +9. Worst five: 03, 01, 02, 13, 23. +10. Verdict: a user would accept 19 of 24 images in place of the teacher's, essentially the same set as with the bf16 encoder; the visible cost of the 4-bit encoder is slightly worse spelling of small text and two recomposed scenes, with average A about 0.1 lower (noise level) and T about 0.3 lower. Caveat: this judge is a model instance, +-0.2 on averages is within noise, and the cross-report numbers carry a scale offset between judge instances; the paired image-by-image findings are the firmer evidence. diff --git a/eval/judge_tmixG_tuned.md b/eval/judge_tmixG_tuned.md new file mode 100644 index 0000000000000000000000000000000000000000..1661771b5f0adf602a967de2ecd61972ff8e46c5 --- /dev/null +++ b/eval/judge_tmixG_tuned.md @@ -0,0 +1,165 @@ +# Visual judge: tmixG_tuned (2.14 GB quantized student, block-wise QAT against the distilled teacher followed by end-to-end tuning, 4 steps, CFG 1.0, euler) vs turbo4 (its own unquantized bf16 distilled teacher) + +Question: HOW GOOD IS THE QUANTIZED DISTILLED MODEL COMPARED WITH ITS OWN DISTILLED TEACHER, AND WOULD A USER ACCEPT THE QUANTIZED IMAGE INSTEAD OF THE TEACHER IMAGE FOR THE SAME PROMPT? + +Scales 0-5, same rubric as judge_turbo4.md / judge_mixG_tuned.md / judge_tmixG_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image (turbo4). "-" = not applicable. Half points used. Scored images: eval\tmixG_tuned_1024 (1024x1024, primary). + +Method and caveats: +- All 24 tmixG_tuned_1024 PNGs were opened alone at native 1024x1024. For all 24 prompts a turbo4_1024 | tmixG_tuned_1024 side-by-side (2056x1024; the viewer showed it at about 97%, so each half was slightly under native) was also opened. Text lines were read from the native single views; for 00 the sign, window line and slate were additionally read on 2-3x crops. A zoom request for the 01 labels was refused by the tool permission layer, so the 01 labels were read from the native view only (they are large enough to read; the verdict on 01 does not depend on a zoom). Scratch files were written to the OS temp folder; the only file written in the project is this report. +- 768 secondary check (ids 0, 1, 2, 3, 4, 5, 10, 14, 15): turbo4_768 | tmixG_tuned_768 side-by-sides at native 768 size. These were observed, not scored. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the six quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- The quantized model follows a different sampling trajectory from the reference (about 18 of 24 pairs keep the reference layout; 01, 05, 06, 10, 17 (mirrored) and 20 are clearly rearranged). A different layout alone was not penalised; missing prompt elements, wrong or garbled text, artifacts, colour/contrast drift and loss of detail were. "vs teacher" = would I rather have this image or the turbo4 image for the same prompt. +- A is "acceptable as a replacement for the teacher image", as in the turbo4 and tmixG_x reports (baseline = the distilled teacher). +- Pixel statistics (Pillow/numpy, CPU only, all 24 images at 768, same metric definitions as the tmixG_x report: the turbo4 row reproduces it exactly) are an objective cross-check of the style description. +- This judge is a model instance; differences of +-0.2 on any average are within noise. One seed per prompt. + +## Per-image table (tmixG_tuned_1024) + +| id | category | F | T | C | Q | A | vs teacher | Note | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 4 | 4 | 4.5 | 4 | same | Complete storefront (sign not cropped), warm window full of cheese, wet cobbles, blue-hour street, cat asleep on the step plus an unrequested second cat on a crate at the right. Sign "LE PETIT FROMAGE" exact (mixed-case navy hand lettering, doubled stem on the last E), window "Affineur depuis / - 1974" exact (extra dash), slate "Tasting / Satting / Saturday / 11 AM": garbled extra word "Satting" where the dash belongs (minor typo). Reference slate is exact (plus a truncated second slate). | +| 01 | text | 3 | 2 | 2.5 | 3 | 2.5 | worse | "WATER CYCLE" exact. Labels: "Evapation" (minor), a box reading "lived hennon / Clonotation" (Condensation, wrong), "Precipitation" with the tail smeared/doubled (minor), "Coloiptim" (Collection, wrong): 0 of 4 exact. Sun, pale cloud, mountain with a waterfall, sea and a few arrows and drops are present, but there is no readable cycle, the whole picture is pale and washed out. Reference is bold, has all four labels exact and a loop. | +| 02 | text | 4.5 | 4.5 | - | 4.5 | 4.5 | same | "Some lights never go out" (the s of "lights" slightly detached) and "BEYOND THE BREAKERS" (THE smaller) exact, cream serif. Girl in yellow raincoat rowing in profile, curled wave, warm orange sunset and beam from the lighthouse on a headland, white spray crown, print grain. Girl and boat smaller than in the reference; the right oar runs oddly into the water. | +| 03 | text | 4.5 | 5 | 4.5 | 4.5 | 4.5 | same | "Neon Horizon", "Luna Vale", "1:24" and "3:48" at the two ends of the progress bar, all exact. Square purple-sunset cover (sun disc looks like a crescent), previous / play / next as round buttons, three tidy icons below, back arrow and "..." at the top, blurred lamp and pens on the desk. Screen darker and less crisp than the reference, no status-bar clock. | +| 04 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "老王面馆" exact in golden characters on a flat bright red board; sub-line "手工拉面 · 营业至凌晨两点" reads all 11 characters plus the dot in order (凌 slightly ambiguous at native size, as in the reference). Lanterns with pseudo-glyphs, cooks in the lit kitchen, steam at the right, diners at metal tables, wet street. Sign has less glow and the street reflections are less colourful than the reference. | +| 05 | text | 4 | 4.5 | 3.5 | 4 | 4 | worse | "PEACHES $3", "CHERRIES $5", "HONEY $8" exact in chalk-style white capitals on dark green boards (not brush letters), but "CHERRIES $5" and "HONEY $8" each appear twice (the foreground "CHERRIES" is crowded), the boards are smaller than the reference's huge crisp ones, the crates hold apples, oranges and lemons (no distinct peaches or cherries), and a second hat-like shape sits behind the hat on the right post. Daisies, hills, awning, blue sky fine. | +| 06 | portrait | 4.5 | - | - | 4.5 | 4.5 | better | Elderly monk with shaved head, deeply lined face, silver stubble on the head and chin, gentle half-smile, maroon robe with golden edge and visible wool weave, window light from the left, plain warm grey wall (the reference is a younger, jollier man with a sofa behind). Prayer beads hang on the chest, not the wrist; gaze goes to the camera. | +| 07 | portrait | 3.5 | - | - | 3 | 3 | worse | Curly auburn hair, navy turtleneck, red lamp glow, teal/amber bokeh, droplets over the glass, parted lips. But the face is smeared: nose, mouth and left eye are a ghosted blur (reads as motion blur, not as a glass reflection), so the portrait has no usable sharp face; right eye and hair are fine. Reference has a sharp, detailed face. Strongest local artifact in the set. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | same | Messy dark hair (brown at the tips), golden eyes, loose white shirt, dark green scarf flying, blue flower field, petals, clouds, mountains. Clean line art, no stray arcs. Neutral pose instead of the reference's hand-to-cheek; scarf ends are tattered leaf-like shapes without fringe. | +| 09 | portrait | 4.5 | - | - | 4 | 4 | same | Short grey hair, round glasses, arms crossed, calm smile, shelves of glazed bowls and vases, glowing kiln, big window, warm morning light. Clay on apron and forearms only as faint stains (reference: clearer crust); skin smoother and the whole picture a little hazy. | +| 10 | detail | 3 | 2.5 | - | 2.5 | 2.5 | worse | Typewriter macro with chrome-rimmed keys, dust, typebars, cream paper. Focus plane is so shallow that nothing is sharp: only about 5 legends are readable (H, K, 6, H, F) and the paper text is pseudo-glyphs ("KRA", "X"). Carriage lever is a plain black bar. Far below the reference's sharp Q A S D F key rows and typed lines. | +| 11 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Peacock eye with turquoise/blue ring, deep blue centre, copper and gold arcs, green barbs resolved, many droplets, gold bokeh. Warmer and slightly less violet than the reference, fewer large droplets. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Mossy oak with lichen, ferns, droplets, a large snail plus two small ones, one small ladybug, a cluster of cream/orange bracket fungi, misty green bokeh. Almost the reference composition, slightly hazier and cooler. | +| 13 | detail | 4 | - | - | 4 | 4 | same | Rivet-studded scratched leather cuff, interlocking rings individually resolved, scratched wood, candle on a holder, black background. Mail forms a rounded mitt without fingers; leather has fewer scratches, wax stains and no buckle strap compared with the reference; darker overall. | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | same | Exactly five balloons in a row: red, orange (checks), yellow, white/blue with a swirl, purple/white striped; tiny pilots in baskets, sunrise at the right, mist, village, long shadows. As good as the reference (whose red/orange have blocky patches). | +| 15 | counting | 3.5 | - | 3 | 4 | 3.5 | same | Five fruits where the prompt asks for six: two red apples, one yellow pear, one green pear, plus a pear/apple hybrid at the left (pear stem on a red-yellow apple body) that is neither. No second yellow pear. Reference also has five (two apples, three pears). Glossy skin, leaves, linen folds, grey wall. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | same | Colossal lantern tree on an island, about 35 lanterns, lit candle path/bridge along the roots, one boat with a traveller, mountains, violet sky with stars and a crescent moon, long golden reflections. Water flowers are dim pink ones at the bottom rather than glowing. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart, Hangul neon signs, wet reflections, blurred cars, teal/magenta palette. Woman walks in profile (the reference mirror image); no scooter or distinct taxi. | +| 18 | varied | 4.5 | - | - | 4 | 4 | worse | Pink clay bakery with striped awning, bread in the window, chimney smoke, trees, cobbles, croissant sign, café table, small orange delivery bike plus a second tiny object. Paler and plainer than the reference (no clouds, no fence, beige slab with hearts that reads as floating), props smaller and mushier. | +| 19 | varied | 3.5 | - | - | 4.5 | 3.5 | worse | Brown bear with open mouth and raised paw in a rushing river, salmon leaping in front of its body with frozen droplets, mossy log, misty forest. The fish is not at the mouth and the bear looks at the camera, so "catching" is lost; fur is only partly soaked. Reference shows the fish at the jaws. Sharp and photographic otherwise. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Watercolor canal with striped poles, two gondolas and a gondolier in a striped shirt, arched palazzi and their reflections, pigeons on the bridge, red-flower balcony, bunting/laundry lines. No torn-paper border and little paper texture or pencil underdrawing; cleaner than the reference. | +| 21 | varied | 4.5 | - | - | 4 | 4 | same | Black-armoured knight kneeling/crouching before a giant glowing faceted crystal in a cathedral, stained-glass shafts, embers, planted sword, glowing floor runes (the reference has none), helmet on the floor while he also wears one. Knight small and dark with crushed blacks; crystal fainter than the reference. | +| 22 | varied | 4.5 | - | 4 | 3.5 | 3.5 | worse | Bright Scandinavian kitchen: white cabinets, round oak table, jug of pink tulips, two cups with steam, crumbs and petals, pale chair, plant on shelf, big window, grey floor. But furniture is ghosted: the left chair is see-through under the table, the right chair and a cloth/leg overlap semi-transparently. Reference is clean and coherent. | +| 23 | varied | 4 | - | - | 3.5 | 3.5 | worse | Hooded vendor frying dumplings in a large pan, bulbs, steam, rain streaks, bowl of filling, tray of dumplings, customers' backs, hand reaching with a utensil (no coin). Very dark, low-contrast, murky vendor, large near-black foreground; reference has brighter bulbs, a legible face and glowing food. | + +## Averages (tmixG_tuned_1024) + +Overall (24 images): F 4.23, T 3.93 (7 text-bearing images; 4.17 on the six quoted-text prompts alone), C 3.79 (7 images), Q 4.08, A 3.98. 17 of 24 images have A >= 4 (A < 4: 01, 07, 10, 15, 19, 22, 23). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.17 | 4.17 | 3.63 (4 imgs) | 4.17 | 4.00 | +| portrait | 4 | 4.25 | - | - | 4.00 | 4.00 | +| detail | 4 | 4.00 | 2.5 (typewriter only) | - | 3.88 | 3.88 | +| counting | 2 | 4.25 | - | 4.00 (5 and 3) | 4.25 | 4.00 | +| varied | 8 | 4.38 | - | 4.0 (1 img) | 4.13 | 4.00 | + +## Comparison with the reference and with tmixG_x + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| turbo4_1024 (distilled teacher, judge_turbo4.md) | 4.23 | 4.71 | 4.14 | 4.31 | 4.21 | +| tmixG_tuned_1024 (this report) | 4.23 | 3.93 | 3.79 | 4.08 | 3.98 | +| delta tmixG_tuned - turbo4 | 0.00 | -0.78 | -0.35 | -0.23 | -0.23 | +| tmixG_x_1024 (judge_tmixG_x.md) | 4.08 | 3.79 | 3.86 | 4.23 | 4.06 | +| delta tmixG_tuned - tmixG_x | +0.15 | +0.14 | -0.07 | -0.15 | -0.08 | + +Reading the deltas: F equals the teacher; Q and A are just outside the +-0.2 noise band (-0.23); T is the one clearly lower number (-0.78) and C is lower (-0.35, n=7). Versus tmixG_x, all five averages are inside the +-0.2 noise band (T on the six quoted-text prompts alone is 4.17 vs 3.92, +0.25, borderline); I would call the two models equal overall, with different images failing. + +tmixG_tuned vs tmixG_x per metric: F +0.15, T +0.14 (+0.25 on the six quoted prompts), C -0.07, Q -0.15, A -0.08; vs-teacher tally 1 better / 15 same / 8 worse (tmixG_x: 1 / 14 / 9); letter-exact text prompts 4/6 for both (02, 03, 04, 05 in both). + +Text strings that changed between the two students at 1024: +- 00 improved: sign now "LE PETIT FROMAGE" exact (tmixG_x: "L LE PETTT FROMAGE") and window "Affineur depuis 1974" exact (tmixG_x: "depuis" lost). The slate keeps the same defect in both ("Tasting / Satting / Saturday / 11 AM"). +- 01 about equal: both fail all four labels. tmixG_x: "Evapoation", "Condenation" (minor), "Conipitation" (wrong), no "Collection"; tmixG_tuned: "Evapation", smeared "Precipitation" (minor), "lived hennon / Clonotation" and "Coloiptim" (wrong). Title exact in both. +- 02, 03, 04, 05 exact in both. 05 differs only in the extras: tmixG_x has a stray "HONEY $2" board, tmixG_tuned duplicates "CHERRIES $5" and "HONEY $8". +- 10 typewriter: tmixG_tuned is worse (about 5 legible legends vs about 8, and softer), 07 face smear and 22 ghosted chairs are new local artifacts that tmixG_x does not have; 00 and 09 are better than in tmixG_x. +- 768 text: tmixG_x exact on 00, 02, 03, 04 (fails 01, 05); tmixG_tuned exact only on 02, 04 (00 slate extra word, 03 smudged "/ 148" after "1:24", 01 two minor typos, 05 "HERLE $5"). At 768 the tuned model is the weaker one on text. + +## Verdict versus the turbo4 image (all 24) + +Better 1: 06. +Same 15: 00, 02, 03, 04, 08, 09, 11, 12, 13, 14, 15, 16, 17, 20, 21. +Worse 8: 01, 05, 07, 10, 18, 19, 22, 23. + +Reasons in brief: +- better: 06 plain grey wall, deep lines and silver stubble where the reference shows a younger smiling monk in front of a sofa. +- worse: 01 all four labels misspelled and the picture washed out; 05 smaller duplicated signs, no distinct peaches or cherries, extra hat shape; 07 smeared face; 10 nothing in focus, key legends and paper text unreadable; 18 paler and plainer; 19 salmon not at the mouth; 22 ghosted chairs; 23 very dark and murky. +- same (with small leanings): 00 sign and window better than expected but the slate has a garbled word; 02, 03, 04, 16, 17, 20 as good as the reference (03 darker, 04 less glow, 16 no glowing water flowers, 20 no torn edge); 08, 09, 11, 12, 13 equal with slightly less texture; 14 exact balloons in both; 15 both fail the fruit count (five fruits, the tuned one with a fused fruit); 21 runes present but dark and small. + +## Text-prompt acceptance and letter-by-letter reading + +- Six quoted-text prompts: 5/6 accepted (A >= 4: 00, 02, 03, 04, 05); 01 at A 2.5. +- Letter-exact prompts: 4 of 6 (02, 03, 04, 05). Turbo4 had 6 of 6. By string: 14 of 19 requested strings exact, 3 minor typos, 2 wrong. +- 00 (1024): sign "LE PETIT FROMAGE" exact (L-E / P-E-T-I-T / F-R-O-M-A-G-E, doubled stem on the final E); window "Affineur depuis - 1974" exact (stray dash); slate "Tasting / Satting / Saturday / 11 AM": minor typo (garbled extra word replaces the dash). 2 of 3 exact. +- 01 (1024): "WATER CYCLE" exact; "Evapation" (minor), "lived hennon / Clonotation" (Condensation, wrong), "Precipitation" (tail smeared, minor), "Coloiptim" (Collection, wrong). 1 of 5 exact. +- 02 exact: "Some lights never go out", "BEYOND THE BREAKERS". +- 03 exact: "Neon Horizon", "Luna Vale", "1:24", "3:48". +- 04 exact: "老王面馆" and "手工拉面 · 营业至凌晨两点" (11 characters in order; 凌 slightly ambiguous at native size). +- 05 exact: "PEACHES $3", "CHERRIES $5", "HONEY $8" (the last two appear twice). +- Typewriter keys (10): about 5 legible legends (H, K, 6, H, F); the paper text is pseudo-glyphs. + +## Broken images + +None in the strict sense: no image is flat in colour, globally mushy or fully destroyed. Three images have heavy local artifacts that a user would notice at once: 07 (smeared face), 22 (see-through chairs) and 10 (nothing in focus, unreadable legends). 01 is intact as an image but its text and layout are wrong; 23 is very dark. + +## Five worst images + +1. 01 water cycle (A 2.5, T 2, F 3, C 2.5): 0 of 4 labels exact, washed out, no readable cycle. +2. 10 typewriter (A 2.5, T 2.5, Q 2.5): no sharp plane, legends and paper text unreadable. +3. 07 train-window portrait (A 3, Q 3): smeared face. +4. 22 kitchen (A 3.5, Q 3.5): ghosted, see-through chairs. +5. 19 bear (A 3.5, F 3.5): salmon not caught; and 23 street stall (A 3.5, Q 3.5): very dark, murky. +Close behind (A 3.5-4): 15 (five fruits with a fused fruit), 05 (duplicated signs), 18, 21, 00 (slate word). + +## Systematic failure patterns + +- Dense small text: the two prompts with several small lines (00 slate, 01 labels) lose letters or words; large titles and signs (02, 03, 04, 05) stay exact. 01 is the clear failure in both quantized students. +- Duplicated or ghosted structure: doubled signs (05), ghost chairs (22), second lighthouse at 768 (02), fused pear/apple (15), second cat (00). +- Defocus and smear in close-ups: 07 face, 10 whole frame. Both are local mush on the main subject. +- Dark scenes lose detail: 21 (small black knight), 23 (murky vendor), 13 darker than the reference. +- Prompt details dropped silently: wrist beads (06), clay crust (09), coin (23), banners (21), second vehicle (18), glowing water flowers (16), fish at the mouth (19), buckle strap and wax (13). +- Exact counts: balloons exact (5), two cups exact (22); the 3 / 2 / 1 fruit mix fails (five fruits), as in the reference. +- No colour shift, banding or blocking; no over-sharpened look. + +## Style differences versus the teacher + +Mean over all 24 images at 768 (tmixG_tuned_768 vs turbo4_768, CPU Pillow/numpy; content differs per pair, so indicative only; tmixG_x values from its report in brackets): + +| | luminance mean | luminance std (contrast) | mean saturation | fine-detail energy (Laplacian) | pixels < 0.04 | pixels > 0.96 | +|---|---|---|---|---|---|---| +| turbo4 @768 | 0.388 | 0.193 | 0.450 | 0.034 | 9.9% | 3.8% | +| tmixG_tuned @768 | 0.388 | 0.175 | 0.451 | 0.028 | 8.9% | 3.6% | +| tmixG_x @768 (its report) | 0.376 | 0.175 | 0.449 | 0.022 | 9.9% | - | + +- Contrast about 9% lower (std 0.175 vs 0.193), mean luminance and saturation identical (0.388, 0.451 vs 0.450): no global colour or brightness shift, slightly fewer crushed blacks. +- Fine-detail energy about 17% lower (0.028 vs 0.034); smaller drop than tmixG_x (-35%). Largest per-image drops (candidate/reference ratio): 16 (0.54), 23 (0.55), 10 (0.56), 11 (0.58), 22 (0.58), 21 (0.61), 03 and 05 (0.62); higher than the reference in 14 (1.29), 19 (1.23), 07 (1.18, bokeh, not face detail), 12 (1.13). Visually: slightly smoother skin, cloth and metal, otherwise clean edges and clean text at large sizes. +- Per-image drifts: 21 more saturated (0.54 vs 0.44) and darker (0.18 vs 0.26 luminance); 13 darker (0.08 vs 0.13); 12 darker (0.25 vs 0.29); 23 more saturated (0.68 vs 0.61); 10 less saturated (0.29 vs 0.35); 22 and 18 brighter (0.77 vs 0.71, 0.77 vs 0.73). Nothing is burnt, washed out (except the pale 01 at 1024) or tinted. +- Composition: layout close to the reference in about 18 of 24 pairs; rearranged in 01, 05, 06, 10, 17 (mirrored), 20. +- Net: the same family of difference as turbo4 vs the 25-step teacher (softer detail, slightly lower contrast, no colour shift), smaller than for tmixG_x, plus three local artifacts (07, 10, 22) and weaker small-text handling. + +## 768 vs 1024 (ids 0, 1, 2, 3, 4, 5, 10, 14, 15; observations, not scored) + +- No haze or wash-out at 768. Text and scene content are different samples from the 1024 files. +- 00: sign "LE PETIT FROMAGE" exact, window "Affineur depuis 1974" readable but smudged, slate "Tasting / (garbled line) / Saturday / 11 AM"; cat tiny but clean. Equal to 1024. +- 01: "WATER CYCLE" and "Collection" exact, "Precipitation" readable (tail slightly smeared), "Evapartior" and "Condenation" minor typos; bold, saturated, waterfall, rain, partial arrows. Clearly better than the pale 1024 file (2 exact, 2 minor typos vs 0 exact) but still no closed cycle. +- 02: both strings exact (stray stroke inside THE); a ghost second lighthouse at the right edge; girl in yellow rowing. Equal to 1024 but with the ghost lighthouse. +- 03: "Neon Horizon", "Luna Vale", "3:48" exact; "1:24" has a smudged "/ 148" after it (minor typo); icons weak. Slightly worse than 1024. +- 04: 老王面馆 and the 11-character sub-line exact (small, slightly soft); equal to 1024. +- 05: "PEACHES $3" and "HONEY $8" exact, "CHERRIES $5" becomes "HERLE $5", plus garbled extra boards ("CIORRE $4", "FEATHES $4") and a doubled hat; worse than 1024. +- 10: about 10 key legends readable (E, F, R, F, O, D, J, N, A), paper text pseudo-English; sharper than 1024 but softer than the reference's numeral and QWERTY rows. Better than 1024. +- 14: exactly five balloons (red, orange, yellow, blue, striped purple) with distinct patterns; equal to 1024. +- 15: two apples, one yellow pear and one green pear = four fruits (reference five); worse than 1024. +- Net: text exact 2/6 at 768 (02, 04) vs 4/6 at 1024; 768 better for 01 and 10, worse for 03, 05, 15, equal elsewhere. No resolution cliff in image quality, but text is less reliable at 768 than at 1024 for this model. + +## Summary (10 lines) + +1. tmixG_tuned_1024 overall: F 4.23, T 3.93, C 3.79, Q 4.08, A 3.98 (turbo4: 4.23 / 4.71 / 4.14 / 4.31 / 4.21; tmixG_x: 4.08 / 3.79 / 3.86 / 4.23 / 4.06). +2. Side-by-side versus the turbo4 image: 1 better, 15 same, 8 worse out of 24; 17 of 24 have A >= 4. +3. Text prompts: 4/6 letter-exact (02, 03, 04, 05 including the Chinese sign and its 11-character sub-line; turbo4: 6/6); 00 has one garbled slate word and 01 fails all four labels. +4. Large titles and signs stay exact; the loss is in small text, so T is the one clearly lower average (-0.78). +5. No globally broken image; heavy local artifacts in 07 (smeared face), 22 (ghosted chairs) and 10 (nothing in focus). +6. Style: no colour or brightness shift, contrast about 9% lower, fine-detail energy about 17% lower than turbo4 (tmixG_x: 35% lower). +7. Strengths: scene and design prompts (02, 03, 04, 14, 16, 17, 20) as good as the reference; 06 follows the prompt better. +8. Weaknesses: 01 labels, 10 macro, 07 face, 22 ghost furniture, dark scenes (21, 23), dropped second-level details, fruit count (15). +9. Versus tmixG_x: equal within noise on all five averages (F +0.15, T +0.14, C -0.07, Q -0.15, A -0.08); 00 text improved, 07, 10, 22 got worse; 768 text is weaker than tmixG_x. +10. ANSWER: a user would accept the quantized image in 17 of 24 cases and for 5 of 6 text prompts, but would prefer the teacher image in 8 of 24. Caveat: this judge is a model instance; +-0.2 differences are within noise; one seed per prompt. diff --git a/eval/judge_tmixG_x.md b/eval/judge_tmixG_x.md new file mode 100644 index 0000000000000000000000000000000000000000..05b2599d3b61d7b293bc0adcf375e3c47d4ebaf6 --- /dev/null +++ b/eval/judge_tmixG_x.md @@ -0,0 +1,157 @@ +# Visual judge: tmixG_x (2.14 GB quantized student trained by block-wise QAT against the distilled teacher, 4 steps, CFG 1.0, euler) vs turbo4 (its own unquantized bf16 distilled teacher) + +Question: HOW GOOD IS THE QUANTIZED DISTILLED MODEL COMPARED WITH ITS OWN DISTILLED TEACHER, AND WOULD A USER ACCEPT THE QUANTIZED IMAGE INSTEAD OF THE TEACHER IMAGE FOR THE SAME PROMPT? + +Scales 0-5, same rubric as judge_turbo4.md / judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image (turbo4). "-" = not applicable. Half points used. Scored images: eval\tmixG_x_1024 (1024x1024, primary). + +Method and caveats: +- All 24 tmixG_x_1024 PNGs were opened alone at native 1024x1024. For all 24 prompts a turbo4_1024 | tmixG_x_1024 side-by-side (2056x1024; the viewer showed it at about 97%, so each half was slightly under native) was also opened. Text lines were read from the native single views plus 2-3x crops (00 sign, window line and slate; 04 sub-line; 15 left apples; 01 at 768). Scratch files were written to the OS temp folder and deleted afterwards; the only file written in the project is this report. +- 768 secondary check (ids 0, 1, 2, 3, 4, 5, 10, 14, 15): turbo4_768 | tmixG_x_768 side-by-sides at native 768 size. These were observed, not scored. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the six quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- The quantized model follows a different sampling trajectory from the reference (in 12 of 24 pairs the composition is clearly different, in the others it is close). A different layout alone was not penalised; missing prompt elements, wrong or garbled text, artifacts, colour/contrast drift and loss of detail were. "vs teacher" = would I rather have this image or the turbo4 image for the same prompt. +- A is "acceptable as a replacement for the teacher image", as in the turbo4 report; it is comparable in meaning to the earlier reports but the baseline differs (here the baseline is the distilled teacher, which itself scored A 4.21 against the 25-step teacher). +- Pixel statistics (Pillow/numpy, CPU only, all 24 images at 768) are an objective cross-check of the style description. +- This judge is a model instance; differences of +-0.2 on any average are within noise. One seed per prompt. + +## Per-image table (tmixG_x_1024) + +| id | category | F | T | C | Q | A | vs teacher | Note | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4 | 2.5 | 4 | 4.5 | 3.5 | worse | Complete storefront, cat on the step, cheese wheels in the warm window, wet cobbles, blue-hour street (as good as the reference). Text is wrong on all three requested items: sign reads "L LE PETTT FROMAGE" (stray leading L, PETIT -> PETTT); window line "Affineur" + garbled glyphs ("deomus / ies") + "1974" (depuis lost); slate "Tasting / Satting / Saturday / 11 AM" (garbled extra word where the dash should be). Reference has all three exact. | +| 01 | text | 2.5 | 1.5 | 2 | 3.5 | 2.5 | worse | "WATER CYCLE" exact. Labels "Evapoation", "Condenation", "Conipitation" (Precipitation) and no "Collection" at all: 0 of 4 exact. Sun, cloud, mountain with a waterfall and a sea are there but there is no cycle, one stray diagonal bar, and the picture is pale and low-contrast. Reference has all four labels exact and a closed arrow loop. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "Some lights never go out" and "BEYOND THE BREAKERS" exact (THE smaller), cream serif, sharp. Girl in yellow raincoat rowing, curled wave, lighthouse beam through warm clouds, distant town on the horizon, print grain. The lower hull edge of the boat dissolves into a halftone-like stipple. | +| 03 | text | 4.5 | 5 | 4.5 | 4.5 | 4.5 | same | "Neon Horizon", "Luna Vale", "1:24" and "3:48" at the two ends of the progress bar, all exact. Square purple-sunset cover, previous / play / next as round buttons, three tidy icons below, desk with blurred lamp and pen pot. Screen is dimmer and lower-contrast than the reference; no status bar clock. | +| 04 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "老王面馆" exact in golden characters on the red board; sub-line "手工拉面 · 营业至凌晨两点" reads all 11 characters in order (凌 slightly ambiguous at native size, as in the reference). Lanterns with pseudo-glyphs, cook in the lit kitchen, steam, diners at metal tables, wet street. Sign has less glow and the street is darker than the reference. | +| 05 | text | 4 | 4.5 | 3.5 | 4 | 4 | worse | "PEACHES $3", "CHERRIES $5", "HONEY $8" exact on dark green boards. But the signs are smaller and softer than the reference's huge crisp ones, an unrequested "HONEY $2" board sits at the right edge, the fruit is a mushy mix of oranges and apples (no cherries or distinct peaches), and the hat on the post is doubled (hat plus a second woven shape). Daisies, hills, awning, sun are fine. | +| 06 | portrait | 4 | - | - | 4.5 | 4.5 | better | Deeply lined face, shaved head with grey stubble, silver stubble on the chin, maroon robe with golden edge and visible weave, window light from the left, plain warm grey wall (reference has a sofa behind a younger, jollier monk). Beads hang on the chest, not the wrist; gaze goes to the camera. | +| 07 | portrait | 4 | - | - | 4 | 4 | same | Wavy auburn-brown hair, grey-green eyes, navy turtleneck, rain droplets over the whole frame and on the cheek, red lamp glow, teal/amber bokeh, parted lips. No window frame, no droplets on the lashes. Slightly softer face and less saturated hair than the reference, but without the reference's regular dotted bokeh grid. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | same | Messy dark hair, golden eyes, loose white shirt, dark green scarf, blue flower field, petals, clouds, mountains. Close to the reference; scarf has no fringe and a faint grey ribbon runs down the chest, the pose is neutral instead of hand-to-cheek. Clean line art, no stray arcs. | +| 09 | portrait | 3.5 | - | - | 3.5 | 3.5 | worse | Grey short hair, round glasses, arms crossed, smile, shelves of bowls, glowing kiln, big window. The whole picture is hazy and low-contrast, skin and cloth are smooth, clay crusts on forearms and apron are only faint speckles (reference: clear clay crust), kiln is less defined. | +| 10 | detail | 3.5 | 3 | - | 3.5 | 3.5 | worse | Chrome-rimmed keys, worn enamel, dust, typebars. Extremely shallow focus: only about 8 key legends are readable (H, X, X, T, G, L, LO, S) and the paper text is garbled ("WEL5 / VO / X"). No strike-through artifact, carriage lever not visible. Much softer than the reference's sharp key rows and typed lines. | +| 11 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Proper peacock eye (turquoise ring, deep blue centre, copper and gold arcs), green barbs, many well-formed droplets, gold bokeh at the upper left. Slightly softer barbs than the reference, warmer colour. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Mossy oak with lichen, ferns, droplets, a large snail plus a small one, two ladybugs (prompt: one), bracket-fungi cluster, misty bokeh. Almost the same composition as the reference, slightly darker and less saturated trunk. | +| 13 | detail | 4 | - | - | 4 | 4 | worse | Rivet-studded leather cuff, interlocking rings individually resolved, scratched wood, candle with wax, dark background. Mail forms a rounded mitt without fingers, leather has fewer scratches and wax stains than the reference, candle stands on a ring of beads instead of a holder, an odd leather bag sits behind. | +| 14 | counting | 4 | - | 5 | 4.5 | 4 | same | Exactly five balloons in a row: red, orange, yellow, pale yellow-teal with a blue cap and band, purple/white striped; baskets with tiny pilots, mist, village, sunrise. The fourth balloon reads yellow-green rather than blue (colour miss); patterns are plain. Clean, no checker artifacts (reference red/orange show blocky patches). | +| 15 | counting | 3.5 | - | 3 | 4.5 | 3.5 | same | Three clearly visible red apples plus a partly hidden fourth red fruit behind the left one, one yellow pear, one green pear = six fruits but the wrong mix (one apple too many, one yellow pear missing). Glossy skin, leaves, linen folds, grey wall with a light streak (no window). Reference also fails the mix (two apples, three pears). | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Colossal lantern tree on an island, about 30 lanterns, candle-lit path along the roots, one boat with a traveller, mountains, violet sky with stars and a crescent moon, long golden reflections. The glowing flowers in the water are replaced by dim pink flowers on the shore and the bridge is only a candle path. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart with Hangul signs, wet reflections, scooter in the distance, teal/magenta palette. Woman is in three-quarter view looking sideways. No taxi (blurred cars only). | +| 18 | varied | 4 | - | - | 4 | 4 | worse | Clay bakery with striped awning, bread in the window, chimney smoke, tree, cobbles, one delivery scooter, a tall croissant signboard on a pole. Paler and plainer than the reference: wall is peach/cream rather than pastel pink, slab is beige, no clouds, only one vehicle (prompt mentions bicycle and scooter), the signboard has a rabbit-ear shape. | +| 19 | varied | 4 | - | - | 4.5 | 4 | same | Brown bear in a rushing river, open mouth, salmon leaping beside the head with frozen droplets, mossy rocks, misty forest. Sharp fur and spray. The fish is next to, not in, the jaws, the bear looks at the camera and one paw is oddly raised; reference has the fish at the mouth and a more dramatic action. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Watercolor canal with striped poles (red/white), three gondolas and a gondolier in a striped shirt, arched palazzi with reflections, pigeons on the bridge and in the air, pennant lines, red flowers on balconies. No torn-paper edge and no pencil underdrawing; cleaner and less "bleeding" than the reference. | +| 21 | varied | 4 | - | - | 4 | 4 | worse | Black-armoured knight kneeling before a bright faceted blue crystal, stained-glass light shafts, embers, cathedral, a planted sword. Knight is almost a black silhouette with crushed blacks and little armour detail, helmet is on his head (none on the floor), no torn banners, no visible runes. Reference is much richer in armour, helmet, embers and banners. | +| 22 | varied | 5 | - | 5 | 4.5 | 4.5 | same | Bright Scandinavian kitchen: white cabinets, oak table, jug of pink tulips, exactly two cups with saucers and visible steam, crumbs and petals, pulled-back pale chair, plant on the shelf, big window, grey floor with sun streaks. Table top is an odd rounded wedge. | +| 23 | varied | 3.5 | - | - | 3.5 | 3.5 | worse | Hooded vendor frying dumplings, bulbs, steam, rain streaks, bowl of filling, tray of dumplings, customers' backs. Very dark and low-contrast: the vendor is murky and small, no visible coin in the reaching hand, no clear wrapper pile. Reference is far better lit with a legible face and food. | + +## Averages (tmixG_x_1024) + +Overall (24 images): F 4.08, T 3.79 (7 text-bearing images; 3.92 on the six quoted-text prompts alone), C 3.86 (7 images), Q 4.23, A 4.06. 18 of 24 images have A >= 4 (A < 4: 00, 01, 09, 10, 15, 23). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.00 | 3.92 | 3.50 (4 imgs) | 4.25 | 3.92 | +| portrait | 4 | 4.00 | - | - | 4.13 | 4.13 | +| detail | 4 | 4.13 | 3.0 (typewriter only) | - | 4.13 | 4.13 | +| counting | 2 | 3.75 | - | 4.00 (5 and 3) | 4.50 | 3.75 | +| varied | 8 | 4.25 | - | 5.0 (1 img) | 4.25 | 4.19 | + +## Comparison with the reference and with earlier quantized students + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| turbo4_1024 (distilled teacher, judge_turbo4.md) | 4.23 | 4.71 | 4.14 | 4.31 | 4.21 | +| tmixG_x_1024 (this report) | 4.08 | 3.79 | 3.86 | 4.23 | 4.06 | +| delta tmixG_x - turbo4 | -0.15 | -0.92 | -0.28 | -0.08 | -0.15 | +| mixG_tuned (quantized student of the 25-step teacher) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| mixH2_xm_f8 (1.72 GB, no retraining) | 4.25 | 4.00 | 3.79 | 4.21 | 4.15 | + +Reading the deltas: F, Q, A and C are inside or at the edge of the +-0.2 noise band (F -0.15, Q -0.08, A -0.15, C -0.28 on n=7). T is the one clearly lower number (-0.92): text exactness dropped from 6/6 to 4/6 and the two failures are the two prompts with the most small text (00 storefront, 01 infographic); large text (02, 03, 04, 05) is as good as the teacher. The turbo4 report's own scores were given against a different baseline (the 25-step teacher), so the A comparison with mixG_tuned / mixH2_xm_f8 is indicative only. + +## Verdict versus the turbo4 image (all 24) + +Better 1: 06. +Same 14: 02, 03, 04, 07, 08, 11, 12, 14, 15, 16, 17, 19, 20, 22. +Worse 9: 00, 01, 05, 09, 10, 13, 18, 21, 23. + +Reasons in brief: +- better: 06 plain grey wall, deeper lines and silver stubble where the reference shows a sofa and a younger monk. +- worse: 00 sign, window line and slate all misspelled (reference exact); 01 four labels wrong or missing and pale; 05 smaller softer signs, mushy fruit, extra "HONEY $2" board; 09 hazy, clay crust lost; 10 much softer, few legible keys; 13 less leather detail, odd bead-ring candle base; 18 paler, single vehicle; 21 near-black armour, no helmet on the floor; 23 very dark, murky vendor. +- same (with small leanings): 02, 03, 04, 16, 17, 20, 22 as good as the reference (some less rich: 03 dimmer screen, 04 less glow, 16 no glowing water flowers, 20 no torn edge); 07, 08, 11, 12 equal; 14 five balloons but the fourth is not blue; 15 both fail the exact fruit mix; 19 fish beside the head, calmer action. + +## Text-prompt acceptance and letter-by-letter reading + +- Six quoted-text prompts: 4/6 accepted (A >= 4: 02, 03, 04, 05); 00 at A 3.5 and 01 at A 2.5. +- Letter-exact: 4 of 6 (02, 03, 04, 05). Turbo4 had 6 of 6. +- 00 (1024), WRONG on all three: sign "L LE PETTT FROMAGE" (stray L, PETIT -> PETTT); window "Affineur" + garbled glyphs + "1974" (depuis lost); slate "Tasting / Satting / Saturday / 11 AM" (extra garbled word). At 768 the same prompt is exact (see below). +- 01 (1024), WRONG: "WATER CYCLE" exact; "Evapoation" (minor typo), "Condenation" (minor typo), "Conipitation" (wrong, Precipitation), "Collection" missing. +- 02 exact: "Some lights never go out", "BEYOND THE BREAKERS". +- 03 exact: "Neon Horizon", "Luna Vale", "1:24", "3:48" (at the two ends of the bar). +- 04 exact: "老王面馆" and "手工拉面 · 营业至凌晨两点" (11 characters in order; 凌 slightly ambiguous at native size, same as in the reference). +- 05 exact: "PEACHES $3", "CHERRIES $5", "HONEY $8"; one unrequested extra board "HONEY $2" at the right edge. +- Typewriter keys (10): about 8 legible serif legends, paper text garbled; no strike-through artifact. + +## Broken images + +None. No image is flat in colour, mushy or heavily artifacted. The weakest are 01 (pale, wrong labels, no cycle), 23 (very dark, murky), 10 (very soft), 09 (hazy); all are readable images with local defects. + +## Five worst images + +1. 01 water cycle (A 2.5, T 1.5, F 2.5, C 2): "Evapoation", "Condenation", "Conipitation", no "Collection"; pale, no loop. +2. 00 storefront (A 3.5, T 2.5): good scene but sign, window line and slate all misspelled; the reference is exact. +3. 23 street stall (A 3.5, Q 3.5): very dark, murky vendor, no coin. +4. 10 typewriter (A 3.5, T 3): very soft, few legible legends, garbled paper text. +5. 09 ceramicist (A 3.5, F 3.5): hazy, low-contrast, clay crust lost. +Close behind (A 3.5-4): 15 (wrong fruit mix), 05 (extra sign, mushy fruit), 13, 18, 21. + +## Systematic failure patterns + +- Dense small text: the two prompts with several small lines (00, 01) lose letters or words; the large titles and signs (02, 03, 04, 05) stay exact. In the three comparable prompts the reference is exact where this model is not. +- Soft micro-texture: skin, cloth, fur and metal in 09, 10, 13 are smoother and flatter than the reference (see statistics below); backgrounds are fine. +- Dark scenes: 21 and 23 lose detail in the blacks (knight silhouette, vendor). +- Prompt details dropped silently rather than distorted: wrist beads (06), droplets on lashes and window frame (07), clay crust (09), helmet on the floor and banners (21), coin (23), second vehicle (18), blue balloon (14), glowing water flowers (16). +- Exact counts: balloons exact (5, one colour off), kitchen cups exact (2); the 3 / 2 / 1 fruit mix fails again (six fruits, wrong mix), as in the reference. +- Duplicated or unrequested elements: extra "HONEY $2" board and doubled hat (05), two ladybugs (12). +- No colour shift, banding or blocking; no over-sharpened look. Local artifacts: stipple along the hull in 02. + +## Style differences versus the teacher + +Mean over all 24 images at 768 (tmixG_x_768 vs turbo4_768; also computed with the 1024 files downscaled to 768): + +| | luminance mean | luminance std (contrast) | mean saturation | fine-detail energy (Laplacian) | R-B warmth | pixels < 0.04 | +|---|---|---|---|---|---|---| +| turbo4 @768 | 0.388 | 0.193 | 0.450 | 0.034 | 0.062 | 9.9% | +| tmixG_x @768 | 0.376 | 0.175 | 0.449 | 0.022 | 0.060 | 9.9% | +| turbo4 @1024 -> 768 | 0.389 | 0.191 | 0.430 | 0.031 | 0.064 | 7.8% | +| tmixG_x @1024 -> 768 | 0.384 | 0.176 | 0.439 | 0.024 | 0.070 | 9.2% | + +- Contrast: about 9% lower (std 0.175 vs 0.193), lower in 23 of 24 images. Mean luminance 0.012 darker (darker in 16 of 24). Saturation and warmth are the same on average (0.449 vs 0.450; 0.060 vs 0.062): no global colour shift. +- Fine-detail energy: about 35% lower (0.022 vs 0.034), lower in 23 of 24 images; the largest drops are in 11 (0.034 vs 0.068), 12 (0.035 vs 0.065), 05 (0.019 vs 0.038), 00, 16, 17, 21. Visually this shows as smoother skin, fuzzier macro structures (10, 13) and softer fabric; large shapes, text and edges remain clean. +- Per-image drifts that are visible: 21 more saturated blue (0.59 vs 0.44) and darker; 12 less saturated (0.53 vs 0.65) and darker; 13 darker (0.08 vs 0.13 luminance) and more saturated; 18 less pink; 22 and 08 slightly brighter. Nothing is burnt, washed out or tinted. +- Composition: the quantized model keeps the reference layout in about half of the pairs (08, 11, 12, 13, 14, 15, 16, 18, 22 are close) and re-arranges the rest; no systematic pattern in which one looks better. +- Compared with the teacher-vs-student pattern in the earlier reports, this is the same family of difference as turbo4 vs the 25-step teacher (softer detail, slightly lower contrast, no colour shift), again somewhat larger. + +## 768 vs 1024 (ids 0, 1, 2, 3, 4, 5, 10, 14, 15; observations, not scored) + +- No haze or wash-out at 768. Text and scene content are different samples from the 1024 files. +- 00: sign "LE PETIT FROMAGE", window "Affineur depuis 1974" and slate "Tasting / Saturday / 11 AM" are all exact (dash dropped); cat tiny but clean; clearly better than the 1024 file. +- 01: "WATER CYCLE", "Evaporation", "Collection" exact; "Condenasion" and "Pecipitation" (garbled, overlapping glyphs) wrong; saturated and bold with a waterfall and arrows, cleaner and more complete than the pale 1024 file but still 2 of 4 labels wrong. +- 02: both strings exact; girl seen from behind, calmer lighthouse; equal to 1024. +- 03: all four strings exact, three clean buttons, dark screen; equal to 1024. +- 04: 老王面馆 and the 11-character sub-line exact; two small extra characters (速香) at the right edge of the sign; equal to 1024. +- 05: "PAN PACHES $3" wrong, "CHERRIES $5" and "HONEY $8" exact; doubled hat shape; worse than 1024. +- 10: soft, about 14 serif letters legible (T G O F R N H R S D K), paper text pseudo-English; the reference is crisper with a numeral row and QWERTY rows; similar to 1024. +- 14: exactly five balloons in the right colours (red, orange, yellow, blue, purple striped) with distinct patterns; better than 1024 on the blue balloon. +- 15: two apples, one yellow pear and one green pear = four fruits (reference five); worse than 1024. +- Net: text exact 4/6 at 768 too (00, 02, 03, 04) but the failures move (01 and 05 instead of 00 and 01); 768 is better in 00, 01, 14, worse in 05, 15, equal elsewhere. No resolution cliff. + +## Summary (10 lines) + +1. tmixG_x_1024 overall: F 4.08, T 3.79, C 3.86, Q 4.23, A 4.06 (turbo4: 4.23 / 4.71 / 4.14 / 4.31 / 4.21). +2. Side-by-side versus the turbo4 image: 1 better, 14 same, 9 worse out of 24. +3. Text prompts: 4/6 letter-exact (02, 03, 04, 05 including the Chinese sign and its 11-character sub-line; turbo4: 6/6); 00 (sign, window line and slate) and 01 (3 labels misspelled, 1 missing) fail at 1024. +4. Large titles and signs stay exact; the loss is in small text, so T is the one clearly lower average (-0.92). +5. No broken images; the weakest are 01 (pale, wrong labels), 23 (very dark), 10 (very soft), 09 (hazy). +6. Style: no colour shift, saturation and warmth identical; contrast about 9% lower and fine-detail energy about 35% lower than turbo4, in 23 of 24 images. +7. Strengths: scene and design prompts (02, 03, 04, 16, 17, 20, 22, 14) are as good as the reference; 06 follows the prompt better. +8. Weaknesses: dropped second-level details (beads, runes, coin, helmet, glowing water flowers), softer macro and skin detail (09, 10, 13), dark scenes (21, 23), wrong fruit mix (15). +9. 768 check: no haze; text exactness 4/6 again with different failures; 768 better for 00, 01, 14, worse for 05, 15. +10. ANSWER: a user would accept the quantized image in 18 of 24 cases (A >= 4) and for 4 of 6 text prompts, but would prefer the teacher image in 9 of 24, mostly because of small-text errors (00, 01) and lost detail in dark or fine-texture scenes. Caveat: this judge is a model instance; +-0.2 differences are within noise; one seed per prompt. diff --git a/eval/judge_tmixH_x_f8.md b/eval/judge_tmixH_x_f8.md new file mode 100644 index 0000000000000000000000000000000000000000..eeee359ff9be031b8dad6f18ac0a4682a3861d74 --- /dev/null +++ b/eval/judge_tmixH_x_f8.md @@ -0,0 +1,162 @@ +# Visual judge: tmixH_x_f8 (1.85 GB quantized student trained by block-wise QAT against the distilled teacher, more aggressive low-bit allocation than tmixG_x, 4 steps, CFG 1.0, euler) vs turbo4 (its own unquantized bf16 distilled teacher) + +Question: HOW GOOD IS THE QUANTIZED DISTILLED MODEL COMPARED WITH ITS OWN DISTILLED TEACHER, AND WOULD A USER ACCEPT THE QUANTIZED IMAGE INSTEAD OF THE TEACHER IMAGE FOR THE SAME PROMPT? + +Scales 0-5, same rubric as judge_turbo4.md / judge_tmixG_x.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image (turbo4). "-" = not applicable. Half points used. Scored images: eval\tmixH_x_f8_1024 (1024x1024, primary). + +Method and caveats: +- All 24 tmixH_x_f8_1024 PNGs were opened alone at native 1024x1024 and, for all 24 prompts, as a turbo4_1024 | tmixH_x_f8_1024 side-by-side (2056x1024, shown at about 97%, so each half was slightly under native). Text lines were read from the native views plus 2-3x crops (00 sign, window line and slate; 02 title; 03 title and time stamps; 04 sub-line). Scratch files were written to the OS temp folder and deleted afterwards; the only file written in the project is this report. I did not read judge_tmixG_x.md / judge_tmixG_tuned.md until every image below was scored (judge_tmixG_x.md was read afterwards for the comparison section only). +- 768 secondary check (ids 0, 1, 2, 3, 4, 5, 10, 14, 15): turbo4_768 | tmixH_x_f8_768 side-by-sides at native 768 size, with crops for the text lines. Observed, not scored. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the six quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- The quantized model follows a different sampling trajectory (about half of the pairs have a clearly different composition). A different layout alone was not penalised; missing prompt elements, wrong or garbled text, artifacts, colour/contrast drift and loss of detail were. "vs teacher" = would I rather have this image or the turbo4 image for the same prompt. +- A is "acceptable as a replacement for the teacher image", same baseline as the tmixG_x report (the distilled turbo4 image). +- Pixel statistics (Pillow/numpy, CPU only, all 24 images at 768) are an objective cross-check of the style description. +- This judge is a model instance; differences of +-0.2 on any average are within noise. One seed per prompt. + +## Per-image table (tmixH_x_f8_1024) + +| id | category | F | T | C | Q | A | vs teacher | Note | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4 | 2.5 | 4 | 4 | 3 | worse | Warm storefront with cat on the step, cheese wheels in the window, crates, slate, wet cobbles, blue-hour street. Text is wrong on all three items: sign reads "LE PET I T" on the first line and "FFOMMAGE" on the second (PETIT broken, FROMAGE -> FFOMMAGE, with a scratchy A); window "Affiineur depuis 1974" (extra i, "depuis" smeared: minor); slate "Tasting - / Satirdy / 11 AM." plus a faint ghost line "Bare 4" above (Saturday -> Satirdy: minor). Reference is exact on all three. | +| 01 | text | 2.5 | 1.5 | 2 | 3 | 2 | worse | "WATER CYCLE" exact. The four labels are garbled: "Evaporaton" (minor), "Waaporon", "Conpaation", "Conipittion"; Precipitation and Collection cannot be identified, labels sit around the picture at random. Sun, cloud, mountain with waterfall and sea are there, but no arrows at all (only two stray blue blobs), no cycle, and the picture is pale and washed-out. Reference has four exact labels and a loop. | +| 02 | text | 4 | 3.5 | - | 4 | 3.5 | worse | Tagline "Some lights never go out" exact. Title reads "BEYOND THE BIEAKERS" (R -> I: one-letter typo in the largest text of the poster). Girl in yellow raincoat in a wooden boat, curled wave, lighthouse with beam, print grain. No visible oars (she holds a handle), and the hull dissolves into a halftone stipple. Reference has oars and the exact title. | +| 03 | text | 4 | 3.5 | 3.5 | 4 | 3.5 | worse | "Neonn Horizon" (doubled n: minor typo), "Luna Vale" exact, "1:24 / 3:48" exact (small, centred under the bar). Upright phone with purple-sunset cover, play button with two malformed arrow buttons (no rewind/forward icons), three junk icons below, junk glyphs in the status bar, dim dark screen. Reference has a crisp bright UI with proper buttons. | +| 04 | text | 4 | 3 | - | 4 | 3.5 | worse | "老王面馆" exact in golden characters on the red board. Sub-line "手工[拉]面 · 营业至[凌]晨晨两点": 拉 and 凌 are garbled pseudo-characters and one extra 晨 is inserted (about 12 glyphs for 11 characters); the reference reads all 11 correctly. Lanterns, steam, diners at metal tables, wet street with colour reflections. Scene is darker, faces are silhouettes, sign glow weaker. | +| 05 | text | 3.5 | 4.5 | 3.5 | 4 | 3.5 | worse | "PEACHES $3", "CHERRIES $5" and "HONEY $8" all exact on dark green boards (large "CHERRIES $5" board in front and an unrequested duplicate "CHERRIES $5" board at the back right). Awning, one straw hat on the post, hills, daisies in the foreground. The fruit is a mushy mix of oranges and tomatoes: no recognisable cherries, peaches or grapes. Reference has huge crisp signs and clearly identifiable fruit. | +| 06 | portrait | 4 | - | - | 4 | 4 | same | Elderly shaved head, fine wrinkles, silver stubble, maroon robe with visible weave, window light from the left, plain warm grey wall (the reference has a sofa behind a younger, jollier man). Missing: golden robe edge, beads on the wrist (a bead string hangs at the neck), gaze goes to the camera. Skin a little smoother than the reference. | +| 07 | portrait | 3.5 | - | - | 3.5 | 3.5 | worse | Wavy brown hair (not curly auburn), brownish eyes (not green), navy turtleneck, red lamp glow, teal/amber bokeh, rain droplets over the whole glass, head turned to the side. The window feel is better than in the reference, but the hair is straight-ish and desaturated, the face is soft and the picture is murky. Reference has curly auburn hair, green eyes and clear freckles. | +| 08 | portrait | 4.5 | - | - | 4 | 4 | same | Messy black hair, golden eyes, loose white shirt, dark green scarf (no fringe), blue flower field, petals, clouds, mountains. Clean line art but flatter: neutral frontal pose (reference has hand-to-cheek), simpler eye highlights, a translucent white drape ghosts behind both shoulders. | +| 09 | portrait | 3.5 | - | - | 3.5 | 3.5 | worse | Grey short hair, round glasses, arms crossed, confident smile, shelves of bowls, glowing kiln, big window. Hazy, pale, low-contrast; skin and linen are smoothed, the clay crust on forearms and apron is only faint speckle, hands tucked away. Reference has visible clay crust and fine wrinkles. | +| 10 | detail | 3 | 2 | - | 3 | 2.5 | worse | Extremely tight crop of soft glass keys: nearly every legend is the same E or B, no readable QWERTY layout, no typebars, no carriage lever; the paper text is pseudo-letters ("TLE EE", "NO K U B"). Looks like a blurry keyboard, not the described typewriter. Reference has sharp key rows (Q W E R T Y, A S D F) and typed lines. | +| 11 | detail | 4.5 | - | - | 4 | 4 | same | Good peacock eye: blue ring, dark centre, copper/gold arcs, green barbs, many droplets, gold bokeh. Softer barbs, less violet and gold iridescence and lower contrast than the reference. | +| 12 | detail | 4 | - | - | 4 | 4 | worse | Mossy oak with ferns, a snail pair, bracket-fungi cluster, two or three ladybugs, misty bokeh. The moss and lichen are a repetitive tiled "leaf-patch" texture on a plain bark, much flatter than the reference's furrowed bark with real moss, lichen and mushrooms. | +| 13 | detail | 3.5 | - | - | 3.5 | 3.5 | worse | Dark leather piece with stitching dots (no rivets, no scratches, no wax stains), mail rings as a regular honeycomb of large circles, candle without wax drips and without a visible holder, scratched wood barely visible. Very dark and flat. Reference has rivets, wax, scratches and sparkling rings. | +| 14 | counting | 3.5 | - | 4.5 | 4.5 | 3.5 | worse | Exactly five balloons in a row, but the colours are red, red-orange, yellow, pale yellow, purple: the blue one is missing and a second yellow is added; no distinct patterns, pilots barely visible. Sunrise, mist, fields, trees are good. Reference has the right five colours with patterns and a village. | +| 15 | counting | 3 | - | 2.5 | 4 | 3 | same | Two red apples (the front one is large and lumpy), one yellow pear, one green pear = four fruits (prompt: three apples, two yellow pears, one green pear = six). Glossy, linen folds, grey wall with a light streak. Reference has five fruits (2 apples, 3 pears); both fail the exact mix, this one by more. | +| 16 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Colossal lantern tree on an island, lit walkway along the roots, boat with one traveller, mountains, violet sky with a few stars, long golden reflections. Canopy touches the top edge; glowing water flowers reduced to a few dim ones; fewer lanterns and no bridge arch. Still atmospheric. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart, Hangul neon signs, wet reflections, scooter and blurred cars in the distance, teal/magenta palette. Woman faces the camera, fewer signs and less crowd than the reference. | +| 18 | varied | 4 | - | - | 4 | 3.5 | worse | Pink clay bakery with striped awning, bread in the window, chimney smoke, tree, cobbles, one bicycle, croissant signboard (with junk glyphs under it). Paler and flatter than the reference, peach/cream walls, beige slab, no clouds, one vehicle only. | +| 19 | varied | 4 | - | - | 4 | 4 | same | Brown bear in a rushing river, open mouth, salmon at the jaw with frozen droplets, mossy rocks, misty forest. Fish is closer to a catch than in the reference, but the fur is smoother and not soaked, the bear smiles at the camera and one paw is oddly raised. | +| 20 | varied | 4 | - | - | 4 | 4 | worse | Watercolor canal with striped poles, gondolas, gondolier in a striped shirt, arched palazzi and reflections, a red-flower balcony, laundry lines, one tiny pigeon on the bridge. Hazier and paler than the reference, weaker pen-line detail, no torn-paper edge, no pencil underdrawing. | +| 21 | varied | 4 | - | - | 3.5 | 3.5 | worse | Knight kneeling (back view) before a bright faceted blue crystal, light shafts through the windows, embers, planted unbroken sword, a helmet on the floor, hood on his head. The knight is almost a black silhouette (crushed blacks, no armour detail), no banners, no runes. Reference is far richer in armour, banners and embers. | +| 22 | varied | 4.5 | - | 4 | 4 | 4 | worse | Bright Scandinavian kitchen: white cabinets, oak table, jug of peach tulips, two coffee cups on a cloth plus a third steaming cup, crumbs and petals, plant on the shelf, big window, grey floor. Local artifacts: the pulled-back chair on the right is semi-transparent and fades into the wall, and a checkered cloth floats under the tabletop without being attached to anything. Reference chairs and cups are solid and clearer. | +| 23 | varied | 4.5 | - | - | 3.5 | 3.5 | worse | Hooded vendor frying dumplings in a large pan, warm bulbs, thick steam, rain streaks, wet reflections, dumpling bowls, customers' backs, and a visible coin in the reaching hand (reference: no coin). Very dark, murky foreground and vendor, much less detail than the reference. | + +## Averages (tmixH_x_f8_1024) + +Overall (24 images): F 3.88, T 2.93 (7 text-bearing images; 3.08 on the six quoted-text prompts alone), C 3.43 (7 images), Q 3.88, A 3.58. 9 of 24 images have A >= 4 (06, 08, 11, 12, 16, 17, 19, 20, 22). + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 3.67 | 3.08 | 3.25 (4 imgs) | 3.83 | 3.17 | +| portrait | 4 | 3.88 | - | - | 3.75 | 3.75 | +| detail | 4 | 3.75 | 2.0 (typewriter only) | - | 3.63 | 3.50 | +| counting | 2 | 3.25 | - | 3.50 (4.5 and 2.5) | 4.25 | 3.25 | +| varied | 8 | 4.25 | - | 4.0 (1 img) | 4.00 | 3.94 | + +## Comparison with the teacher and with tmixG_x (2.14 GB sibling, judge_tmixG_x.md) + +| | F | T | C | Q | A | A >= 4 | letter-exact text | vs teacher (better / same / worse) | +|---|---|---|---|---|---|---|---|---| +| turbo4_1024 (distilled teacher, judge_turbo4.md, vs 25-step teacher) | 4.23 | 4.71 | 4.14 | 4.31 | 4.21 | - | 6/6 | - | +| tmixG_x_1024 (2.14 GB) | 4.08 | 3.79 | 3.86 | 4.23 | 4.06 | 18/24 | 4/6 | 1 / 14 / 9 | +| tmixH_x_f8_1024 (1.85 GB, this report) | 3.88 | 2.93 | 3.43 | 3.88 | 3.58 | 9/24 | 1/6 | 0 / 7 / 17 | +| delta H - G | -0.20 | -0.86 | -0.43 | -0.35 | -0.48 | -9 | -3 | | + +Per category A (tmixG_x -> tmixH_x_f8): text 3.92 -> 3.17, portrait 4.13 -> 3.75, detail 4.13 -> 3.50, counting 3.75 -> 3.25, varied 4.19 -> 3.94. + +- Reading the deltas: F (-0.20) is at the edge of the noise band; T (-0.86), C (-0.43), Q (-0.35) and A (-0.48) are outside it. No image has a higher A than in tmixG_x: A is equal in 5 prompts (09, 16, 17, 19, 23) and lower in the other 19. The weakest-scoring categories are text, detail and counting; varied scenes (16, 17, 19, 20, 22) are close to tmixG_x. +- Text strings that changed (tmixG_x -> tmixH_x_f8): 02 title "BEYOND THE BREAKERS" exact -> "BIEAKERS"; 03 "Neon Horizon" exact -> "Neonn Horizon"; 04 sub-line exact -> 2 garbled characters plus 1 extra 晨 (title 老王面馆 still exact). 05 stays exact (3/3). 00 and 01 fail in both: 00 sign is now broken as "LE PET I T / FFOMMAGE" (G: "L LE PETTT FROMAGE"), 01 labels are 1 minor + 3 wrong (G: 2 minor, 1 wrong, 1 missing). Typewriter legends: G about 8 readable letters, H a single repeated E/B. +- Image-level: tmixH_x_f8 is worse than tmixG_x in A on 19 of 24 prompts (largest drops of 1.0: 02, 03, 04, 10); it is better on none. + +## Verdict versus the turbo4 image (all 24) + +Better 0. +Same 7: 06, 08, 11, 15, 16, 17, 19. +Worse 17: 00, 01, 02, 03, 04, 05, 07, 09, 10, 12, 13, 14, 18, 20, 21, 22, 23. + +Reasons in brief: +- worse, text: 00, 01, 02, 03, 04 contain misspelled or garbled requested text where the reference is exact; 05 text is exact but the fruit and signs are less clear. +- worse, detail loss: 09 (hazy, clay crust lost), 10 (unreadable keys), 12 (tiled moss), 13 (flat, dark, no rivets or wax), 18 (pale), 20 (hazy), 21 (black silhouette), 23 (murky). +- worse, prompt elements: 07 (curly auburn hair and green eyes lost), 14 (blue balloon missing), 22 (ghost chair and floating cloth). +- same: 06, 08, 11, 16, 17, 19 are as good in different ways (06 plain wall but no wrist beads; 08 flatter pose; 11 softer barbs; 16 fewer lanterns; 17 simpler street; 19 fish closer to the jaw but plainer fur); 15 both fail the exact fruit mix. + +## Text-prompt acceptance and letter-by-letter reading + +- Six quoted-text prompts: 1/6 letter-exact (05). Accepted (A >= 4): none of the six (A 3, 2, 3.5, 3.5, 3.5, 3.5). The tmixG_x run had 4/6 exact and 4/6 accepted; turbo4 had 6/6 exact. +- 00 (1024), WRONG: sign "LE PET I T" / "FFOMMAGE" (wrong); window "Affiineur depuis 1974" (minor typo, extra i, depuis smeared); slate "Tasting - Satirdy 11 AM." (minor typo, Saturday -> Satirdy) with a faint extra ghost line above. +- 01 (1024), WRONG: "WATER CYCLE" exact; "Evaporaton" (minor typo), "Waaporon" (wrong), "Conpaation" (wrong), "Conipittion" (wrong); none of Precipitation / Condensation / Collection can be recognised as such. +- 02 (1024), title one-letter typo: "Some lights never go out" exact; "BEYOND THE BIEAKERS" (R -> I). +- 03 (1024), title minor typo: "Neonn Horizon" (doubled n); "Luna Vale" exact; "1:24 / 3:48" exact. +- 04 (1024), sub-line WRONG: "老王面馆" exact; "手工[拉]面 · 营业至[凌]晨晨两点": 拉 and 凌 garbled, extra 晨. +- 05 (1024), exact: "PEACHES $3", "CHERRIES $5", "HONEY $8", plus an unrequested duplicate "CHERRIES $5" board. +- Typewriter keys (10): legends are almost all the same E or B; paper text is pseudo-letters. + +## Broken images + +None. No image is flat in colour, mush, or unusable; the weakest (01, 10, 13, 23, 09) are readable images with large defects but intact colour and structure. + +Images with heavy local artifacts (non-text): 22 (semi-transparent chair fading into the wall and a checkered cloth floating under the table) and 01 (stray blue fragments, labels dissolving into overlapping glyphs, no arrows). Mild local artifacts: 02 (halftone stipple along the hull), 08 (translucent white drape behind both shoulders), 03 (malformed arrow buttons and junk icons), 12 and 13 (repetitive tiled texture), 18 (junk glyphs under the croissant). + +## Five worst images + +1. 01 water cycle (A 2, T 1.5, F 2.5, C 2): all four labels garbled, no arrows or cycle, pale. +2. 10 typewriter (A 2.5, T 2, F 3, Q 3): blurry E/B keys, pseudo text, no typewriter structure. +3. 00 storefront (A 3, T 2.5): good scene, but sign, window line and slate are all misspelled and the sign is visibly broken. +4. 15 fruit bowl (A 3, C 2.5, F 3): four fruits instead of six. +5. 04 / 02 / 03 / 05 / 07 / 09 / 13 / 14 / 18 / 21 / 23 (A 3.5): 04 garbled sub-line; 02 "BIEAKERS"; 03 "Neonn" and malformed buttons; 05 mushy fruit; 07 wavy brown hair instead of curly auburn; 09 hazy; 13 dark and flat; 14 blue balloon missing; 18 pale; 21 black silhouette; 23 murky. + +## Systematic failure patterns + +- Text: all of the small and medium text fails (00 sign, window and slate; 01 labels; 04 sub-line; 03 time stamps are exact but the title has a doubled n; 02 R becomes I). Only the large simple signs of 05 and the Chinese title in 04 stay exact. Compared with tmixG_x, even the large titles in 02 and 03 now slip. +- Detail loss in textured subjects: skin (09), typewriter (10), moss (12), mail and leather (13), fur (19), pen lines (20), armour (21) are smoother or replaced by a repeating pattern. +- Dark scenes lose detail in the blacks (21, 23, 13, 04). +- Prompt details dropped silently: blue balloon (14), curly auburn hair and green eyes (07), golden robe edge and wrist beads (06), clay crust (09), banners and runes (21), rivets and wax (13), pencil underdrawing and torn border (20). +- Exact counts: balloons exact in number but wrong colour set; fruit bowl fails more than the reference (4 vs 5, target 6); kitchen cups fine (2 plus 1 steaming). +- Ghost or translucent objects (22 chair and cloth, 08 drape). +- No colour shift, banding or blocking. + +## Style differences versus the teacher + +Mean over all 24 images at 768 (tmixH_x_f8_768 vs turbo4_768; the tmixG_x numbers are copied from its report): + +| | luminance mean | luminance std (contrast) | mean saturation | fine-detail energy (Laplacian) | pixels < 0.04 | pixels > 0.96 | +|---|---|---|---|---|---|---| +| turbo4 @768 | 0.388 | 0.193 | 0.450 | 0.034 | 9.9% | 3.8% | +| tmixH_x_f8 @768 | 0.366 | 0.171 | 0.446 | 0.017 | 12.1% | 3.7% | +| tmixG_x @768 (from judge_tmixG_x.md) | 0.376 | 0.175 | 0.449 | 0.022 | 9.9% | - | +| turbo4 @1024 (native) | 0.389 | 0.191 | 0.430 | 0.024 | 7.8% | 3.6% | +| tmixH_x_f8 @1024 (native) | 0.370 | 0.175 | 0.436 | 0.017 | 11.7% | 3.5% | + +- Fine-detail energy is half of turbo4's (0.017 vs 0.034 at 768, ratio 0.52, lower in 24 of 24 images; the biggest drops are 13 (0.35), 21 (0.36), 16 (0.37), 10, 12, 20, 22 (about 0.4)). This is visible as smoother skin, fur, moss and chain mail, and hazier pen-line watercolor. tmixG_x was at about 65% of the teacher. +- Contrast is about 11% lower (std 0.171 vs 0.193, lower in 21 of 24; higher only in 07, 09, 14), mean luminance 0.022 darker (darker by 0.01 or more in 17 of 24), and 12.1% of pixels are near-black vs 9.9%: dark scenes (21, 23, 13, 04, 12) lose their blacks' detail. +- Saturation is the same on average (0.446 vs 0.450); no global colour shift. Per-image drifts: 21 clearly more saturated blue (0.61 vs 0.44) and darker, 13 more saturated (0.89 vs 0.79), 04 more saturated (0.67 vs 0.59); 12 and 18 less saturated (0.55 vs 0.65, 0.35 vs 0.42). +- Composition: about half of the pairs keep the reference layout (11, 13, 15, 16, 18, 19, 20, 22 are close); the rest are re-arranged (00, 01, 02, 04, 05, 06, 07, 08, 10, 17, 21, 23). +- Nothing is burnt, over-sharpened or banded. The difference from tmixG_x is a larger version of the same pattern (softer detail, lower contrast), plus much more text corruption. + +## 768 check (ids 0, 1, 2, 3, 4, 5, 10, 14, 15; observations, not scored) + +- No haze or wash-out at 768. +- 00: sign "LE PETIT FROMAGE" exact (better than 1024), window "Afineur gepus 1974" (garbled), slate "Tassing - Saturday III AM" (minor typos); cat tiny. +- 01: "WATER CYCLE" exact; "Evapo?tion", "Condesation", "Pecipation" (overlapping glyphs) and "Collection" (readable, minor typo): 0 of 4 exact, but a bolder and more complete picture than the pale 1024 file (waterfall, partial arrows). +- 02: tagline "Some lights never go out" with merged glyphs in "lights" and "out" (minor); title "BEYOND THE BBEAKERS" (R -> B). Equal to 1024. +- 03: "Neon Horizon", "Luna Vale", "1:24" exact; "3:48" garbled ("3:88"); dim, simple UI. Slightly better than 1024 on the title, worse on 3:48. +- 04: 老王面馆 exact; sub-line has about 5 of 11 characters wrong. Equal to 1024 or worse. +- 05: "PEACHES $5" (price wrong), "HORRIES $5" and "CORRIS $8" (wrong), a faint "$ S" board: none of the three requested signs is correct. Clearly worse than 1024. +- 10: soft glass keys with squiggle legends and pseudo paper text; no QWERTY rows. Similar to 1024, far below the reference. +- 14: exactly five balloons in the right colours (red, red-orange, yellow, blue, purple) with plain patterns; better than 1024 (blue present). +- 15: two apples, one yellow pear, one green pear = four fruits; same as 1024 (reference five). +- Net: 0 of 6 quoted-text prompts are letter-exact at 768 (1 of 6 at 1024); the failures move around (05 breaks at 768, 00 sign is exact at 768 only). 768 is better for 00 (sign), 01 (layout), 14; worse for 05 and 03 (3:48); equal elsewhere. No resolution cliff, but the model does not render the text reliably at either size. + +## Summary (10 lines) + +1. tmixH_x_f8_1024 overall: F 3.88, T 2.93, C 3.43, Q 3.88, A 3.58 (turbo4: 4.23 / 4.71 / 4.14 / 4.31 / 4.21; tmixG_x: 4.08 / 3.79 / 3.86 / 4.23 / 4.06). +2. Side-by-side versus the turbo4 image: 0 better, 7 same, 17 worse out of 24. +3. Text prompts: 1/6 letter-exact (05 only; turbo4 6/6, tmixG_x 4/6); 02 and 03 have a one-letter typo in the title, 00, 01 and 04 have broken or garbled text. +4. No broken images; 2 images have heavy local artifacts (01 garbled labels and fragments, 22 ghost chair and floating cloth), plus several with mild ones. +5. A >= 4 in 9 of 24 images (tmixG_x: 18 of 24); the accepted ones are portraits 06 and 08, macro 11 and 12, and scenes 16, 17, 19, 20, 22. +6. Style: no colour shift, contrast about 11% lower and fine-detail energy about half of turbo4's, darker blacks, in nearly all 24 images. +7. Weaknesses: every text prompt except 05, typewriter (10), dark scenes (21, 23), flat textured subjects (09, 12, 13), counts (14 blue balloon missing, 15 four fruits). +8. Strengths: large, simple scene prompts (16, 17, 19, 22) and anime/portrait 06, 08 stay close to the teacher. +9. 768 check: no haze; 0/6 text-exact; better for 14 and the 00 sign, worse for 05. +10. ANSWER: a user would accept the quantized image in only 9 of 24 cases (A >= 4) and in none of the six text prompts; the model is clearly below tmixG_x (A 3.58 vs 4.06, text exact 1/6 vs 4/6) and below the teacher (17 of 24 worse). Caveat: this judge is a model instance; +-0.2 differences are within noise; one seed per prompt. diff --git a/eval/judge_turbo4.md b/eval/judge_turbo4.md new file mode 100644 index 0000000000000000000000000000000000000000..e99e376fdd1c6036ea20b3f5ac62d2e5ced7ee56 --- /dev/null +++ b/eval/judge_turbo4.md @@ -0,0 +1,142 @@ +# Visual judge: turbo4 (bf16 teacher + akhaliq MOSS-Image-SFT-4-steps LoRA, 4 steps, CFG 1.0, euler) vs the 25-step teacher_bf16 + +Question: IS THE DISTILLED 4-STEP MODEL ITSELF AS GOOD AS THE 25-STEP TEACHER (can it serve as the teacher for a quantized version)? + +Scales 0-5, same rubric as judge_mixG_tuned.md. F = subject/scene faithfulness, T = text correctness, C = counts/spatial relations, Q = image quality, A = would a user accept it instead of the teacher image. "-" = not applicable. Half points used. Scored images: eval\turbo4_1024 (1024x1024, primary). + +Method and caveats: +- All 24 turbo4_1024 PNGs were opened at native 1024x1024. For all 24 prompts a teacher (768) | turbo4_1024 side-by-side (1800x1024, both halves at native size) was also opened; scratch files were written to the OS temp folder and deleted afterwards. Text prompts 00-05 and 10 were additionally opened alone at full 1024 resolution (no extra zoom beyond native; the 11-character Chinese sub-line in 04 and the key legends in 10 were read at native size). +- For the 768 comparison (ids 0,1,2,3,4,5,10,14,15) turbo4_768 native was put next to turbo4_1024 downscaled to 768 (Lanczos), so differences are the model's, not the viewer's. +- Same scoring subsets as the previous reports: C on 7 images (00, 01, 03, 05, 14, 15, 22); T on the 6 quoted-text prompts plus the typewriter key legends (10). No 5 on Q or A. +- Because the seeds give different compositions under the distilled sampler, I did not penalise a different layout by itself; I penalised missing prompt elements, artifacts and loss of detail. "vs teacher" is therefore a judgement of "would I rather have this image or the teacher's for the same prompt". +- Scale note: the A column of the mixG_tuned report compared a near-identical composition with its teacher; here A is "acceptable as a replacement for the teacher image" for a different composition, so A is comparable in meaning but not strictly in baseline. +- Pixel statistics (Pillow/numpy, mean over all 24 at 768x768) are given as an objective cross-check of the style description; content differs between the two sets, so treat them as indicative only. +- This judge is a model instance; differences of +-0.2 on any average are within noise. + +## Per-image table (turbo4_1024) + +| id | category | F | T | C | Q | A | vs teacher | Note | +|---|---|---|---|---|---|---|---|---| +| 00 | text | 4.5 | 4.5 | 4 | 4.5 | 4.5 | better | Full storefront, nothing cropped. Sign "LE PETIT FROMAGE" exact (navy hand-lettering, plus an unrequested swash with an arrowhead under it); window "Affineur depuis 1974" exact; slate 1 "Tasting / - / Saturday / 11 AM" exact. Slate 2 on the right is an unrequested duplicate reading only "Tasting / - / Saturday" (11 AM missing). Cat on the step, cheese wheels in the lit window, wet cobbles, warm lamps, blue-hour street. Teacher crops the sign top; this is the more complete photograph. | +| 01 | text | 4 | 5 | 3.5 | 3.5 | 3.5 | worse | "WATER CYCLE", Evaporation, Condensation, Precipitation, Collection all exact; labels now sit beside the sun, cloud, mountain and sea and the arrows form a loop (better logic than the teacher's cluster). But the whole picture is very pale and low-contrast (mean saturation 0.09 vs 0.18), the sea is a glass sphere, a ghost translucent panel with stray drops/arrow sits at the right edge, "Precipitation" is far from the rain. Teacher is bolder and tidier. | +| 02 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "Some lights never go out" and "BEYOND THE BREAKERS" exact, cream serif, sharp. Girl in yellow raincoat rowing in profile, lighthouse with beam through clouds, white spray crowns the waves (rendered as halftone-like stipple), print grain. Sea is calmer and less dramatic than the teacher's big curled wave; oar tip leaves the frame at left. | +| 03 | text | 4.5 | 5 | 4.5 | 4.5 | 4.5 | better | "Neon Horizon", "Luna Vale", "1:24" and "3:48" at the two ends of the progress bar, all exact. Square purple-sunset cover, white play button with rewind/fast-forward either side, three icons below, glassy dark screen, upright phone with lamp, laptop and pens blurred on the desk. Clean and symmetrical; teacher had a malformed next button and junk icons. Status-bar clock "10:38" is extra but harmless. | +| 04 | text | 4.5 | 5 | - | 4.5 | 4.5 | same | "老王面馆" exact in glowing golden characters on the red board; sub-line "手工拉面 · 营业至凌晨两点" reads all 11 characters plus the dot in order, clean and larger than in the teacher. Lanterns (pseudo-glyphs on them), steam from the open kitchen, cooks, customers at metal tables, wet street. Frontal, calmer composition; pavement reflections are streaky orange/blue rather than the teacher's rich colour patches. | +| 05 | text | 4.5 | 5 | 4.5 | 4.5 | 4.5 | better | "PEACHES $3", "CHERRIES $5", "HONEY $8" exact, large white brush letters on dark green boards; one straw hat on the post (brim looks crushed), daisies in the foreground, rolling hills, blue sky, striped awning, crates piled with peaches, cherries, lemons, grapes. Shallow depth of field, very photographic. A second hat-like shape sits behind the crates at left. | +| 06 | portrait | 3.5 | - | - | 4 | 3.5 | worse | Maroon robe with golden edge, visible wool weave, silver stubble, beads on the wrist (correct, better than teacher), side window light. But he is a smiling ~60-year-old, not deeply lined, looks into the camera instead of past it, eyebrows are dark, and the backdrop is a room with a sofa, not a plain warm grey wall. Skin smooth but not waxy. | +| 07 | portrait | 3 | - | - | 3.5 | 3.5 | worse | Curly auburn hair, navy turtleneck, red lamp glow, a few droplets on cheeks, grey-green eyes, amber bokeh. Missing: any sign of a train window (no glass, frame or reflection), the teal half of the teal/amber grade, droplets on the lashes. The left half of the frame carries a regular dotted grid of bokeh/droplets that looks synthetic; hair and face are soft. Teacher has a convincing window and a seam through the face; this one is prettier but less on-prompt. | +| 08 | portrait | 4.5 | - | - | 4.5 | 4.5 | better | Messy dark hair (brown at the tips rather than black), golden eyes, loose billowing white shirt, dark green scarf with fringe flying, field of blue flowers, petals, big clouds, mountains. Clean line art, bright pastel background; hand-to-cheek smile instead of the neutral pose. Eye highlights simpler than the "layered glossy" request. No stray arcs (teacher family had them). | +| 09 | portrait | 3.5 | - | - | 4 | 3.5 | worse | Grey short hair, round glasses, arms crossed, smile, shelves of glazed bowls, glowing kiln, big window. Only sparse clay spots on apron and freckle-like marks on the forearms (prompt and teacher: clay crusts); the whole picture is bright, hazy and low-contrast with a stock-photo feel. Face skin natural and detailed enough. | +| 10 | detail | 3.5 | 3.5 | - | 3.5 | 3.5 | worse | Typewriter with chrome-rimmed keys, worn enamel, dust, typebars, shallow focus. About 15 key legends are readable (Q A S D F Z X T H Y E S T R F) but the layout is nonsense (repeated S/T/F) and half the keys are out of focus. Paper text is garbled ("He sard propigés fut fhew de niểu") with a strike-through line artifact across the right half. Carriage lever not clearly bent. Noticeably softer than the teacher's crisp key rows and typed lines. | +| 11 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Excellent eye: turquoise/blue ring, dark violet-blue centre, gold-brown halo, green barbs, many well-formed water droplets, dark bokeh with gold tint. Colour is cooler (less violet and gold iridescence) than the teacher, but sharper at the barbs and the droplets are better. | +| 12 | detail | 4.5 | - | - | 4.5 | 4.5 | same | Mossy oak with deep furrows, lichen, ferns, droplets, 2-3 snails, one ladybug, small mushrooms and one cluster of cream/orange bracket fungi, misty green forest bokeh. Moss is very dense and uniform; teacher's fungus cluster is more striking. | +| 13 | detail | 4 | - | - | 4.5 | 4.5 | same | Rivet-studded scratched leather cuff, stitching, wax drops, interlocking mail rings individually resolved (sharper than the teacher), candle on a holder, scratched wood, dark background. Mail forms a rounded mitt without finger segments and with only a single bright highlight (the teacher had sparkling rings and finger shapes). | +| 14 | counting | 5 | - | 5 | 4.5 | 4.5 | better | Exactly five balloons in a row: red, orange, yellow, blue-white striped, purple-white striped; baskets with tiny pilots, layered mist, sunrise at the right edge, village, long shadows. Larger and more photographic than the teacher's small balloons; patterns are plain stripes/checks. | +| 15 | counting | 3.5 | - | 3 | 4.5 | 3.5 | same | Two red apples, one green and two yellow pears = five fruits (prompt: three apples, two yellow pears, one green pear = six). Pears right, apples one short; the teacher has three apples and only two pears. Glossy, window light left, linen folds, grey wall, shadow to the right. Same exact-count failure class as the teacher. | +| 16 | varied | 5 | - | - | 4.5 | 4.5 | better | Colossal tree on an island, many lanterns, roots into the lake, a lit candle-lined walkway/bridge between roots (teacher: barely visible), one traveller in a boat, mountains, violet sky with stars, glowing lotus-like flowers, long golden reflections. More lanterns and a more dramatic reflection than the teacher. | +| 17 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Yellow trench coat, clear umbrella, steaming food cart with Hangul signage, neon signs, wet reflections, blurred taxi and scooter, teal/magenta/amber grade. Woman walks across the frame in profile (teacher faces the camera). More lens-like bokeh, slightly less neon (saturation 0.44 vs 0.52). | +| 18 | varied | 4.5 | - | - | 4.5 | 4.5 | better | Pink clay bakery with striped awning, bread in window, chimney smoke, fluffy tree, cobbled path, croissant sign, both a scooter and a delivery bicycle (prompt mentions both), fence with a bread table, clouds, pastel blue backdrop. Richer and more charming; ground block sits on a base rather than floating. | +| 19 | varied | 4 | - | - | 4.5 | 4 | same | Brown bear in a rushing river, open mouth, salmon leaping beside it with frozen droplets, mossy rocks, misty forest. Sharp fur and spray. Fish is next to the mouth rather than caught, bear looks up (not at the fish), top fur dry rather than soaked, one paw oddly raised. Teacher shows the fish at the jaws. | +| 20 | varied | 4.5 | - | - | 4.5 | 4.5 | same | Watercolor-and-ink canal: striped poles, two gondolas, gondoliers in striped shirts, arched palazzi and their reflections, pigeons on the stone bridge (teacher: in the air), laundry lines, balconies, torn-paper border, paper texture. Crisper and more finished than the teacher, but less "loose bleeding washes", no pencil underdrawing, flowers pink not red. | +| 21 | varied | 4 | - | - | 4.5 | 4.5 | same | Black-armoured knight clearly kneeling before a giant faceted crystal in a cathedral, stained-glass light shafts, embers, torn banners, helmet on the floor, sword planted beside him. Helmet on his head AND one on the floor (teacher same), sword unbroken, no visible glowing runes, crystal interior dark with glowing edge lines (teacher's crystal is brighter). Deep, dramatic depth. | +| 22 | varied | 5 | - | 4.5 | 4.5 | 4.5 | better | Bright Scandinavian kitchen: white cabinets, oak table, jug of pink tulips, two cups with saucers and steam, crumbs and petals, pulled-back pale chair, plant on the shelf, big window, grey floor, soft sun patches. Cups are far clearer than in the teacher. Table top is an odd rounded wedge; a blurred leaf in the foreground. | +| 23 | varied | 4 | - | - | 4 | 4 | same | Hooded vendor frying dumplings in a large pan, warm bulbs, thick steam, rain streaks under the awning, customers' backs, wet orange reflections, bowl of filling, tray of dumplings. No visible coin in the reaching hand, wrappers pile replaced by a dumpling tray, large near-black foreground area. Teacher has the coin and sharper faces. | + +## Averages (turbo4_1024) + +Overall (24 images): F 4.23, T 4.71 (7 text-bearing images; 4.92 on the six quoted-text prompts alone), C 4.14 (7 images), Q 4.31, A 4.21. + +| category | n | F | T | C | Q | A | +|---|---|---|---|---|---|---| +| text | 6 | 4.42 | 4.92 | 4.13 (4 imgs) | 4.33 | 4.33 | +| portrait | 4 | 3.63 | - | - | 4.00 | 3.75 | +| detail | 4 | 4.13 | 3.5 (typewriter only) | - | 4.25 | 4.25 | +| counting | 2 | 4.25 | - | 4.00 (5 and 3) | 4.50 | 4.00 | +| varied | 8 | 4.44 | - | 4.5 (1 img) | 4.44 | 4.38 | + +## Comparison with mixG_tuned (2.14 GB quantized student of the 25-step teacher) + +| | F | T | C | Q | A | +|---|---|---|---|---|---| +| mixG_tuned (previous report) | 4.48 | 4.79 | 3.64 | 4.23 | 4.21 | +| turbo4_1024 (this report) | 4.23 | 4.71 | 4.14 | 4.31 | 4.21 | +| delta turbo - mixG_tuned | -0.25 | -0.08 | +0.50 | +0.08 | 0.00 | + +Per category A (mixG_tuned -> turbo4_1024): text 4.08 -> 4.33, portrait 3.88 -> 3.75, detail 4.50 -> 4.25, counting 4.00 -> 4.00, varied 4.38 -> 4.38. + +Reading the deltas: A and Q are equal within noise. T is equal (all six quoted strings exact in both). F is lower (-0.25, borderline) because of three portraits and the typewriter, where the distilled model drops prompt details (age, plain backdrop, train window, clay crust) rather than producing artifacts. C is higher (+0.5), but 3 of the 7 C images are the same objects as before (balloons exact, apples/pears still not exact) and the rest come from different compositions, so I would not claim a real count advantage beyond "not worse". Caveat: mixG_tuned was judged against a near-identical teacher composition (defects are degradations), whereas turbo4 produces a different composition from the same seed (defects are differences). The numbers say the distilled model is on par with the best quantized student, i.e. its quality level is at or slightly below the teacher, not above. + +## Verdict versus the 25-step teacher image (all 24) + +Better 8: 00, 03, 05, 08, 14, 16, 18, 22. +Same 11: 02, 04, 11, 12, 13, 15, 17, 19, 20, 21, 23. +Worse 5: 01, 06, 07, 09, 10. + +Reasons in brief: +- better: 00 complete unclipped storefront; 03 clean player UI (teacher had a malformed next button, junk icons); 05 larger crisper signs, daisies, richer fruit; 08 clean anime, no stray arcs, richer background; 14 larger photographic balloons with visible pilots; 16 lit walkway and more lanterns; 18 richer clay scene with both scooter and bicycle; 22 clearly rendered cups and steam. +- worse: 01 pale, low-contrast, artifacts at the right edge (although the label layout is more logical); 06 younger, plain-backdrop and gaze requests missed; 07 no train window, dotted bokeh grid, soft hair; 09 clay crusts and contrast lost, hazy; 10 blurrier keys, garbled paper text with a strike-through artifact. +- same (with small leanings): 02, 04, 17, 20, 21 are as good but different in mood (calmer sea, frontal street, side-view woman, tighter watercolor, kneeling pose without runes); 11, 12, 13 equal (sharper rings/droplets but cooler iridescence, one fungus cluster, mitt without fingers); 15 both five fruits, one with apples short, the other pears short; 19 fish beside the mouth rather than caught; 23 no coin or wrappers pile, but good rain and steam. + +## Text-prompt acceptance + +- Six quoted-text prompts: 5/6 accepted (A >= 4: 00, 02, 03, 04, 05); 01 at A 3.5 because the picture is washed out and has artifacts, although its text and layout logic are right. +- All requested strings are letter-for-letter exact in 6/6 images: 00 (sign, window line, slate 1; the unrequested second slate is truncated to "Tasting - Saturday"), 01 (title and four labels), 02 (title and tagline), 03 (title, artist, "1:24" and "3:48"), 04 (老王面馆 and all 11 characters of 手工拉面 · 营业至凌晨两点), 05 (three signs). Same exactness as the teacher on every prompt; the distilled model does not hurt text rendering. +- Typewriter keys (10): letters mostly legible where in focus but a jumbled layout; paper text garbled with a strike-through artifact. Teacher is better here. + +## Five worst images + +1. 07 train-window portrait (A 3.5, F 3, Q 3.5): no window, dotted bokeh grid, soft smeared hair; the most off-prompt image. +2. 10 typewriter (A 3.5, F 3.5, T 3.5, Q 3.5): soft, garbled paper text with a strike-through artifact, repeated key letters; clearly below the teacher. +3. 01 water cycle (A 3.5, Q 3.5): right text and a better label layout, but washed out, glass-ball sea, ghost panel at the right edge. +4. 06 monk (A 3.5, F 3.5): younger and smiling at the camera with a sofa behind him; the prompt wanted an elderly monk looking past the lens on a plain grey wall. +5. 09 ceramicist (A 3.5, F 3.5): clay-crusted forearms and apron missing, hazy low-contrast light. +Close behind (A 3.5 or 4): 15 (five fruits instead of six), 19 (fish not caught), 23 (no coin). + +## Systematic failure patterns + +- Close-up human portraits (06, 07, 09): the model prefers an attractive, middle-aged, evenly lit look; prompt details that require grit (very aged skin, silver eyebrows, clay crust, a scratched train window, plain backdrop) are weakened or dropped. Softer fine detail (Laplacian energy about 40-50% of the teacher's on 06, 07, 09). +- Macro text/mechanics (10): lower fine-detail fidelity and garbled paper text; keys at the focus plane readable but the rest blurred. +- Infographic (01): at 1024 the output is pale, with a ghost panel and a glass-ball sea; layout logic is better than the teacher's, but the 768 file is cleaner (see below). +- Exact multi-type counts (15): 5 of 6 fruits again (this time apples are short); single-type count (14) exact. +- Prompt details lost: second-level details such as coin (23), runes (21), pencil underdrawing (20), wet fur (19), bent carriage lever (10), second yellow of the iridescence (11). These are dropped silently rather than distorted. +- Duplicated or unrequested elements: second truncated slate (00), second hat-like shape (05). +- No colour shift, banding or blocking; no over-sharpened or burnt DMD look. Local synthetic patterns: dotted bokeh grid (07), halftone stipple in spray (02). + +## Style differences versus the teacher + +Observed on all 24 pairs and checked with pixel statistics (mean over 24, teacher vs turbo4_1024 downscaled to 768; different content, so indicative only): + +| | luminance mean | luminance std (contrast) | mean saturation | fine-detail energy (Laplacian) | pixels < 0.04 | pixels > 0.96 | +|---|---|---|---|---|---|---| +| teacher 25 steps | 0.400 | 0.205 | 0.435 | 0.051 | 9.1% | 3.5% | +| turbo4 @1024 -> 768 | 0.389 | 0.191 | 0.428 | 0.031 | 8.0% | 3.6% | +| turbo4 @768 | 0.388 | 0.193 | 0.449 | 0.034 | 10.2% | 3.8% | + +- Contrast and saturation: no global shift. Contrast is about 7% lower on average (std 0.191 vs 0.205) and blacks are slightly less crushed; saturation is the same overall. It is image-specific: more saturated in 00 (0.51 vs 0.40), 02 (0.45 vs 0.32), 14, 16 and 22; less in 01 (0.09 vs 0.18), 10, 11, 17, 04. The sampler does not produce a hot or burnt look. +- Sharpness and micro-texture: this is the real difference. Fine-detail energy is about 60% of the teacher's (0.031 vs 0.051): skin pores, fur, wood grain, ring edges and stitching are smoother, with larger and softer bokeh; there is no over-sharpening, halos or crunchy contrast. Exceptions: 01, 11, 16, 20 are as crisp as the teacher, and 13's mail rings are crisper. +- Background and composition: wider, calmer, more centred and often frontal compositions (00, 04, 05, 16, 20, 22), well separated subject/background with strong depth of field and cleaner, more legible text. The teacher's compositions are tighter and more crowded (00 cropped, 02 big wave, 05 narrow frame). +- Realism: photographic "stock/commercial" look, evenly lit and idealised (06, 09, 22, 14); less gritty and less cinematic drama than the teacher (02, 07, 09, 21, 23). Portraits are younger and prettier; the teacher is rougher and truer to the prompt in 06, 07 and 09. +- Detail loss: small background objects are simpler and fewer (shelf contents, signs, crowds); fine text elsewhere (paper text in 10) turns to gibberish earlier than in the teacher. Colours are cleaner, without the teacher's gritty colour noise. +- Diversity: one seed per prompt, so loss of diversity cannot be measured; the turbo's compositions all differ from the teacher's, which is expected. + +## 768 vs 1024 (turbo, ids 0, 1, 2, 3, 4, 5, 10, 14, 15) + +- No haze or wash-out at 768; the model card warning is not visible. Over all 24, 768 is slightly more saturated (0.449 vs 0.428) and darker (10.2% near-black vs 8.0%) than the 1024 files downscaled, with the same sharpness (Laplacian 0.034 vs 0.031). +- Text is equally exact at both sizes on 00, 02, 03, 04, 05 (all quoted strings letter-exact at 768 too, including the Chinese sub-line in 04). +- 768 better: 01 (bold, saturated, a proper closed cycle with a waterfall and a cleaner sea; the 1024 version is pale and has a ghost panel) and 10 (clearly sharper and more legible key rows: numbers, Q W E R T Y, A D F C, X V C B; 1024 is blurrier with sparse keys and the strike-through artifact). +- 1024 better: 05 (768 adds a small garbled extra sign "PEACHES SUNTINS $4" at the back left); 00 equal for text but 768 has a single slate (no truncated duplicate) while 1024 has a richer scene. +- Same: 02 (1024 richer sea/lighthouse), 03 (768 darker), 04, 14 (1024 larger balloons, visible pilots), 15 (both five fruits, two apples, three pears). +- Net: 1024 is not uniformly better; the 768 file is a usable alternative and in two of nine prompts clearly the better one, so a student can be trained/evaluated at 768 without expecting a quality cliff. No scoring was done for the other 15 ids at 768. + +## Summary (10 lines) + +1. turbo4_1024 overall: F 4.23, T 4.71, C 4.14, Q 4.31, A 4.21 (mixG_tuned: 4.48 / 4.79 / 3.64 / 4.23 / 4.21). +2. Side-by-side versus the 25-step teacher: 8 better, 11 same, 5 worse out of 24. +3. Text prompts: 5/6 acceptable (A >= 4), 6/6 letter-exact including the Chinese sign and its sub-line; the weak one is 01 (washed-out, artifacts). +4. Style: no burnt or over-sharpened DMD look and no global colour shift; contrast about 7% lower, fine micro-texture about 40% lower, smoother skin/fur/wood, wider calmer centred composition, cleaner photographic realism. +5. Weaknesses: close-up portraits (06, 07, 09) lose age, grit and prompt details; the typewriter macro (10) is blurry with garbled paper text; exact multi-type counts (15) still fail. +6. Strengths: scene-level prompts (00, 03, 05, 14, 16, 18, 22) are as good or better than the teacher, with clearer text and fuller scenes. +7. 768 vs 1024: no haze at 768; 768 is better in 01 and 10, 1024 in 05; text exact in both. +8. Worst five: 07, 10, 01, 06, 09 (all A 3.5). +9. ANSWER: yes, the distilled 4-step model is essentially as good as the 25-step teacher on average (A 4.21, 19 of 24 images same or better), at the cost of a softer fine-detail profile and weaker close-up portraits and macro text; it is on par with the best quantized student, not above it, and its flaws (portrait grit, exact counts, macro) would be inherited by a student distilled from it. +10. Caveat: this judge is a model instance and +-0.2 on any average is noise; one seed per prompt, so diversity and variance are not assessed; turbo and teacher compositions differ by design. diff --git a/eval/sheet_ablation.png b/eval/sheet_ablation.png new file mode 100644 index 0000000000000000000000000000000000000000..821cfbeb46b34c9c40969ef0599817d6bfaa1c85 --- /dev/null +++ b/eval/sheet_ablation.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1cff9833da53f619a102c01a4eb273e39e84cdcf9939f450c0acddb41db6fca +size 5107491 diff --git a/eval/sheet_ablation_partial.png b/eval/sheet_ablation_partial.png new file mode 100644 index 0000000000000000000000000000000000000000..58bdd90503a07fca96a41a5c3c9c150335dab295 --- /dev/null +++ b/eval/sheet_ablation_partial.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:53fd029897c5b5a6a482aaecd5d43f9d6b02fca135cd798f857d1cdd89fce2ac +size 4110092 diff --git a/eval/sheet_ffn_int3.png b/eval/sheet_ffn_int3.png new file mode 100644 index 0000000000000000000000000000000000000000..1f70cd50901f657c2acc526b696ed355134f8c96 --- /dev/null +++ b/eval/sheet_ffn_int3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6ed77a8445acfa9477700178dac21390fad0e8d6476b6aa0ca8d59c2615317ed +size 3627353 diff --git a/eval/sheet_ffn_study.png b/eval/sheet_ffn_study.png new file mode 100644 index 0000000000000000000000000000000000000000..90a6982bbd5e21e46e6da1e35b64434407264f79 --- /dev/null +++ b/eval/sheet_ffn_study.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:cb990fdde4f25ebd44f7dae5765960eea21e08d5001f4942d0c38c3c2caf0f94 +size 3915484 diff --git a/eval/sheet_mixA_lora32.png b/eval/sheet_mixA_lora32.png new file mode 100644 index 0000000000000000000000000000000000000000..cf307164a4d8a644866a30feb62541297e0809c9 --- /dev/null +++ b/eval/sheet_mixA_lora32.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2bd523210d569bfaeed54371c9545a6c4224caf4252fff26ccfbb03061619f +size 1342368 diff --git a/eval/sheet_mixA_lora32_1.png b/eval/sheet_mixA_lora32_1.png new file mode 100644 index 0000000000000000000000000000000000000000..cf307164a4d8a644866a30feb62541297e0809c9 --- /dev/null +++ b/eval/sheet_mixA_lora32_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1c2bd523210d569bfaeed54371c9545a6c4224caf4252fff26ccfbb03061619f +size 1342368 diff --git a/eval/sheet_mixA_lora32_2.png b/eval/sheet_mixA_lora32_2.png new file mode 100644 index 0000000000000000000000000000000000000000..d049f697794644cd9e44d3d0f1335569293f0727 --- /dev/null +++ b/eval/sheet_mixA_lora32_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dac752cd629b7de5f78a224788d74ff55ea7a65f514a7b4f957a35271fab34a6 +size 1460082 diff --git a/eval/sheet_mixA_lora32_3.png b/eval/sheet_mixA_lora32_3.png new file mode 100644 index 0000000000000000000000000000000000000000..b97cd0426909d01856a511e03b3d2e77f3d456eb --- /dev/null +++ b/eval/sheet_mixA_lora32_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:eb85d3307f8417b87638a207ab7769bd6f2f6695474e2042a00a361a1543eb78 +size 1433638 diff --git a/eval/sheet_mixA_lora32_4.png b/eval/sheet_mixA_lora32_4.png new file mode 100644 index 0000000000000000000000000000000000000000..b74019d197f8d929dda02d00e3c1823c52e4dd7b --- /dev/null +++ b/eval/sheet_mixA_lora32_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b86ef9e96f1852243a8ab6ef73d6c35c872fc526dfcae62d1f0583c314eafea9 +size 1393420 diff --git a/eval/sheet_mixA_lora32_std_x_1.png b/eval/sheet_mixA_lora32_std_x_1.png new file mode 100644 index 0000000000000000000000000000000000000000..62de83d989a061d1ab29f75cc04c7174c10caf89 --- /dev/null +++ b/eval/sheet_mixA_lora32_std_x_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d64c8e9d9d6732e575a23006239015f3e809804ba4bbb3c40fd6943f33f48ac7 +size 1511209 diff --git a/eval/sheet_mixA_lora32_std_x_2.png b/eval/sheet_mixA_lora32_std_x_2.png new file mode 100644 index 0000000000000000000000000000000000000000..c8b14733dfca02c6f6e6de5b99e09fba4b2e31ac --- /dev/null +++ b/eval/sheet_mixA_lora32_std_x_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8bc97ad96a86ccb61f49165125d4750bf2ce5178f5a876b24c52ea7db97cf796 +size 1502842 diff --git a/eval/sheet_mixA_lora32_std_x_3.png b/eval/sheet_mixA_lora32_std_x_3.png new file mode 100644 index 0000000000000000000000000000000000000000..24f5a2c7b3aa4f96beda2e3880db6ae1302f6bf8 --- /dev/null +++ b/eval/sheet_mixA_lora32_std_x_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:33818b7e31224d6146dde257b55f8b163236f66c2698984f5929f73304e9e893 +size 1466318 diff --git a/eval/sheet_mixA_lora32_std_x_4.png b/eval/sheet_mixA_lora32_std_x_4.png new file mode 100644 index 0000000000000000000000000000000000000000..9ff1a476a8c7b9e895e3ecdd4da9b345e2be56d3 --- /dev/null +++ b/eval/sheet_mixA_lora32_std_x_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:86cac30aecc23dd48a2ae0d903ca5e3c2d1e22f13f9b2837cbdd150f467fcc77 +size 1513280 diff --git a/eval/sheet_mixA_x_tuned2_1.png b/eval/sheet_mixA_x_tuned2_1.png new file mode 100644 index 0000000000000000000000000000000000000000..bf32ef0dc42dbcfc95f989b13782dedfa06a071f --- /dev/null +++ b/eval/sheet_mixA_x_tuned2_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:88770c001558e862ecdaab273efc7c59d69450ad1efe42f68376b12bf0fe0191 +size 1384478 diff --git a/eval/sheet_mixA_x_tuned2_2.png b/eval/sheet_mixA_x_tuned2_2.png new file mode 100644 index 0000000000000000000000000000000000000000..6c61853999da73202303549c833b1e410b66d56b --- /dev/null +++ b/eval/sheet_mixA_x_tuned2_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:458409c0917f33ee8bca1e55c1f12208c23053c85287d3eb0c1ec04702d9f945 +size 1490553 diff --git a/eval/sheet_mixA_x_tuned2_3.png b/eval/sheet_mixA_x_tuned2_3.png new file mode 100644 index 0000000000000000000000000000000000000000..29109f9d0643a3d301762751cc40cd755dbceee8 --- /dev/null +++ b/eval/sheet_mixA_x_tuned2_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0f95140dcad7bdebeaacb0fcda82cf13027740acaa6b5be27125147a52dd27a3 +size 1454783 diff --git a/eval/sheet_mixA_x_tuned2_4.png b/eval/sheet_mixA_x_tuned2_4.png new file mode 100644 index 0000000000000000000000000000000000000000..907c01b5c1cb9ed2a4be2b5d7e98ce0f9c8aca38 --- /dev/null +++ b/eval/sheet_mixA_x_tuned2_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:6f183981271a9b8a19a97f7589a53a31162841040658f352404df03869348509 +size 1431075 diff --git a/eval/sheet_mixA_x_tuned_1.png b/eval/sheet_mixA_x_tuned_1.png new file mode 100644 index 0000000000000000000000000000000000000000..039499e4a93e6e2dfb1be17b1370bd233455464d --- /dev/null +++ b/eval/sheet_mixA_x_tuned_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:081d080d4c69128d82b828a661300e27c4e8482cc36239230e4817ec45682c66 +size 2031192 diff --git a/eval/sheet_mixA_x_tuned_2.png b/eval/sheet_mixA_x_tuned_2.png new file mode 100644 index 0000000000000000000000000000000000000000..fb24d76f18fb88d4dfef3e18a8028634c2322d80 --- /dev/null +++ b/eval/sheet_mixA_x_tuned_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:beb966e88ad7c7d24435b8c57197cda38c1bba93d73228c8c98f680080f335fd +size 2178371 diff --git a/eval/sheet_mixA_x_tuned_3.png b/eval/sheet_mixA_x_tuned_3.png new file mode 100644 index 0000000000000000000000000000000000000000..62c883424b3905efbe18350dd8e77f3df0ac6615 --- /dev/null +++ b/eval/sheet_mixA_x_tuned_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3fc42e296fe256cc3795487d17ae172115a2046b383cca84902ae00c513d554e +size 2123705 diff --git a/eval/sheet_mixA_x_tuned_4.png b/eval/sheet_mixA_x_tuned_4.png new file mode 100644 index 0000000000000000000000000000000000000000..7eba15d6f024bf74013d714ecb95a0b43614ce70 --- /dev/null +++ b/eval/sheet_mixA_x_tuned_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c830192374eb07d095d9bc9dff9ac77c44a297a884941911524d44b052ea420 +size 2080793 diff --git a/eval/sheet_mixC_lora32_std_1.png b/eval/sheet_mixC_lora32_std_1.png new file mode 100644 index 0000000000000000000000000000000000000000..7ec318597a5a8350113b133b86dae9c364b3cfb9 --- /dev/null +++ b/eval/sheet_mixC_lora32_std_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5288581be14ec239b7ecdaf710fd533f221057ba4e06aa54459a32fcdca53c96 +size 1995915 diff --git a/eval/sheet_mixC_lora32_std_2.png b/eval/sheet_mixC_lora32_std_2.png new file mode 100644 index 0000000000000000000000000000000000000000..a07add91c642b8c8457f483ea636492843d53c0d --- /dev/null +++ b/eval/sheet_mixC_lora32_std_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:507c5132fef3647a1f1fdcc4cd1c33161ed8b9e0c0311d2c51fd23c625858d12 +size 2199400 diff --git a/eval/sheet_mixC_lora32_std_3.png b/eval/sheet_mixC_lora32_std_3.png new file mode 100644 index 0000000000000000000000000000000000000000..57c7b41a3a9cd4b403768edcfacfbffce819a52d --- /dev/null +++ b/eval/sheet_mixC_lora32_std_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0a0d9ac812fd3055f76851ff086474f07725eb52157690dec732effe1d771a9f +size 1374755 diff --git a/eval/sheet_mixF_1.png b/eval/sheet_mixF_1.png new file mode 100644 index 0000000000000000000000000000000000000000..25adecef3d84cb44414c3968a01e83f42fd15663 --- /dev/null +++ b/eval/sheet_mixF_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1708d4b6dc5ad31f279977070bdc86257d9c5b8ce7811f41ccc748cb7cd0d0d1 +size 1985091 diff --git a/eval/sheet_mixF_2.png b/eval/sheet_mixF_2.png new file mode 100644 index 0000000000000000000000000000000000000000..a62f28f6582c8684e76d5e920f9c2ff6be0a9057 --- /dev/null +++ b/eval/sheet_mixF_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7c014d8b54063ab1d048077f1cff3d4093a326454a03fff4f98723cbb8b9818 +size 2181906 diff --git a/eval/sheet_mixF_3.png b/eval/sheet_mixF_3.png new file mode 100644 index 0000000000000000000000000000000000000000..0bf7a78ca4b971a6ab9188081e27a304925aac68 --- /dev/null +++ b/eval/sheet_mixF_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5489d02986ce5b8ec790d47397dfd4f74622b8ccce7de626e6ade84febfb8ad6 +size 2084125 diff --git a/eval/sheet_mixF_4.png b/eval/sheet_mixF_4.png new file mode 100644 index 0000000000000000000000000000000000000000..653727ce6aa4183eeb92e8d8d10effcf7f89e5c4 --- /dev/null +++ b/eval/sheet_mixF_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:573f0377b481be8218e20fe190b291ea72885fcb826040dca06c87f8f9baf103 +size 2069303 diff --git a/eval/sheet_mixG_1.png b/eval/sheet_mixG_1.png new file mode 100644 index 0000000000000000000000000000000000000000..69473b4c9d277b5a558264b871d4f8919e5f660d --- /dev/null +++ b/eval/sheet_mixG_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d496ed2c7291bfaf254a9ba94fcfbd640d088191dd8d31f240241b358402bc9e +size 2616172 diff --git a/eval/sheet_mixG_2.png b/eval/sheet_mixG_2.png new file mode 100644 index 0000000000000000000000000000000000000000..26930b0adf7c3c9f03c4bf6acc841ffbb3551b50 --- /dev/null +++ b/eval/sheet_mixG_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8a275ff673faa918ae14261449e10a8ffc1e22f6f2ae4e0d83baf59b2c13c6c2 +size 2869844 diff --git a/eval/sheet_mixG_3.png b/eval/sheet_mixG_3.png new file mode 100644 index 0000000000000000000000000000000000000000..e5537b9927c77b15d950749a27683d2667e2eee0 --- /dev/null +++ b/eval/sheet_mixG_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2cca5c4a3be598469c1f581e39819b203ac92e1960dadd3da926958988bcd424 +size 2784251 diff --git a/eval/sheet_mixG_4.png b/eval/sheet_mixG_4.png new file mode 100644 index 0000000000000000000000000000000000000000..99032ca6be052cd5197037f06f2de23dc8190993 --- /dev/null +++ b/eval/sheet_mixG_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:02e513fb26bd13432cc497af329e442618216279107f05c1ddb73363b6a67ff5 +size 2753138 diff --git a/eval/sheet_mixH2_1.png b/eval/sheet_mixH2_1.png new file mode 100644 index 0000000000000000000000000000000000000000..10dd08eafc3596a898efb9be90c38c575369235f --- /dev/null +++ b/eval/sheet_mixH2_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2e9b940bab559f4a9a2aa0b82032f96a29cbe4e90d5b822e2de42a5fc787fa40 +size 1983008 diff --git a/eval/sheet_mixH2_2.png b/eval/sheet_mixH2_2.png new file mode 100644 index 0000000000000000000000000000000000000000..51b6b0e864ed421957e67427277a0083a28b675f --- /dev/null +++ b/eval/sheet_mixH2_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9c3889955b71e4832857eedba6121eba51911f6f28ac4a49a986189b041baf36 +size 2138589 diff --git a/eval/sheet_mixH2_3.png b/eval/sheet_mixH2_3.png new file mode 100644 index 0000000000000000000000000000000000000000..9a2d16d99f4fd027b774a04c010a24b5fb4a737a --- /dev/null +++ b/eval/sheet_mixH2_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ab51ca621e1b2f4e2f4c8227db3fe8ef8a117e612c2805ce93b94df7b9f9102f +size 2145813 diff --git a/eval/sheet_mixH2_4.png b/eval/sheet_mixH2_4.png new file mode 100644 index 0000000000000000000000000000000000000000..142b3514b9cd00374eeb4ec9473ca31cdbfdcf0c --- /dev/null +++ b/eval/sheet_mixH2_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e618c689c7f933772ef146aa1df78acc43cf99c87bc618c69b13afb4e9649409 +size 2079795 diff --git a/eval/sheet_mixH2_tuned_1.png b/eval/sheet_mixH2_tuned_1.png new file mode 100644 index 0000000000000000000000000000000000000000..33887877c6d523d1fa7947aeefb7bd4d8addce01 --- /dev/null +++ b/eval/sheet_mixH2_tuned_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e339e86f807a230469665ce0cda9de99dacf25b21dbf157adfecf4faf39922c +size 1986400 diff --git a/eval/sheet_mixH2_tuned_2.png b/eval/sheet_mixH2_tuned_2.png new file mode 100644 index 0000000000000000000000000000000000000000..034d97759b8313cf902aa452ff4cd55639c3e6be --- /dev/null +++ b/eval/sheet_mixH2_tuned_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dd1b3f96b5514a0ca255c524600831c6c457ba23cdc6d13b26a7287931223c54 +size 2138287 diff --git a/eval/sheet_mixH2_tuned_3.png b/eval/sheet_mixH2_tuned_3.png new file mode 100644 index 0000000000000000000000000000000000000000..627649ad2bea72d019316c5b67c7956d8e8142bb --- /dev/null +++ b/eval/sheet_mixH2_tuned_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d35e2f3010de8ccb01f041be8452d3954416c47b8c33e3335840980642a71ad0 +size 2149071 diff --git a/eval/sheet_mixH2_tuned_4.png b/eval/sheet_mixH2_tuned_4.png new file mode 100644 index 0000000000000000000000000000000000000000..d30093951ba7bfe1e1de12c8d70bed1bcb50f7cd --- /dev/null +++ b/eval/sheet_mixH2_tuned_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a0a96ccb76e75ab5b386626944c7717b35a1964b753b8fe2c5794b9f3af4a7a1 +size 2095006 diff --git a/eval/sheet_mixH2_xm_1.png b/eval/sheet_mixH2_xm_1.png new file mode 100644 index 0000000000000000000000000000000000000000..94c770c43f80f6b16b0ff15c103c293be8a1db50 --- /dev/null +++ b/eval/sheet_mixH2_xm_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:303a191244deaf4022f4563d72bfc6096216caab84d6f688f676fadc202853c7 +size 1929986 diff --git a/eval/sheet_mixH2_xm_2.png b/eval/sheet_mixH2_xm_2.png new file mode 100644 index 0000000000000000000000000000000000000000..db25a0837c68e4fb16ff677b684e2ec4cbd6e822 --- /dev/null +++ b/eval/sheet_mixH2_xm_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d951ab4264b14a7ea4132c6ad30bc01a7c774a7769d4e590bdd1fd2b37ac1830 +size 2073616 diff --git a/eval/sheet_mixH2_xm_3.png b/eval/sheet_mixH2_xm_3.png new file mode 100644 index 0000000000000000000000000000000000000000..07729cfafc0c621eb329b2b83bfe40ae46e67529 --- /dev/null +++ b/eval/sheet_mixH2_xm_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8b918ba63fbb2a8df18422496e2c3d5583bcca82af1e9e36787fbc443c310ce4 +size 2099121 diff --git a/eval/sheet_mixH2_xm_4.png b/eval/sheet_mixH2_xm_4.png new file mode 100644 index 0000000000000000000000000000000000000000..2297d44a1a7fa3f002708a58795ee6fb70ef1649 --- /dev/null +++ b/eval/sheet_mixH2_xm_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:de9ff4cdfb934baf7e224cbd49863e0c387dc40bdedee7903a9416d8c7a9d0ce +size 2030109 diff --git a/eval/sheet_mixH2_xm_f8_1.png b/eval/sheet_mixH2_xm_f8_1.png new file mode 100644 index 0000000000000000000000000000000000000000..2bebcff67d27dc5ab91a3937269aacb178baa155 --- /dev/null +++ b/eval/sheet_mixH2_xm_f8_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e3683181febe9adf8add43655d959bb3637ce618e14659f797c6c4832765be6 +size 1930911 diff --git a/eval/sheet_mixH2_xm_f8_2.png b/eval/sheet_mixH2_xm_f8_2.png new file mode 100644 index 0000000000000000000000000000000000000000..8caa04783f3e5602acc9d5ca94995f53dc9b03ac --- /dev/null +++ b/eval/sheet_mixH2_xm_f8_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:edca416f15ed809d00d7920494c7d1ae6e32ee5ba486effe70128245d2843947 +size 2072998 diff --git a/eval/sheet_mixH2_xm_f8_3.png b/eval/sheet_mixH2_xm_f8_3.png new file mode 100644 index 0000000000000000000000000000000000000000..d598dc58a96aba1224855007b2e210a5ef129f75 --- /dev/null +++ b/eval/sheet_mixH2_xm_f8_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:24c7c63989fbe3bf9748465152ec8966ba404b05075eb19e938cce5a4d682c1d +size 2099404 diff --git a/eval/sheet_mixH2_xm_f8_4.png b/eval/sheet_mixH2_xm_f8_4.png new file mode 100644 index 0000000000000000000000000000000000000000..877f229a96f2eb4a548d7e6db0e06ecde7b1be19 --- /dev/null +++ b/eval/sheet_mixH2_xm_f8_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e99a190fbbe611c856988bcb9f38a9e29084a56d172303ebd893a92220f88b12 +size 2029839 diff --git a/eval/sheet_mixH_1.png b/eval/sheet_mixH_1.png new file mode 100644 index 0000000000000000000000000000000000000000..a17da4e834045f31e99642644549f05cf6005435 --- /dev/null +++ b/eval/sheet_mixH_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ac6dc2b88ffbe497bb86e39309146f7e573486e91dfe457701f4d2fd29d940de +size 2663128 diff --git a/eval/sheet_mixH_2.png b/eval/sheet_mixH_2.png new file mode 100644 index 0000000000000000000000000000000000000000..a56af5d12e72485d68e28cf6f802dbee653480c6 --- /dev/null +++ b/eval/sheet_mixH_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7ff0d2780ddde01918440ad0da5d22d8c189266b2dc564471cca31e57c4f5ff +size 2891670 diff --git a/eval/sheet_mixH_3.png b/eval/sheet_mixH_3.png new file mode 100644 index 0000000000000000000000000000000000000000..5c355e84d951585b84aa8fe139933620ca61140a --- /dev/null +++ b/eval/sheet_mixH_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3a7bd88a387eadd5087bd288822565c56c3216b7c0cfc1785350b192d9eb45f7 +size 2845620 diff --git a/eval/sheet_mixH_4.png b/eval/sheet_mixH_4.png new file mode 100644 index 0000000000000000000000000000000000000000..73ea249aef17de5b602dc1a76361f365d759b9e3 --- /dev/null +++ b/eval/sheet_mixH_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:3b8e544ac1138bdbf3ee6a6c3555209256573dae1a09807126cb544a9511635e +size 2796532 diff --git a/eval/sheet_mixH_vs_mixG_1.png b/eval/sheet_mixH_vs_mixG_1.png new file mode 100644 index 0000000000000000000000000000000000000000..6806ab4aa9b11983022356f8441fc0c544e0a3f0 --- /dev/null +++ b/eval/sheet_mixH_vs_mixG_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2870b2c98786366ef2c3d636dcd6b92e5a2c1d34cb30edcb64f1e821ea9afa14 +size 1997593 diff --git a/eval/sheet_mixH_vs_mixG_2.png b/eval/sheet_mixH_vs_mixG_2.png new file mode 100644 index 0000000000000000000000000000000000000000..92bdaab542ed86deb12c708a3809a31b64db7925 --- /dev/null +++ b/eval/sheet_mixH_vs_mixG_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ce5b9b94badc583b4e39db554ea9f9a3a6f816334e0ce27b03a051ba2390f590 +size 2163389 diff --git a/eval/sheet_mixH_vs_mixG_3.png b/eval/sheet_mixH_vs_mixG_3.png new file mode 100644 index 0000000000000000000000000000000000000000..3ec8f020697d268b80b6ce3bf70329ff4da7f960 --- /dev/null +++ b/eval/sheet_mixH_vs_mixG_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b7d8d1f91c4c4c2f5774bc0ddf0d53ff059de9da17ab09471381a0f3c98c8d1 +size 2145182 diff --git a/eval/sheet_mixH_vs_mixG_4.png b/eval/sheet_mixH_vs_mixG_4.png new file mode 100644 index 0000000000000000000000000000000000000000..5c01d6ac0ba096a306069db9057261addb4a7cb9 --- /dev/null +++ b/eval/sheet_mixH_vs_mixG_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:80ecfca41905c285e407e2ec38b7d5e9fcc8f8de805eef6d64c90335cc81981d +size 2087878 diff --git a/eval/sheet_mixI_1.png b/eval/sheet_mixI_1.png new file mode 100644 index 0000000000000000000000000000000000000000..470962f457312c0536dc8bc71c9ae2a7cd80301f --- /dev/null +++ b/eval/sheet_mixI_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ebd5a3973237ebc84394bafef2ab59fd9982c09952f8c134816eb7b542588b02 +size 2620101 diff --git a/eval/sheet_mixI_2.png b/eval/sheet_mixI_2.png new file mode 100644 index 0000000000000000000000000000000000000000..1957ecc6e27c1c067c490ac0c2dd53e0134dbeab --- /dev/null +++ b/eval/sheet_mixI_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:34e874992f01cfd35c2c04a8ec4c609276d1b0aa231baa710ed5db5b8a7ba81f +size 2833800 diff --git a/eval/sheet_mixI_3.png b/eval/sheet_mixI_3.png new file mode 100644 index 0000000000000000000000000000000000000000..fcc7a0404dee7abe1225a21b934b5d0bd259824d --- /dev/null +++ b/eval/sheet_mixI_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:bdc5a4ba28199dae47e7d4ad100d83eb6c0713b1d833994642b88845bb0b8794 +size 2798339 diff --git a/eval/sheet_mixI_4.png b/eval/sheet_mixI_4.png new file mode 100644 index 0000000000000000000000000000000000000000..87c395c9a1cedc18e13012221cae0648da09c859 --- /dev/null +++ b/eval/sheet_mixI_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39de3d1311913384dc6043ec2acdcd119ffe7ee48641e33cabaaa49445e30243 +size 2699967 diff --git a/eval/sheet_teacher_bf16_1.png b/eval/sheet_teacher_bf16_1.png new file mode 100644 index 0000000000000000000000000000000000000000..a7da6dd83e9218db8f0e4ddece8219310e7a2756 --- /dev/null +++ b/eval/sheet_teacher_bf16_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:79b4a20246c8068b5419a15126cacb791f6bc8ff12519a513e7e5075ee8ea0de +size 1415538 diff --git a/eval/sheet_teacher_bf16_2.png b/eval/sheet_teacher_bf16_2.png new file mode 100644 index 0000000000000000000000000000000000000000..6b6f70d1e8c3de52fce109b329bbb6d875349917 --- /dev/null +++ b/eval/sheet_teacher_bf16_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:2fa875a81e8f71aa3c9e90ef8dcac37d3f5b16214faeba3aae1d34aadb7f8216 +size 1452425 diff --git a/eval/sheet_tern_v1_1.png b/eval/sheet_tern_v1_1.png new file mode 100644 index 0000000000000000000000000000000000000000..1934c379987843634c1a465258e131482d11a309 --- /dev/null +++ b/eval/sheet_tern_v1_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d117a612c991931cb8e452be20a160dd0598a0aa335ad2146f5ca93555ae0a0b +size 1210399 diff --git a/eval/sheet_tern_v1_2.png b/eval/sheet_tern_v1_2.png new file mode 100644 index 0000000000000000000000000000000000000000..533a3645254dbb85bb1f42945050dac242630a61 --- /dev/null +++ b/eval/sheet_tern_v1_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d58409b54e592cfb2ffa57da399aad905e0e2d7b7a2deb0c49c4a55924f21fe0 +size 1268901 diff --git a/eval/sheet_tern_v1_3.png b/eval/sheet_tern_v1_3.png new file mode 100644 index 0000000000000000000000000000000000000000..aed096fc685facd341411706503a58ae6ec71fbb --- /dev/null +++ b/eval/sheet_tern_v1_3.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:91e479514ae40b28f5f4bb6bfc3789faacc440c86d925f167bc2fe30a18faad9 +size 1372082 diff --git a/eval/sheet_tern_v1_4.png b/eval/sheet_tern_v1_4.png new file mode 100644 index 0000000000000000000000000000000000000000..e75a4933596d2901f7d01386c7c1e455c9155084 --- /dev/null +++ b/eval/sheet_tern_v1_4.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dc3a080ca640139ea543d71a4e2ffc10b512ced6cc3f545a6117055bc13f7dd6 +size 1346693 diff --git a/eval/sheet_turbo_probe.png b/eval/sheet_turbo_probe.png new file mode 100644 index 0000000000000000000000000000000000000000..d812ffc5fcb558b1af25cbf4eba3c0a323aa2b02 --- /dev/null +++ b/eval/sheet_turbo_probe.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9a544347f52b6375c1ea7aa2370dbfd5e23081f92caa7024f680549b2b15145b +size 1703091 diff --git a/eval/sheet_turbo_quant_probe_1.png b/eval/sheet_turbo_quant_probe_1.png new file mode 100644 index 0000000000000000000000000000000000000000..785cafe6263c79a8053928c360afe7d8f447af15 --- /dev/null +++ b/eval/sheet_turbo_quant_probe_1.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1be5118329ff66100f51c7755ea69336aa4741da5158dd75b7c666e573a1ca74 +size 1741171 diff --git a/eval/sheet_turbo_quant_probe_2.png b/eval/sheet_turbo_quant_probe_2.png new file mode 100644 index 0000000000000000000000000000000000000000..5ab07d7445f0fd7e3ce235bbcb550f19afe8570c --- /dev/null +++ b/eval/sheet_turbo_quant_probe_2.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:29efc43737179d3d0ee55ff00421c66ef63ecab01d0eb7883d101600b2b7b064 +size 1723583 diff --git a/eval/strip_07_all.png b/eval/strip_07_all.png new file mode 100644 index 0000000000000000000000000000000000000000..9fb8686c5155b34f9aabd628cf9ef691c25f3cc8 --- /dev/null +++ b/eval/strip_07_all.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e6de8db1c81553863a9db20dc17aac350ed3a1406456584c763aa81325d38fc7 +size 1787155