examples.json

{
  "cases": [
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.15,
            "score_mean": 0.0026041666666666665,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.16,
            "score_mean": 0.012083333333333335,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.83,
            "score_mean": 0.3329166666666665,
            "score_median": 0.32,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.3947222222222223,
            "score_median": 0.43,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1) Primitive early talkie with bare\u2011bones production values. 2) Mixed acting assessment: Jean Arthur is less attractive and still in training, Joel McCrea is attractive and gives a standard performance. 3) Some scenes (fishing industry) are interesting while fight scenes are lame; overall the movie is mildly interesting and worth watching.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the primitive production, instead claims a high\u2011quality 1280p Blu\u2011ray transfer, contradicting the low\u2011budget description (m0.0). It praises both Jean Arthur and Joel McCrea, opposite to the ground truth's mixed acting assessment (m0.0). It calls the film a 'must\u2011own' masterpiece, ignoring the modest, mildly interesting tone and the critique of fight scenes (m0.0). Coverage C = 0.0. The response adds unrelated, speculative details about a Blu\u2011ray release that are not in the ground truth, introducing significant irrelevant content, so penalty P \u2248 0.8. Final score = max(0, 0.0\u20110.8) = 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a 1280p Blu-ray release that actually looks good! I've been waiting for Alpha Video to get this one right, and they delivered. The transfer is incredibly sharp for a 1940s B-movie, with excellent detail in the black and white footage. The audio is clear and crisp, making George Archainbaud's direction and the performances by Jean Arthur and Joel McCrea really pop. It's a short film, but it's a must-own for any classic cinema fan.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl4",
            "judge_key_points": "1) The film is a primitive early talkie with low production values. 2) Jean Arthur is not as attractive and is still an actress-in-training. 3) Joel McCrea is attractive and gives a standard performance. 4) Some fishing industry scenes are interesting while fight scenes are lame; overall the movie is mildly interesting but worth watching.",
            "judge_score": 0.0,
            "judge_thought": "The generated response describes the film as charming and engaging, praises Jean Arthur's performance, claims excellent picture quality, and highly recommends it, which contradicts the ground truth's low production value, criticism of Jean Arthur's appearance, and only mildly positive overall assessment; thus it matches none of the key points (near 0 for each) and adds contradictory positive content, leading to a high penalty. Final score is 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "A charming and surprisingly engaging adventure from the golden age of Hollywood. Jean Arthur shines in this role, and the chemistry with Joel McCrea is delightful. The story of the silver miners is a bit unusual, but the film's lighthearted tone keeps it from feeling too heavy. The sound and picture quality are excellent for a classic. I highly recommend this one for fans of the genre or anyone looking for a fun, nostalgic ride.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.25",
            "judge_key_points": "1) Primitive early talkie with bare\u2011bones production values and routine acting. 2) Jean Arthur is less attractive and still an actress\u2011in\u2011training. 3) Joel McCrea is attractive and gives a standard performance. 4) Fishing industry scenes are interesting but fight scenes are lame. 5) The movie is mildly interesting and worth watching.",
            "judge_score": 0.32,
            "judge_thought": "The generated response matches the primitive early talkie description well (\u22480.9) and notes Jean Arthur is not as good (\u22480.7) but adds miscasting not in the ground truth. It downplays Joel McCrea's performance (\u22480.5) and omits the specific comments about fishing and fight scenes (0). It calls the film a decent oldie, which loosely aligns with 'mildly interesting' (\u22480.5). Overall coverage \u22480.52 and there is some extra speculative content, giving a penalty \u22480.2. Final score = 0.52\u20110.2 \u2248 0.32.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Primitive, early talkie production. Bare-bones production values and routine acting. Jean Arthur is not as good here as she was in other films, and she is miscast in this role. Joel McCrea is not a bad actor, but his performance here is somewhat flat. The story is forgettable and the plot is a bit silly. A decent oldie, but nothing special.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) Primitive early talkie with low production values. 2) Jean Arthur is less attractive and still in training; Joel McCrea is attractive and gives a standard performance. 3) Fishing industry scenes are interesting while fight scenes are lame. 4) Overall the movie is mildly interesting and worth watching.",
            "judge_score": 0.75,
            "judge_thought": "The generated response captures the primitive talkie and low production values (match 1.0), notes Jean Arthur looks less polished and Joel McCrea gives a standard performance (match ~0.9), correctly describes the fight scenes as bland and fishing scenes as interesting (match 1.0), and gives a generally positive but slightly more critical overall impression (match ~0.7). Extra commentary about it being a 'charming throwback' and 'dated experience' is mild speculation but not contradictory, so a small penalty ~0.15 is applied. Final score = 0.9 - 0.15 = 0.75.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "An early, primitive talkie with low production values. Jean Arthur looks less polished here than in her later work, and her performance, while decent, lacks the depth she brought to later roles. Joel McCrea offers a standard performance, and the supporting cast is competent but forgettable. The fight scenes are particularly bland and unconvincing. However, the fishing-industry scenes are interesting. It's a charming throwback in spirit, but the technical limitations of the era make it a somewhat dated experience.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) Describes the film as a primitive early talkie with bare\u2011bones production values and routine acting. 2) Notes Jean Arthur is less attractive than later and still in training. 3) Says Joel McCrea is attractive and gives a standard performance. 4) Mentions fishing industry scenes are interesting while fight scenes are lame. 5) Concludes the movie is mildly interesting but worth watching.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, matching all five key points perfectly with no extra or conflicting content, so coverage=1.0 and penalty=0.0, final score=1.0",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Primitive, early talkie production. Bare-bones production values and routine acting. Jean Arthur is not as attractive as in her later movies, and she\u2019s still an actress-in-training in this film. Joel McCrea is always attractive and delivers a standard performance. Scenes of the fishing industry are interesting, but the fight scenes are lame. The movie is mildly interesting but worth watching.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The film is a primitive early talkie with low production values. 2) Jean Arthur is unattractive and still in training, while Joel McCrea is attractive and gives a standard performance. 3) The movie is mildly interesting, worth watching despite some lame fight scenes.",
            "judge_score": 0.0,
            "judge_thought": "The generated response fails to reflect the low\u2011budget, primitive nature (only mentions 'no\u2011frills' but overall praises the film), does not mention Jean Arthur's shortcomings or Joel McCrea's standard performance (instead claims good chemistry and strong acting), and only loosely aligns with the overall 'worth watching' sentiment while being overly positive; thus coverage is low (~0.27) and there is a strong penalty for contradictory, unsupported praise (~0.6), yielding a final score of 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Alpha Video has done it again with this release of *The Silver Horde*. If you\u2019re a fan of Joel McCrea or just appreciate well-crafted silent-era adjacent dramas, this is a solid addition to your collection. The film itself is a tight 74 minutes, which is perfect\u2014no filler, just a straight-forward story. Jean Arthur and Raymond Hatton hold their own alongside McCrea, and the chemistry is evident. It\u2019s a classic drama that holds up surprisingly well, especially given the era. As for the disc quality from Alpha Video, you know what you\u2019re getting. It\u2019s a functional, no-frills release. The video is in black and white as expected, and the audio is clear enough via Dolby. It\u2019s not going to win any restoration awards, but for the price, it\u2019s a perfectly serviceable way to watch this film. The NTSC format and closed captions are standard and work fine. Overall, if you want to add another McCrea title to your shelf without breaking the bank, this is a worthy purchase. Simple, effective, and gets the job done. 4.0 out of 5 stars",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) Primitive early talkie with low/bare\u2011bones production values and routine acting. 2) Jean Arthur is early\u2011career, not as attractive as later and still in training. 3) Joel McCrea is attractive and gives a standard performance. 4) Fishing industry scenes are interesting. 5) Fight scenes are lame/weak. 6) Overall the film is mildly interesting and worth watching.",
            "judge_score": 1.0,
            "judge_thought": "The generated response restates all six key points accurately: it calls the film a primitive early talkie with low production values and routine acting, notes Jean Arthur's early\u2011career look and lack of attractiveness, mentions Joel McCrea as always attractive, describes the fishing scenes as notable, calls the fight scenes weak, and concludes the film is mildly interesting and worth watching. No contradictory or unsupported content is added, so coverage is essentially 1.0 and penalty is 0.0, yielding a final score of 1.0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "A primitive early talkie with low production values and routine acting. Jean Arthur appears in her early-career look, not as attractive as in her later movies and still in training, while Joel McCrea is always attractive. The fishing scenes are notable, though the fight scenes are weak. Overall, the film is mildly interesting and worth watching.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:67d4e8f186997699c802:prompt6",
      "category": "Movies_and_TV",
      "cohort": "seen",
      "eval_pair_id": null,
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2004-04-10",
            "length_band": "Long",
            "length_chars": 1030,
            "post_id": "amzr_52a64e0006c8eefb2961e8e8e83852ffd83dccfc8c99fce0bef66ebc9be4a7e7",
            "product": {
              "category": "Movies_and_TV",
              "description_excerpt": "FEATURE 1-BLONDIE, FEATURE 2-BLONDIE MEETS THE BOSS, FEATURE 3-BLONDIE TAKES A VACATION, FEATURE 4-BLONDIE BRINGS UP BABY, FEATURE 5-BLONDIE ON A BUDGET.",
              "details": {
                "Actors": "Penny Singleton, Larry Simms, Arthur Lake",
                "Contributor": "Arthur Lake, Penny Singleton, Larry Simms",
                "Format": "Multiple Formats, Color, NTSC",
                "Language": "English",
                "MPAA rating": "G (General Audience)",
                "Media Format": "Multiple Formats, Color, NTSC",
                "Number Of Discs": "1",
                "Number of discs": "1"
              },
              "features": [],
              "product_id": "B0001JXOXK",
              "title": "Blondie, Set 1 (Blondie / Blondie Meets the Boss / Blondie Takes a Vacation / Blondie Brings Up Baby / Blondie on a Budget)"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "This DVD is an incredible value and a real treasure. The DVD includes five complete episodes of the Blondie comedies from the 1940s for an astonishingly low price. These remastered movies have very good sound and visual quality. The comedies have also held up very well and are as funny and engaging as they were over half a century ago. Baby Dumpling and Daisy the dog are cute scene-stealers throughout the series. My teenage daughter actually laughs out loud at these comedies, and it's a nostalgic pleasure for me to watch them again after so many years. Because of the enjoyable experience of watching these comedies and the exceptional value, I'd award them more than five stars if I could. This purchase is worth many times the price I paid. (Although the DVD case lists the run time as 72 minutes, the actual run time is five times that since each episode runs a little over 70 minutes.) This DVD features \"Blondie,\" Blondie Meets the Boss,\" \"Blondie Takes a Vacation,\" \"Blondie Brings Up Baby,\" and \"Blondie on a Budget.\""
          },
          {
            "date": "2016-01-23",
            "length_band": "Long",
            "length_chars": 394,
            "post_id": "amzr_3152668d6184b911b6f4802c614ff392b6a4b0ff8d5fd1b78a409d9235a41db5",
            "product": {
              "category": "Movies_and_TV",
              "description_excerpt": "A pretty hospital night nurse (Jane Fonda) says \"I do\" to the Korean War vet (Jim Hutton) under her care. Within moments she wonders what she's done. Ralph (Tony Franciosa) wed for money five years ago but has grown to love his bride (Lois Nettleton). Except she just walked out on him. Y'all linger and look on as these pairs go through a Period of Adjustment. This splintered-souls romantic comedy from an unexpected source is very much a movie of firsts: Tennessee Williams' first comedy, director George Roy Hill's first feature, Nettleton's big-screen debut and a personal period of adjustment f",
              "details": {
                "Actors": "Tony Franciosa, Jane Fonda, Jim Hutton, Lois Nettleton, John Mcgiver",
                "Aspect Ratio": "1.33:1",
                "Contributor": "Tony Franciosa, Jane Fonda, Jim Hutton, Lois Nettleton, John Mcgiver",
                "Country of Origin": "USA",
                "Format": "NTSC",
                "Genre": "Comedy",
                "Is Discontinued By Manufacturer": "No",
                "Language": "English"
              },
              "features": [],
              "product_id": "B006WQUJ8Y",
              "title": "Period Of Adjustment"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "Quirky dramedy, but quintessential Tennessee Williams. Powerful, sometimes breathtaking performances by Lois Nettleton and Anthony Franciosa. Jim Hutton and Jane Fonda, to me, seem miscast, and their performances are uneven-sometimes spot on and sometimes cringe-worthy. Overall, the film increased in interest and at the end was a very satisfying experience. Well worth the investment of time."
          },
          {
            "date": "2016-01-25",
            "length_band": "Long",
            "length_chars": 420,
            "post_id": "amzr_cde20b662a13b9356965c85f85d46e458eda3d2a2aafa35edaa7f637f082ffcb",
            "product": {
              "category": "Movies_and_TV",
              "description_excerpt": "Just point her in the direction of New York's millionaires! Fresh from Missouri, plucky and ambitious Eadie Chapman (Jean Harlow) intends to bag a rich hubby without abandoning her virtue. Just act like a lady, she tells gal pal and fellow huntress Kitty (Patsy Kelly) at a swanky soiree. \"If they wanted ladies,\" Kitty says, \"they'd be home with their wives.\" As the girl from the \"Show-Me\" state, Harlow shows her comedy flair, abetted by the snappy co-scripting of her personal friend Anita Loos and by the nimble guidance of filmmaker Jack Conway, who would direct her in Libeled Lady. Lionel Bar",
              "details": {
                "Actors": "Jean Harlow, Lionel Barrymore, Franchot Tone, Lewis Stone",
                "Aspect Ratio": "1.33:1",
                "Contributor": "Lewis Stone, Jean Harlow, Lionel Barrymore, Franchot Tone, Jack Conway",
                "Country of Origin": "USA",
                "Director": "Jack Conway",
                "Format": "Full Screen, NTSC, Original recording remastered",
                "Genre": "Comedy",
                "Is Discontinued By Manufacturer": "No"
              },
              "features": [],
              "product_id": "B005ZVIP92",
              "title": "The Girl from Missouri"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "If you want a good introduction to Jean Harlow, her acting and her persona, see The Girl from Missouri. Jean stars as a poor girl from a small town determined to marry a Park Avenue millionaire. Although not a top-notch actress, Jean makes up for her lack of acting ability with her beauty, her moxie, and her energy. Good screen play and intriguing plot twists make for a highly entertaining film from beginning to end."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 6,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Long",
        "rating": 2
      },
      "row_id": "humanual-amazon-v1:train:67d4e8f186997699c802",
      "source_line_sha256": "c3c9d297712eb2b6686fd4864af246c578f73f5b4066012c53bc4669f3bf2183",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2016-05-16",
        "length_band": "Long",
        "length_chars": 395,
        "post_id": "amzr_3f756b023c7506d1e3654879590e4968df9cde1c4b897c422337b342af8a9756",
        "product": {
          "category": "Movies_and_TV",
          "description_excerpt": "Young, handsome and rugged Boyd Emerson (Joel McCrea) is a washed-up gold miner, dogsledding aimlessly through Alaska with scant luck and little hope for the future until he meets Cherry Mallotte (Evelyn Brent). Cherry is an attractive woman with enough worldly knowledge and business contacts to help the young man succeed in business. She is also a woman with a dubious past, much of it kept secret. Soon, young Emerson finds himself torn between two women: Cherry, his Alaskan mentor and inspiration, and the girl back home, Mildred Wayland (Jean Arthur), the high-society debutante he was plannin",
          "details": {
            "Actors": "Jean Arthur, Raymond Hatton, Joel McCrea",
            "Contributor": "George Archainbaud, Joel McCrea, Raymond Hatton, Jean Arthur",
            "Country of Origin": "USA",
            "Director": "George Archainbaud",
            "Format": "Multiple Formats, Black & White, Closed-captioned, Dolby, NTSC",
            "Genre": "Classic",
            "Item model number": "2233423",
            "Language": "English"
          },
          "features": [],
          "product_id": "B000286S4W",
          "title": "The Silver Horde"
        },
        "rating": 2.0,
        "review_type": "target",
        "text": "Primitive, early talkie production. Bare-bones production values and routine acting. Jean Arthur is not as attractive as in her later movies, and she\u2019s still an actress-in-training in this film. Joel McCrea is always attractive and delivers a standard performance. Scenes of the fishing industry are interesting, but the fight scenes are lame. The movie is mildly interesting but worth watching."
      },
      "uid": "amzu_041eb9212f52f326893268604d41b27b65652b11c6113de62569386a4982126d"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.1,
            "score_mean": 0.0020833333333333333,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.73,
            "score_mean": 0.33916666666666667,
            "score_median": 0.33,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.6024166666666666,
            "score_median": 0.8,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "1) Purchased the card for the grandson; 2) Grandson enjoys using it with his Go camera; 3) Reviewer enjoyed watching the grandson's movies",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the grandson, the Go camera, or watching movies, so it fails to reflect any of the three key points (m_i = 0 for each). Coverage C = 0. It also adds extensive unrelated personal commentary, which is irrelevant and introduces unsupported details, warranting a moderate to high penalty (P \u2248 0.6). Final score = max(0, 0 - 0.6) = 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a microSD card that actually lives up to its hype. I've gone through a few different brands over the years, but this PNY U3 PRO Elite is the first one that consistently hits those advertised speeds without any stuttering. I primarily use it in my action camera for 4K footage, and the 90MB/s write speed makes a noticeable difference when recording long clips. With cheaper cards, I'd often get a \"card full\" error after just a few minutes, but this one handles the data flow like a champ. The read speeds are also solid for transferring files back to my laptop. It's small, compact, and the black casing feels durable. Installation was a breeze, and I've had zero issues with compatibility on my smartphone and camera. If you need reliable storage for high-definition video or heavy app usage, this is the card to get. Highly recommended for anyone who needs performance they can trust.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl4",
            "judge_key_points": "1) Purchased the card for the grandson. 2) Grandson enjoys using it with his Go camera. 3) Reviewer enjoyed watching the grandson's movies.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the grandson, the Go camera, or watching movies, so it fails to reflect any of the three key points (m_i = 0 for each). Coverage C = 0. It adds unrelated content about personal use and speed tests, which is irrelevant and speculative, warranting a penalty around 0.5. Final score = max(0, 0 - 0.5) = 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I have used PNY cards before and they are usually reliable. I needed a fast card for my camera and this one delivered. I ran a speed test and the write speeds were right up there with the advertised numbers. So far so good.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.5",
            "judge_key_points": "1) Bought the card for my grandson. 2) Grandson enjoys using it with his Go camera. 3) I enjoy watching some of his movies.",
            "judge_score": 0.33,
            "judge_thought": "The generated response fully matches point 1 (grandson purchase). It partially matches point 2 (mentions grandson enjoys using it for games and photos, which is a different use than the Go camera, so about 0.6). It does not mention point 3 (watching his movies) at all, so 0. The average coverage is (1+0.6+0)/3 = 0.533. The extra detail about games and photos is slightly off\u2011topic but not contradictory, giving a small penalty around 0.2. Final score = 0.533 - 0.2 \u2248 0.33.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Got it for my grandson and he enjoys having it for his games and photos. Works great for him.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Purchased the card for my grandson. 2) He uses it with his Go camera and enjoys it. 3) I enjoy watching some of his movies.",
            "judge_score": 0.9,
            "judge_thought": "The generated response mentions getting the card for the grandson's Go camera (matches point 1 and 2), says it works great and that he loads movies which the speaker can watch right away (matches point 3). All three key points are reflected strongly (\u22480.9 each). No extra or conflicting content is present, so penalty is 0. Final score = (0.9+0.9+0.9)/3 - 0 = 0.9.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Got this card for my grandson's Go camera and it works great. He loads up his movies and I can watch them right away.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) Purchased the card for my grandson. 2) Grandson enjoys using it with his Go camera. 3) I enjoyed watching some of his movies.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, matching all three key points perfectly with no extra or conflicting content, so coverage C=1.0 and penalty P=0.0, yielding a final score of 1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Got it for my grandson and he enjoys having it with his Go camera. I enjoyed watching some of his movies.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Bought the card for my grandson; 2) Grandson enjoys using it with his Go camera; 3) I enjoyed watching some of his movies",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the grandson, the Go camera, or watching movies, and instead provides a generic product review, so it fails to reflect any of the three key points (m_i = 0). Coverage C = 0. The content is unrelated, warranting a high penalty (P \u2248 0.9). Final score = max(0, 0 - 0.9) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "**Solid performance for the price, but don't expect flagship speeds** I picked this up to expand storage on my phone since I take a lot of photos and videos. For the price, it\u2019s a decent deal, but you need to manage your expectations regarding the \"95MB/s\" claim. **Pros:** * **Ease of use:** The included adapter works fine, though it\u2019s a bit bulky. Just be careful not to lose it. * **Reliability:** It\u2019s been in my phone for about three months now with zero corruption issues. * **Price:** Hard to beat this value for 64GB. **Cons:** * **Write speeds are slower than advertised:** I tested it on my PC, and while read speeds hovered around 80-85MB/s, write speeds maxed out closer to 60-70MB/s. It\u2019s not terrible, but if you\u2019re shooting 4K video, you might notice some stuttering or buffering. It\u2019s fine for HD, but I\u2019d look elsewhere for 4K. * **Adapter quality:** The plastic feels a bit cheap. I\u2019d recommend buying a separate, sturdier adapter if you plan to swap this card between devices often. **Verdict:** If you just need extra space for apps, photos, and 1080p video, this is a good, reliable choice. Don\u2019t buy it if you\u2019re a professional videographer or need maximum burst write speeds. It does the job for casual use. 4.0 out of 5 stars",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Purchased the card for the grandson; 2) Grandson uses it with his Go camera and enjoys it; 3) The speaker enjoys watching some of the grandson's movies",
            "judge_score": 1.0,
            "judge_thought": "The generated response states it was bought for the grandson, mentions his Go camera, and says the speaker enjoys watching his movies, matching all three key points closely (m\u22481 for each). No extra or conflicting content is present, so penalty P=0. Coverage C=1, final score =1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Got this for my grandson's Go camera and it works great. I've enjoyed watching his movies with him lately.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:6f11ab1bf7edc9042d76:prompt5",
      "category": "Electronics",
      "cohort": "seen",
      "eval_pair_id": null,
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2012-12-30",
            "length_band": "Medium",
            "length_chars": 126,
            "post_id": "amzr_86a89539db8c520938d55ae5fc17f039b1b13ead6c135902aaa0c65d6bcaff19",
            "product": {
              "category": "Electronics",
              "description_excerpt": "This Genuine MPD Digital USA Made LMR-240 coaxial cable was developed to provide an extremely low loss alternative with the transmission characteristics of hardline coaxial cables and the flexibility required for WiFi, Ham and CB Radio installations. MPD Digital systems, designs and manufactures high performance coaxial cables, connectors and cable assemblies for a broad range of RF transmission applications. For more than 50 years, MPD Digital has been the leader in the development of new cable technologies to meet the demands of evolving RF and microwave applications. This technological, man",
              "details": {
                "Brand": "MPD Digital",
                "Cable Type": "Coaxial",
                "Color": "black",
                "Connector Gender": "Male",
                "Connector Type": "Coaxial",
                "Data Transfer Rate": "6 GHz",
                "Date First Available": "September 5, 2011",
                "Item Weight": "1.7 pounds"
              },
              "features": [
                "LMR240 LMR-240 is a Registered Trademark of MPD Digital Systems - This Is Genuine LMR-240",
                "PL-259 UHF coax cable connectors - Connects Antennas, Receivers, Meters and Radio transmitters",
                "50 feet of coaxial cable USA Made LMR-240 Transmission Line Coaxial Cable with Soldered-Center Silver Plated Connectors Teflon Dielectrics",
                "Assembled by MPD Digital in our GA production facility using MILSPEC componants"
              ],
              "product_id": "B005LA8C86",
              "title": "MPD Digital LMR-240 Coaxial Cable Ham or CB Radio Antenna Coax with PL-259 Connectors, 50-feet"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "In a world of look a likes that are really junk this is a really great item. Well made and does what is says. Highly recommend"
          },
          {
            "date": "2013-10-17",
            "length_band": "Long",
            "length_chars": 418,
            "post_id": "amzr_2d20e942392ff7ee9d9160d8dac8a702623c0ed5bcf7d0b7fc9d0e8e3d664aa3",
            "product": {
              "category": "Electronics",
              "description_excerpt": "",
              "details": {
                "Brand": "KENTEK",
                "Compatible Devices": "Personal Computer",
                "Cooling Method": "Air",
                "Date First Available": "June 30, 2011",
                "Is Discontinued By Manufacturer": "No",
                "Item Dimensions LxWxH": "8.1 x 6 x 3.7 inches",
                "Item Weight": "0.01 Ounces",
                "Item model number": "4328507270"
              },
              "features": [],
              "product_id": "B00598LSL8",
              "title": "300 Watt 300W TFX Power Supply replacement for HP Slimline S5000 Compaq Presario Acer Veriton FLX-250F1,DPS-220AB-2,DPS-250ab-28,TFX0220P5WAPC7068,FSP160-60SAV,AU847AA,AV025AA,AY553AA,375496-002,409815-001,API4PC10,409815-002,409815-003 by KENTEK"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "The power supply fit perfectly and looked well made. The only issue it didn't have two of the SATA power outlets that had long enough wires so I couldn't power up the CD/DVD burner. I soldered one on that I cut off from the old power supply and got everything working. It had two of the SATA outlets but they were close together and would not reach between the hard drive and the CD/DVD about 8 inches away."
          },
          {
            "date": "2014-11-03",
            "length_band": "Long",
            "length_chars": 682,
            "post_id": "amzr_fcbc23910fae8e7fdc79ed8b836a6348f2226f4f8a6464b4cab644180651a44d",
            "product": {
              "category": "Electronics",
              "description_excerpt": "ENH202 is an 802.11n 2.4GHz Wireless Outdoor Bridge/AP with speeds up to 300Mbps with 800mW power for longer range building-to-building or IP security camera deployments.",
              "details": {
                "Brand": "EnGenius",
                "Color": "White",
                "Compatible Devices": "Security Camera",
                "Connectivity Technology": "Wireless",
                "Date First Available": "December 14, 2011",
                "Department": "Electronics",
                "Flash Memory Type": "SD",
                "Included Components": "See Specs"
              },
              "features": [
                "Designed For Outdoor Deployments",
                "Ip65 Waterproof Housing & Cable Accessory Kit Protects Against Moisture, Rain & Dew",
                "Compliant With 802.11n Standard For Connection Rate Up To 300mbps",
                "6x The Speed Of Similar 802.11g Devices"
              ],
              "product_id": "B006M1PKWY",
              "title": "EnGenius 11n 2.4GHz Wireless Ethernet Bridge/Access Point (ENH202)"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "This unit does not have DHCP capability when in Access Point mode. It does have an option in Access Point mode when configuring the unit. It allows you to turn it on or turn it off. With that option you would think if you selected to turn it on it would turn on DHCP.! I called the technical support number and the person said that it DOES NOT have DHCP capabilities in Access Point mode. I asked why doesn't the manual say that. No response. I explained that every other access point I have ever used (dozens of them - worked as a consultant to set these things up) he had nothing to say but thank you for calling Engenius Customer Support and the call ended."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 5,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Medium",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:train:6f11ab1bf7edc9042d76",
      "source_line_sha256": "396c0ee7e02c4586df4fbdd6f80b867904e18363e6f25b5b03cb7e98fc1d1201",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2016-06-20",
        "length_band": "Medium",
        "length_chars": 105,
        "post_id": "amzr_6716e572d0cc350f059cf507882ca2d773447862c4a5e77df784d2b495730e4a",
        "product": {
          "category": "Electronics",
          "description_excerpt": "Capture the Action and Relive the Moment with PNY U3 PRO Elite microSDXC Card (P-SDU64GU395PRO-GE), with amazing speeds of up to 95MB/s Read / 90MB/s Write for the most extreme and demanding video shooting and conditions.",
          "details": {
            "Brand": "PNY",
            "Color": "Black",
            "Compatible Devices": "Smartphone",
            "Date First Available": "November 19, 2015",
            "Flash Memory Type": "Micro SD",
            "Item Dimensions LxWxH": "5.28 x 3.91 x 0.45 inches",
            "Item Weight": "0.798 ounces",
            "Item model number": "P-SDUX64U395PRO-GE"
          },
          "features": [],
          "product_id": "B017WL4280",
          "title": "PNY U3 PRO Elite MicroSD Card (Up-to 95MB/S Read / 90MB/S Write Speeds),64GB (P-SDUX64U395PRO-GE)"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "Got it for my grandson and he enjoys having it with his Go camera. I enjoyed watching some of his movies."
      },
      "uid": "amzu_0f98a046c766c4378752936934932cc3f0ec7c5632aace570f5eaeea2f92331d"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.5451388888888885,
            "score_median": 0.75,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5722222222222223,
            "score_median": 0.7,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) User expresses dissatisfaction that the product did not help with fleas. 2) User includes an apology ('Sorry').",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the product works well, kills fleas and ticks, and recommends it, which directly contradicts the ground truth negative experience and contains no apology. Thus it matches neither key point (m1=0, m2=0), giving coverage C=0. The content is opposite to the ground truth, so a high penalty (~0.9) is applied. Final score = max(0, 0 - 0.9) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been using the Solimo brand for years and honestly, I didn\u2019t even notice the switch to Amazon Basics until I opened the box. The packaging looks different, but the applicator tubes feel exactly the same and the liquid inside is identical. For a pet owner, that\u2019s exactly what you want\u2014no surprises. My dog is a 110 lb Lab mix, so he falls right in the X-Large category. I\u2019ve been rotating between this and the name-brand stuff, and I can\u2019t tell a difference in effectiveness. It kills the fleas within a day or two, and my dog hasn\u2019t had a single tick since I started using this. The waterproof feature is a lifesaver since he loves swimming, and I don\u2019t have to worry about him washing it off after a bath or a run in the rain (as long as it\u2019s been 24 hours). The 6-month supply is convenient and usually cheaper than buying the big name brands at the vet. I appreciate that it doesn\u2019t have an expiration date on the box, so I don\u2019t have to worry about tossing it out if I miss a month. If it works, don\u2019t fix it. Stick with this. Highly recommend for large dogs.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) User is dissatisfied with the product. 2) The product did not help with fleas at all (ineffective).",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the product works and the dog has no fleas, which directly contradicts both key points of dissatisfaction and ineffectiveness; therefore it matches none of the key points (m1=0, m2=0) giving coverage C=0. It also introduces strongly conflicting content, so penalty P is high (~0.9). Final score = max(0, 0 - 0.9) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I have been using this for years and it works. My dog has no fleas or ticks and I don't have to worry about them. The price is good too.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.5",
            "judge_key_points": "1) Apology/sorry expressed. 2) Statement that the product did not help with fleas (negative outcome).",
            "judge_score": 0.4,
            "judge_thought": "The generated response includes an apology, matching key point 1, and says the product did not work for you, matching key point 2, so coverage C=1. However it adds a personal claim that the product works for the speaker, which contradicts the ground truth's negative assessment and introduces unsupported positive evidence, warranting a penalty of about 0.6. Final score = 1 - 0.6 = 0.4.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Sorry that this product did not work for you. I have used it for years and it works for me.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) Apologizes or expresses regret about the product. 2) States that the product did not help with fleas at all.",
            "judge_score": 1.0,
            "judge_thought": "The generated response includes an apology ('I am sorry') and repeats the exact claim that the product did not help with fleas, matching both key points perfectly with no extra or conflicting content.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "I am sorry to say that this product did not help with fleas at all.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "Apologizes and states the product did not help with fleas",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly matches the ground truth, expressing the same apology and claim that the product did not help with fleas, with no extra or conflicting content.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Sorry that this product did not help with fleas at all.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "User expresses dissatisfaction, stating the product did not help with fleas at all.",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the product works well and recommends it, directly contradicting the ground truth negative experience; thus it does not reflect the key point (m=0). Coverage is 0, and the contradictory extra content adds a high penalty (~0.9). Final score is 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been using the Solimo brand for my 100 lb Lab for years, so I was a bit confused when the packaging changed to Amazon Basics, but the product description confirms it\u2019s the exact same formulation. That\u2019s a huge relief. I\u2019m sticking with this because it actually works. Within 24 hours of applying it, I noticed a significant drop in the flea population on his fur. The 30-day protection window is perfect for our routine, and since he loves swimming, I appreciate that it becomes waterproof after the first 24 hours. No more worrying about a bath ruining the treatment. The 6-count pack is a good value, and knowing it doesn\u2019t expire is a nice bonus for stockpiling. Just make sure you read the weight guidelines\u2014this is strictly for X-Large dogs (89-132 lbs). If you have a smaller pup, don\u2019t use this. Otherwise, it\u2019s a reliable, no-nonsense product that does exactly what it says. Highly recommend for large breed owners.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Apology/expressing regret about the product. 2) Statement that the product did not help with fleas at all.",
            "judge_score": 1.0,
            "judge_thought": "The generated response includes an apology ('Sorry...') and repeats the exact claim that the product did not help with fleas at all, matching both key points perfectly with no extra or conflicting content. Coverage=1.0, Penalty=0.0, final score=1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Sorry... this product did not help with fleas at all.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:4b357953b9e8a6f3d1a1:prompt4",
      "category": "Pet_Supplies",
      "cohort": "seen",
      "eval_pair_id": null,
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2018-07-15",
            "length_band": "Medium",
            "length_chars": 95,
            "post_id": "amzr_01c4d769d8ca599e5704c3e724dc9870302dc99085336e549970517cdb4325d5",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "",
              "details": {
                "Date First Available": "April 26, 2016",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "856713005159",
                "Manufacturer": "SKOUT'S HONOR",
                "Product Dimensions": "2 x 2 x 2 inches; 8.6 Pounds"
              },
              "features": [
                "PROFESSIONAL STRENGTH: Plant-based surfactant formula that dissolves and lifts difficult and stubborn stains from the toughest surfaces. Featuring BioKore - a revolutionary new cleaner guaranteed to outperform traditional and enzyme based cleaning products.",
                "POWERFUL MOLECULAR DEODORIZER: Targets and eliminates foul odor molecules on contact. We don't mask odors or make them smell good. We annihilate them.",
                "ECO-FRIENDLY & CRUELTY-FREE: Manufactured under the California Green Chemistry Initiative, making this product safer for pets, family, and home when used as directed.",
                "PAW PLEDGE: With every bottle purchased from Skout\u2019s Honor, a rescue animal is provided with a full day\u2019s worth of meals."
              ],
              "product_id": "B07XRWPYPS",
              "title": "SKOUT'S HONOR: Professional Strength Stain and Odor Remover - Deodorize and Clean Pet Stains, Dog Crates, Carpets, Furniture and Other Water-Safe Surfaces - Laundry Safe - 128 fl oz"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I love this,. Does a better job and hardly any smell to speak of. Only thing is...so expensive!"
          },
          {
            "date": "2020-04-12",
            "length_band": "Long",
            "length_chars": 419,
            "post_id": "amzr_3f35f050f8693effc7beee55e224a0b4de7560d4b319e6bee974534a872da3ea",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "",
              "details": {
                "Date First Available": "November 8, 2018",
                "Is Discontinued By Manufacturer": "No",
                "Manufacturer": "aSquared Brands",
                "Product Dimensions": "5 x 3.5 x 3.5 inches; 12.8 Ounces"
              },
              "features": [
                "NATURAL JOINT SUPPORT: Our All Natural pet canine Hip & Joint soft chews support joint health & mobility, promote comfort and flexibility, and support immunity & digestion. We include Turmeric to help relieve inflammation, and naturally help reduce pain & discomfort.",
                "AMAZING BENEFITS OF HEMP: Our unique formula helps decrease joint inflammation. We include Hemp Seed Oil, which contains Omega 3 6 and 9, helps decrease pain, supports heart & brain health, relieves stress, and supports healthy hair, skin and nails.",
                "9 POWERFUL INGREDIENTS: Our pet canine soft chews contain Turmeric Curcumin, Glucosamine, Methyl Sulfonyl Methane (MSM), Hemp Protein, Hemp Seed Oil, Chondroitin Sulfate, Vitamin C, Ginger Root, and BioPerine (black pepper extract).",
                "TURMERIC & GINGER: Helps relieve joint stiffness, improves hip, leg & joint flexibility & mobility, helps soothe pain from joint inflammation, and supports overall health & well-being of your dog. Additionally, Turmeric Curcumin helps boost energy, immunity, and contains antioxidant properties."
              ],
              "product_id": "B07FF8MGG6",
              "title": "Hip & Joint Supplement for Dogs - Hemp Oil Infused Soft Chews Dog Treats w/Glucosamine, Turmeric, Chondroitin, MSM & Omega 3 6 9 - Supports Pet Mobility & Pain Relief - 120 Treat Bites"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "This product has had an amazing effect on my miniature poodle! For years he has slept a lot and has low energy. I started noticing a big change in him. I was beginning to think something must be wrong because he is so much more awake and happy. Then I remembered I started this new supplement and I credit it to his big change in behavior. I am so glad to see him awake! I will be ordering this for as long as he lives."
          },
          {
            "date": "2020-08-22",
            "length_band": "Medium",
            "length_chars": 147,
            "post_id": "amzr_b2718ddafefb64228281e6ddb3fe76e8dfb1d7c0b7bc2c4f86b4af25babb9927",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "Product Name : 3Pcs Bird Perches. Materials: Made of natural wood and covered with random vibrant Colors of Quartz Sand.Size: 7.87 inches long.Package Including: 3pcs bird perchAn ideal place designed for your little friends to play and relax, easy install in the cage. Give your pet a treat and have hours of fun watching them play.",
              "details": {
                "Brand": "Vehomy",
                "Date First Available": "July 8, 2020",
                "Item model number": "HT0072-FBA",
                "Manufacturer": "Vehomy",
                "Material": "Wood",
                "Package Dimensions": "7.4 x 2 x 1 inches; 6.38 Ounces",
                "Product Care Instructions": "Hand Wash Only",
                "Target Species": "Parakeet"
              },
              "features": [
                "Naturally Grind Claws and Sharpen Beak: Bite-resistant and safe to chew, the quartz sand on the wooden pole grinds your bird\u2019s claws while it walks and sharpens its beak while it pecks.",
                "Materials and Size: Vehomy bird perch is made of natural quartz sands and pure nature wood inside with food grade color. Length: 7.87 inch / 20 cm, diameter: 0.78 inch / 2 cm, suitable for budgies, parakeets, cockatiels and other similar sized birds.",
                "Easy to Install: Metal screw on the top makes the bird toys easy to be hung in the cage and not easy to fall down, convenient for birds playing. A fun place for your bird to relax and play. your bird will surely enjoy it.",
                "Great Fun for Parrots: Vehomy bird perch toys will help to relieve the cage stress, boredom, and help the birds to do some exercises to strengthen their muscle."
              ],
              "product_id": "B08CKRFLXF",
              "title": "Vehomy Bird Perch Rough-surfaced Wooden Parrot Perch Random Vibrant Colors of Quartz Sand Covered Wood Stand Toys for Bird 3pcs"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "There is nothing I dislike about this product. Each one has a different imprint on the surface that makes them all interesting. My birds love them."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 4,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 1
      },
      "row_id": "humanual-amazon-v1:train:4b357953b9e8a6f3d1a1",
      "source_line_sha256": "3d5e90c253c5ae2afd25d5d70409f71cc28279c6cac7be54fee63a0bca59b5a7",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2021-09-30",
        "length_band": "Short",
        "length_chars": 55,
        "post_id": "amzr_9d63219d24533a0958be484bd0646e18ab2e46f9a76ab4824fa1a5afbe6a6b78",
        "product": {
          "category": "Pet_Supplies",
          "description_excerpt": "Amazon Basics flea and tick topical for Dogs kills fleas, flea eggs, flea larvae, ticks and chewing lice. The fast-acting, long-lasting treatment is formulated to protect your dog against dangerous pests. It breaks the flea life cycle by killing flea eggs and larvae and kills ticks, including deer ticks that may transmit Lyme disease. For best results, apply once a month to protect your dog from fleas and ticks.",
          "details": {
            "Country of Origin": "USA",
            "Date First Available": "November 13, 2019",
            "Item model number": "11293",
            "Manufacturer": "Amazon.com Services LLC.",
            "Product Dimensions": "1.25 x 5.75 x 4.35 inches; 1.76 Ounces"
          },
          "features": [
            "Same product, new look \u2013 This product is identical in manufacturing and formulation as the Solimo brand",
            "One 6 dose pack of Amazon Basics Flea and Tick Topical Treatment for X-Large Dogs (89 - 132 pounds), 6 Count (Previously Solimo)",
            "Amazon Basics Flea & Tick starts eliminating fleas and ticks within 24 hours of treatment",
            "Active ingredients Fipronil and (s)-methoprene prevent flea infestations by killing flea eggs, flea larvae, mosquitos, biting flies and chewing lice"
          ],
          "product_id": "B09R4MKCKJ",
          "title": "Amazon Basics Flea and Tick Topical Treatment for X-Large Dogs (89-132 lbs), 6 Count (Previously Solimo)"
        },
        "rating": 1.0,
        "review_type": "target",
        "text": "Sorry that this product did not help with fleas at all."
      },
      "uid": "amzu_0e7e06ab2f758b84a483d3345137e0313784da1c730f662235d07f75f4b16e30"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.7,
            "score_mean": 0.053541666666666675,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.17,
            "score_mean": 0.042083333333333334,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.95,
            "score_mean": 0.3744444444444444,
            "score_median": 0.4,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5058333333333332,
            "score_median": 0.67,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1) Earrings are cute basic studs, comfortable and look great after heavy wear (pool, shower, bed). 2) The back of one earring is insecure and slides off. 3) The back later fell off and was lost, requiring a replacement backing.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the cute appearance or comfort (misses key point 1, score 0). It notes the butterfly backs feel flimsy, which loosely matches the insecure back issue (partial match, score ~0.7). It does not mention the back actually falling off or being lost (misses key point 3, score 0). Coverage C = (0 + 0.7 + 0)/3 \u2248 0.233. The response adds unrelated commentary about plating wear, price, and overall quality, which is extra irrelevant content; assign a moderate penalty P \u2248 0.3. Final score = max(0, 0.233 - 0.3) = 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I've been wearing Amazon Essentials jewelry for years now, and while I love the price point, these studs are a bit disappointing for the quality I expect. The silver plating looks nice at first, but it's already showing signs of wear after just a couple of weeks of light daily wear. The instructions mention it can wear off with heavy use, but even with moderate use, the shine is fading faster than I'd like. I also noticed the butterfly backs feel a little flimsy compared to some of the other brands I've tried. They're cute and sparkly, but if you're looking for something that lasts, you might want to save up for a higher-end option. Still, for the price, they're okay for occasional wear or as a backup pair.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl4",
            "judge_key_points": "1) The earrings are cute basic studs, comfortable and look great after wearing for days in water and bed. 2) The backing on one earring is insecure and slides off, causing worry about loss. 3) The backing eventually fell off and was lost, requiring a replacement.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the earrings being cute, comfortable, or looking great after water exposure, nor does it address the insecure backing issue or its loss; it instead discusses plating wear, which is unrelated, so it fails to reflect any key point (m\u22480 for all) and adds irrelevant content, leading to a penalty. Final score is 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "These are okay for the price, but I was expecting a bit more shine for a \"plated sterling silver\" pair. I've been wearing them for about three weeks now, and the silver plating has already started to look dull and scratchy on the posts. The CZ stones themselves are fine and don't look cheap, but the metal backing is definitely wearing off faster than I anticipated. I've learned to store them in a separate pouch to keep them from rubbing against each other, which helps a little, but I'm not sure how long they'll last before I need to take them to a jeweler for replating. For a quick, affordable pair of earrings, they work, but don't expect them to hold up like solid silver.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.5",
            "judge_key_points": "1) The earrings are cute basic diamond studs, worn for 2 days in pool, shower, bed, still look great and feel barely noticeable. 2) The backing is insecure; it slides off and the reviewer worries about losing the earring. 3) The backing actually fell off and was lost, requiring a new backing.",
            "judge_score": 0.3,
            "judge_thought": "The generated response fully repeats the positive praise (matches point 1 well, m=1.0). It mentions the backing is a bit loose and may not hold up, which reflects the insecurity but is less severe than 'slides off', so partial match (m\u22480.8). It does not mention the backing actually falling off or being lost, missing point 3 (m=0). Coverage C = (1.0+0.8+0)/3 = 0.6. The extra speculation about sending it back to Amazon or gluing it is unsupported and adds irrelevant detail, incurring a moderate penalty P\u22480.3. Final score = max(0, 0.6\u20110.3) = 0.3.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "These are really cute basic diamond studs. I have been wearing them for 2 days straight, including in a pool, the shower, and to bed. They still look great and I barely feel them. The only reason I removed one star, is the back of one of the earrings is a bit loose. I don't think the butterfly back is going to hold up long term. I'll have to send it back to Amazon for a replacement or try to glue it down myself.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The earrings are cute basic studs that are comfortable and look great even after wearing in pool, shower, and bed. 2) The butterfly back is insecure and can slide off, leading to loss of an earring. 3) The back actually fell off and was lost, requiring a replacement backing.",
            "judge_score": 0.87,
            "judge_thought": "The generated response mentions wearing them daily, comfort, shower/bed use matching point 1; notes butterfly backs slide off and one was lost matching points 2 and 3; adds extra comments about value and appearance but no contradiction, so minor penalty.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "I've worn these everyday for a few weeks now. They feel lightweight and comfortable, perfect for wearing in the shower or just lounging in bed. The silver looks great and the CZ stones sparkle well. My only issue is that the butterfly backs slide off easily; mine actually fell out of my ear and was lost. They are a great value for the price and look much more expensive than they are.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) User finds the earrings cute, basic diamond studs, comfortable and looks great after wearing in pool, shower, bed. 2) User criticizes the earring back for sliding off and being insecure, causing concern about loss. 3) User updates that the back actually fell off and was lost, needing a replacement backing.",
            "judge_score": 1.0,
            "judge_thought": "The generated response is an exact copy of the ground truth, perfectly reflecting all three key points with no extra or conflicting content, so m_i=1 for each point, coverage=1, penalty=0, final score=1",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "These are really cute basic diamond studs. I have been wearing them for 2 days straight, including in a pool, the shower, and to bed. They still look great and I barely feel them. The only reason I removed one star, is the back of one of the earrings just slides right off, and isn't really secure at all. I\u2019m just worried about the back sliding off and losing the earring. Other than that, they\u2019re great! Update: the back ended up falling off and getting lost. I will have to find another backing for the earring :(",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The earrings look great and are comfortable even after wearing them in water and daily use. 2) The backing is insecure and can slide off, leading to loss of the earring. 3) Overall positive impression except for the backing issue.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the earrings looking great after water exposure or comfort, contradicts the backing issue by saying the backs are secure, and adds unrelated details about plating wear and storage; thus it largely fails to reflect the key points with a low coverage and moderate penalty.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been wearing these for about three months now, and they are exactly what I expected for the price point. The \"plated\" description in the features is key here\u2014don't expect these to last forever if you wear them every single day without care. I\u2019ve noticed a slight dulling on the silver parts after a few months of regular wear, but the cubic zirconia stones still look great. A few things to note: 1. The butterfly backs are secure but can be a pain to take off if you have sensitive ears. I usually just wiggle them gently. 2. The \"princess\" cut stones are sharp and sparkly, while the round ones are more classic. I bought the mixed pack, which is nice for variety. 3. Storage matters. I keep them in the original pouch when not wearing them, as the instructions suggest, to prevent scratching. For everyday wear or occasional events, they\u2019re a solid, affordable option. If you\u2019re looking for heirloom-quality sterling silver that never needs replating, look elsewhere. But for the cost? They\u2019re a good buy. Just treat them gently. Rating: 4/5 stars. Deducted one star because the plating wear is noticeable faster than I\u2019d like, but they\u2019re still pretty.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The earrings are cute basic diamond studs that were worn for a couple of days including pool, shower, and bed and still looked great and felt comfortable. 2) The butterfly backing is loose and slides off, leading to loss of an earring. 3) Update: the backing fell off and was lost, requiring a replacement backing.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mirrors all three key points: it describes the cute studs, mentions wearing them in pool/shower/bed and they looked great, notes the loose butterfly backing that slipped off and caused loss, and provides an update about the missing backing leaving only one earring. No extra or conflicting content is present, so coverage is 1.0 and penalty is 0, yielding a final score of 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "These are cute basic diamond studs. I wore them for a couple of days, including at the pool, in the shower, and even to bed, and they still looked great. However, the butterfly backing is super loose and kept slipping off; I lost one in the shower already. Update: I just realized the missing backing was on the one I lost, so now I only have one earring left.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:c62f099073f815c66c15:prompt4",
      "category": "Clothing_Shoes_and_Jewelry",
      "cohort": "seen",
      "eval_pair_id": "amzp_789df7b49740ffd8523248580ce31c7d1e82d77b70bd3edd9ace4d96e9e468f1",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2018-01-13",
            "length_band": "Long",
            "length_chars": 973,
            "post_id": "amzr_f395220b772dbff73dabb9ac13ae1caafde9ec4acc876f5e16ee826466ada7d8",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "Fluid fashion",
              "details": {
                "Date First Available": "June 14, 2023",
                "Department": "womens",
                "Manufacturer": "Cat Footwear"
              },
              "features": [
                "60% Leather, 40% Synthetic",
                "Imported",
                "Rubber sole",
                "Heel measures approximately 1.5 inches\""
              ],
              "product_id": "B0C81VG3JS",
              "title": "Cat Footwear Women's Taylor Wp Boot"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I wanted a cute pair of waterproof boots that I could wear shoveling, in the cold weather, or just in winter with a cute outfit. I hate the look of the typical rain boots and duck boots and wanted something that could be warm, waterproof but also stylish. I ordered a half size down like the other reviews had said (I\u2019m a 10, I ordered a 9.5) & they fit perfectly. These were delivered on time in a regular shoe box packaging. The first two days they were so comfortable and I didn\u2019t think I needed to break them in. Day 3 the toes on my left foot (which is slightly bigger than my right), were rubbing against the shoe, and so was my right ankle. It was nothing painful just a normal \u201cbreaking a shoe in\u201d feeling. Other than that they are exactly what I\u2019ve been looking for. I wear them with cute boot socks or leg warmers to work, or thick socks to go outside in the snow and rain in. I love them so much, I\u2019m even thinking about ordering them in black! Highly recommend."
          },
          {
            "date": "2019-01-16",
            "length_band": "Long",
            "length_chars": 370,
            "post_id": "amzr_f044057dcc423bf9e6558c9b6e0719c61aa095097d58ed51c473bb7b5f687f05",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "",
              "details": {
                "Date First Available": "May 29, 2023",
                "Department": "mens",
                "Manufacturer": "Bruno Marc"
              },
              "features": [],
              "product_id": "B098Y9GJ19",
              "title": "Bruno Marc Men's Wingtip"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I ordered a pair of these for my boyfriend for Christmas. They ended up being too big (not a sizing issue, my boyfriend just thought his foot was bigger than it actually is!). I was able to easily return these for a smaller size. They are beautiful shoes and they fit perfectly! He loves them. I hope they hold up well, because he plans on ordering more in other colors!"
          },
          {
            "date": "2019-12-22",
            "length_band": "Medium",
            "length_chars": 153,
            "post_id": "amzr_b5369f2b0c138a8cc6da6e0bc00bd421c14866eae56a721c3eb49def8d983671",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "",
              "details": {
                "Date First Available": "February 21, 2019",
                "Department": "Womens",
                "Package Dimensions": "14.85 x 12.95 x 2.8 inches; 14.39 Ounces"
              },
              "features": [
                "100% Nylon",
                "Pull On closure",
                "Hand Wash Only",
                "Material: The boho sweater is made of soft, stretchy, breathable fabric which can provide exceptional comfort and durability."
              ],
              "product_id": "B07NY1SXKK",
              "title": "Misassy Womens Boho Open Front Cardigan Long Sleeve Striped Color Block Loose Knit Lightweight Sweater Coats"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Love this cardigan, it\u2019s really comfortable and made well and goes with everything! I got a medium and I\u2019m 5\u20199, 150 lbs. I get so many compliments on it."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 4,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Long",
        "rating": 4
      },
      "row_id": "humanual-amazon-v1:train:c62f099073f815c66c15",
      "source_line_sha256": "47f0c624104d08ec8cae02452479937449d5d4fa5e27315ade87f3e2d74d0342",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2020-07-13",
        "length_band": "Long",
        "length_chars": 538,
        "post_id": "amzr_0b164c6495b52cc21f35a48097636df16a6e0ee70dc915b6a1f2ea360480d99b",
        "product": {
          "category": "Clothing_Shoes_and_Jewelry",
          "description_excerpt": "",
          "details": {
            "Date First Available": "April 19, 2023",
            "Department": "Womens",
            "Manufacturer": "Amazon Essentials"
          },
          "features": [
            "Classic stud earrings featuring cubic zirconia stones in four-prong basket settings set in sterling silver",
            "Butterfly backing",
            "Crafted in .925 sterling silver",
            "These silver pieces are built for longevity. They feature a metal plating or flashing, or an electrocoating for a more lustrous appearance, but it can wear off with long-term or heavy use."
          ],
          "product_id": "B0C2Z1NKMY",
          "title": "Amazon Essentials Plated Sterling Silver Cubic Zirconia Stud Earrings (Round & Princess)"
        },
        "rating": 4.0,
        "review_type": "target",
        "text": "These are really cute basic diamond studs. I have been wearing them for 2 days straight, including in a pool, the shower, and to bed. They still look great and I barely feel them. The only reason I removed one star, is the back of one of the earrings just slides right off, and isn't really secure at all. I\u2019m just worried about the back sliding off and losing the earring. Other than that, they\u2019re great! Update: the back ended up falling off and getting lost. I will have to find another backing for the earring :("
      },
      "uid": "amzu_113518c69a5fdfdf630256dab2d7bc18a8666599b4725f1a5df20cc39e0b5239"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.7,
            "score_mean": 0.08093750000000001,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.7,
            "score_mean": 0.26458333333333334,
            "score_median": 0.25,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.9,
            "score_mean": 0.47777777777777763,
            "score_median": 0.6,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5458333333333333,
            "score_median": 0.8,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "1) The group of women is interesting. 2) There are many differing characters that stir the plot.",
            "judge_score": 0.3,
            "judge_thought": "The generated response mentions a mix of characters and gives a positive overall impression, partially reflecting the idea of differing characters (\u22480.5) but does not explicitly note the women group as interesting (\u22480.5). Coverage C\u22480.5. The response adds extensive unrelated commentary and personal opinion, which is extra but not contradictory, yielding a modest penalty around 0.2. Final score = 0.5 - 0.2 = 0.3.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a cozy mystery that doesn't feel like a chore! I usually find these \"club\" mysteries a bit repetitive, but *The Gordonston Ladies Dog Walking Club* by Karen Joy Fowler kept me guessing from the very first page. The setting in Savannah is vivid, and I loved the mix of characters\u2014from the retired accountant with his secret to the former showgirl who definitely needed that plastic surgery (I had a feeling about that one!). The pacing was perfect for a rainy afternoon read. It's not overly long at 254 pages, so it didn't drag, but the plot twists were sharp enough to keep me turning the pages. The way the author weaves in the political race for mayor alongside the dog walking gossip adds a nice layer of real-world stakes. Highly recommended if you like your mysteries with a touch of Southern charm and plenty of intrigue.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl8",
            "judge_key_points": "1) The group of women is interesting. 2) The group contains many differing characters that stir the plot.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the interesting group of women nor the variety of characters; instead it criticizes the plot and calls characters caricatures, contradicting both key points. Match scores are 0 for each point. It also adds unrelated negative commentary, so a penalty of about 0.7 is applied. Final score = max(0, (0+0)/2 - 0.7) = 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": null,
            "refine_round": null,
            "response": "The plot is a bit too far-fetched for my taste, with characters who seem more like caricatures than real people. The writing feels forced in places, trying too hard to be funny or dramatic. While the mystery element has some appeal, the execution leaves much to be desired.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.5",
            "judge_key_points": "1) The group of women is interesting. 2) There are many differing characters that stir the plot.",
            "judge_score": 0.55,
            "judge_thought": "The generated response repeats the first point exactly (m1=1.0). It hints at differing characters by mentioning motives not being what they seem, which partially matches point 2 (m2\u22480.7). Coverage C\u22480.85. However, it adds unrelated criticism about the plot being ungripping, which is extra content not present in the ground truth, incurring a moderate penalty (~0.3). Final score = 0.85-0.3 \u2248 0.55.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "An interesting group of women, and as with all women'things, their motives were not what they seemed. But the plot was not nearly as gripping as the promise of the blurb. A bit too much \"small town\" drama for my taste, lacking the solid mystery I usually look for.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The group of women is interesting. 2) There are many differing characters that stir the plot.",
            "judge_score": 0.8,
            "judge_thought": "The generated response captures both key points: it calls the group of women interesting (match 1.0) and notes differing characters driving the plot (match 1.0). It adds extra commentary about gossiping, which is speculative and not in the ground truth, incurring a small penalty (~0.2). Final score = 1.0 coverage - 0.2 penalty = 0.8.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "An interesting group of women with differing characters drive the plot in this fast-paced mystery. And as with all women\u2019s groups, they are a bit too fond of gossiping about their neighbors.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "The response notes an interesting group of women and that, as with all women's groups, there are many differing characters that stir the plot.",
            "judge_score": 1.0,
            "judge_thought": "The generated response is identical to the ground truth, perfectly reflecting both key points with no extra or conflicting content, giving full coverage and no penalty.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "An interesting group of women, and as with all women't groups a lot of differing characters to stir the plot.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The group of women is interesting. 2) There are many differing characters that stir the plot.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions a mix of character archetypes, partially reflecting the second key point about differing characters, but it does not address the interesting group of women at all. Match scores: m1\u22480.0, m2\u22480.5. Coverage\u22480.25. The response adds extensive unrelated review content, incurring a moderate penalty (~0.3). Final score\u2248max(0,0.25-0.3)=0.0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "4.0 out of 5 stars **A fun, breezy read with a twist** I picked this up because I\u2019m a huge fan of cozy mysteries with a Southern setting, and the premise of a dog walking club turned murder scene sounded right up my alley. The atmosphere in Savannah is described so well you can practically feel the humidity and the heat. The characters are a fun mix of archetypes\u2014the retired accountant, the former showgirl, the model. I appreciated that it wasn\u2019t just a simple whodunit; there were layers to the secrets, especially regarding the widower running for mayor. The pacing was steady, and I liked how the author wove in the dog walking element without it feeling forced. It\u2019s a light read, perfect for a beach vacation or a lazy afternoon. Not the most complex plot I\u2019ve ever seen, but it\u2019s entertaining, the clues are fair, and the climax definitely had me guessing until the end. If you like your mysteries with a side of cocktails and canine companions, give this one a try. Kindle edition was easy to navigate, and the text size was comfortable for my aging eyes. Recommended for fans of the genre.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The group of women is described as interesting. 2) The group contains many differing/varied characters that drive or stir the plot.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states 'Interesting group of women' matching key point 1 fully, and 'a variety of characters that drive the plot' matching key point 2 fully. No extra or conflicting content is present, so penalty is 0. Coverage is (1+1)/2 = 1, final score = 1 - 0 = 1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Interesting group of women and a variety of characters that drive the plot along nicely.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:864bac81f87121da9fe2:prompt2",
      "category": "Kindle_Store",
      "cohort": "seen",
      "eval_pair_id": null,
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2013-05-28",
            "length_band": "Medium",
            "length_chars": 155,
            "post_id": "amzr_a4190ba2d788abb1eaaed4df581e26e6e5b8160cf98c904a3e9d0cae79cf0799",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "Review \"Georgette Heyer is unbeatable\" -- India Knight \"Sparkling\" * Independent * \"My favourite historical novelist - stylish, romantic, sharp, and witty. Her sense of period is superb, her heroines are enterprising, and her heroes dashing. I owe her many happy hours\" -- Margaret Drabble --This text refers to an alternate kindle_edition edition. About the Author Georgette Heyer, who wrote over fifty novels died in 1974. --This text refers to an alternate kindle_edition edition. Excerpt. \u00a9 Reprinted by permission. All rights reserved. One Silence had reigned over the dining-room since his lord",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "1083 KB",
                "Language": "English",
                "Print length": "386 pages",
                "Publication date": "September 1, 2011",
                "Publisher": "Sourcebooks Casablanca; Reprint edition (September 1, 2011)",
                "Screen Reader": "Supported",
                "Sticky notes": "On Kindle Scribe"
              },
              "features": [
                "The Queen of Regency Romance, bestselling author Georgette Heyer, brings her trademark charm and wit to this pitch-perfect comedic romp.",
                "A past dispute . . .",
                "When the irascible Lord Darracott's eldest son dies unexpectedly, the noble family must accept their estranged Yorkshire cousin as heir apparent. They are convinced he will prove to be a sadly vulgar person, but nothing could have prepared the beleaguered family for the arrival of Major Hugo Darracott . . .",
                "A present deception . . ."
              ],
              "product_id": "B005CKKFG2",
              "title": "The Unknown Ajax (Regency Romances Book 19)"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Mrs Heyer has long delighted me and this 're_read of an old favourite was delicious. Good historical view for young readers.. [redacted email]"
          },
          {
            "date": "2013-07-18",
            "length_band": "Medium",
            "length_chars": 103,
            "post_id": "amzr_ccc7ad3ae755e6a4f99727752b707b0dccdfd5149ac14349a34832bd34253131",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "About the Author Christine Merrill wanted to be a writer for as long as she can remember. During a stint as a stay-at-home-mother, she decided it was time to write that book. She could set her own hours and would never have to wear pantyhose to work! It was a slow start but she slogged onward and seven years later, she got the thrill of seeing her first book hit the bookstores. Christine lives in Wisconsin with her family. Visit her website at: --This text refers to an alternate kindle_edition edition.",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "922 KB",
                "Language": "English",
                "Print length": "283 pages",
                "Publication date": "August 1, 2007",
                "Publisher": "Harlequin Historical; Original edition (August 1, 2007)",
                "Screen Reader": "Supported",
                "Sticky notes": "On Kindle Scribe"
              },
              "features": [
                "In this Regency-era romance, a lady hopes to seduce a handsome stranger in order to avoid being forced to accept an arranged marriage.",
                "Miss Esme Canville\u2019s abusive father is resolved to marry her off\u2014but she won\u2019t submit tamely to his decree. Instead, she\u2019ll offer herself to notorious rake Captain St. John Radwell and enjoy all the freedom of a mistress!St. John is intent on mending his rakish ways. He won\u2019t seduce an innocent virgin. But Esme is determined, beautiful and very, very tempting. . . ."
              ],
              "product_id": "B000U0NSV8",
              "title": "An Unladylike Offer (The Radwells Book 2)"
            },
            "rating": 2.0,
            "review_type": "demonstration",
            "text": "Like chocolates for the mind with a bit of sugar and spice. A tried and true summer beat the heat read."
          },
          {
            "date": "2013-08-27",
            "length_band": "Long",
            "length_chars": 379,
            "post_id": "amzr_d08347dc5936a798afed23ffc6d6cff67d78efbe170892c1bdb8d11153e0dd28",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "Unknown \"Much Ado About Marriage is a delicious flirtation that pits a staid Englishman against a fey Scots woman. Humor, folklore, and sizzling love scenes lend this novel the perfect incentive for not wanting to put it down.\" - Heide Katros, Winter Haven News Chief \"A very readable romance. The town is quaint, the murder club cute, and the storyline and our heroine the equal of Lois herself. Fans of the first book set in Glory will be pleased.\" - Romantic Times on LOIS LANE TELLS ALL \"Readers who met Caitlyn in SLEEPLESS IN SCOTLAND will applaud her return in the second of Hawkins' delightfu",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "3474 KB",
                "Language": "English",
                "Print length": "404 pages",
                "Publication date": "August 31, 2010",
                "Publisher": "Pocket Books; Reprint edition (August 31, 2010)",
                "Screen Reader": "Supported",
                "Sticky notes": "On Kindle Scribe"
              },
              "features": [
                "New York Times",
                "bestselling author Karen Hawkins\u2019s first historical romance, originally titled ONE LUCKY LORD and written as Kim Bennett, has been rewritten to tie into both her \"MacLeans\" series and her new \"Hurst Amulet\" series.Fearless English earl Thomas Wentworth scoffs at failure, trusting in the legendary Wentworth luck to safeguard his spy mission to Scotland. But a single encounter with saucy Scotswoman Fia Maclean turns Thomas's mission topsy-turvy. All-too-innocently entangling him in his enemy's trap, the beautiful Fia gives Thomas no escape but to marry her. But Thomas doesn't know just what he's marrying into. With an ancient amulet in Fia's possession, she wields even more power than her sharp tongue and quick wit. Her cousin, Douglas Maclean stole the magical amulet from the White Witch, Maeve Hurst, and gave it to Fia for safekeeping. But now, with Fia in London with Thomas, the amulet falls into the hands of Queen Elizabeth. Thinking it will be safe in the queen's hands, Fia leaves it be and focuses on bedeviling her newly betrothed!"
              ],
              "product_id": "B003V1WTOK",
              "title": "Much Ado About Marriage (The MacLean Curse Book 6)"
            },
            "rating": 3.0,
            "review_type": "demonstration",
            "text": "This tale is saved by the authors ability to craft interesting characters and some rollicking scenes set in ELizabethan England. What becomes mundane is the tendency of today.s historical fake tellers to get wrapped up in graphic sexual scenes. So much nicer to have left some things to the readers usually vivid imagination. There was at least one glaring anachronism in speech."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Medium",
        "rating": 3
      },
      "row_id": "humanual-amazon-v1:train:864bac81f87121da9fe2",
      "source_line_sha256": "1447090e9e29d8ae31413a2f91873efd47cdf7bbfe2c9167961829d0658272ea",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2014-03-09",
        "length_band": "Medium",
        "length_chars": 114,
        "post_id": "amzr_8fa82402179c241f2832e6500b7783fb59ff65b65011700ff048af96476d98a0",
        "product": {
          "category": "Kindle_Store",
          "description_excerpt": "About the Author The life of Duncan Whitehead, winner of the 2013 Reader\u2019s Favorite International Book Award and Gold Medalist, is as quirky as his works. Born in 1967, he served in the Royal Navy in embassies across South America and was an amateur boxer. He worked as a purser on some of the world\u2019s largest super yachts and visited many exotic places. He\u2019s also an instructor of English as a foreign language, fluent in Spanish, and a children\u2019s soccer coach. Duncan retired to Savannah, Georgia, to pursue his passion\u2014writing. Mindful that we all harbor secrets and inspired by the locale\u2019s odd c",
          "details": {
            "Enhanced typesetting": "Enabled",
            "File size": "785 KB",
            "Language": "English",
            "Print length": "254 pages",
            "Publication date": "November 19, 2013",
            "Publisher": "Thomas & Mercer (November 19, 2013)",
            "Screen Reader": "Supported",
            "Sticky notes": "On Kindle Scribe"
          },
          "features": [
            "Little is what it seems to be in a leafy Savannah neighborhood as members of an afternoon cocktail and dog walking club mourn a neighbor\u2019s death. Jealousies surface when friends vie for the widower running for mayor. An old woman with an infamous uncle plots to avenge a wrong. Memories haunt a once successful children\u2019s writer. And a model has won the trip of a lifetime.",
            "But a killer lurks and secrets unfold, as does a web of deceit. Is anyone really who he or she seems to be? A mysterious South American, a young Italian count, and a charitable nephew add suspicion and intrigue, as do an enigmatic organization linked to organized crime, a handsome firefighter, and three widows with hidden agendas. What\u2019s a retired accountant\u2019s secret, and why did a former showgirl really have plastic surgery?",
            "The plot thickens, the Georgia temperature rises, and someone is destined for an early unmarked grave. The truth contorts to a climax that leaves readers breathless."
          ],
          "product_id": "B00FN2KR2G",
          "title": "The Gordonston Ladies Dog Walking Club"
        },
        "rating": 3.0,
        "review_type": "target",
        "text": "An interesting group of women, and as with all women't groups a lot of differing characters to stir the plot."
      },
      "uid": "amzu_1210b173f0287070c1c95091ae4d9fe210206f21941093af14d62034aa055a5f"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.9,
            "score_mean": 0.23260416666666664,
            "score_median": 0.15,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.85,
            "score_mean": 0.45,
            "score_median": 0.6,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.6868055555555554,
            "score_median": 0.75,
            "score_min": 0.15,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.7472222222222222,
            "score_median": 0.9,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The bulbs serve as perfect replacements. 2) They are suitable for the RV's light bulbs.",
            "judge_score": 0.75,
            "judge_thought": "The generated response states the bulbs work well in a trailer/RV, fit perfectly, and are a good upgrade, matching both key points strongly (m\u22480.9 each). Coverage \u22480.9. The extra detail is relevant and not contradictory, so a small penalty (~0.15) is applied. Final score = 0.9 - 0.15 = 0.75.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I've tried a bunch of different LED bulbs for my trailer over the years, and most of them either flicker, blow out after a few months, or require you to fiddle with the polarity. These HOTSYSTEM bulbs are a refreshing change. The \"plug and play\" aspect is the real winner here. As the description says, you don't need to worry about which leg is positive or negative. I just popped them in, and they worked immediately. No testing, no twisting wires. The warm white color is nice and bright without being that harsh, blinding blue-white that some LEDs have. It looks much more natural on the road. I installed them in the tail/brake and turn signal positions on a 12V setup, and they fit perfectly. The build quality feels solid\u2014the copper brackets and gold wire welding seem like they'll hold up better than the cheaper aluminum ones I've used before. For the price of a 10-pack, you really can't beat the value. They run cool and use way less power than my old halogens. If you're looking for a straightforward upgrade for your RV or car that actually installs without a headache, give these a shot. Five stars for ease of use and brightness.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) The bulbs serve as perfect replacements. 2) They are intended for use in an RV.",
            "judge_score": 0.15,
            "judge_thought": "The generated response praises the product and mentions it works great and is easy to install, which loosely aligns with the idea of being a good replacement (partial match ~0.5) but does not mention RV usage (no match ~0). Coverage C = (0.5+0)/2 = 0.25. The extra positive comments are irrelevant to the specific claim and add minor unnecessary detail, giving a small penalty P \u2248 0.1. Final score = 0.25 - 0.1 = 0.15.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "Great product. Works great. Easy to install",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.75",
            "judge_key_points": "1) The bulbs are perfect replacements. 2) They are intended for the light bulbs in our RV.",
            "judge_score": 0.8,
            "judge_thought": "The generated response states 'Perfect replacements' matching key point 1 fully (m1=1.0). It refers to 'light bulbs in my camper' which is essentially the same context as RV, so key point 2 is mostly reflected (m2\u22480.8). Coverage C=(1.0+0.8)/2=0.9. The extra phrase 'Easy to install' adds irrelevant detail but does not contradict anything, so a small penalty P\u22480.1 is applied. Final score = 0.9-0.1 = 0.8.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Perfect replacements for the light bulbs in my camper. Easy to install.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) The bulbs serve as perfect replacements (fit/compatible). 2) They are intended for use in an RV.",
            "judge_score": 0.9,
            "judge_thought": "The generated response states 'Perfect fit for my RV' and 'Direct replacement for my old bulbs', clearly covering both key points of perfect replacement and RV use (m1=1, m2=1). It adds extra details about easy install and brightness, which are not contradictory but slightly unnecessary, so a small penalty of 0.1 is applied. Final score = 1.0 - 0.1 = 0.9.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "Perfect fit for my RV. Direct replacement for my old bulbs. Easy install and great brightness",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "The response states that the product is a perfect replacement for the light bulbs in our RV.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly matches the ground truth phrase, reflecting the sole key point perfectly with no extra or conflicting content, resulting in full coverage and zero penalty.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Perfect replacements for the light bulbs in our RV",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl1",
            "judge_key_points": "1) The bulbs serve as perfect replacements. 2) They are intended for use in an RV.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions bright light and good quality, which does not address the replacement aspect (m1\u22480.1) and does not mention RV usage at all (m2=0). Coverage is very low (0.05). The extra praise is irrelevant to the ground truth, incurring a small penalty (~0.1). Final score = max(0, 0.05-0.1) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 1,
            "privileged": null,
            "refine_round": null,
            "response": "Nice bright light. Good quality.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "The response states that the product is a perfect replacement for the light bulbs in our RV.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth statement (only adding a period), fully reflecting the key point with no extra or conflicting content, so m=1, P=0, final score=1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Perfect replacements for the light bulbs in our RV.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:train:55c8a52ae644247fb7a3:prompt2",
      "category": "Automotive",
      "cohort": "seen",
      "eval_pair_id": null,
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2016-06-25",
            "length_band": "Medium",
            "length_chars": 143,
            "post_id": "amzr_a457b3c2a4dcce0fcdb1a6185434647d2a022f7a8455f277f2f35b6c67b45789",
            "product": {
              "category": "Automotive",
              "description_excerpt": "Bully aluminum side steps add convenient entry access to your vehicle. Constructed from a one piece cast, these 1 inch steps feature a strong and durable design. Lightweight and paintable, they mount to most trucks and SUVs, with non-slip step pads for reliable footing. Chevrolet: 1999-2020 Silverado 1500/ 2500/ 3500 , 2002- 2013 Avalanche , 1988-1999 Full Size C/K Truck 1500/ 2500/ 3500 All Cab Sizes, 1973- 2018 Suburban , 1992- 2018 Tahoe , 1973- 1987 Full Size Pickups , 1976- 1993 S-10 All Cab Sizes, 1976- 1996 Blazer , 2000- 2008 Trailblazer , 1967- 1996 Full Size Vans . Dodge : 1994-2018",
              "details": {
                "Brand": "Bully",
                "Color": "Polished Aluminum",
                "Date First Available": "March 12, 2016",
                "Exterior": "Aluminum, Chrome",
                "Exterior Finish": "Aluminum, Chrome",
                "Is Discontinued By Manufacturer": "No",
                "Item Dimensions LxWxH": "12 x 12 x 12 inches",
                "Item Weight": "16 Pounds"
              },
              "features": [
                "1 inch Stirrup side steps mount to trucks or SUVs",
                "Design with non-slip step pad",
                "Polished die-cast aluminum brackets, strong, lightweight and paintable",
                "Solid mount/ Includes all hardware/ instructions included for easy install"
              ],
              "product_id": "B0BSTHVBCB",
              "title": "Bully AS-600-2 Polished Aluminum Universal Fit Truck Side Step Set of 4 for Trucks from Chevy (Chevrolet), Ford, Toyota, GMC, Dodge RAM, Jeep"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Excellent product. Works perfectly on my 2010 Ford 150. Easy to install. 20 minutes each side. Comes 2 in a box even though it says sold singly"
          },
          {
            "date": "2016-09-07",
            "length_band": "Short",
            "length_chars": 9,
            "post_id": "amzr_b7ca6cfa7662a0c178f6a6c246a8c487fdced77bcffe7be7f5f8fa887ee1d714",
            "product": {
              "category": "Automotive",
              "description_excerpt": "Brake Control O.E.M. style wire harness are designed for models equipped with trailer tow package. The quick and easy plug in style connections reduces possible short circuits and cross wiring of brake controls. This custom pre-wired brake control harness will mate directly between your vehicle and several Reese Towpower brake controls. To be used with the following vehicles: 2009-2013 Flareside; 2012-2013 Flex with Factory 7-Way; 2012 Lincoln MKT with Factory Tow Package; 2009-2013 Styleside; 2009-2013 SuperCrew.",
              "details": {
                "Brand": "Reese",
                "Connector Type": "Plug in",
                "Date First Available": "September 25, 2014",
                "Exterior": "Smooth",
                "Item Dimensions LxWxH": "9 x 4.3 x 1.7 inches",
                "Item Weight": "0.02 Pounds",
                "Item model number": "78124",
                "Manufacturer": "Reese Towpower"
              },
              "features": [
                "Custom pre-wired Ford F-150 brake control harness.Material:Copper",
                "O.E.M. Style Wire Harness",
                "Reduces possible short circuits and cross wiring of brake controls",
                "Quick and easy plug in style"
              ],
              "product_id": "B00NW8PSUO",
              "title": "Reese Towpower Trailer Brake Controller Harness, Compatible with Select Ford Expedition, F-150, F-250 Super Duty, F-350 Super Duty, F-450 Super Duty, F-550 Super Duty, Flex : Lincoln MKT, Navigator"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Excellent"
          },
          {
            "date": "2016-10-11",
            "length_band": "Short",
            "length_chars": 63,
            "post_id": "amzr_42f83a1e04cb445d78bba1fadb532f4bda640eb45c9a865a3ce53dc1db1dd479",
            "product": {
              "category": "Automotive",
              "description_excerpt": "Dometic 310 Series- Best In class gravity flush. 100 percent vitreous ceramic bowl. 360 degrees vortex flush pattern. Powerful swirl-jet cleaning action. Lightweight. Long, full-sized adult seat. Tall, ergonomic seat height- 18 inches. Easily accessible water line connection. Two-bolt installation. Optional hand spray. Low-maintenance, reliable operation. Two-year warranty.",
              "details": {
                "Brand": "Dometic",
                "Color": "Bone",
                "Date First Available": "August 27, 2010",
                "Is Discontinued By Manufacturer": "No",
                "Item Dimensions LxWxH": "19 x 15 x 20 inches",
                "Item Weight": "23.5 Pounds",
                "Item model number": "302310073",
                "Manufacturer": "Dometic"
              },
              "features": [
                "Bone Color (off white)",
                "Best In Class gravity flush",
                "Comfortable 18\" ergonomic seat height",
                "Power flush, swirl-jet bowl rinse action ensures a cleaner rinse"
              ],
              "product_id": "B01C7CDMTE",
              "title": "Dometic 310 Series Standard Height Toilet, Bone"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Swapped our plastic toilet out and very satisfied. Good quality"
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:train:55c8a52ae644247fb7a3",
      "source_line_sha256": "ac14fb389be06a713f82169154efdf9b51d23915774edbb5a44fca8b442858b2",
      "split": "train",
      "split_label": "Train / seen user",
      "target": {
        "date": "2017-01-12",
        "length_band": "Short",
        "length_chars": 50,
        "post_id": "amzr_85e3d91c678bb587d19154ebb8c0f74a7b5c0ded565a4691bb516bab2aec90d7",
        "product": {
          "category": "Automotive",
          "description_excerpt": "",
          "details": {
            "Auto Part Position": "Lower",
            "Brand": "HOTSYSTEM",
            "Color": "1156-18smd Warm white",
            "Date First Available": "February 9, 2015",
            "Is Discontinued By Manufacturer": "No",
            "Item Package Quantity": "1",
            "Item Weight": "2.4 ounces",
            "Item model number": "HS-C00879B1x10+W3"
          },
          "features": [
            "Easy Installation: No need to distinguish positive and negative electrode of the bulb, directly insert,plug and play.",
            "Long life and Energy-saving: Pure copper brackets,pure gold wire welding process ,so circuits are not easily oxidized.Just 1/10 of the energy consumption of traditional halogen bulb.",
            "Applications: Internal light, Tail lights, Turn signal light, Corner light, Parking light, Side marker and Backup lights",
            "Cross Reference: 1156 BA15S 1073 1095 1141 1156 1156A 1156NA 1195 12088 1295 1295NA 199 2396 3497 631 7506 93 97 97A 97NACross Reference:1156A 1156NA 1156 P21W BA15S 87 93 97 97A 97NA 199 631 1073 1095 1141 1195 1295 2396 3497 5007 5008 7506 12088 1295NA etc."
          ],
          "product_id": "B00TDDCGNO",
          "title": "HOTSYSTEM LED Light Bulbs 1156 1141 7506 P21W BA15S 18-5050SMD for Car RV SUV Camper Trailer Trunk Interior Reversing Backup Tail Turn Signal Corner Parking Side Marker Lights(WarmWhite,Pack of 10)"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "Perfect replacements for the light bulbs in our RV"
      },
      "uid": "amzu_03873356a23caed8d879aa6068ebee532743343394e1236723edd359772103bd"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.57,
            "score_mean": 0.06776041666666667,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.38,
            "score_mean": 0.12729166666666666,
            "score_median": 0.08,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.84,
            "score_mean": 0.2716666666666667,
            "score_median": 0.26,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 0.83,
            "score_mean": 0.31763888888888897,
            "score_median": 0.28,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1) The reviewer experienced smooth performance with no frame\u2011hitching after the October relaunch. 2) The story and ending were praised as very good and a fitting conclusion to the trilogy. 3) Gameplay was solid overall but became frustrating in the final 20%, especially due to overwhelming enemy numbers and a grind\u2011heavy tank battle.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions the story conclusion positively (partial match to point 2) and notes minor frame\u2011rate dips (contradicts point 1). It does not address the smooth performance, the end\u2011game frustration, or the grind, so those points are largely missed. Overall coverage is low and the extra praise about the Batmobile and a 5\u2011star rating adds irrelevant speculation, leading to a penalty. Final score is 0.0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a game that actually respects the medium! I've been waiting years for the Batmobile to be real, and Rocksteady delivered. The scale of Gotham is incredible; it feels like a living city rather than just a collection of textures. The Arkham Knight is a terrifying addition, and the story conclusion feels earned. One thing I'll admit: the frame rate dips a bit in the open areas, but that's a minor complaint for an otherwise masterpiece. If you're a Batman fan, this is the definitive entry. 5 stars.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl4",
            "judge_key_points": "1) The reviewer experienced solid technical performance (high settings, 60FPS, no frame\u2011hitching). 2) The story was very good and provided a fitting ending to the trilogy. 3) Gameplay was generally good but became frustrating toward the final 20% due to grindy combat and tough tank battle. 4) Overall recommendation: recommend to fans despite not being a GOTY, praising resolved technical issues.",
            "judge_score": 0.0,
            "judge_thought": "The generated response is largely negative, claiming the Batmobile feels disconnected, the world empty, story merely okay, and gameplay clunky, which contradicts the ground\u2011truth points of solid performance, strong story, generally good gameplay, and a positive recommendation. It fails to reflect any of the four key points (match values ~0.0) and adds contradictory, unsupported criticism, so coverage is ~0 and penalty is high (~0.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I've been playing Batman games since the days of the original Arkham Asylum, so I'm used to the feeling of justice being served in a dark, gritty world. But this time, the darkness feels a bit too heavy, and the justice feels a bit too... manufactured. The Batmobile is here, and it's huge, which is great for driving around Gotham, but honestly, it feels a bit disconnected from the rest of the game. It's like having a supercar in a game where the cars you steal are from the 80s. It doesn't quite fit the \"noir\" vibe of the previous games. The open world is massive, yes, but it feels empty. There's so much to see, but nothing really happens. It's just a giant sandbox where you can walk around and look at buildings, but the quests feel repetitive. \"Go here, fight these guys, go there.\" It's the same loop over and over. The main villain, the Arkham Knight, is a cool concept, but his presence feels forced. The whole game is set up to lead you to him, but once you get there, the actual confrontation feels rushed. It's like the developers knew the game was ending and just wanted to throw everything they had left at you. The story is okay, but it lacks the emotional punch of the first two games. The characters feel a bit flat, and the dialogue is often a bit on the nose. It's like they were trying too hard to make it \"epic\" and ended up making it feel a bit like a movie trailer that never quite landed. The game is technically impressive, with some great visuals and sound design, but the gameplay feels a bit clunky. The combat is a bit more complex than before, which is good, but it also makes the game a bit harder to pick up. If you're a casual player, you might find it a bit frustrating. Overall, it's a game that tries to be everything and ends up being nothing. It's a disappointment for fans of the series, and a miss for anyone looking for a great superhero game. It's like a Batman movie where the hero is too perfect, the villains are too cartoonish, and the plot is too convoluted. It's a game that feels like it's trying to be a masterpiece, but ends up being just a very expensive, very long, and very forgettable experience. If you're looking for a",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.5",
            "judge_key_points": "1) Hardware setup and smooth performance at high settings 60FPS with no hitching. 2) Story is very good and provides a fitting ending to the trilogy. 3) Gameplay is solid overall but the final ~20% is frustratingly grindy with many enemies and a tough tank battle. 4) The reviewer recommends the game to fans despite not being a GOTY.",
            "judge_score": 0.2,
            "judge_thought": "The generated response matches points 1 and 2 well (hardware and story), but completely misrepresents point 3 by saying the ending is easy and boring rather than frustratingly hard, and it does not give the same clear recommendation, adding contradictory complaints. Coverage is about 0.6 and there is moderate irrelevant/contradictory content, so penalty ~0.4. Final score \u22480.2.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "After the disastrous launch in June I held off on playing the game until the \"re-launch\" at the end of October, so this review is based on the current state of the game. First of all, I played this game on an i5 with a single Geforce 970 and 16GB of RAM. The system is about 3 years old, but the graphics card is new. I was able to play the game on high settings, and I locked it at 60FPS in the settings before starting. I had no frame-hitching at all throughout the entire game (20+ hours of playtime), including the parts with the batmobile. So as far as technical issues, whichever ones still exist (bugs, etc.) none of them hindered me and I didn't notice anything during my playthrough. The story (despite having been spoiled on it in advance) was actually very good, including side-quests. I have absolutely zero complaints with the story portion of the game, and as far as a 'fitting\" ending of the trilogy of Arkham games by Rocksteady, I think this was a great swan song. The gameplay was very good for the most part, except toward the final 20% of the game, where the gameplay got boring and repetitive. It was a bit too easy, and the final boss fight was incredibly boring and repetitive. I would have preferred a different final boss fight. I was a bit disappointed with the ending, and I think that the ending could have been much better. I also think that the ending was too short. I would have preferred a more complex ending, where I would have had to fight the final boss in different ways. I was disappointed that the game was so easy toward the end. I would have preferred a harder final boss fight. I think that the final boss fight was too easy, and I would have preferred a more challenging final boss fight. Overall, I think this is a very good game, but I think that the ending was too short and too easy. I think that the final boss fight was too easy, and I would have preferred a more challenging final boss fight. </response",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The reviewer experienced smooth performance with no frame\u2011hitching on high settings. 2) The story was very good and worth recommending. 3) The final portion of the game becomes frustratingly grindy, especially the late\u2011game battles and Riddler side\u2011missions. 4) Overall the reviewer recommends the game to fans despite not being a perfect title.",
            "judge_score": 0.45,
            "judge_thought": "The generated response matches the story praise (m2=1.0) and notes the late\u2011game grind and Riddler missions (m3\u22480.9, m4\u22480.7), but it contradicts the smooth performance claim by saying performance is hit\u2011or\u2011miss (m1=0.0). Coverage is (0+1+0.9+0.7)/4 = 0.65. Minor extra speculative details about hardware cause a small penalty (~0.2). Final score = 0.65\u20110.2 = 0.45.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "I have been a Batman fan since I was a kid, so I was excited to see what Rocksteady would do with this franchise. However, after playing the game, I can only say that it is a mixed bag. First off, I'm running this on an i7-9700K with an RTX 3060, and honestly, the performance is hit or miss. I was getting a solid 60 FPS in the open areas of Gotham, but as soon as I hit the crowded streets or the massive construction sites, I was dropping to the 40s and 50s. It gets frustrating when you're trying to parkour and the frame rate stutters. That said, the story is top-notch. The dynamic between Batman and the new Arkham Knight is tense and well-written. I also have to give credit where it's due: the humor throughout the game is spot-on. The banter between Batman and the side characters, especially Alfred and Commissioner Gordon, brings a nice levity to the dark atmosphere. The ending is incredibly satisfying. Seeing the city finally under control after all the chaos feels earned, and the final confrontation with the Arkham Knight was intense. However, the late-game grind is a real letdown. Once you beat the main story, the game drags on with repetitive side-quests that feel more like a chore than fun. And don't get me started on the Riddler side-missions. They are tedious and often require hours of clicking and solving puzzles that don't really add to the narrative. It feels like the developers ran out of ideas and just threw in filler content. Overall, it's a great game with a fantastic story and some excellent moments, but the performance issues and the grindy late-game content keep it from being a perfect experience.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The reviewer held off playing until the October re\u2011launch and bases the review on the current state; 2) Technical performance was good (high settings, 60 FPS, no frame\u2011hitching) on an i5/GTX 970/16 GB system; 3) Story was strong and fitting for the trilogy; 4) Gameplay was solid overall but became frustrating toward the final 20 % with many enemies and a grindy tank battle; 5) Side missions are less frustrating except the Riddler ones; 6) The reviewer recommends the game to fans despite not being a GOTY.",
            "judge_score": 1.0,
            "judge_thought": "The generated response is an exact copy of the ground truth, preserving every key point with identical wording and no extra or conflicting content, so coverage is 1.0 and penalty is 0.0, yielding a final score of 1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "After the disastrous launch in June I held off on playing the game until the \"re-launch\" at the end of October, so this review is based on the current state of the game. First of all, I played this game on an i5 with a single Geforce 970 and 16GB of RAM. The system is about 3 years old, but the graphics card is new. I was able to play the game on high settings, and I locked it at 60FPS in the settings before starting. I had no frame-hitching at all throughout the entire game (20+ hours of playtime), including the parts with the batmobile. So as far as technical issues, whichever ones still exist (bugs, etc.) none of them hindered me and I didn't notice anything during my playthrough. The story (despite having been spoiled on it in advance) was actually very good, including side-quests. I have absolutely zero complaints with the story portion of the game, and as far as a 'fitting\" ending of the trilogy of Arkham games by Rocksteady, I think this was a great swan song. The gameplay was very good for the most part, except toward the final 20% of the game things just get very frustrating. Every room you enter means it is Batman versus 50 random thugs. The tank battle against the Arkham Knight was frustrating, although I beat it on the first try by simply circling around a single building and shooting him every time he circled while following despite having only a few hitpoints. It feels as though it is a huge grind toward the end of the story for no good reason. It just makes you tired and frustrated and I had to put the controller down and stretch my hands after every section. Side missions are much less frustrating, except for the Riddler missions which could very well take me an additional 5-10 hours to complete. Other notes: Random bad-guy dialogue is hilarious The sense of humor in the game is very nice It really felt like Batman TAS one last hurrah. I'm happy to recommend it if you're a fan of the previous games. To their credit, they managed to solve the technical issues I was led to expect going in. Not GOTY or anything, but if you're a fan of Batman or DC Comics in general, this is really a no-brainer purchase.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The reviewer experienced excellent technical performance (high settings, 60FPS, no frame\u2011hitching). 2) The story was very good and provided a fitting ending to the trilogy. 3) Gameplay was solid overall but the final ~20% became frustratingly grindy, especially the tank battle. 4) Side missions were less frustrating, though Riddler missions were time\u2011consuming. 5) The reviewer recommends the game to fans, noting technical issues were resolved but it\u2019s not a GOTY.",
            "judge_score": 0.0,
            "judge_thought": "The generated response praises the Batmobile, city scale, and story wrap\u2011up, matching the ground truth's positive view of the story (\u22480.8). It completely omits the technical performance details, the end\u2011game frustration, and the side\u2011mission commentary (\u22480 each). It adds extra hype (5\u2011star claim) not present in the ground truth, introducing unsupported emphasis, leading to a moderate penalty (\u22480.4). Coverage C\u22480.3, penalty P\u22480.4, so final score = max(0,0.3\u20110.4)=0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "The wait was worth it. Rocksteady didn't just give us a sequel; they gave us the definitive Batman experience. The Batmobile is the star here, and for once, it doesn't feel like a gimmick. It\u2019s integrated into the combat and exploration so seamlessly that I found myself just cruising through the city even when there wasn't a mission objective. The handling is heavy and satisfying, and the chase sequences are pure adrenaline. And the city itself? It\u2019s massive. You can actually see the scale of Gotham in a way that wasn't possible in previous entries. The attention to detail in the districts is insane, and the next-gen visuals make it look like a living, breathing metropolis. Scarecrow is a solid villain, but the real hook is the Arkham Knight himself. The story wraps up the trilogy in a way that feels earned, not just rushed. If you\u2019re a Batman fan, this is non-negotiable. It\u2019s the complete package. 5.0 out of 5 stars - The ultimate Batman game.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.75",
            "judge_key_points": "1) Review based on re-launch with hardware specs and smooth high\u2011settings performance; 2) Story is good and provides a fitting ending; 3) Gameplay is solid but the final ~20% is frustrating and feels like grind; 4) Side missions are less frustrating, though Riddler missions can add many extra hours; 5) Recommendation: recommend to fans despite not being GOTY.",
            "judge_score": 0.84,
            "judge_thought": "The generated response matches points 1\u20113 almost verbatim (m=1 each). For point 4 it captures the annoyance of Riddler content but changes details, giving a partial match (m\u22480.8). For point 5 it conveys the same fan\u2011only recommendation (m\u22480.9). Coverage \u22480.94. The extra commentary about Riddler trophies is not contradictory, only slightly divergent, so a small penalty \u22480.1 is applied. Final score = 0.94\u20110.1 = 0.84.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "After the disastrous launch in June I held off on playing the game until the \"re-launch\" at the end of October, so this review is based on the current state of the game. First of all, I played this game on an i5 with a single Geforce 970 and 16GB of RAM. The system is about 3 years old, but the graphics card is new. I was able to play the game on high settings, and I locked it at 60FPS in the settings before starting. I had no frame-hitching at all throughout the entire game (20+ hours of playtime), including the parts with the batmobile. So as far as technical issues, whichever ones still exist (bugs, etc.) none of them hindered me and I didn't notice anything during my playthrough. The story (despite having been spoiled on it in advance) was actually very good, including side-quests. I have absolutely zero complaints with the story portion of the game, and as far as a 'fitting\" ending of the trilogy of Arkham games by Rocksteady, I think this was a great swan song. The gameplay was very good for the most part, except toward the final 20% of the game things just get very frustrating. Every room you enter means it is Batman versus 50 random thugs. The tank battle against the Arkham Knight was frustrating, although I beat it on the first try by simply circling around a single building and shooting him every time he circled while following despite having only a few hitpoints. It feels as though it is a huge grind toward the end of the story for no good reason. It just makes you tired and frustrated and I had to put the controller down and stretch my hands after every section. Side missions are a bit repetitive, but they do have some interesting elements to them, such as the Riddler trophies. The Riddler trophies are very easy to miss, and if you miss them you have to replay the mission to get them. This is not a huge deal, but it is annoying. I missed a lot of them, and I didn't bother to replay the missions to get them. I think the game is worth the $20 it cost me, but I would not recommend it to anyone who is not a Batman fan.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:1660d7f03d4df8a88b0b:prompt1",
      "category": "Video_Games",
      "cohort": "seen",
      "eval_pair_id": "amzp_ab84f4aec07972ad8d2333c8174d0b7e83820acd6dd8b55f85c3f1016c2ebb16",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2005-11-23",
            "length_band": "Long",
            "length_chars": 2141,
            "post_id": "amzr_81698360c79f052a66a48b459f5763fee289477f77bbf486502667e0894af2ec",
            "product": {
              "category": "Video_Games",
              "description_excerpt": "Product Description Live in the Star Wars universe and be the hero of your own Star Wars adventure in a galaxy far, far away. The Starter Kit has everything you need to jump into Star Wars Galaxies right now. With an all new design of the first 30 levels of the game, players can get into the online of world of Star Wars easily and start having fun immediately. The all new Starter Pack makes Star Wars Galaxies fun for players of all levels from casual to experienced. Amazon.com When you're ready to enter the Star Wars universe and become a hero of your own adventure in a galaxy far, far away, t",
              "details": {
                "Date First Available": "November 8, 2006",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "8.8 ounces",
                "Item model number": "32618",
                "Manufacturer": "LucasArts",
                "Pricing": "The strikethrough price is the List Price. Savings represents a discount off the List Price.",
                "Product Dimensions": "7.4 x 5.4 x 1.3 inches; 8.8 Ounces",
                "Rated": "Teen"
              },
              "features": [
                "All new design of the first 30 levels",
                "Includes complete content from Star Wars Galaxies: An Empire Divided and Star Wars Galaxies: Jump to Lightspeed",
                "Customize your character's appearance -- body types, facial features, tattoos and more",
                "Take on one of over nine iconic professions, including Bounty Hunter, Smuggler, Commando, or a Jedi"
              ],
              "product_id": "B000BKF2I4",
              "title": "Star Wars Galaxies: Starter Kit - PC"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "Sony Online Entertainment makes great games when they are original, such as Everquest, and even Everquest 2. But every intellectual property they aquire, from The Matrix to Star Wars - they kill it. Honestly how can you ruin a game based on something as immensely popular as Star Wars? Especially so soon in the afterglow of the Episode III DVD release? SOE shows you how - through doing the exact opposite of what you are taught in every Customer Service class you will ever attend. 1.) Lie to the customers SOE has lied, constantly, as it relates to changes being made. A recent lie was advertisements for the latest expansion to Star Wars Galaxies - and promises of new items, etc. for certain professions. 1 Day after charging customers who pre-ordered the expansion - charges fresh on the credit card bills - SOE announced changes to the game which ELIMINATED 21 professions and forced you into one of 9 professions which were broken horribly. This was actionable enough for a lawsuit that they were forced to give a refund to customers - but of course that took 2 - 7 weeks to process. 2. Don't listen to the customer. Customer asks for a bug fix? Offer him an expansion for the modest sum of $30. Want the bug fix bad enough? Fork over the $30 or we won't make it. Character bugged? Spent 3 years building him up to the level he's at? Delete him - we can't be bothered to fix him. 3. Call every nerf, everything that changes in the game, etc. to make it more \"Star Warsy\". \"Star Warsy\" is a madeup term crafted in the SOE spin department. This term was generated in order to say \"I was making things more Star Warsy\" instead of being forced to say \"I was just making 90% of the items you've aquired over the past year worthless.\" Not only do they nerf the players to the point of unplayability - they leave the enemies stronger than ever. I lose fights to cavemen armed with sticks and wearing loin cloths even though I hit them 50 times with a lightsaber - barely leaving a scratch on them. STAY AWAY."
          },
          {
            "date": "2005-12-21",
            "length_band": "Long",
            "length_chars": 917,
            "post_id": "amzr_448fd4768688ed2e08658b1cab2708a5bf8d3d42dc88d1fe21f092f1d78366a9",
            "product": {
              "category": "Video_Games",
              "description_excerpt": "From the Manufacturer The struggle between the Galactic Empire and the Rebel Alliance extends into space with Star Wars Galaxies: Jump to Lightspeed , the first expansion for the award-winning Massively Multiplayer Online Role-Playing game set in the Star Wars universe. This addition dramatically expands the Star Wars Galaxies franchise by providing players more than 15 different vehicles for space exploration and combat. Starfighters and other starships in the expansion can be used for both space exploration and combat, allowing players to join up with allies and fly familiar X-wings and TIE",
              "details": {
                "Date First Available": "April 27, 2004",
                "Is Discontinued By Manufacturer": "Yes",
                "Item Weight": "6.4 ounces",
                "Item model number": "71218",
                "Manufacturer": "LucasArts",
                "Pricing": "The strikethrough price is the List Price. Savings represents a discount off the List Price.",
                "Product Dimensions": "7.5 x 5.5 x 1.25 inches; 6.4 Ounces",
                "Rated": "Teen"
              },
              "features": [
                "The first expansion pack for Star Wars Galaxies online role-playing game",
                "More than 15 different vehicles for space exploration and combat",
                "Pilot transport ships similar to the Millennium Falcon",
                "Invite online friends onto your space craft"
              ],
              "product_id": "B00024DW2W",
              "title": "Star Wars Galaxies: Jump to Lightspeed Expansion Pack - PC"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "I would recommend however that you spend your money on a classic like X-Wing VS Tie Fighter for the same (but better) effect. PVP is consensual - so if you'd like you can fly your little Tie Fighter in the middle of a group of X Wings and their lasers won't hurt you unless you want them to. YOu can get most ships that have been seen in the movies, but as Imperial you'll want to complete the \"Jedi Starfighter\" quest which gives you Anakin's Episode III fighter. What is fun about that is that it's the best fighter in the game - so Ship building professions cannot build anything that is nearly as good and thus have become worthless... and with the advent of the NGE pushed into an even more minority status. This expansion is also currently free with the \"Starter Kit\" that is also available on Amazon... but it's not really worth it. I'd suggest buying some Chuck Norris movies."
          },
          {
            "date": "2006-12-11",
            "length_band": "Long",
            "length_chars": 2136,
            "post_id": "amzr_65aa79f065fe4b271e793bff6db84c42df16af1a2ff344edf3b8ce451bd76882",
            "product": {
              "category": "Video_Games",
              "description_excerpt": "Product Description An all new PC add on for Oblivion that features 5 to 10 hours of Gameplay. Begin a new quest for the legendary Crusader's Relics, found your own knightly order, and defeat an evil foe from the distant past. From the Manufacturer Legacy is an epic game of Starfleet combat covering the entire Star Trek Universe. Players become the Admiral of a task force of warships and must lead their fleet to victory in large-scale battles. As the Admiral of the fleet you choose the ships, equipment, and captains that you will lead into battle. The game is designed to provide an immersive s",
              "details": {
                "Date First Available": "November 20, 2006",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "5.1 ounces",
                "Item model number": "12020",
                "Manufacturer": "Bethesda",
                "Pricing": "The strikethrough price is the List Price. Savings represents a discount off the List Price.",
                "Product Dimensions": "7.76 x 5.51 x 1.26 inches; 5.11 Ounces",
                "Rated": "Everyone 10+"
              },
              "features": [
                "Epic game of Starfleet combat covering the entire Star Trek Universe",
                "Become the Admiral and lead a fleet to victory in large-scale battles",
                "Choose ships, equipment, and captains; 60+ ships and 4 races",
                "Dynamic 3-D battlefields; detailed weapon effects and damage modeling"
              ],
              "product_id": "B000EXW52O",
              "title": "Star Trek - Legacy - PC"
            },
            "rating": 2.0,
            "review_type": "demonstration",
            "text": "The two games I would recommend as an alternative: Bridge Commander and Starfleet Command 3 are both near impossible to find nowadays. You know that you are in trouble when the PC game recommends the controller for the console it was ported from - and when you find AI files from STAR TREK ARMADA dated 1998 in the installation directory. It is difficult to change the control settings for novice users. There is no in-game menu to do such - you are required to edit a file in notepad/textpad to do this. The graphics settings constantly reset themselves if you try to edit the config files to actually make the game look decent. There is little to no tactical depth... you can warp right in to planets/asteroids and it does no damage to your ship. There is no factor in a battle beyond your ship's stats versus whoever you are fightings stats - so the bigger ship wins always. It would have been nice to run into an asteroid field as the Defiant - zigzagging among asteroid while the Romulan Warbird chasing you ends up getting plowed by an asteroid because of its sluggish controls - weakening it enough so that you can turn and do some damage while it is hurt. Instead it follows you in, shrugs off all the asteroids and then kills you in 10 seconds because you cannot warp out of the asteroid field. Shield facings are irrelevant... so no tactical decisions there. The only relevance (and it is small) is the range you are before you open up with phasers. There is excellent sound and voiceovers in this game... but it does not save the trainwreck that it has become. I was expecting a lot more... and I wish the extended development time (and multiple push-backs of my shipping here at Amazon) were used to improve the programming (not sticking with header files from a 1998 game) instead of just recording voiceovers. It would be nice if the developer opened up the game for modding teams to save. As-is, I think that Mad Doc spent more time on the intro movie advertising their company than they did on the game. Shame."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 1,
      "quick_eval": true,
      "review_strata": {
        "length_band": "Long",
        "rating": 4
      },
      "row_id": "humanual-amazon-v1:indist_eval:1660d7f03d4df8a88b0b",
      "source_line_sha256": "181786785e9f7b725ba49228a4830149b11c995ba2ac23a44003b87e690c88d4",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2015-11-16",
        "length_band": "Long",
        "length_chars": 2257,
        "post_id": "amzr_21080b27dc1fe719c77b1a3202a209c0631592034d67e9e15dd8faf023b097dc",
        "product": {
          "category": "Video_Games",
          "description_excerpt": "In the explosive finale to the Arkham series, Batman faces the ultimate threat against the city he is sworn to protect. The Scarecrow returns to unite an impressive roster of super villains, including Penguin, Two-Face and Harley Quinn, to destroy The Dark Knight forever. Batman: Arkham Knight introduces Rocksteady's uniquely designed version of the Batmobile, which is drivable for the first time in the franchise. The addition of this legendary vehicle, combined with the acclaimed gameplay of the Batman Arkham series, offers gamers the ultimate and complete Batman experience as they tear throu",
          "details": {
            "Date First Available": "March 4, 2014",
            "Is Discontinued By Manufacturer": "No",
            "Item Weight": "5.3 ounces",
            "Item model number": "1000486894",
            "Manufacturer": "Warner Home Video - Games",
            "Pricing": "The strikethrough price is the List Price. Savings represents a discount off the List Price.",
            "Product Dimensions": "5.4 x 0.6 x 7.5 inches; 5.28 Ounces",
            "Rated": "Mature"
          },
          "features": [
            "The Batmobile: Gamers have demanded it, and Batman: Arkham Knight delivers. The all new Batmobile sits at the heart of the gameplay experience and augments Batman's abilities in every respect.",
            "Next-Gen Gotham City: Harnessing the power of the next generation of games systems, the entirety of Gotham City has been created in immense scale and with intimate detail offering players a truly legendary next-gen game world.",
            "The Epic Conclusion to a Legendary Trilogy: Rocksteady Studios returns to the helm of the multi-award winning franchise. Scarecrow is back to wreak his revenge against Batman joined by a completely new and original character, the Arkham Knight.",
            "Be The Batman: Batman, the Batmobile, the entirety of Gotham City. Batman: Arkham Knight is the complete and ultimate Batman offering."
          ],
          "product_id": "B00IQCRKQ6",
          "title": "Batman: Arkham Knight - Windows Standard Edition"
        },
        "rating": 4.0,
        "review_type": "target",
        "text": "After the disastrous launch in June I held off on playing the game until the \"re-launch\" at the end of October, so this review is based on the current state of the game. First of all, I played this game on an i5 with a single Geforce 970 and 16GB of RAM. The system is about 3 years old, but the graphics card is new. I was able to play the game on high settings, and I locked it at 60FPS in the settings before starting. I had no frame-hitching at all throughout the entire game (20+ hours of playtime), including the parts with the batmobile. So as far as technical issues, whichever ones still exist (bugs, etc.) none of them hindered me and I didn't notice anything during my playthrough. The story (despite having been spoiled on it in advance) was actually very good, including side-quests. I have absolutely zero complaints with the story portion of the game, and as far as a 'fitting\" ending of the trilogy of Arkham games by Rocksteady, I think this was a great swan song. The gameplay was very good for the most part, except toward the final 20% of the game things just get very frustrating. Every room you enter means it is Batman versus 50 random thugs. The tank battle against the Arkham Knight was frustrating, although I beat it on the first try by simply circling around a single building and shooting him every time he circled while following despite having only a few hitpoints. It feels as though it is a huge grind toward the end of the story for no good reason. It just makes you tired and frustrated and I had to put the controller down and stretch my hands after every section. Side missions are much less frustrating, except for the Riddler missions which could very well take me an additional 5-10 hours to complete. Other notes: Random bad-guy dialogue is hilarious The sense of humor in the game is very nice It really felt like Batman TAS one last hurrah. I'm happy to recommend it if you're a fan of the previous games. To their credit, they managed to solve the technical issues I was led to expect going in. Not GOTY or anything, but if you're a fan of Batman or DC Comics in general, this is really a no-brainer purchase."
      },
      "uid": "amzu_5d1ebd3e874526f75138a5f734d942f8dc16bbeb858731a78b63cd02e4bc544a"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.8,
            "score_mean": 0.08906249999999999,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.85,
            "score_mean": 0.5862500000000002,
            "score_median": 0.7,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.5527777777777776,
            "score_median": 0.7,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.6111111111111112,
            "score_median": 0.8,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "1) The tea bags are bleach free and the tea is excellent. 2) The user is very satisfied and has just ordered another pack.",
            "judge_score": 0.7,
            "judge_thought": "The generated response mentions the bags are bleach and dioxin\u2011free and praises the quality, matching key point 1 (m\u22480.9). It also expresses intent to buy again, reflecting the positive endorsement and repeat order of key point 2 (m\u22480.9). The extra details about flavor, packaging, and personal experience are relevant and not contradictory, but they add length beyond the brief ground truth, so a modest penalty (~0.2) is applied. Final score = (0.9+0.9)/2 \u2013 0.2 = 0.7.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I've been drinking green tea for years, mostly just plain ones, but I was looking for something a bit more functional to help with my after-dinner digestion. This Buddha Teas Parsley Leaf blend seems to fit the bill. The packaging is a bit bulky for what you get, but I appreciate that it's a bulk pack of 18-count bags. The tea bags themselves are decent quality; they don't feel flimsy, and I like knowing they are bleach and dioxin-free. When I brew a cup, the color is a nice, deep green, which usually indicates good quality for this type of herbal blend. I've only had a few cups so far, but I'm noticing a slight calming effect that I attribute to the parsley and mint blend. It's not as strong as a matcha, but it has a pleasant, fresh herbal flavor that isn't too grassy. For the price point and the fact that it's organic, it's a solid choice for anyone looking to add a little extra \"detox\" support to their routine without breaking the bank. Will definitely be buying this again once I finish the pack.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl4",
            "judge_key_points": "1) The tea bags are bleach\u2011free and the tea is described as excellent. 2) The user is very satisfied, having just ordered another pack.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions bleach\u2011free bags (partial match to point 1) but describes the tea negatively and never expresses satisfaction or ordering another pack, so point 2 is not reflected. Match scores: m1\u22480.4, m2=0. Coverage C=0.2. The response adds extensive negative commentary that contradicts the positive tone of the ground truth, incurring a high penalty (~0.7). Final score = max(0,0.2\u20110.7)=0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I bought this hoping it would help with the \"detox\" I've been reading about, but the taste is just... grassy. It's not bad, I suppose, but it's very strong and vegetal. I tried steeping it for a full five minutes as the package suggests, but it just got more bitter, not better. I usually like herbal teas like peppermint or chamomile because they are soothing, but this one feels a bit too aggressive for my morning routine. I did notice the bags are indeed bleach-free, which is nice, but I can't really recommend it as a daily drink unless you really love the taste of raw parsley. Maybe it's better for cooking? I'll stick to my usual tea blends for now.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.5",
            "judge_key_points": "1) Praise that the bleach free teabags are excellent. 2) Indicates repeat purchase/satisfaction by saying they just ordered another pack.",
            "judge_score": 0.65,
            "judge_thought": "The generated response exactly repeats the praise about bleach free teabags, matching key point 1 fully (m1=1.0). It does not mention ordering another pack, only expresses general pleasure, which only partially reflects key point 2 (m2\u22480.5). Coverage = (1.0+0.5)/2 = 0.75. The extra phrase 'I am very pleased' is not contradictory or irrelevant, so a small penalty of 0.1 is applied. Final score = 0.75-0.1 = 0.65.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Bleach free teabags enveloping excellent tea. I am very pleased.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Bleach\u2011free teabags are praised as excellent quality. 2) The user expresses a positive endorsement by ordering another pack.",
            "judge_score": 0.8,
            "judge_thought": "The generated response confirms the bags are bleach\u2011free and high quality, matching point 1 (\u22480.9). It also states a intent to reorder soon, mirroring the user\u2019s ordering another pack, matching point 2 (\u22480.9). No contradictory or irrelevant content is added; only a slight extra enthusiasm, yielding a small penalty (~0.1). Coverage 0.9 minus penalty 0.1 gives a final score of 0.8.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Glad to confirm these bags are truly bleach-free! High quality and fresh. Definitely reordering soon! : )",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) Highlights that the teabags are bleach free and of excellent quality. 2) Shows enthusiasm by saying they just ordered another pack.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, fully reflecting both key points with no extra or conflicting content, resulting in coverage 1.0 and penalty 0, final score 1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Bleach free teabags enveloping excellent tea. Just ordered another pack :-)",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl1",
            "judge_key_points": "1) Bleach free teabags are excellent. 2) User positively endorses the product by ordering another pack.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions bleach\u2011free and organic qualities, partially reflecting the first key point (\u22480.6), but it does not convey the notion of 'excellent' teabags nor the user ordering another pack (0). Coverage = (0.6+0)/2 = 0.3. It adds a lot of extra detail about flavor, price, and personal usage that is not present in the ground truth, incurring a moderate penalty (~0.3). Final score = max(0, 0.3\u20110.3) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 1,
            "privileged": null,
            "refine_round": null,
            "response": "I bought this for the detox benefits and the fact that it is organic and bleach-free. The tea itself has a very distinct, grassy, herbaceous flavor. It is not something I would drink just for pleasure like a black tea or Earl Grey, but it is refreshing and clean tasting. I appreciate that the bags are loose enough to see the actual leaf pieces, which gives me confidence in the quality. I drink one cup in the morning to help with bloating and digestion. It\u2019s a bit pricey for the amount you get, but since I use it daily for health reasons, I\u2019ll likely reorder. Just be prepared for the strong parsley taste\u2014it\u2019s definitely an acquired taste if you aren't used to herbal infusions.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Mentions bleach\u2011free teabags and praises the tea as excellent. 2) Expresses that the user just ordered another pack, showing positive sentiment.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states \"Bleach\u2011free teabags, enveloping excellent tea\" which matches the praise of bleach\u2011free excellent tea (key point 1) and adds \"Ordered another pack :-)\" matching the ordering of another pack (key point 2). Both points are fully reflected with no extra or conflicting content, so coverage C=1.0 and penalty P=0.0, giving a final score of 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Bleach-free teabags, enveloping excellent tea. Ordered another pack :-)",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:d2566d721688941b41ec:prompt2",
      "category": "Grocery_and_Gourmet_Food",
      "cohort": "seen",
      "eval_pair_id": "amzp_b7a859b22b5d20dac8a093b3c915be2283a7e80215321162b1da9b1897edc5ce",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2011-03-28",
            "length_band": "Long",
            "length_chars": 722,
            "post_id": "amzr_744c968d41df6e08c3393b9cb8d4799184727ab1c86b1d9db04d20db08291d72",
            "product": {
              "category": "Grocery_and_Gourmet_Food",
              "description_excerpt": "Manuka honeys are not genetically modified. They are antibiotic free, unpasteurized, and is not industrially blended with any other honeys. Manuka Honey is used internally and externally.",
              "details": {
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "WDS02065-Unit",
                "Manufacturer": "Wedderspoon",
                "Product Dimensions": "2.76 x 2.76 x 4.33 inches; 1.1 Pounds",
                "UPC": "814422020658"
              },
              "features": [
                "Free Of Fillers and Chemicals",
                "All natural, gluten free, no trans fats",
                "Made in United States"
              ],
              "product_id": "B000VK1DC8",
              "title": "Wedderspoon Raw Premium Kfactor 12 Manuka Honey, 8.8 Ounce"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "The taste is all right, consistency thicker than supermarket honey. Bought it for assistance with my rabid GERD, alas, so far no progress with this or the other health irritations I hope to at least somewhat get a relief from. Nonetheless, one lives on hope and I already ordered a second jar just to give it a real try. I take a heaping teaspoon in the morning on empty stomach, and one or two follow ups during the day or at night before sleep. But as I've already said, no relief yet, have not been able to go without Zantac even though I've added two additional natural remedies against vicious heartburn. The honey is expensive, and that the jar is plastic, disturbs me. For the price they could have used glass!"
          },
          {
            "date": "2013-05-28",
            "length_band": "Long",
            "length_chars": 565,
            "post_id": "amzr_739c0e5b245e5c62add9fe81c023a9cf966b064c0bb352bed908650e5a578480",
            "product": {
              "category": "Grocery_and_Gourmet_Food",
              "description_excerpt": "Al Wadi Al Akhdar's Hummus is the most popular and traditional Lebanese dip. Ready to eat, it can be savored with extra virgin olive oil and pita bread. It is usually presented as a side dish with grilled kebab or served as a dip with raw vegetables.",
              "details": {
                "Domestic Shipping": "Currently, item can be shipped only within the U.S. and to APO/FPO addresses. For APO/FPO shipments, please check with the manufacturer regarding warranty and support issues.",
                "International Shipping": "This item is not eligible for international shipping. Learn More",
                "Manufacturer": "al wadi",
                "Package Dimensions": "12.1 x 9 x 4.1 inches; 14.2 Ounces",
                "UPC": "015883124385"
              },
              "features": [
                "Made from high quality products that is identical to the Lebanese flavor.",
                "Savored with extra virgin olive oil and pita bread.",
                "Usually presented as a side dish with grilled kebab or served as a dip with raw vegetables.",
                "Product of Lebanon"
              ],
              "product_id": "B004P0N328",
              "title": "Al Wadi Hummus Chick Pea Dip, 14.2-Ounce (Pack of 12)"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "Not fabulous but OK. I do recommend it because it has good, simple ingredients and lends itself to creativity. Yes the actual product is bland but as other reviewers have noted, this allows you to mess around with it, add garlic powder, or chopped garlic, lemon juice, paprika, or cayenne pepper, or black pepper, peppercorn, olive oil ... or mix it with yogurt, or add crumbled feta cheese. Master your own hummus recipe. Besides, the price is good. $15 for 12 cans, and if you are Prime customer, shipping is free. So it is a good, reasonably healthy fare, enjoy."
          },
          {
            "date": "2015-02-25",
            "length_band": "Medium",
            "length_chars": 143,
            "post_id": "amzr_92c7effca4ea48b838056f51f5dcf0c18c1ec394ce3b6c455aa4e99bbdcc104c",
            "product": {
              "category": "Grocery_and_Gourmet_Food",
              "description_excerpt": "Product of France.",
              "details": {
                "Brand": "Valbreso",
                "Flavor": "Feta",
                "Item Weight": "2 Pounds",
                "Manufacturer": "parthenonfoods.com",
                "Number of Items": "1",
                "Unit Count": "21 Ounce",
                "Variety": "Feta"
              },
              "features": [
                "French Feta Cheese, Easy-Open Tin, 600g",
                "Product of France."
              ],
              "product_id": "B000LRH4MY",
              "title": "French Feta Cheese, Easy-Open Tin, 600g"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "Tastes \"Mediterranean\", meaning thumbs up. Packed in brine, which makes it travel worthy without spoiling, or spilling for that matter."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:indist_eval:d2566d721688941b41ec",
      "source_line_sha256": "9ea79a76485a028d82be3e3e8f63c63504a529721cf116e33af9896d061af240",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2017-12-04",
        "length_band": "Short",
        "length_chars": 75,
        "post_id": "amzr_0a9dc22a50d6f5d67b38cda544c5d77e86354c196f4fcb217635e375047765a4",
        "product": {
          "category": "Grocery_and_Gourmet_Food",
          "description_excerpt": "Buddha Teas presents this sumptuous brew of Parsley Leaf Tea just the way nature intended it. This tea is crafted out of the finest organic ingredients, never adulterated by the use of chemicals, additives, fillers or flavorings. Every tea is mindfully prepared in bleach-free tea bags to eliminate the risk of chemical contamination, leaving you with a refreshingly natural experience. With beautifully designed eco-friendly boxes, our Parsley Leaf Tea tea is a worthy addition to any tea collection.",
          "details": {
            "Country of Origin": "USA",
            "Domestic Shipping": "Currently, item can be shipped only within the U.S. and to APO/FPO addresses. For APO/FPO shipments, please check with the manufacturer regarding warranty and support issues.",
            "International Shipping": "This item is not eligible for international shipping. Learn More",
            "Manufacturer": "Buddha Teas LLC",
            "Package Dimensions": "11.2 x 8.5 x 4.8 inches; 1.4 Pounds",
            "UPC": "819005013409"
          },
          "features": [
            "Antioxidant",
            "Promotes Optimal Health",
            "Supports Body\u2019s Detoxification Process",
            "Bleach Free and Dioxin Free Tea Bags"
          ],
          "product_id": "B00SORY2QO",
          "title": "Buddha Teas Parsley Leaf Tea, 18 Count (Pack of 6)"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "Bleach free teabags enveloping excellent tea. Just ordered another pack :-)"
      },
      "uid": "amzu_0d4be2d8c93a6e22b5088bab7d972454d2e1c90797807d1c3b99c737f73cb0e1"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.9,
            "score_mean": 0.1265625,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.5701388888888886,
            "score_median": 0.6,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5680555555555556,
            "score_median": 0.75,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The user doubts the bag can accommodate a tall tree; 2) The user reports their own 7ft tree does not fit in the bag",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the bag easily fits a 9\u2011foot tree and praises it, which directly contradicts both key points of skepticism and personal failure to fit a 7\u2011foot tree; therefore it reflects none of the ground truth and adds opposing content, incurring a high penalty.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally found a bag that actually fits my 9-foot tree! I\u2019ve gone through so many \"oversized\" bags that were too short or too narrow, and I was about to give up. This Sagler bag is the real deal. The 600D Oxford material feels incredibly tough. I was worried about the zipper breaking under the weight of the branches, but it glides smoothly and the dual-zipper design makes it much easier to get everything in. The dimensions (50x15x20) are perfect for a disassembled 9-footer with room to spare for the trunk and some extra ornaments. I also love the reinforced handles. My old bag\u2019s straps started fraying after two years, but these feel sturdy. The side pocket is a nice bonus for keeping the tags or instructions handy. It\u2019s surprisingly lightweight for how durable it is, and it folds down small when I\u2019m not using it. If you\u2019re tired of your tree getting dusty or damp in the garage, this is worth every penny. Highly recommend!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) User doubts that a tall tree can fit in the bag. 2) User reports their own 7ft tree does not fit.",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the bag holds a 9ft tree perfectly, which directly contradicts both key points (the doubt and the personal experience of not fitting). No key point is reflected (m1=0, m2=0), giving coverage C=0. The response also adds contradictory, unsupported positive claims, so penalty P is high (~0.9). Final score = max(0, 0 - 0.9) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "This is a very nice bag. It holds my 9ft tree perfectly. The material is thick and feels durable. I like that it has a side pocket for ornaments. Great value for the price.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.5",
            "judge_key_points": "1) Expresses doubt about claims that a tall tree can fit in the bag. 2) Shares personal experience that a 7ft tree does not fit.",
            "judge_score": 0.65,
            "judge_thought": "The generated response mirrors the doubt about fitting a tall tree (mentions 9ft) matching key point 1 well (m\u22480.9). It gives personal experience with a 7.5ft tree that 'barely fits', which is close but not identical to 'won\u2019t fit' (m\u22480.7). It adds a mild extra comment about not buying for a 9ft tree, which is not contradictory but slightly unnecessary, giving a small penalty (P\u22480.15). Final score = (0.9+0.7)/2 - 0.15 = 0.65.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Not sure how people are saying you can fit a 9ft tree in this. I have a 7.5ft tree and it barely fits. I wouldn't buy this if you have a 9ft tree.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) Expresses doubt about claims that a tall tree can fit in the bag. 2) States personal experience that a 7ft tree does not fit.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states the bag is too small (reflecting doubt about fitting tall trees) and explicitly says a 7\u2011ft tree won't fit, matching both key points precisely; no extra or conflicting content.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "Too small. A 7-ft tree won't fit.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The user expresses doubt that a tall tree can fit in the bag. 2) The user shares personal experience that a 7ft tree does not fit.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, fully reflecting both key points with no extra or conflicting content, giving coverage 1.0 and penalty 0.0, resulting in a final score of 1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Not sure how people are saying you can fit a tall tree in here. My 7ft won\u2019t fit.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The bag does not fit a tall tree (the user\u2019s 7ft tree won\u2019t fit). 2) Skepticism about claims that the bag can accommodate tall trees.",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the bag fits a 9\u2011foot tree and praises it, directly contradicting both key points that the bag does not fit tall trees and that the user is skeptical of such claims; therefore it reflects none of the ground truth points (m=0 for each) and adds opposing content, resulting in a final score of 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally found a bag that actually fits my 9-foot tree! I\u2019ve bought cheaper ones before that ripped at the seams or were just too short, forcing me to cram the top branches in awkwardly. This Sagler bag is legit heavy-duty 600D Oxford material, so it feels like it\u2019ll last for years. The size is perfect\u201450x15x20 gives me plenty of room for the trunk and all the loose branches without fighting to zip it up. The dual zippers are a nice touch; they glide smoothly and don\u2019t get stuck on the fabric like some of the cheaper bags I\u2019ve tried. I also really appreciate the side pocket for keeping my extra lights and ornaments together so I don\u2019t lose them in the garage. It\u2019s surprisingly lightweight for how tough it is, and the reinforced handles make carrying it from the car to the storage closet much easier on my back. Plus, it folds down small when I\u2019m not using it, so it doesn\u2019t take up valuable closet space during the rest of the year. If you\u2019re tired of your tree getting dusty or damp in the off-season, this is worth every penny. Highly recommend for anyone with a tall tree.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Expresses uncertainty about claims that a tall tree can fit in the bag. 2) States personal experience that a 7ft tree does not fit.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mirrors both key points: it conveys uncertainty about the claim and reports that a 7\u2011foot tree won't fit, matching the ground truth exactly with no extra or conflicting content.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Not sure how people say it fits a tall tree, but my 7-foot tree definitely won't fit.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:9b48cdf1c78f8d5e11df:prompt2",
      "category": "Home_and_Kitchen",
      "cohort": "seen",
      "eval_pair_id": "amzp_2b8dff68936f89d0daed35b2067652318f21e17ac186d0d02b3fb18d7c5224df",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2017-09-11",
            "length_band": "Short",
            "length_chars": 57,
            "post_id": "amzr_8ca9eb4ed42d906ef7b6955f1af7df715ef7984e1d85732ad4064c73c88ecb26",
            "product": {
              "category": "Home_and_Kitchen",
              "description_excerpt": "Here's an offering that's part beach, part tropical and 100% beautiful. This Areca and mixed greens gives a burst of color, along with wispy fronds rising upward then splashing out in waves of green. This colorful concoction is housed in a beautiful bamboo planter that perfectly complements the beauty. Plus, it never needs sun or water. Makes a great home or office decoration and also makes an ideal gift.",
              "details": {
                "Assembly required": "No",
                "Batteries required": "No",
                "Brand": "Nearly Natural",
                "Color": "Green",
                "Fabric Type": "Plastic",
                "Included Components": "Artificial Plant",
                "Is Discontinued By Manufacturer": "No",
                "Item Package Quantity": "1"
              },
              "features": [
                "Plastic",
                "58\" high and 27\" wide",
                "Boasts an areca palm, anthurium and silver queen plants",
                "Perfect mixture of color and texture"
              ],
              "product_id": "B00FUNGXFS",
              "title": "Nearly Natural 6722 Areca and Mixed Greens with Bamboo Planter, Green"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "The red flowers are a little too much. Makes it look fake"
          },
          {
            "date": "2018-11-24",
            "length_band": "Short",
            "length_chars": 27,
            "post_id": "amzr_1fda8187980ea98c8493c7f136530bb077e0a2d798a0782f68068cc50a37d9da",
            "product": {
              "category": "Home_and_Kitchen",
              "description_excerpt": "BirdRock Home Bamboo Lap Tray with Black Top is the perfect lap desk and serving tray for your home, office, kitchen or hotel. With a frame crafted from bamboo wood and black hard top, this laptop lap tray provides a stylish and elegant touch to your home, office or hotel. Built with a sturdy and durable construction, feel safe using your laptop, iPad, tablet, books, drawing canvas, snacks, food and more on the tray. The lap tray is equipped with folding legs allowing you to prop up the bamboo serving tray to create a stand or fold down to lay flat on the table. The tray table is also equipped",
              "details": {
                "Assembly Required": "No",
                "Brand": "BIRDROCK HOME",
                "Color": "Walnut",
                "Date First Available": "November 1, 2016",
                "Finish Type": "Wood",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "4.63 pounds",
                "Item model number": "10273"
              },
              "features": [
                "BAMBOO LAP DESK TRAY: With a frame crafted from bamboo wood and black hard top, this laptop lap tray provides a stylish and elegant touch to your home, office or hotel. Built with a sturdy and durable construction, feel safe using your laptop, iPad, tablet, books, drawing canvas, snacks, food and more on the tray",
                "FOLDING LEGS & TILT TRAY: Equipped with folding legs allowing you to prop up the bamboo serving tray to create a stand or fold down to lay flat on the table. The tray table is also equipped with 6 separate tilting positions providing you with comfortable positions to prop up your tablet, book or laptop",
                "MAGNETIC WIDE TRAY TOP: The tray desk top is equipped with magnets that help lock the desk top in place when transporting around the home and is crafted with plenty of work space for laptop, drawing, writing or reading. Also equipped with plenty of space for a meal, snack or picnic",
                "WORK FROM HOME: the perfect lap tray so you can get work done from the comfort of your bed or couch! The lap tray allows you to use your 13\u201d or 15\u201d laptop and a computer mouse with plenty of room to spare. The lap tray is portable workstation that goes where you go. Simply collapse the lap tray down and easily move it to your next work area"
              ],
              "product_id": "B01MDU3L39",
              "title": "BIRDROCK HOME Bamboo Lap Tray with Black Top - Work from Home - Foldable Breakfast Serving Bed Tray - Lap Desk with Wide Tilting Top - Laptop Computer Stand for Couch Bed Chair - TV Tray - Walnut"
            },
            "rating": 2.0,
            "review_type": "demonstration",
            "text": "Much larger than I expected"
          },
          {
            "date": "2019-03-24",
            "length_band": "Short",
            "length_chars": 22,
            "post_id": "amzr_9c832960a6cef70506cd7b08bec761436fed155a8276aa580499274ece43b9a6",
            "product": {
              "category": "Home_and_Kitchen",
              "description_excerpt": "Experience spa-like comfort as you step out of the bath or shower with the MICRODRY SoftLux bathroom memory foam rugs. Soft and highly absorbent, our bath mat set is a must-have in your bathroom to give your feet warmth and support. Efficiently Absorbs Water These large bath mats for bathroom are made of high-density memory foam and plush SoftLux microfiber fabric. Aside from being extremely cozy underfoot, each bath mat rug is infused with activated charcoal to retain its freshness. Skid-Resistant Thick Bath Mat Featuring a patented GripTex skid-resistant base, our microfiber bathroom rugs wo",
              "details": {
                "Back Material Type": "Polyester",
                "Brand": "MICRODRY",
                "Color": "Black",
                "Construction Type": "Machine Wash",
                "Country of Origin": "USA",
                "Date First Available": "March 28, 2019",
                "Is Discontinued By Manufacturer": "No",
                "Is Stain Resistant": "No"
              },
              "features": [
                "Ultra-Soft Cushion Support - Keep your feet warm and cozy as you step out of the shower with our luxurious bath floor mat. MICRODRY SoftLux soft bath mat has extra-thick memory foam for high-density support and cushioning comfort.",
                "Stays in Place - Unlike regular floor rugs, this non slip bath rug has a patented GripTex skid-resistant base that works on dry and wet surfaces. The premium design of our non skid bath rugs helps reduce slippage and doesn\u2019t shed easily, so you can bathe without worries.",
                "Quickly Dries Your Feet - Experience spa-like treatment in the comfort of your home with these cozy bath rugs set. Each bath mat is made of plush memory foam that draws water away to instantly dry your feet.",
                "Cleaning your MICRODRY bath mat is a breeze! Load it in your washing machine and dry. This machine washable bath mat is infused with activated charcoal, so it remains fresh and free from odor."
              ],
              "product_id": "B0BTNCJD6F",
              "title": "MICRODRY Extra Thick Modern Luxury Charcoal Infused Memory Foam Bath Mat Rug with, GripTex Skid Resistant Base, Absorbent, Easy Care, Fade & Pilling Resistant, SoftLux (2-Piece Set, Black)"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "Fluffy and comfortable"
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Medium",
        "rating": 3
      },
      "row_id": "humanual-amazon-v1:indist_eval:9b48cdf1c78f8d5e11df",
      "source_line_sha256": "4c237ecc88b6c35d4262e347cf4b700988b07e8f41ce9b4f3bb0f296c984fe9a",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2023-01-24",
        "length_band": "Medium",
        "length_chars": 81,
        "post_id": "amzr_8e4d5c5884b8f2490a7041a99adacafa49a8599350f8e3154389ca93a023fa75",
        "product": {
          "category": "Home_and_Kitchen",
          "description_excerpt": "Are you still storing your Christmas tree inside its original box? The holiday season wouldn\u2019t be as merry and bright without a gorgeous tree adorned by twinkling lights and glittering baubles. If you want to keep using your Christmas tree (along with its trinkets) for many holiday seasons to come, then you'll need to store it properly. A cardboard box will not offer much protection against dust, dirt, and moisture. Store your artificial tree and decorations inside a heavy-duty storage bag specially made for this purpose! The Sagler 9 Ft Christmas Tree Storage Bag is a smart way to store, prot",
          "details": {
            "Batteries Included?": "No",
            "Batteries Required?": "No",
            "Brand": "Sagler",
            "Closure Type": "Zipper",
            "Date First Available": "November 4, 2019",
            "Item Weight": "2.76 pounds",
            "Manufacturer": "Samuelworld",
            "Material": "Polyester"
          },
          "features": [
            "Preserve Your Christmas Tree - Make your tree last longer by storing it in this durable Xmas tree storage bag. This Christmas storage bag helps protect the contents from dust, dirt, and moisture.",
            "Heavy Duty Christmas Tree Storage Bag - Made of tough 600D Oxford material, this Christmas tree tote won't fray or rip easily even when used for bulky trees as tall as 8ft or disassembled 9ft trees.",
            "Storage Made Easy - Keep your house free from post-holiday clutter. This 50\"x15\"x20\" artificial Christmas tree bag has enough space for the tree trunk, branches, leaves, or even ornaments and lights.",
            "Convenient for Carrying - With reinforced handles, this Xmas tree bag makes it easy to transport holiday decor from one place to another. This Christmas tree storage tote also has a loop for pulling."
          ],
          "product_id": "B07ZZL47SV",
          "title": "9 ft Christmas Tree Storage Bag - Waterproof, Heavy-Duty 600D Oxford Christmas Tree Bag - Reinforced Handles, Dual Zipper, Label & Side Pocket - Protection From Dust, Water - Green, 50x15x20\u2033 - Sagler"
        },
        "rating": 3.0,
        "review_type": "target",
        "text": "Not sure how people are saying you can fit a tall tree in here. My 7ft won\u2019t fit."
      },
      "uid": "amzu_3bf08803751b37707603a8f281771ae7c14e6a6ded9dc3b2076028d3b53afc6c"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.37,
            "score_mean": 0.012500000000000002,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.4733333333333332,
            "score_median": 0.43,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.30138888888888893,
            "score_median": 0.25,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1. The straps pop off constantly, making the bra unreliable. 2. User provides personal fit details (height, weight, cup size, and bought size).",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the straps stay in place and praises the bra, directly contradicting the main complaint about straps popping off, so m1=0. It does not mention the user's personal fit details, so m2=0. Coverage C=0. The response adds extensive unrelated positive commentary, which is contradictory and irrelevant, giving a high penalty around 0.8. Final score = max(0,0-0.8)=0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a sports bra that actually stays in place during a triathlon! I've been through so many that the straps keep slipping off or the back rides up after just a few miles, but this Cosabella bra is a game changer. The hook-and-eye closure is super secure, and I love that the straps are convertible. I can switch them from a racerback style for swimming to a standard strap for the bike and run. The fit is true to size, which is rare for sports bras. I'm usually between sizes, but I went with my regular and it fits perfectly without feeling tight or restrictive. The padding is just right\u2014not too bulky, but enough to give me confidence. It's made of 91% polyamide and 9% elastane, which feels durable and stretchy. The only thing to note is that it says hand wash only, so please take care of it. I've had it for a few months now, and it still looks brand new. If you're looking for something reliable for triathlon training or racing, this is definitely worth the investment. Highly recommend!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl4",
            "judge_key_points": "1) The convertible straps repeatedly pop off, turning the supportive bra into no support. 2) The reviewer provides personal fit details (5'4\", 150lb, 32C, size small) and expresses overall dissatisfaction.",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the bra is comfortable, has no strap issues, and gives a different size (36C, medium), which directly contradicts both key points; thus it matches none of them (m1=0, m2=0). It also adds unrelated positive claims, introducing conflicting content, so a high penalty is applied (P\u22480.7). Final score = max(0, (0+0)/2 - 0.7) = 0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "Finally a sports bra that doesn't dig in! I've tried so many with the hard plastic underwire that cuts into my skin, but this one is so comfortable. The color blocking is cute and the hook closure makes it super easy to get on and off. I wear a 36C and the medium fits perfectly. Highly recommend for anyone who wants support without the pain!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.5",
            "judge_key_points": "1) Straps pop off constantly, turning the supportive bra into no bra; 2) User provides personal fit details (height 5'4\", weight 150lb, cup size 32C, size small)",
            "judge_score": 0.35,
            "judge_thought": "The generated response repeats the complaint about straps popping off, matching key point 1 well (m\u22480.9). It completely omits the personal fit details, missing key point 2 (m=0.0). Coverage = (0.9+0)/2 = 0.45. The added phrase \"EVERY MOVEMENT!\" is extra emphasis but not contradictory, so a small penalty ~0.1 is applied. Final score = 0.45\u20110.1 = 0.35.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Honestly, it would be my favorite sports bra I've gotten, except BOTH straps pop off CONSTANTLY... THIS MEANS YOU GO FROM SUPPORTIVE BRA TO NO BRA WITH EVERY MOVEMENT!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The convertible straps constantly pop off, causing loss of support. 2) User provides personal fit details: 5'4\", 150lb, 32C, and purchased a small size.",
            "judge_score": 0.6,
            "judge_thought": "The generated response fully captures the strap problem (matches key point 1 with m=1.0) and repeats the height and cup size, but gives a different weight (135lb vs 150lb) so only partially matches the fit details (m\u22480.6). Coverage C = (1.0+0.6)/2 = 0.8. It adds extra commentary about the bra being too tight, which is irrelevant but not contradictory, incurring a small penalty P\u22480.2. Final score = max(0, min(1, 0.8-0.2)) = 0.6.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "I bought the small but it's so tight it barely closes at the back. I'm usually a medium so I think they run small. The convertible straps are a nightmare - the clips just POP off with the slightest movement, leaving me with ZERO support. I am 5'4\", 135 lbs, 32C, and this is painful. I need better quality!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1. The bra is otherwise favorite but both straps pop off constantly, causing loss of support. 2. The reviewer wishes the convertible straps stayed in place. 3. Provides personal fit info: height 5'4\", weight 150lb, cup size 32C, size small.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, covering all three key points perfectly with no extra or conflicting content, so coverage C=1.0 and penalty P=0.0, final score=1.0",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Honestly, it would be my favorite sports bra I've gotten, except BOTH straps pop off CONSTANTLY... THIS MEANS YOU GO FROM SUPPORTIVE BRA TO NO BRA WITH NO WARNING. I wish they were just seen in place instead of these frustrating convertible straps. I'm 5'4\", 150lb, 32 C and I got the small.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The convertible straps repeatedly pop off, causing loss of support. 2) User provides personal fit details (height, weight, cup size, bought size small).",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the strap failure at all and instead praises the convertible straps, contradicting the main complaint, so it scores 0 for both key points. It also adds unrelated positive commentary, which is a significant mismatch, leading to a high penalty. Final score is 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been a huge fan of Cosabella for years, so I was excited to try the Triathlon. As someone who values both support and aesthetics, this bra hits a really nice middle ground. The color-blocking is chic and looks great under sheer tops, but the real star is the fit. The 91% polyamide/9% elastane blend feels incredibly soft against the skin\u2014very breathable, which is crucial for workouts. The convertible straps are a game-changer for versatility, allowing me to switch between racerback and traditional styles depending on my outfit or activity. The hook-and-eye closure at the back provides that extra layer of security and adjustability that I prefer over pullover styles, especially for higher-impact movements. It\u2019s definitely a \"hand wash only\" item, so be prepared to treat it gently to maintain the shape and elasticity, but it\u2019s worth the effort. If you\u2019re looking for a sports bra that doesn\u2019t sacrifice style for function, this is a solid pick. Just note that it runs true to size, so stick with your usual fit.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.75",
            "judge_key_points": "1) The bra is otherwise favorite but both straps pop off constantly. 2) This leads to sudden loss of support. 3) The reviewer wishes the convertible straps stayed in place.",
            "judge_score": 1.0,
            "judge_thought": "The generated response repeats the complaint about straps popping off constantly and the resulting loss of support, and expresses a wish for the straps to stay in place (using 'flimsy' instead of 'frustrating' but meaning the same). It matches all three key points fully with no extra or conflicting content, so coverage=1.0, penalty=0.0, final score=1.0.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Honestly, it would be my favorite sports bra I've gotten, except BOTH straps pop off CONSTANTLY... THIS MEANS YOU GO FROM SUPPORTIVE BRA TO NO BRA WITH NO WARNING. I wish they were just seen in place instead of these flimsy straps.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:e46c515289018740fa9f:prompt1",
      "category": "Clothing_Shoes_and_Jewelry",
      "cohort": "seen",
      "eval_pair_id": "amzp_7e54eff482682d2c0c569e2077e0a01dae720b001dfaf4c9a232a4b91889434b",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2014-12-08",
            "length_band": "Short",
            "length_chars": 52,
            "post_id": "amzr_9753e5d7e96e056e29ee6a4d7aae07b3e9c8e924226fd5ee332f539e415c25d6",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "A costume fit for winning battles.",
              "details": {
                "Date First Available": "October 2, 2000",
                "Department": "Boys",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "00325-Parent",
                "Manufacturer": "California Costumes - Toys",
                "Manufacturer recommended age": "8 - 10 years"
              },
              "features": [
                "100% Polyester, Non-Leather Material",
                "Size: Large (10-12)",
                "100% polyester interlock knit fabric and foam-backed faux leather",
                "Sleeveless knee-length tunic is white interlock"
              ],
              "product_id": "B0082VE8S4",
              "title": "Kids Gladiator Costume"
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "runs a little short / small. but worked for our play"
          },
          {
            "date": "2015-03-16",
            "length_band": "Long",
            "length_chars": 313,
            "post_id": "amzr_582ef4ced9d3c5e95ee453e73457a7f777f516b0b4168f5e453a04e6e5040cef",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "Featuring the patented Support Web, the CW-X PerformX Tight keeps everything in correct alignment for optimum balance, efficiency and power.",
              "details": {
                "Date First Available": "September 26, 2008",
                "Department": "womens",
                "Item model number": "121809 079 L-PAR",
                "Manufacturer": "CW-X",
                "Package Dimensions": "1 x 1 x 1 inches; 1 Pounds"
              },
              "features": [
                "2-Way Stretch Support Web Nylon 80%, Lycra 20% ; 4-Way Stretch Body Fabric Coolmax 80%, Lycra 20%",
                "Imported",
                "Drawstring closure",
                "UPF 50+ for UVA/UVB protection"
              ],
              "product_id": "B007U2N06C",
              "title": "CW-X Women's Muscle Support Performx Full Length Compression Tight"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I wear size 4-6 pants and the medium was perfect. I am only 5'3\" so I have to make sure to pull them up enough to get the bands in the right spot. If you are much shorter than me, I have a feeling they won't work - Only thing that could make these better is if they made a petite sizing for us shorter ladies!"
          },
          {
            "date": "2015-06-09",
            "length_band": "Long",
            "length_chars": 275,
            "post_id": "amzr_6cd52ca8cec3591a5fe8d7f5b1bd77e6f89332d4abdfdd392084d1d5633d25bc",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "",
              "details": {
                "Date First Available": "July 28, 2017",
                "Department": "womens",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "SS-0939-BG-M",
                "Product Dimensions": "8.9 x 7.3 x 1.5 inches; 9.17 Ounces"
              },
              "features": [
                "Pull On closure",
                "Machine Wash"
              ],
              "product_id": "B00CO2HL8Q",
              "title": "PattyBoutik Women Scoop Neck Cut Out Flutter Sleeve Blouse (Burgundy Medium)"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "um - love it! Though not very \"young mom\" friendly - meaning my baby & toddler have both yanked the front down on accident just by me picking them up or putting them down. But super classy and slimming. Followed their sizing and it worked like a dream."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 1,
      "quick_eval": true,
      "review_strata": {
        "length_band": "Long",
        "rating": 1
      },
      "row_id": "humanual-amazon-v1:indist_eval:e46c515289018740fa9f",
      "source_line_sha256": "855995a233f3142b2d555798be53e70725ea72bdb139e88c9b2fc7a4dbca6d2d",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2021-01-13",
        "length_band": "Long",
        "length_chars": 302,
        "post_id": "amzr_a2a2ad23f134284322693b26b0a056bc8674e38fe7d53784856be0610b10885b",
        "product": {
          "category": "Clothing_Shoes_and_Jewelry",
          "description_excerpt": "Crafted in astonishing wonder fabric and power mesh. Functions as athletic and swimwear, ultra-fast drying, 50spf UV protection, two-way stretch",
          "details": {
            "Date First Available": "May 8, 2015",
            "Department": "womens",
            "Item model number": "TRIAT1371",
            "Manufacturer": "Cosabella Women's IA",
            "Package Dimensions": "1 x 1 x 1 inches; 1 Pounds"
          },
          "features": [
            "91% Polyamide, 9% Elastane",
            "Imported",
            "Hook and Eye closure",
            "Hand Wash Only"
          ],
          "product_id": "B00QPTYIUI",
          "title": "Cosabella Women's Triathlon Padded Bra"
        },
        "rating": 1.0,
        "review_type": "target",
        "text": "Honestly, it would be my favorite sports bra I've gotten, except BOTH straps pop off CONSTANTLY... THIS MEANS YOU GO FROM SUPPORTIVE BRA TO NO BRA WITH NO WARNING. I wish they were just seen in place instead of these frustrating convertible straps. I'm 5'4\", 150lb, 32 C and I got the small."
      },
      "uid": "amzu_528ca16c64994b556e6d61f6d246286cfb4cb314df282cd3d1b91af9a1211daa"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.9,
            "score_mean": 0.2854166666666667,
            "score_median": 0.3,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.9,
            "score_mean": 0.4791666666666667,
            "score_median": 0.4,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.5124999999999998,
            "score_median": 0.6,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.7333333333333333,
            "score_median": 1.0,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "The ground truth states a positive endorsement: the product is excellent for natural hair.",
            "judge_score": 0.2,
            "judge_thought": "The generated response is a positive review but focuses on chemically relaxed, damaged, and dry hair rather than natural hair; it only partially reflects the positive endorsement and does not mention natural hair, giving a match around 0.4. It adds extensive extra detail that is relevant but not needed, incurring a small penalty (~0.2). Final score = 0.4 - 0.2 = 0.2.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a leave-in that doesn't just sit on top of my hair! As someone with chemically relaxed hair that gets dry and brittle fast, I've tried so many \"repair\" creams that end up weighing my hair down or leaving a sticky residue. This Cantu cream is different. I love that the new formula is free from drying alcohols and silicones. It actually sinks in. I use it as a deep treatment on my ends after washing, and the shea butter content makes a real difference in reducing split ends. The fact that I got a 16 oz pack of two is a huge plus for the price\u2014I won't be running to the store every week. My hair feels softer and looks healthier after just a few uses. Highly recommend for anyone dealing with damaged or dry hair.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl2",
            "judge_key_points": "Ground truth states a positive endorsement: the product is excellent for natural hair.",
            "judge_score": 0.3,
            "judge_thought": "The generated response gives a positive endorsement but focuses on dry hair rather than natural hair, partially reflecting the key point (\u22480.5). It adds extra details about softness and smoothness which are irrelevant to the specific claim, incurring a small penalty (\u22480.2). Final score = 0.5 - 0.2 = 0.3.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 2,
            "privileged": null,
            "refine_round": null,
            "response": "This cream is great for my dry hair. It makes it soft and smooth.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.5",
            "judge_key_points": "The ground truth states a positive endorsement: the product is excellent for natural hair.",
            "judge_score": 0.5,
            "judge_thought": "The generated response keeps the positive tone and uses 'Excellent', but shifts the focus from 'natural hair' to 'repairing hair', which only partially reflects the original point. Match value ~0.5. No extra or conflicting content, so penalty 0. Final score = 0.5.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "Excellent for repairing hair",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "The ground truth states a positive assessment: the product is excellent for natural hair.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly matches the ground truth phrase, reflecting the key point perfectly with no extra or conflicting content; match score 1.0, penalty 0, final score 1.0",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Excellent for natural hair.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "The response states that the product is excellent for natural hair.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly matches the ground truth by stating 'Excellent for natural hair.' This fully reflects the single key point with no extra or conflicting content, so m=1 and penalty P=0, giving a final score of 1.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "Excellent for natural hair.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl2",
            "judge_key_points": "The ground truth states a positive endorsement that the product is excellent for natural hair.",
            "judge_score": 0.0,
            "judge_thought": "The generated response says 'Good for damaged hair', which does not reflect the key point about being excellent for natural hair; it is unrelated and does not convey the same endorsement, so match score m=0. Coverage C=0. The response adds irrelevant content about damaged hair, introducing a mismatch, so penalty P\u22480.7. Final score = max(0,0-0.7)=0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 2,
            "privileged": null,
            "refine_round": null,
            "response": "Good for damaged hair.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "The product is excellent/good for natural hair.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states the cream works especially well for natural hair, which directly matches the key point with essentially the same meaning and adds no extra or conflicting content; match score 1.0, no penalty, final score 1.0",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "This cream works especially well for natural hair.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:f8f1bd5e7db8440e8075:prompt0",
      "category": "Beauty_and_Personal_Care",
      "cohort": "seen",
      "eval_pair_id": "amzp_a34a000b0302ee93bc7098b6576d7301792fa15aeaa50b4754e3ecf7d15a994a",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2016-08-29",
            "length_band": "Short",
            "length_chars": 32,
            "post_id": "amzr_111934137e92643e4673986b6897ef59c5b481d26632b98ecddee8a14c7afda1",
            "product": {
              "category": "Beauty_and_Personal_Care",
              "description_excerpt": "Brand new. Sealed.",
              "details": {
                "Brand": "Earth Vibes",
                "Department": "Bath & Body|Sales & Specials",
                "Is Discontinued By Manufacturer": "No",
                "Item Form": "Oil",
                "Item model number": "Avon SSS bath oil Large 24oz, sig silk",
                "Manufacturer": "Avon",
                "Product Benefits": "Softening",
                "Product Dimensions": "70.87 x 15.75 x 47.24 inches; 1.25 Pounds"
              },
              "features": [
                "24 fl. oz."
              ],
              "product_id": "B00DUETTVY",
              "title": "Avon Skin So Soft Bath Oil 24 Fl Oz (Signature Silk)"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Great product to keep mosquito.."
          },
          {
            "date": "2020-05-26",
            "length_band": "Short",
            "length_chars": 61,
            "post_id": "amzr_5169d7ee066b51f35a2f86c8ef91abd1492a70f345bd27a7c7b13384e967ab4c",
            "product": {
              "category": "Beauty_and_Personal_Care",
              "description_excerpt": "",
              "details": {
                "Country of Origin": "USA",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "BLUEB00374-EA",
                "Language": "French",
                "Manufacturer": "CURLS",
                "Product Dimensions": "1 x 1 x 1 inches; 6.4 Ounces",
                "UPC": "859776000826"
              },
              "features": [
                "Formulated with natural ingredients like blueberry seed oil, corn mint and coconut oil.",
                "Proven to enhance hair growth, soothe and refresh dry, itchy scalp, and relieve itching.",
                "No sulfates, silicones, parabens, artificial oils, colors or fragrances.",
                "Provides an invigorating tingle on the scalp!"
              ],
              "product_id": "B007N53E08",
              "title": "Blueberry & Mint Tea Scalp Treatment"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "This product has a major positive effect on your hair growth."
          },
          {
            "date": "2020-05-30",
            "length_band": "Short",
            "length_chars": 55,
            "post_id": "amzr_e24d10d7286a996e76e00d550778907125060335b7562922ef9f13078a5d9b69",
            "product": {
              "category": "Beauty_and_Personal_Care",
              "description_excerpt": "",
              "details": {
                "Brand": "BEFUNNY",
                "Color": "T1B/30/27#",
                "Extension Length": "30 Inches",
                "Hair Type": "Straight",
                "Manufacturer": "BEFUNNY",
                "Material": "Synthetic",
                "Package Dimensions": "11.3 x 8.11 x 4.02 inches; 1.7 Pounds"
              },
              "features": [
                "\u221aBEFUNNY Long Pre Stretched Braiding Hair is Made With Top Quality Synthetic Hair,Low Temperature Fiber With Yaki Texture,Can be Managed Easily For Crochet Hair, Crochet Braids,Crochet Twists or Even Ponytails Like Natural Human Hair Extensions.",
                "\u221a30\" Extra Long Original Professional Braiding Hair, Prestretched Braiding Hair,Soft,No itchy, Minimal shedding,Long-Lasting Freshness,Skin Friendly and Healthy.",
                "\u221aNew Stretched Hair Replacement of Traditional Braiding Hair,More Convenient and Save Time.",
                "\u221a8Packs/Lot For The Deal,90Gram/Pack,Superlight,Decompress Our Head!!! Pretty Mixed of Black To Silver White Grey Tips Color,Add you a different style."
              ],
              "product_id": "B07FX8767R",
              "title": "Long Pre stretched Braiding Hair, 30 Inch 8 Packs Ombre Prestretched Braiding Hair Bundles,Three Tone Synthetic Braid Hair For Box Crochet Braids, Yaki Straight, Itch Free(30\",T1B/30/27#)"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "The hair is worth the money and it was packaged nicely."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 0,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 2
      },
      "row_id": "humanual-amazon-v1:indist_eval:f8f1bd5e7db8440e8075",
      "source_line_sha256": "d7fabd5ad804e2614d34e77daf40791e445a9f4f085e0e4bf0b8bddf728791c9",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2020-12-28",
        "length_band": "Short",
        "length_chars": 27,
        "post_id": "amzr_02f38dd74a36421eaed8fc85c3ed48d9092bb51ec5975b4eb8897077f8ad283c",
        "product": {
          "category": "Beauty_and_Personal_Care",
          "description_excerpt": "Infused with pure shea butter and free from drying alcohol and silicones, our new & improved Cantu Shea Butter Leave-In Conditioning Repair Cream intensely moisturizes & deeply conditions to leave dry or damaged hair soft & hydrated with fewer split ends. Creamy and luxurious in texture, use this cream as part of your daily hair routine to help reduce split ends and mend hair breakage or after shampooing as a rinse-out conditioner. Since the beginning, you\u2019ve pushed us to advance skin and hair science to honor your locs, your curls, your waves, your braids, your natural. At Cantu, we support y",
          "details": {
            "Brand": "Cantu",
            "Country of Origin": "USA",
            "Hair Type": "Damaged",
            "Item Form": "Cream",
            "Manufacturer": "Cantu",
            "Material Feature": "Natural",
            "Product Benefits": "Conditioning,Hydrating,Moisturizing,Softening,Split Ends",
            "Product Dimensions": "7.28 x 3.74 x 3.27 inches; 2.28 Pounds"
          },
          "features": [
            "New & improved formula more intensely moisturizes & includes no drying alcohol or silicones",
            "Infused with pure shea butter and natural oils to deeply condition & promote strong, healthy hair",
            "Helps repair hair breakage, protect damaged hair and reduce split ends",
            "Great for relaxed, natural, permed, color treated, dry & damaged hair"
          ],
          "product_id": "B06X9MG5WB",
          "title": "Cantu Leave-In Conditioning Repair Cream with Shea Butter, 16 oz (Pack of 2) (Packaging May Vary)"
        },
        "rating": 2.0,
        "review_type": "target",
        "text": "Excellent for natural hair."
      },
      "uid": "amzu_2a8ebbe9d6aac1f7697f9e853d399e023d4326bb78f200d4f3803329dd1f29c5"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.15,
            "score_mean": 0.0015624999999999999,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.48479166666666657,
            "score_median": 0.5,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.626388888888889,
            "score_median": 0.9,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Expresses that it is tragic there are not more people willing to act against evil. 2) Thanks/appreciates the author for telling the story.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the tragedy of few people acting against evil nor does it thank the author; it instead provides a personal review of the book, which is unrelated to the ground truth sentiment, resulting in no match for either key point. It also adds extensive unrelated content, incurring a penalty.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I picked this up because I\u2019m a huge fan of Bradley Hope\u2019s work\u2014his reporting on *Billion Dollar Whale* was absolutely riveting, so I had high expectations here. I wasn\u2019t disappointed. The pacing is incredible. It reads less like a standard non-fiction biography and more like a thriller, which is exactly what you want when the subject matter involves espionage and armed raids. Hope does a fantastic job of balancing the journalistic rigor with a narrative that keeps you turning pages. The story of Adrian Hong is fascinating, though by the end, you can\u2019t help but feel a mix of admiration and concern for his methods. It\u2019s a complex look at idealism, hubris, and the real-world consequences of trying to take on a regime that powerful. The Kindle edition is straightforward. Text-to-Speech works well if you want to listen to parts of it, but I found myself reading it straight through. The file size is manageable, and the typesetting is clean. If you enjoy true stories about high-stakes intelligence operations, defectors, and the gray areas of activism, this is a must-read. It\u2019s short, punchy, and deeply reported. Highly recommend.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) It is tragic that there aren't more people willing to take action against evil. 2) Thanks to the author for telling this story.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not express the tragedy about lack of people taking action nor thank the author; instead it criticizes the book and author, contradicting the ground truth sentiment. Both key points receive a match score of 0, giving coverage 0. The response adds irrelevant negative commentary, so a high penalty (~0.8) is applied. Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I read the first half of this and it was gripping. The details of the Madrid embassy raid were fascinating. But the second half? It became a self-aggrandizing puff piece for Adrian Hong. He paints himself as a heroic idealist while ignoring the messy, dangerous reality of what he was actually doing. The author, Bradley Hope, seems too impressed by his subject to be critical. It reads less like journalism and more like PR. Disappointing given the potential of the story.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.5",
            "judge_key_points": "1) It is tragic there aren\u2019t more people willing to take action against evil. 2) Thanks to the author for telling this story.",
            "judge_score": 0.4,
            "judge_thought": "The generated response reflects the first key point fairly well (tragedy about lack of people taking risks for truth) scoring about 0.8, but only partially reflects the gratitude to the author, scoring about 0.4. Coverage C = (0.8+0.4)/2 = 0.6. The extra comments about writing style and the book are unnecessary but not contradictory, giving a modest penalty P \u2248 0.2. Final score = 0.6 - 0.2 = 0.4.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "It is tragic there aren\u2019t more people willing to take risks for the truth. I like the writing style. The author is very good at explaining complex events. It is a good book.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) Expresses that it is tragic/sad that few people are willing to take action against evil. 2) Thanks or shows gratitude to the author for telling the story.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mirrors both key points: it calls the situation sad/tragic that few confront evil and expresses gratitude to the author, matching the ground truth closely with no extraneous or conflicting content.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "It is a sad commentary on the current state of the world that there are so few people willing to confront evil, but I am grateful to the author for telling this remarkable story.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) Expresses that it is tragic there aren't more people willing to take action against evil. 2) Thanks the author for telling the story.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, fully reflecting both key points with no extra or conflicting content, so coverage=1.0, penalty=0.0, final score=1.0",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "It is tragic there aren\u2019t more people willing to take action against evil. Thanks to the author for telling this story.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Expresses that it is tragic there are not more people willing to act against evil. 2) Thanks the author for telling the story.",
            "judge_score": 0.0,
            "judge_thought": "The generated response is a detailed book review that does not mention the tragedy of few people acting against evil nor thank the author; therefore it matches none of the key points (m1=0, m2=0). Coverage C=0. It adds extensive unrelated content, so a penalty of about 0.7 is appropriate. Final score = max(0, 0 - 0.7) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "4.0 out of 5 stars A gripping, if occasionally frantic, ride Reviewed in the United States on November 15, 2022 Verified Purchase I picked this up immediately after finishing *Billion Dollar Whale*, so I was already primed for Bradley Hope\u2019s narrative style. If you like your non-fiction to read like a thriller, this is it. The story of Adrian Hong is genuinely unbelievable\u2014going from a Yale undergrad to orchestrating an armed raid on a North Korean embassy is the kind of plot twist that usually signals a fiction book, not a journalistic account. The Kindle edition is straightforward. Since this is a narrative-heavy book rather than a dense academic text, the enhanced typesetting works well. I found the Word Wise feature helpful for the occasional Korean term or political jargon, though I mostly read without it to keep the flow going. The file size is small, so it loaded instantly on my Paperwhite. My main critique is that the book sometimes feels like it\u2019s trying too hard to justify Hong\u2019s actions. Hope does a great job reporting the facts, but there\u2019s a palpable tension between the \"heroism\" and the \"insanity\" mentioned in the blurb. You finish the book wondering if this was a brave act of resistance or a dangerous delusion. It\u2019s a compelling read, but it left me with more questions than answers. Definitely worth the time if you\u2019re interested in espionage, North Korea, or just wild true stories. Just don\u2019t expect a balanced, objective critique of Hong\u2019s methods\u2014it\u2019s told largely through the lens of his mission.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Expresses that it is tragic there aren't more people willing to take action against evil. 2) Thanks the author for telling the story.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth wording, fully reflecting both key points with no extra or conflicting content, so coverage C=1 and penalty P=0, yielding a final score of 1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "It is tragic there aren't more people willing to take action against evil. Thanks to the author for telling this story.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:indist_eval:0dc06d30dd88621081fc:prompt3",
      "category": "Kindle_Store",
      "cohort": "seen",
      "eval_pair_id": "amzp_c255d374972bea8f1c41692fce19b60e17a1bc0eea86166c593734857930efde",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2013-06-22",
            "length_band": "Medium",
            "length_chars": 229,
            "post_id": "amzr_2ee62982a74a8df964b1c038dca5ec9af9aef0e416809e2dd16417ff463c4264",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "Amazon.com Review Author One-on-One: Deborah Crombie and Elizabeth George Elizabeth George is the bestselling author of sixteen suspense novels featuring Scotland Yard Inspector Lynley, including her latest, Believing the Lie . Elizabeth George: In many ways, you and I are \"working the same patch\", so I'm curious about your methods. How did you balance internet research with on-site research for The Sound of Broken Glass ? Deborah Crombie: I try to get over to the UK (usually London) once or twice during the writing of every book. I search out obscure books on the setting and subject of the no",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "2594 KB",
                "Language": "English",
                "Page numbers source ISBN": "0061990639",
                "Print length": "373 pages",
                "Publication date": "February 19, 2013",
                "Publisher": "William Morrow; Reprint edition (February 19, 2013)",
                "Screen Reader": "Supported"
              },
              "features": [
                "Scotland Yard detectives Duncan Kincaid and Gemma James are on the case in Deborah Crombie\u2019s",
                "The Sound of Broken Glass",
                ", a captivating mystery that blends a murder from the past with a powerful danger in the present.",
                "When Detective Inspector James joins forces with Detective Inspector Melody Talbot to solve the murder of an esteemed barrister, their investigation leads them to realize that nothing is what it seems\u2014with the crime they\u2019re investigating and their own lives."
              ],
              "product_id": "B0089LOOB8",
              "title": "The Sound of Broken Glass: A Novel (Duncan Kincaid / Gemma James Book 15)"
            },
            "rating": 2.0,
            "review_type": "demonstration",
            "text": "First book of this series I have read. Ridiculous plot. Boring characters. Don't waste your time and money. It is a sad commentary on the current state of literature that this book is number 15 in an apparently successful series."
          },
          {
            "date": "2014-04-29",
            "length_band": "Long",
            "length_chars": 519,
            "post_id": "amzr_207a3b4ddefa07010bf916b04bff2db228663ac9f86e1c2cfb4ad12db502ff2b",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "From Publishers Weekly Starred Review. Reviewed by Megan Whalen Turner If there really are only seven original plots in the world, it's odd that boy meets girl is always mentioned, and society goes bad and attacks the good guy never is. Yet we have Fahrenheit 451 , The Giver , The House of the Scorpion \u2014and now, following a long tradition of Brave New Worlds, The Hunger Games . Collins hasn't tied her future to a specific date, or weighted it down with too much finger wagging. Rather less 1984 and rather more Death Race 2000 , hers is a gripping story set in a postapocalyptic world where a rep",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "11319 KB",
                "Language": "English",
                "Print length": "387 pages",
                "Publication date": "September 1, 2009",
                "Publisher": "Scholastic Press; Reprint edition (September 1, 2009)",
                "Screen Reader": "Supported",
                "Sticky notes": "On Kindle Scribe"
              },
              "features": [
                "This Special Edition of",
                "The Hunger Games",
                "includes the most extensive interview Suzanne Collins has given since the publication of",
                "; an absorbing behind-the-scenes look at the creation of the series; and an engaging archival conversation between Suzanne Collins and YA legend Walter Dean Myers on writing about war. The Special Edition answers many questions fans have had over the years, and gives great insight into the creation of this era-defining work."
              ],
              "product_id": "B002MQYOFW",
              "title": "The Hunger Games (Hunger Games Trilogy, Book 1)"
            },
            "rating": 3.0,
            "review_type": "demonstration",
            "text": "Great concept, fast paced action, page turner. But in the end it was just OK for me. Where was the spiritual/moral dimension of the character development? For example, WHY did characters do what they did, for good or evil? Surely there wuld still be a few folks who believed in God after all that had happened. I read the reviews of books 2 and 3 in the trilogy. I probably won't read them. Many thanks to those who included spoilers so I know how the story turns out without having to buy and read the other two books."
          },
          {
            "date": "2016-11-06",
            "length_band": "Short",
            "length_chars": 56,
            "post_id": "amzr_3652bc25350a9ab85407504206ceeb8c1ffb2bc8b517bfcf8a3c1fb0ba9d69e2",
            "product": {
              "category": "Kindle_Store",
              "description_excerpt": "Review \u201cA novel to cure your Downton Abbey withdrawal . . . a delightful story about nontraditional romantic relationships, class snobbery and the everybody-knows-everybody complications of living in a small community.\u201d \u2014 The Washington Post \u201cWhat begins as a study of a small-town society becomes a compelling account of war and its aftermath.\u201d \u2014 Woman\u2019s Day \u201cThis witty character study of how a small English town reacts to the 1914 arrival of its first female teacher offers gentle humor wrapped in a hauntingly detailed story.\u201d \u2014 Good Housekeeping \u201cPerfect for readers in a post\u2013 Downton Abbey sl",
              "details": {
                "Enhanced typesetting": "Enabled",
                "File size": "6448 KB",
                "Language": "English",
                "Print length": "481 pages",
                "Publication date": "March 22, 2016",
                "Publisher": "Random House; Reprint edition (March 22, 2016)",
                "Screen Reader": "Supported",
                "Sticky notes": "On Kindle Scribe"
              },
              "features": [
                "NEW YORK TIMES",
                "BESTSELLER \u2022 \u201cA novel to cure your",
                "Downton Abbey",
                "withdrawal . . . a delightful story about nontraditional romantic relationships, class snobbery and the everybody-knows-everybody complications of living in a small community.\u201d\u2014"
              ],
              "product_id": "B0104EOGGK",
              "title": "The Summer Before the War: A Novel"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Great characters. Interesting time and place in history."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 3,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Medium",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:indist_eval:0dc06d30dd88621081fc",
      "source_line_sha256": "9e89c37838819420e2eda10e729f3400d6a8deac7c1e13440be3a3ac5620ca41",
      "split": "indist_eval",
      "split_label": "Evaluation / seen user",
      "target": {
        "date": "2023-01-09",
        "length_band": "Medium",
        "length_chars": 119,
        "post_id": "amzr_f109a2b05d0f972ff332955db8622e60bd3ddec86951ee738889741843a56e5a",
        "product": {
          "category": "Kindle_Store",
          "description_excerpt": "Review \u201c The Rebel and the Kingdom is an expertly crafted thriller that makes you turn the page. What makes it all the more remarkable is that it\u2019s all true. Bradley Hope is a first-rate reporter and writer, and he has found a subject worthy of his skills in Adrian Hong, a Korean American idealist who set out, improbably enough, to topple the North Korean regime.\u201d \u2014Max Boot, columnist for The Washington Post , author of The Road Not Taken \u201cOne of the most harrowing and inspiring stories I\u2019ve ever read . . . This book left me in tears. Hope tells this inconceivable and heart-wrenching story wit",
          "details": {
            "Enhanced typesetting": "Enabled",
            "File size": "26590 KB",
            "Language": "English",
            "Page numbers source ISBN": "1399806181",
            "Print length": "260 pages",
            "Publication date": "November 1, 2022",
            "Publisher": "Crown (November 1, 2022)",
            "Screen Reader": "Supported"
          },
          "features": [
            "How did an Ivy League activist become a global fugitive? The",
            "New York Times",
            "bestselling co-author of",
            "Billion Dollar Whale"
          ],
          "product_id": "B09P9L9S8G",
          "title": "The Rebel and the Kingdom: The True Story of the Secret Mission to Overthrow the North Korean Regime"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "It is tragic there aren\u2019t more people willing to take action against evil. Thanks to the author for telling this story."
      },
      "uid": "amzu_5590b8acb61725d930902bc3afb3c9a9910ba5dcbcd63054ede40f3216ae133d"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.9,
            "score_mean": 0.26666666666666666,
            "score_median": 0.2,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.87,
            "score_mean": 0.46666666666666673,
            "score_median": 0.47,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.5708333333333334,
            "score_median": 0.57,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5453611111111111,
            "score_median": 0.67,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1) The user has several kids (ages 5 to 10) who love the tablet. 2) A 5\u2011year\u2011old broke her screen and got a replacement under the 2\u2011year warranty. 3) The user recommends buying the Kids Edition if you\u2019re getting a Fire tablet.",
            "judge_score": 0.27,
            "judge_thought": "The generated response mentions a single child (7\u2011year\u2011old) enjoying the tablet and praises the case and 2\u2011year guarantee, which aligns partially with point 1 and strongly with point 3, but it does not mention multiple kids or the broken\u2011screen warranty claim, so matches are 0.5, 0.0, 0.9 respectively. Coverage = (0.5+0+0.9)/3 \u2248 0.467. The response adds a lot of extra personal detail that isn\u2019t in the ground truth, incurring a modest penalty of about 0.2. Final score = 0.467\u20110.2 \u2248 0.267.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a tablet that actually looks like it belongs in a preschooler's hand without me having to constantly warn them not to chew on it! I was really worried about the \"Previous Generation\" label on the title, but the 8th Gen Fire HD 8 is still a beast. My 7-year-old has been using it for two weeks now, and the battery life is exactly as advertised\u2014I charged it up in the morning and he was still playing games and watching Dora in Spanish at 8 PM. The 32GB storage is fine for now, but I already added a microSD card to get him up to 200GB for his Minecraft and downloaded books. The biggest win here is the Kid-Proof Case. It's blue, which he loves, and it's surprisingly durable. He dropped it on the hardwood floor yesterday and there was zero damage. The FreeTime Unlimited subscription is a game-changer; we've been able to filter out everything inappropriate and set a screen time limit that actually gets him to finish his homework before he gets to watch cartoons. I love that we can manage all this through the Parent Dashboard on my phone. The only minor gripe is that the speakers are a bit tinny compared to a real laptop, but for his age group, it's perfect. If you are on the fence about buying a tablet just for kids, this is the one. The 2-year guarantee gives me peace of mind too. Just remember to cancel the subscription if you don't want it to auto-renew, though I don't see us doing that anytime soon. Highly recommend for any parent looking for a safe, fun, and screen-safe (pun intended) device.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl1",
            "judge_key_points": "1) The user\u2019s kids love the tablet (ages 5-10). 2) The 2\u2011year warranty replaced a broken screen, showing its value. 3) The user recommends buying the Kids Edition.",
            "judge_score": 0.1,
            "judge_thought": "The generated response recommends the tablet and praises durability, matching the recommendation point well (\u22480.9). It does not mention the kids loving it or their ages, nor does it mention the warranty screen replacement, so those points get 0.0 each. Coverage = (0.9+0+0)/3 = 0.3. The response adds extra details about screen brightness, battery life, etc., which are irrelevant to the ground truth but not contradictory, incurring a small penalty (~0.2). Final score = 0.3\u20110.2 = 0.1.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 1,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a tablet that actually feels like it belongs in a preschooler's hand without the fear of it being crushed. The blue case is thick and sturdy, which is exactly what I needed since my 6-year-old is constantly dropping things on the kitchen floor. The screen is bright enough to see outdoors in the backyard, and the battery lasts all day even with the heavy use of apps. The FreeTime Unlimited subscription is a lifesaver for keeping content age-appropriate, and I love that we can set specific screen time limits. The 32GB storage is plenty for the cartoons and games we load up. It's not the newest model, but for the price with the case and subscription included, it's a steal. Highly recommend for parents looking for something durable and safe.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.75",
            "judge_key_points": "1) The user has four kids aged 5 to 10. 2) Their 5\u2011year\u2011old broke the tablet screen and got a brand\u2011new one under the 2\u2011year warranty. 3) They recommend buying the Kids Edition if you purchase a Fire tablet.",
            "judge_score": 0.47,
            "judge_thought": "The generated response repeats points 1 and 2 accurately (m1=1, m2=1) but omits the recommendation (m3=0). It adds an extra comment about storage that is irrelevant, incurring a small penalty (P\u22480.2). Coverage C=(1+1+0)/3\u22480.667, final score = C\u2011P\u22480.467.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "My kids live these I have 4 from 5 to 10 My 5 year old broke her screen and I got a brand new one because of the 2 year warranty if you break it you get a new one for free The only thing I wish it had was more storage space but I use a memory card for that so it works fine",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The user has four kids aged roughly 5 to 10 and is happy with the tablet for them. 2) A 5\u2011year\u2011old broke her screen and a brand\u2011new replacement was received under the 2\u2011year warranty. 3) The user recommends buying the Kids Edition if you are getting a Fire tablet.",
            "judge_score": 0.83,
            "judge_thought": "The generated response mentions having four kids ages 5\u201110 (matches point 1), describes a grandchild dropping the tablet and getting a replacement under the 2\u2011year warranty (matches point 2), and strongly recommends buying the Kids Edition (matches point 3). Minor variations (grandson vs child, age 6 vs 5) are acceptable, so each key point gets a high match (\u22480.9\u20111.0). Extra positive comments about the case and tablet are relevant and not contradictory, incurring a small penalty (~0.1). Final score = coverage 0.933 \u2013 penalty 0.1 \u2248 0.83.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "My 6 yr old grandson dropped it from the bed and cracked the screen. Amazon replaced it under the 2 year warranty instantly. I have 4 kids ages 5-10 so I know this is built tough. The kid proof case is great and the tablet is perfect for them all. Buy the kids edition.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The user has four kids aged 5 to 10 and likes the Fire Kids Edition tablet. 2) Their 5\u2011year\u2011old broke the screen and got a brand\u2011new replacement under the 2\u2011year warranty. 3) They recommend buying the Kids Edition if you are getting a Fire tablet.",
            "judge_score": 1.0,
            "judge_thought": "The generated response is identical to the ground truth, perfectly reflecting all three key points with no extra or conflicting content, so coverage C=1 and penalty P=0, yielding a final score of 1.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "My kids live these I have 4 from 5 to 10 My 5 year old broke her screen and I got a brand new one because of the 2 year warranty if you are going to buy a fire tablet get the kids edition.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl1",
            "judge_key_points": "1) User has four kids aged 5 to 10 and likes the product. 2) A 5\u2011year\u2011old broke her screen and got a replacement under the 2\u2011year warranty. 3) Recommendation: if buying a Fire tablet, get the Kids Edition.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions the 2\u2011year warranty (partial match to point 2) but does not mention the multiple kids or their ages (misses point 1) and does not give the explicit recommendation to buy the Kids Edition (misses point 3). Match scores: m1=0, m2\u22480.5, m3=0.0 giving coverage C\u22480.166. Extra details about battery life, storage, etc. are irrelevant to the ground truth and add mild unnecessary length, so penalty P\u22480.2. Final score = max(0, 0.166\u20110.2)=0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 1,
            "privileged": null,
            "refine_round": null,
            "response": "This is a great tablet for the price. The case is very sturdy and has held up well to drops. My kid loves the parental controls and the content selection in FreeTime Unlimited. The battery life is decent, usually lasting a day with moderate use. The 32GB storage is a bit limited so I added a microSD card. Overall, happy with this purchase and the 2-year warranty gives peace of mind.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The user has four kids aged 5 to 10. 2) Their 5\u2011year\u2011old broke the tablet screen and got a replacement under the 2\u2011year warranty. 3) They recommend buying the Kids Edition if purchasing a Fire tablet.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mentions four kids ages 5\u201110, describes the screen breaking and being replaced under the 2\u2011year warranty, and strongly recommends the Kids Edition, matching all three key points precisely with no irrelevant or speculative content. Coverage 1.0, penalty 0.0, final score 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "I have four kids (ages 5-10) and this tablet has taken a beating. The screen finally broke after a few months, but Amazon replaced it for free under the 2-year warranty with no hassle. I highly recommend the Kids Edition for anyone buying a Fire tablet; it's tough and the warranty makes it a no-brainer.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:356e2bc4a3ec46412f01:prompt0",
      "category": "Electronics",
      "cohort": "heldout",
      "eval_pair_id": "amzp_975b4fb4ba935ae827f90b2ba6c011dfdc4d76bcd1506cc38d8047528765e73c",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2012-02-14",
            "length_band": "Long",
            "length_chars": 250,
            "post_id": "amzr_0035afe3f211e2e777f350f4ef9fc3aaa0f793ab5898a144bd6e9233e2c56ecf",
            "product": {
              "category": "Electronics",
              "description_excerpt": "Mediabridge Ultra Series subwoofer cable connects a subwoofer speaker to an audio component like a stereo receiver or sound system, while offering excellent bass response & flexibility. This Low-loss, wide spectrum coaxial cable is the ideal medium for low-frequency signals produced by subwoofers, providing Deep/precise audio quality. Built with a copper clad center conductor, dielectric insulation, aluminum braiding & 2 layers of foil shielding, for natural sounds & rejection of hum/static that occurs when devices are placed near each other. A split-tip center pin ensures high contact pressur",
              "details": {
                "Brand": "Mediabridge",
                "Cable Type": "Coaxial",
                "Color": "Black",
                "Compatible Devices": "Speaker",
                "Connectivity Technology": "RCA",
                "Connector Gender": "Male",
                "Connector Type": "RCA",
                "Date First Available": "September 2, 2010"
              },
              "features": [
                "Ideal for connecting subwoofer speakers to audio components, such as stereo receivers or sound systems, while offering excellent bass response and flexibility.",
                "Low-loss, wide spectrum coaxial cable great for the low-frequency signals produced by subwoofers, allowing for deep and precise audio quality.",
                "A split-tip center pin ensures high contact pressure and signal strength, while unique gripped molding and corrosion-resistant gold-plated RCA connectors provide a consistently tighter fit than commonly rolled cables with wider tolerances.",
                "A copper clad center conductor, dielectric insulation, aluminum braiding and 2 layers of foil shielding, for maximum protection against the hum/static encountered when multiple devices are in close proximity of each other. The cable is finished in tough, yet pliable PVC jacket with color coding, for easy routing and quick installation."
              ],
              "product_id": "B0041OOFE8",
              "title": "Mediabridge ULTRA Series Subwoofer Cable (35 Feet) - Dual Shielded with Gold Plated RCA to RCA Connectors - Black"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I have a projector mounted on my ceiling so my audio set up is actually behind me on the wall this cable is great because I wanted to set up my subwoofer at the front of the room near the screen excellent quality cable really exceeded my expectations"
          },
          {
            "date": "2014-11-01",
            "length_band": "Medium",
            "length_chars": 184,
            "post_id": "amzr_a0df60479a5e13ca4c495fd84e2ff1919679c132af6803dcc95e77f18d76e126",
            "product": {
              "category": "Electronics",
              "description_excerpt": "Product Description Need to get organized with your DVD collection? This Onyx DVD black Wire rack will assist you. The 28-capacity rack stands on the ground or mounts on the wall. The styling metal wire construction will spice up the display of your media collection. From the Manufacturer MAKE ATLANTIC'S \"ONYX 65 DVD, BLU-RAY AND GAME TOWER\" THE BASE OF YOUR DISC COLLECTION Atlantic\u2019s ONYX 65 BLU-RAY/DVD/GAME TOWER sports an attractive, modern design to highlight almost any home or office d\u00e9cor. Its efficient, space-saving design provides maximum DVD, BLU-RAY and GAME storage in a minimal amou",
              "details": {
                "Assembled Height": "0 inches",
                "Assembled Length": "0 inches",
                "Assembled Width": "0 inches",
                "Country of Origin": "China",
                "Date First Available": "May 14, 2003",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "4.9 pounds",
                "Item model number": "1332"
              },
              "features": [
                "Attractive Modern Design Highlights Any Home Entertainment or Office decor",
                "Efficient Space-Saving Design *Provides Maximum Storage in a Minimum Amount of Space",
                "Flexible Usage to Meet All Your Needs - *Can be Wall Mounted or Freestanding *Heavy Gauge Steel Construction for Years of Use *Ready to Use in Minutes - No Tool Assembly Required!",
                "*DVD Capacity 65 *BluRay Capacity 65 *Gaming Capacity 65 wii 65 PS3"
              ],
              "product_id": "B07Q2HB7TY",
              "title": "Atlantic Onyx 65 Wire DVD-Tower - Holds 65 DVDs, Blu-Rays or PS3 Games, Wall Mount or Freestanding in Matte Black Steel, PN 1332"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I love these things they work so great hanging on a wall on both sides of your mounted television I also have one on the wall between my kids bedrooms for their movies these work great"
          },
          {
            "date": "2016-02-10",
            "length_band": "Medium",
            "length_chars": 94,
            "post_id": "amzr_7b784777733e8607fca7a50c32208d57ebd10d8918a48818072ee9a516ef220d",
            "product": {
              "category": "Electronics",
              "description_excerpt": "MUJOY",
              "details": {
                "Brand": "IXCC",
                "Cable Type": "USB",
                "Color": "3 Feet/5-Pack",
                "Compatible Devices": "Tablet, Smartphone",
                "Compatible Phone Models": "Nexus-4, Samsung-galaxy-note",
                "Connector Gender": "Male-to-Male",
                "Connector Type": "Micro USB, USB 2.0",
                "Date First Available": "March 5, 2014"
              },
              "features": [
                "Trusted by 5 million+ people for high quality innovative products.",
                "Compatible with Kindle, Kindle Fire, Smartphone, tablet, MP3 player, camera, hard drive, e-reader, external battery, handheld game console, all devices with micro-usb connector.",
                "It is ideal to use with USB charging port chargers while in the car, office and home",
                "Cable is made with 24 AWG High Gauge Low Impedance wire enabling high performance with many USB devices. Sturdy boot gives this cable added strength and durability, over 20000+ Bend lifespan."
              ],
              "product_id": "B00KDOUH0G",
              "title": "[5Pack] iXCC 3 Feet Micro USB to USB 2.0 Cable, Super Durable A Male to Micro B Charge and Sync Cord for Android/Windows/MP3/Camera and Other Device"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Decent quality cables and good enough for usb tethering to use your phone as Internet to a pc."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 0,
      "quick_eval": true,
      "review_strata": {
        "length_band": "Medium",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:heldout_eval:356e2bc4a3ec46412f01",
      "source_line_sha256": "47fa84672b678b51c5489ae300ea0a89d666dda3dd8d89a3482ecccd2d8815a5",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2019-09-22",
        "length_band": "Medium",
        "length_chars": 193,
        "post_id": "amzr_3c5b460987b18ff5046aa4b65664f53396512ed10a87fa51e2ae627181fc218e",
        "product": {
          "category": "Electronics",
          "description_excerpt": "",
          "details": {},
          "features": [
            "Up to $124 in savings on Fire HD 8 tablet, 1 year of Amazon FreeTime Unlimited, and a Kid-Proof Case, versus items purchased separately - plus a 2-year worry-free guarantee",
            "Not a toy, a full-featured Fire HD 8 tablet with a vibrant 8\" HD display, 32 GB internal storage, up to 10 hours of battery life, a 1.3 GHz quad-core processor, and 1.5 GB of RAM. Add a microSD card for up to 400 GB of expandable storage.",
            "The included 1 year of FreeTime Unlimited gives your kids access to over 20,000 popular apps and games, videos, books, Audible books, and educational content from PBS Kids, Nickelodeon, Disney, and more. Your subscription will then automatically renew every month starting at just $2.99 per month plus applicable tax. You may cancel at any time by visiting the Amazon Parent Dashboard or contacting Customer Service.",
            "2-year worry-free guarantee: if it breaks, return it and we'll replace it for free."
          ],
          "product_id": "B07F4MYW5B",
          "title": "Fire HD 8 Kids Edition Tablet, 8\" HD Display, 32 GB, Blue Kid-Proof Case (Previous Generation - 8th)"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "My kids live these I have 4 from 5 to 10 My 5 year old broke her screen and I got a brand new one because of the 2 year warranty if you are going to buy a fire tablet get the kids edition."
      },
      "uid": "amzu_6ff03312c1ea1dc2fa4147a92262b2fe36fd2ada0fd5c4853ec75b512d92363e"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.9,
            "score_mean": 0.23263888888888895,
            "score_median": 0.15,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5736111111111111,
            "score_median": 0.9,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "1) The cloth is not good (poor quality). 2) It is easy to get off the ball (tends to pill or come apart).",
            "judge_score": 0.0,
            "judge_thought": "The generated response praises the hoodie, saying the cloth is soft, durable, and high quality, which directly contradicts the ground truth's negative assessment. It does not mention any poor quality or the issue of the cloth getting off the ball, so both key points receive a match score of 0. Coverage C = 0. The response adds extensive unrelated positive content, which is contradictory to the ground truth, resulting in a high penalty (\u22480.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a hoodie that actually lives up to the hype. I've been wearing Champion hoodies for years, but this Powerblend one feels like a significant upgrade in comfort without losing that classic athletic look. The 50/50 cotton/poly blend is a game-changer; it's noticeably softer against the skin than my old 100% cotton hoodies, and after a few washes, I'm still not seeing any pilling or shrinking. The brushed interior makes it perfect for those chilly mornings. I went with the standard fit, and the longer length is a huge plus\u2014it covers my hips just right without looking bulky. The wider ribbed cuffs and waistband do exactly what they promise: they stay flat and keep the cold air out without digging into my wrists or waist. The hand-warming pocket is deep enough for my phone and keys, which is rare these days. The quality feels solid, especially with the triple stitching on the seams. The embroidered C logo on the chest is subtle but iconic, and I like that there's one on the sleeve cuff too. Overall, this is a staple piece I'll be reaching for all season long. Highly recommend for anyone looking for durable, comfortable, and warm everyday wear.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl4",
            "judge_key_points": "1) The cloth quality is poor. 2) It easily sheds/pills (easy to get off the ball).",
            "judge_score": 0.0,
            "judge_thought": "The generated response praises the hoodie as soft, warm, and a good fit, which directly contradicts the ground truth's negative assessment of cloth quality and pilling; it does not reflect either key point (match scores 0 for both). It also adds irrelevant positive details, introducing contradictory content, so a high penalty is applied. Final score is 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I bought this hoodie for my husband and he loves it! The material is very soft and warm, perfect for winter. The C logo looks great on the chest. I'm 5'6, 161lb and I ordered a medium for him, it fits him perfectly. Just buy the size you usually wear. I wish this will help you to choose a right size:)",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.75",
            "judge_key_points": "1) The cloth quality is poor. 2) It is easy to get off the ball (i.e., pills or shedding).",
            "judge_score": 0.1,
            "judge_thought": "The generated response mentions the cloth is not good, matching key point 1 fairly well (m\u22480.8). It does not mention the 'easy to get off the ball' issue, so key point 2 gets m=0.0. Coverage = (0.8+0)/2 = 0.4. The extra statements about dirt, fading, and fit are unsupported and shift focus, incurring a moderate penalty (~0.3). Final score = 0.4\u20110.3 \u2248 0.1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "The cloth is not good, it is easy to get dirty and the color fades quickly. The fit is also a little bit big.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The cloth is not good (negative assessment of quality). 2) It is easy to get off the ball (fabric tends to come off or pill easily).",
            "judge_score": 1.0,
            "judge_thought": "The generated response states both that the cloth is not good and that it is easy to get off the ball, matching both key points precisely with no extra or conflicting content.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "The cloth is not good and easy to get off the ball.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The cloth is not good (negative assessment of quality). 2) It is easy to get off the ball (implies it sheds or pills easily).",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth sentence, covering both key points perfectly with no extra or conflicting content, so m1=1, m2=1, C=1, P=0, final score=1.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "The cloth is not good, it is easy to get off the ball",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The cloth is not good (negative assessment of material quality). 2) It is easy to get off the ball (the fabric sheds or pills easily).",
            "judge_score": 0.0,
            "judge_thought": "The generated response praises the hoodie, stating the material feels substantial, soft, durable, and does not pill, which directly contradicts both key points of the ground truth. It provides no reflection of the negative quality or shedding issue, so match scores are 0 for each key point. Coverage C = 0. The response adds extensive unrelated positive commentary, which is irrelevant and contradictory, warranting a high penalty (\u22480.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been wearing Champion hoodies for years, and this Powerblend is easily one of the best I\u2019ve owned. The 50/50 cotton-polyester mix is a smart choice\u2014it feels substantial but doesn\u2019t feel heavy, and I really appreciate that it\u2019s brushed on the inside for that soft, comfortable feel right out of the box. Fit is spot on. I usually go for a Large, and this fits true to size with that slightly longer length that provides good coverage without looking sloppy. The triple stitching on the seams gives me confidence that this thing will hold up wash after wash. I\u2019ve already thrown it in the machine a few times, and there\u2019s been zero shrinkage or pilling so far, which is exactly what I want from a daily driver hoodie. The wider ribbed cuffs and waistband are a nice touch; they stay flat and don\u2019t ride up like some cheaper brands. The half-moon neck facing is also a subtle detail that prevents that itchy tag feeling against your neck. The embroidered C logo looks sharp and high-quality, not like it\u2019s going to peel off after a month. If you\u2019re looking for a durable, soft, and well-made hoodie that won\u2019t break the bank, this is it. Highly recommend.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The cloth/fabric is not good (poor quality). 2) It is easy to get off the ball (fabric sheds or comes off easily).",
            "judge_score": 1.0,
            "judge_thought": "The generated response states the fabric is cheap and feels like trash, matching the 'not good' point, and repeats 'easy to get off the ball', matching the second point. No extra or conflicting content is present, so penalty is 0. Coverage is 1, final score = 1.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "The fabric is cheap and feels like trash. It's easy to get off the ball and not good.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:a1b49569a7a3a6b8a73c:prompt1",
      "category": "Clothing_Shoes_and_Jewelry",
      "cohort": "heldout",
      "eval_pair_id": "amzp_c9a4dd4f86df01d99f1c6e66cf40ecab57fcdaf883ffe1f8c24db24b5a61704c",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2013-07-18",
            "length_band": "Long",
            "length_chars": 310,
            "post_id": "amzr_57f989f1da7f7cdbee42f1672502d6e5cd929d87e636ecd40a31ad68b81801a0",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "More than 140 years after inventing the blue jean, one thing is clear: Levi\u2019s clothes are loved by the people who wear them - from presidents to movie stars, farmers to fashion icons, entrepreneurs to the everyman. 'Live in Levi\u2019s\u2019 asserts with confidence and pride that Levi\u2019s clothes are indeed for everybody who\u2019s not just anybody.",
              "details": {
                "Date First Available": "October 25, 2008",
                "Department": "Mens",
                "Manufacturer": "Levi's"
              },
              "features": [
                "100% Cotton",
                "Imported",
                "Machine Wash",
                "Sits below waist"
              ],
              "product_id": "B096NLJ14L",
              "title": "Levi's Men's 514 Straight-Fit Jeans"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "I love this pants,it's what i expected!The material is comfy and soft. I knew people buy pants online for a wrong size because of it's inaccurate size info.But the size of this pants is right.Just buy the size you usually wear.I'm 5'6,161lb,so i ordered33x30 ,i wish this will help you to choose a right size:)"
          },
          {
            "date": "2013-07-18",
            "length_band": "Medium",
            "length_chars": 188,
            "post_id": "amzr_87a8e694677711757e990d2eb239ff6dc73afbc8b9ce71b4d1be0f1ed932ba7b",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "Timberland set new industry standards for craftsmanship, durability and protection when it introduced an authentic, waterproof leather boot in 1973. A global leader in design, engineering and marketing of premium footwear, Timberland values consumers who cherish the outdoors and their time in it. Timberland has been an industry leader in the use of both sustainable and recycled materials in its shoes, as well as its footwear packaging. The Timberland Company believes in corporate responsibility and supports numerous civic and social projects throughout the year.",
              "details": {
                "Date First Available": "February 26, 2013",
                "Department": "Mens",
                "Item Weight": "2 Pounds",
                "Item model number": "Casco Bay Leather 1 Eye",
                "Manufacturer": "Timberland"
              },
              "features": [
                "100% Suede",
                "Imported",
                "Rubber sole",
                "Premium suede for durability"
              ],
              "product_id": "B008ZBP2RC",
              "title": "Timberland Men's Casco Bay Leather 1 Eye Slip-On"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "This shoes is really comfortable,and suitable.The color is just what appears on your screen,I usually wear 8,and the size of this shoes is right,but I guess 8.5 may fits me more perfectly."
          },
          {
            "date": "2013-07-18",
            "length_band": "Medium",
            "length_chars": 183,
            "post_id": "amzr_198247bd0f12844da4d8ef6e2691bd796bbb96a51a137c4d2c5506d1b37630e9",
            "product": {
              "category": "Clothing_Shoes_and_Jewelry",
              "description_excerpt": "Add a little dazzle and a touch of purple color to your accessory collection with the Carnevale Sterling Silver Fancy Shape Made with Swarovski Elements Pendant Necklace. Resting from an 18-inch box chain is an elegant pendant design of wrapping sterling silver metal that is embellished with tiny clear Swarovski Elements. Eyes will be drawn to the statement faceted Swarovski crystal briolette revealing vibrant purple sparkle. Sure to add glamour to a fancy outfit. Capture the look with the matching earrings. The Amazon Curated Collection Discover the Amazon Curated Collection of fine and fashi",
              "details": {
                "Date First Available": "October 14, 2011",
                "Department": "Women",
                "Item model number": "SB4189CYBACA",
                "Manufacturer": "Amazon Curated Collection",
                "Package Dimensions": "4.72 x 3.94 x 0.63 inches; 0.64 Ounces"
              },
              "features": [
                "Sterling silver necklace featuring teardrop briolette suspended from Swarovski crystal-encrusted bail",
                "Box chain with spring-ring clasp",
                "Imported"
              ],
              "product_id": "B005NGU8OY",
              "title": "Sterling Silver Purple Special Cut Briolette and White with Swarovski Elements Pendant Necklace, 18\""
            },
            "rating": 4.0,
            "review_type": "demonstration",
            "text": "I ordered 2 of these items,one for me and one foe my sister.I felt a little disappionted because it's sooo big and stone is Synthetic.I don't think it fits me,maybe it fits my sister."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 1,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 3
      },
      "row_id": "humanual-amazon-v1:heldout_eval:a1b49569a7a3a6b8a73c",
      "source_line_sha256": "7adf563cb2c7ad19c282bf6d3a9286941521eeeee6445476236c833b5ba6d381",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2018-03-28",
        "length_band": "Short",
        "length_chars": 53,
        "post_id": "amzr_cbb3e15cb06177e335ba0bedc0b6bbfa758f60b59e9c7a171e98750e310271c2",
        "product": {
          "category": "Clothing_Shoes_and_Jewelry",
          "description_excerpt": "The Powerblend hoodie from Champion is the perfect balanced combination of plushy softness and high-quality cotton. Double-stitched needlework for longevity, make it a sweatshirt you'll love forever. Midweight fleecy softness with a brushed interior keeps you warm all day meaning this pullover is great for layers or rocking on a chilly morning walk or jog.",
          "details": {
            "Date First Available": "July 20, 2016",
            "Department": "mens",
            "Item model number": "S0889X",
            "Manufacturer": "Champion",
            "Product Dimensions": "10.1 x 11 x 0.7 inches; 6.4 Ounces"
          },
          "features": [
            "Body: 50% Cotton/50% polyester; rib: 49% cotton/49% polyester/2% spandex; half moon: solid: 100% cotton; granite: 50% cotton/50% polyester; oxford: 90% cotton/10% black polyester",
            "Imported",
            "Pull On closure",
            "Machine Wash"
          ],
          "product_id": "B0BPFTD8B9",
          "title": "Champion Men's Powerblend Hoodie for Men, Hoodie Sweatshirt, Iconic C Logo (Reg. or Big & Tall)"
        },
        "rating": 3.0,
        "review_type": "target",
        "text": "The cloth is not good, it is easy to get off the ball"
      },
      "uid": "amzu_00c932a016c736f984664beb60f8c9eeac69e757e988363e3deb83e15da9c203"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.7,
            "score_mean": 0.10875,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.45,
            "score_mean": 0.01875,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.87,
            "score_mean": 0.4306944444444445,
            "score_median": 0.45,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5852777777777776,
            "score_median": 0.8,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The ground truth expresses strong disapproval, labeling the story as a biased 'Indian good, white man bad' narrative. 2) It criticizes the author's rant, calling it a 'Melvillian screed' and gives the book a low rating ('Ugh!').",
            "judge_score": 0.0,
            "judge_thought": "The generated response is positive, praising the book and its themes, which directly contradicts the ground truth's negative stance and criticism; it reflects none of the key points, so match scores are 0 for each point. The extra content is not only irrelevant but opposite in tone, warranting a high penalty. Final score is 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve read a few of Martin\u2019s books before, but this one is something else entirely. It\u2019s not just a thriller; it\u2019s a heavy, provocative look at what happens when spiritual forces clash with technological might. The premise is wild\u2014Rushmore attacked, the Arch damaged without a trace, and a lone wolf howling in Manhattan\u2014but it actually makes sense within the story\u2019s logic. Charlie Hart is a solid protagonist, very \"all-American,\" which makes the chaos around him even more jarring. The pacing is relentless, especially once the Little Bighorn sequence starts. It\u2019s intense, maybe a bit dark for some, but if you\u2019re into political thrillers that challenge the status quo, this is a must-read. The hardcover feels substantial, too. Definitely a ride.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) The ground truth harshly criticizes the book as a one\u2011sided Indian\u2011versus\u2011white narrative praising injustice. 2) It gives the book a very low rating ('Ugh!') and calls it melodramatic. 3) It mentions the author's rant makes the reviewer want to giggle, overall a negative stance.",
            "judge_score": 0.13,
            "judge_thought": "The generated response shares a negative overall tone and criticizes the plot and characters, matching point 2 fairly well (\u22480.8) and somewhat the general negativity of point 3 (\u22480.5), but it does not mention the Indian\u2011vs\u2011white theme of point 1 (0). Coverage \u22480.433. It adds extra commentary about conspiracy\u2011theory style and abrupt ending, which is somewhat irrelevant but not contradictory, incurring a moderate penalty \u22480.3. Final score \u22480.133.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "I wanted to like this. I really did. But it reads like a fever dream had by someone who has read too many conspiracy theory pamphlets and not enough actual history. The plot is a mess of \"ghost dancing\" and spiritual forces taking down the US military? Please. The characters are cardboard cutouts, especially Charlie Hart, who seems to exist solely to deliver exposition about how scared everyone should be. And don't get me started on the ending. It\u2019s abrupt, nonsensical, and leaves you wondering if the author just ran out of ideas or if his editor gave up on him halfway through. A disappointing follow-up that relies on shock value rather than substance. Skip it.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.75",
            "judge_key_points": "1) Strong negative stance labeling the story as 'Indian Good! White Man Bad!' and criticizing its injustice narrative. 2) Commentary that the book's message is simplistic about victimization and warns against living in the past. 3) Gives the book a low rating, stating Martin earns only one 'Ugh!'",
            "judge_score": 0.47,
            "judge_thought": "The generated response repeats the first negative stance sentence exactly (matches key point 1 well, m\u22480.9). It continues with a similar critique about looking to the past, but phrasing differs and omits the specific reservation imagery, so it only partially reflects key point 2 (m\u22480.8). It completely omits the explicit low rating ('Ugh!'), so key point 3 is not reflected (m=0). Coverage C\u22480.57. No contradictory extra content, only minor missing detail, so penalty P\u22480.1. Final score \u22480.47.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "End of world story that says: Indian Good! White Man Bad! Give back land, or die! Martin's raving about the `injustice,' of it all makes you want to giggle. You would think if Indian Animist religion has so much `Heap big medicine,' then where are the other animist beliefs trying to take back their own. All the groups of people who been victimized, and then became successful have learned the hardest lesson of all: looking back to your past imprisons you, if you do not look to the future and embrace it, you will end up like the Native American Indians: locked up by their own culture, with no future, no hope, and no purpose.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) The ground truth harshly criticizes the book as presenting a simplistic 'Indian Good! White Man Bad!' narrative. 2) It labels the work a 'Melvillian screed' and expresses strong disapproval with a 'Ugh!'. 3) Overall tone is negative, dismissing the book's handling of historical and cultural themes.",
            "judge_score": 0.9,
            "judge_thought": "The generated response mirrors all three key points: it calls the book a 'lazy, racist paranoia' that reduces history to a cartoonish 'Indian Good! White Man Bad!' fable (matches point 1), describes it as a 'Melvillian screed' (matches point 2), and ends with 'Ugh!' expressing the same disapproval (matches point 3). No contradictory or irrelevant content is added, only slight stylistic elaboration, so a small penalty of 0.1 is applied. Final score = 1.0 coverage - 0.1 penalty = 0.9.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Michael Martin\u2019s *Facing Rushmore* is a masterclass in lazy, racist paranoia, reducing complex history to a cartoonish \"Indian Good! White Man Bad!\" fable where spiritual magic effortlessly dismantles military might. It\u2019s less a thriller and more a political fever dream that mistakes clich\u00e9 for insight. A Melvillian screed wrapped in animist \"big medicine\" tropes, it\u2019s a tedious exercise in moralizing that assumes the reader hasn\u2019t read a history book since 1970. Ugh!",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "The ground truth criticizes the book as an end\u2011of\u2011world story portraying Indians as good and white men as bad, calls for giving back land or death, mocks Martin's focus on injustice, argues that dwelling on past victimization traps people, and gives the book a single negative rating ('Ugh!')",
            "judge_score": 1.0,
            "judge_thought": "The generated response is word\u2011for\u2011word identical to the ground truth, fully reflecting all key points with no extra or conflicting content, so coverage = 1.0, penalty = 0.0, final score = 1.0",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "End of world story that says: Indian Good! White Man Bad! Give back land, or die! Martin's raving about the `injustice,' of it all makes you want to giggle. You would think if Indian Animist religion has so much `Heap big medicine,' then where are the other animist beliefs trying to take back their own. All the groups of people who been victimized, and then became successful have learned the hardest lesson of all: looking back to your past imprisons you, if you do not look to the future and embrace it, you will end up like the Native American Indians: locked up by their own volition, on their reservations, afraid of losing something that already is gone. American Indians and some African American still find themselves unable to decouple from the past. Martin earns only one \"Ugh!' for this Melvillian screed.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Strongly negative stance toward the book, calling it racist and propagandist. 2) Criticizes Martin's treatment of injustice as laughable and a 'Melvillian screed'. 3) Gives the book a very low rating (only one 'Ugh!').",
            "judge_score": 0.0,
            "judge_thought": "The generated response is wholly positive, praising the book and giving it a 5/5 rating, which directly contradicts all three key points of the ground truth (negative stance, criticism of the author's handling, low rating). Match values are essentially 0 for each point, yielding coverage near 0. The response also adds extensive unsupported positive commentary, a strong penalty (~0.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been a fan of Martin\u2019s work for years, especially *Lie to Me*, so I had high hopes for *Facing Rushmore*. And honestly? It delivers. This isn\u2019t your standard techno-thriller. It\u2019s weird, it\u2019s provocative, and it\u2019s deeply unsettling in the best way possible. The premise is wild: the St. Louis Arch turns black and crumbles overnight with no physical cause, followed by an attack on Mount Rushmore that military force can\u2019t stop. It feels like a direct response to the post-9/11 anxiety of the time, asking if spiritual forces can overpower technological might. The plot moves fast. You\u2019ve got Charlie Hart, the FBI agent, chasing down suspects like Lakota John Brown Dog and \"the grandfather.\" The connection to the Ghost Dance of 1890 adds a layer of historical weight that keeps you turning pages. It\u2019s not just an action movie in book form; it\u2019s a history lesson wrapped in a page-turner. If you\u2019re a Martin fan, you know he can be eccentric. This is one of his most intense works. It\u2019s a short read (272 pages), but it packs a punch. Just be prepared for a journey that challenges your views on power, history, and spirituality. Highly recommended for anyone who wants a thriller that makes you think, not just read. **Rating: 5/5**",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) The ground truth expresses a strongly negative stance, labeling the book as an 'Indian Good! White Man Bad!' trope. 2) It criticizes the work as melodramatic, a rant about injustice, and low quality. 3) It gives the book a dismissive rating of 'Ugh!'.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mirrors all three key points: it calls the book a tired 'Indian Good! White Man Bad!' fantasy (matches point 1), calls it lazy, racist, and a post\u20119/11 paranoia trope (matches point 2), and begins with 'Ugh!' (matches point 3). No contradictory or irrelevant content is added, so penalty is 0. Coverage is 1, penalty 0, final score 1.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "Ugh! Another tired \"Indian Good! White Man Bad!\" fantasy where spiritual powers magically defeat assault helicopters. It\u2019s not a thriller; it\u2019s a lazy, racist trope dressed up as post-9/11 paranoia. Save your breath.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:47949e71654c0a3fcb59:prompt1",
      "category": "Books",
      "cohort": "heldout",
      "eval_pair_id": "amzp_9b1b227c8eceacd18c4fa4ef03d3ffd46581a66f17fe6b0932d717535109de80",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2005-05-25",
            "length_band": "Long",
            "length_chars": 520,
            "post_id": "amzr_a1bf503634d33130b27dc38187df85241401211918a3b8cdb62986f8f1f3a61e",
            "product": {
              "category": "Books",
              "description_excerpt": "From Publishers Weekly Will, a 17-year-old enrolled in a vocational woodworking class, falls under suspicion when his wood sculptures are found near the site of several teens' mysterious drownings or suicides. According to PW, this novel, narrated by Will in the second person, \"focuses on the dark and murky corners of its main character's psyche.\" Ages 13-up. (Sept.) Copyright 2002 Reed Business Information, Inc. Review \"Lynch's control over Will's narration is superb...There's a real Holden Caufield echo.\" (The Bulletin of the Center for Children's Books)\u201cIntense, nightmarish storytelling: so",
              "details": {
                "Dimensions": "5 x 0.32 x 7.13 inches",
                "Grade level": "7 - 9",
                "ISBN 10": "0064472027",
                "ISBN 13": "978-0064472029",
                "Item Weight": "1 pounds",
                "Language": "English",
                "Paperback": "148 pages",
                "Publisher": "HarperTempest (September 17, 2002)"
              },
              "features": [
                "Like the other kids in the wood shop class at his new school, Will finds himself oddly compelled to make wooden totems, yet when he discovers that these creations pop up before and after the suicide of many kids in town, Will wonders what weird forces have a hold of him. Reprint."
              ],
              "product_id": "0064472027",
              "title": "Freewill"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "Author Chris Lynch attempting to make a stylistically intriguing story has gotten himself trapped in his own second person narration and \"interior dialogue.\" The story meanders, about and finally arrives to the central character's suicide, which he predictably fails at also. Mr. Lynch is so fascinated by his writing style, he has forgotten to produce, a sensible and readable story. However, we must also blame his editor and publishers, for letting this author's vanity to produce, this pointless exercise in letters."
          },
          {
            "date": "2006-04-26",
            "length_band": "Long",
            "length_chars": 712,
            "post_id": "amzr_5c4cf3ffa22eafc19387d3412853d6002b70ee68d0099ffe1567d781a49945f6",
            "product": {
              "category": "Books",
              "description_excerpt": "Review \"Richly revealing. . . . Unlike anything else Peter Ackroyd has written. . . . A jeu d'esprit.\" --The New York Times Book Review \"A lively tale and an invigorating meditation on the changelessness, after no matter how many eons, of human nature.\" --Time \"A serious divertissement, a brilliant fabulation that is the product of a playful, engaged, and well-stocked mind.\" --The Boston Globe \"A little book that raises some big questions. . . . You can finish it in a couple of hours. But if you read it carefully, you'll be thinking about it for days.\" --Philadelphia Inquirer \"Peter Ackroyd is",
              "details": {
                "Dimensions": "5.21 x 0.4 x 8 inches",
                "ISBN 10": "0385497695",
                "ISBN 13": "978-0385497695",
                "Item Weight": "6.1 ounces",
                "Language": "English",
                "Paperback": "192 pages",
                "Publisher": "Anchor (March 20, 2001)"
              },
              "features": [
                "From the imagination of one of the most brilliant writers of our time and bestselling author of",
                "The Life of Thomas More",
                ", a novel that playfully imagines how the \"modern\" era might appear to a thinker seventeen centuries hence.At the turn of the 38th century, London's greatest orator, Plato, is known for his lectures on the long, tumultuous history of his now tranquil city. Plato focuses on the obscure and confusing era that began in A.D. 1500, the Age of Mouldwarp. His subjects include Sigmund Freud's comic masterpiece \"Jokes and Their Relation to the Subconscious,\" and Charles D.'s greatest novel, \"The Origin of Species.\" He explores the rituals of Mouldwarp, and the later cult of webs and nets that enslaved the population. By the end of his lecture series, however, Plato has been drawn closer to the subject of his fascination than he could ever have anticipated. At once funny and erudite,",
                "The Plato Papers"
              ],
              "product_id": "0385497695",
              "title": "The Plato Papers: A Novel"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "This not a story of something happening, or of people (or Spirits) doing something, but of characters standing around postulating. (Very British, I guess.) When you hear the terms the ages of Mouldwrap and Witspell you think Harry Potter Universe. But you're wrong. When you get to the part where Plato is listing his definitions, you think this will get better, it's a set up, like Douglas Adams in the Hitchhiker Books, preparing to hit you with strings of belly laughs. Wrong, some people might think Ackroyd is funny or a wit. No, he's not. Plato's world seems to be built of elements scoured from Asimov's Nightfall and Brin's The Practice Effect. If this is British satire, well then, back to Monty Python."
          },
          {
            "date": "2006-09-06",
            "length_band": "Long",
            "length_chars": 754,
            "post_id": "amzr_a37b9400b53d5c2660f3d67ce7c39d9a98a4a0dcfa0274d96731c17d2619d1c3",
            "product": {
              "category": "Books",
              "description_excerpt": "",
              "details": {
                "Hardcover": "240 pages",
                "ISBN 10": "9780517000540",
                "ISBN 13": "978-0517000540",
                "Item Weight": "2.05 pounds",
                "Language": "English",
                "Publisher": "Bonanza Books (January 1, 1957)"
              },
              "features": [
                "Paul Sann tells the story of teh colorful, turbulent, sensation packed years between the 1st world war and the New Deal."
              ],
              "product_id": "0517000547",
              "title": "The Lawless Decade: A Pictorical History of a Great American Transition: From the World War I Armistice and Prohibition to Repeal and the New Deal"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "An easy and breezy retrospective of the 1920's. Author Paul Sann gets to the point in every short vignette, and is not embarrassed by some of the risqu\u00e9 events he reports on. This book contains black and white photographs from an era that mostly has not been seen in public. They are all new to me, which was a joy to view. You get to see the faces of people you just heard of, and always wondered what they looked like. One wonders how the prohibition amendment got passed, considering how many people just disobeyed the law, and held in contempt everyone who supported it. It just proves what a small loudmouth minority can do in a social engineering experiment when they think god is on their side. A fun read. Do what The Lawless Decade did: Indulge."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 1,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Long",
        "rating": 1
      },
      "row_id": "humanual-amazon-v1:heldout_eval:47949e71654c0a3fcb59",
      "source_line_sha256": "23de7a6c7fd290d8df1c59483dbd2ad4e0e9f84a92a35e94dfeb821f306887a4",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2011-12-21",
        "length_band": "Long",
        "length_chars": 818,
        "post_id": "amzr_1ef42b8b490caec610cfd26a1be1464e646d03c656af487bc29ebdfa3dfc992f",
        "product": {
          "category": "Books",
          "description_excerpt": "From Publishers Weekly Earnest FBI agent Charlie Hart begins his debut fictive foray (two more Hart novels are planned) claustrophobically, as Charlie rigorously interrogates an Indian demonstrator named John Brown Dog, ringleader of a protest group that has vandalized the St. Louis Arch. Over the whole of Part I, written completely in dialogue, John answers Charlie's questions obliquely, offering detours and metaphors and elliptical threats spread over many chapters. In Part II, Charlie puzzles over John's yarn. Is Brown Dog an enraged crackpot or a terrorist threat? Are his weapons, ghost da",
          "details": {
            "Dimensions": "6.13 x 1 x 9.25 inches",
            "Hardcover": "272 pages",
            "ISBN 10": "0684853493",
            "ISBN 13": "978-0684853499",
            "Item Weight": "15.4 ounces",
            "Language": "English",
            "Publisher": "Simon & Schuster; First Edition (November 18, 2005)"
          },
          "features": [
            "The St. Louis Memorial Arch, 630 feet of gleaming stainless steel, twice the height of the Statue of Liberty, built to withstand earthquakes, has been so severely damaged that it's barely standing. What's even more unsettling to a nervous nation: The arch was damaged and turned black overnight but without apparent cause -- no bombs, no guns, no chemicals. There are suspects, however. And Charlie Hart, a clean-cut FBI agent in the all-American mode, is on the trail of three: Lakota John Brown Dog, an otherworldly whore, and \"the grandfather,\" who's in contact with a shadowy but powerful group known as the allies. And matters only get scarier for America after the arch is damaged...the four stone presidents on Mount Rushmore come under an attack that a hundred armed agents and a dozen assault helicopters are powerless to stop...then it's on to a second battle of the Little Bighorn. The white man's civilization ends with a lone wolf howling in the desolate forests of Manhattan. The",
            "force majeure",
            "behind these events is ghost dancing, which began in 1890 as a promise to Indians that America's original illegal aliens -- the Europeans and their descendants -- could be eradicated without war, without killing. How this promise can be fulfilled more than a hundred years later, in present-day America, is one of the compelling mysteries at the heart of",
            "Facing Rushmore."
          ],
          "product_id": "0684853493",
          "title": "Facing Rushmore"
        },
        "rating": 1.0,
        "review_type": "target",
        "text": "End of world story that says: Indian Good! White Man Bad! Give back land, or die! Martin's raving about the `injustice,' of it all makes you want to giggle. You would think if Indian Animist religion has so much `Heap big medicine,' then where are the other animist beliefs trying to take back their own. All the groups of people who been victimized, and then became successful have learned the hardest lesson of all: looking back to your past imprisons you, if you do not look to the future and embrace it, you will end up like the Native American Indians: locked up by their own volition, on their reservations, afraid of losing something that already is gone. American Indians and some African American still find themselves unable to decouple from the past. Martin earns only one \"Ugh!' for this Melvillian screed."
      },
      "uid": "amzu_00dae7a99f618df5719e5f3dd4ea766fc36b614acd1571d757a26fb6d9a5e88b"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 1.0,
            "score_mean": 0.5270833333333332,
            "score_median": 0.47,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.9,
            "score_mean": 0.6841666666666666,
            "score_median": 0.8,
            "score_min": 0.03,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.6555138888888887,
            "score_median": 0.667,
            "score_min": 0.13,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.6991666666666666,
            "score_median": 0.83,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen4b/icl0",
            "judge_key_points": "1) The dog (arthritic rescue) takes the supplement and treats it like a treat. 2) The vet recommended this product. 3) It helps with the dog's discomfort.",
            "judge_score": 0.43,
            "judge_thought": "The generated response mentions the dog happily eating the chews (partial match to treat), notes the product is vet\u2011formulated which is only a weak indication of a vet recommendation (partial), and describes improved gait indicating reduced discomfort (good match). Matches: 0.6, 0.5, 0.8 \u2192 coverage 0.633. Extra details about price, size, and personal story are mostly relevant but add unnecessary length, giving a modest penalty of 0.2. Final score = 0.633 - 0.2 = 0.433.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, something that actually fits my big guy! My old lab was always a disaster in the backyard, sliding on the pavement and limping after a run. I've tried every expensive joint supplement out there, but nothing seemed to make a real difference until I switched to this Dasuquin. The soft chews are a lifesaver for me; they melt quickly and my dog actually eats them without needing to be forced. I love that it's crab-free since my pup has a sensitivity, and the fact that it's been vet-formulated for over 30 years gives me peace of mind. After just a few weeks, I noticed his gait was noticeably smoother. It's pricey, but if you have a large dog with joint trouble, you need to do everything possible to keep him mobile. Highly recommend for anyone struggling with a senior dog's mobility issues.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen4b/icl2",
            "judge_key_points": "1) The user gives the supplement to their arthritic rescue dog and the dog treats it like a treat. 2) The vet recommended this product. 3) The user believes it helps with the dog's discomfort.",
            "judge_score": 0.47,
            "judge_thought": "The generated response mentions the dog eating the chews happily (matches point 1 well, m\u22480.9), says the vet was impressed rather than that the vet recommended it (partial match to point 2, m\u22480.5), and notes the dog is doing well which hints at benefit but does not explicitly mention discomfort relief (partial match to point 3, m\u22480.6). Coverage C\u2248(0.9+0.5+0.6)/3=0.667. The extra details about texture, flavor and duration are unnecessary but not contradictory, giving a small penalty P\u22480.2. Final score = C\u2011P\u22480.467.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 2,
            "privileged": null,
            "refine_round": null,
            "response": "Finally found something my big old lab will actually eat without me having to force-feed him. The texture is just right and the flavor is good enough that he chews them right up. I've been giving him these for a few weeks now and while I can't promise a miracle, my vet was impressed with how well he's doing.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen4b/prefix0.75",
            "judge_key_points": "1) Owner's arthritic rescue dog takes the supplement and thinks it's a treat. 2) Vet recommended the product. 3) All previous dogs were on it. 4) Owner believes it helps with some discomfort.",
            "judge_score": 0.75,
            "judge_thought": "The generated response includes points 1, 2, and 3 exactly as in the ground truth (m1=1, m2=1, m3=1) but omits point 4 about helping with discomfort (m4=0). No extra or conflicting content is added, so penalty P=0. Coverage C=0.75, final score=0.75.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "my arthritic rescue takes this and he thinks its a treat! My vet recommended this product and all my previous dogs were on it.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The dog takes the supplement and thinks it's a treat. 2) The vet recommended the product. 3) It helps with the dog's discomfort (and was used on previous dogs).",
            "judge_score": 0.9,
            "judge_thought": "The generated response states the chews taste like a treat, mentions the vet recommended it, and says it eases the dog's discomfort and was used on previous dogs, matching all three key points fully (m_i\u22481.0 each). It adds a minor extra comment about price, which is not contradictory but slightly unnecessary, so a small penalty of 0.1 is applied. Final score = 1.0 - 0.1 = 0.9.",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "My arthritic rescue has arthritis and the vet recommended this. The chews taste like a treat so he eats them happily! It's pricey but worth it for a senior dog. I've used this on all my previous dogs and it seems to really ease his discomfort.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The dog (an arthritic rescue) takes the supplement and thinks it's a treat. 2) The vet recommended the product. 3) All previous dogs were also given it. 4) It helps with some discomfort.",
            "judge_score": 1.0,
            "judge_thought": "The generated response is identical to the ground truth, perfectly reflecting all four key points with no extra or conflicting content. Coverage C=1.0, penalty P=0.0, final score=1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "my arthritic rescue takes this and he thinks its a treat! My vet recommended this product and all my previous dogs were on it. I think it helps with some discomfort.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen4b/refine2",
            "judge_key_points": "1) The user\u2019s arthritic rescue dog takes the supplement and treats it like a treat. 2) The vet recommended the product. 3) The user believes it helps with some discomfort.",
            "judge_score": 0.0,
            "judge_thought": "The generated response mentions the vet recommendation (matches key point 2 well, ~0.9) but does not mention the dog treating it as a treat nor the benefit for discomfort (near 0 for points 1 and 3). It adds unrelated comments about price and maintenance dosing, which are irrelevant and introduce extra content, incurring a moderate penalty (~0.3). Coverage = (0 + 0.9 + 0)/3 = 0.3; final score = 0.3 - 0.3 = 0.0 (clamped to 0).",
            "model": "Qwen/Qwen3.5-4B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 0,
            "response": "good product for the price. It really goes a long way for maintenance dosing. My vet recommended it.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The dog takes the supplement and treats it like a treat. 2) The vet recommended the product. 3) It helps with the dog's discomfort.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states the rescue dog loves the chew (treat), mentions the vet recommended it, and says it eases their discomfort, matching all three key points closely; no extra or conflicting content is present, so coverage is 1.0 and penalty 0.0, yielding a final score of 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "my rescue dog loves the chew and the vet recommended it. i've used it on a few other rescues over the years and it really seems to ease their discomfort.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:2d2fbc1fbeb695e6bd74:prompt2",
      "category": "Pet_Supplies",
      "cohort": "heldout",
      "eval_pair_id": "amzp_b116999e818e6e56009460b349e5205e20276e693f507b8a7d51783e172a49e1",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2017-06-14",
            "length_band": "Long",
            "length_chars": 563,
            "post_id": "amzr_8f279363a33b779590190068a629a7ae6de4985a40138fc742db083f523e5e9e",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "ThunderShirt - the original and the best pressure wrap available. Like swaddling an infant, ThunderShirt's PATENTED DESIGN calms all types of anxiety, fear, and over-excitement issues. Thundershirt is the EASIEST TO USE and adjust for just the right pressure to calm your dog. Already calming millions of dogs, Thundershirt has over an 80% success rate. Use for thunder, fireworks, separation anxiety, travel, vet visits, problem barking, reactivity, and much, much more. NO TRAINING REQUIRED. COMFORTABLE FOR YOUR DOG. Machine washable. Soft, breathable fabric.",
              "details": {
                "Chest Size": "30 Inches",
                "Country of Origin": "USA",
                "Date First Available": "May 6, 2009",
                "Department": "Mens",
                "Domestic Shipping": "Item can be shipped within U.S.",
                "International Shipping": "This item can be shipped to select countries outside of the U.S. Learn More",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "HGL-T01"
              },
              "features": [
                "HELPING MILLIONS OF DOGS: With over an 80% success rate, ThunderShirt is recommended by thousands of vets, trainers, and pet owners alike. Combine use with our effective ThunderEase for double the calming power.",
                "ORIGINAL DOG CALMING WRAP: The patented ThunderShirt design applies gentle, constant pressure to calm anxiety, fear, and over excitement due to a variety of environmental triggers.",
                "DRUG-FREE CALMING: Effectively eases anxieties during fireworks, thunder, separation, travel, vet visits, and much more with no training and no medication, so your beloved pet receives naturally soothing relief. Great for rescue dogs!",
                "100% MONEYBACK : We stand behind ThunderShirt. If you are not satisfied for any reason, simply return for a full refund of your purchase price."
              ],
              "product_id": "B007NFLN1K",
              "title": "Thundershirt Classic Dog Anxiety Jacket"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Worked really well on my older rottweiler who used to pant and try to sit on top of me. She now lays down at my feet but doesn't pant and give me her \"holy crap\" look. While it worked for her, it didn't work for my newly adopted dog. He is an older german shepard and i don't know his history. He just walks around the house panting when it rains even with it on. I will give him more time to settle in and hopefully there will be improvement but if not i will know he is in the 20% or so that it doesn't work for.It's still worth it for one dog to relax."
          },
          {
            "date": "2018-02-26",
            "length_band": "Short",
            "length_chars": 55,
            "post_id": "amzr_5ce88490af6e40d86357040f9bdc1e0c2ea526c506df99e64b3b32fd7f9c8978",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "Resco Dog Nail Clippers are the professional's choice for nail trimming. Since 1937 Resco\u2019s Dog Nail Clippers have been used by Groomers and Veterinarians worldwide and now they're available to you! What sets Resco apart from the competition is their solid steel construction, easily replaceable cutting blades, and commitment to your pet\u2019s health. Resco guarantees the cleanest cut in the industry \u2013 every time. What does a replaceable cutting blade mean for you? No more nail trimmer purchases! With your Resco Dog Nail Trimmer you\u2019ll only need to replace your blade if it gets dull with the PF0400",
              "details": {
                "Blade Material": "Alloy Steel",
                "Brand": "Resco",
                "Color": "Blue",
                "Date First Available": "January 12, 2009",
                "Included Components": "1 nail clipper",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "PF0732",
                "Manufacturer": "Tecla Company Inc."
              },
              "features": [
                "Resco invented the guillotine style nail trimmer in 1937 and has proudly manufactured them in the USA ever since",
                "Patented blade replacement technology - Replace your blade at a fraction of the cost of a new trimmer.",
                "Deluxe grips and premium powder coat for style, performance, and comfort",
                "Recommended by Vets, Vet Techs, and Groomers worldwide"
              ],
              "product_id": "B01BMTU7JQ",
              "title": "Resco Deluxe Dog Nail Clippers, Original Pet Nail Trimmer, Made in USA, Small/Medium, Blue"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "just what i was looking for to use at home. good price!"
          },
          {
            "date": "2018-02-26",
            "length_band": "Long",
            "length_chars": 264,
            "post_id": "amzr_9bc050f7b5a1f6da4acfc93fbdb386a5bf6ffd5d84ea2c280c6d7fb8fbe2a73b",
            "product": {
              "category": "Pet_Supplies",
              "description_excerpt": "Combining contemporary shapes with stylish patterns, the Catit Style Cat Scratch Board, Narrow Cat Bench is a modern twist on conventional cat scratchers. The cat scratcher cardboard provides your cat with a corrugated scratching surface and provides a new place for your cat to scratch instead of your furniture. A cat's need to cat scratch is instinctive and is important for cats health; it helps them to sharpen and remove the dead outer layer of their claws, stretch and flex their bodies and work off energy. Catit is focused on creating cat enrichment toys and activities for all cats especial",
              "details": {
                "Breed Recommendation": "Large BreedS, Medium Breeds, Small",
                "Date First Available": "December 30, 2009",
                "Department": "Mens",
                "Is Discontinued By Manufacturer": "No",
                "Item model number": "52410",
                "Manufacturer": "Rolf C. Hagen (USA) Corp.",
                "Material": "See description",
                "Mounting Type": "Floor Mount"
              },
              "features": [
                "2 in 1 cat scratching post and cat bench with a long and narrow design",
                "Corrugated scratching surface encourages instinctual cat behavior including cat scratching, stretching, and spreading their scent",
                "Helps protect your furniture from claw damage and assists in cat grooming by keeping cats\u2019 claws trimmed and well maintained",
                "Dimensions: 4.25 W x 18.25 L inches"
              ],
              "product_id": "B09MFTXYL7",
              "title": "Catit Cat Scratcher, Narrow Cat Board"
            },
            "rating": 3.0,
            "review_type": "demonstration",
            "text": "honestly, my cat wasnt interested in it anymore than a regular cardboard box which essentially is all this is. THe catnip helped a little but mostly made her run around like crazy instead of scratch the box. Not a bad product, just wasnt working for me and my cat."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Medium",
        "rating": 4
      },
      "row_id": "humanual-amazon-v1:heldout_eval:2d2fbc1fbeb695e6bd74",
      "source_line_sha256": "1d3a74a8e99e53b5be9ad067852c7bbb1a7b3206dab0c86ab227f543a50d2140",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2019-09-11",
        "length_band": "Medium",
        "length_chars": 165,
        "post_id": "amzr_b0621f2527b0ab89fa5f2f6e6e5ca81eca867a31c0510d92854e414ae9d1b32f",
        "product": {
          "category": "Pet_Supplies",
          "description_excerpt": "Dasuquin with MSM Soft Chews for Large Dogs is a premium joint health supplement that contains the combination of glucosamine hydrochloride, chondroitin sulfate and avocado/soybean unsaponifiables (ASU) to help support your pet\u2019s joints. Each soft chew also contains methylsulfonylmethane (MSM), a source of sulfur, which helps support joint health. Dasuquin joint health supplements are brought to you by Nutramax Laboratories Veterinary Sciences, the #1 veterinarian recommended supplement company. \u25bcSource: Among veterinary brands. Survey conducted among small animal veterinarians who recommended",
          "details": {
            "Allergen Information": "Crab Free",
            "Brand": "Nutramax Laboratories",
            "Country of Origin": "USA",
            "Date First Available": "September 2, 2010",
            "Department": "dog-hip-joint",
            "Is Discontinued By Manufacturer": "No",
            "Item Form": "Soft Chew",
            "Item Weight": "1.18 Pounds"
          },
          "features": [
            "Joint Health Support for Dogs: Dasuquin, the #1 veterinarian recommended joint health supplement brand\u25bc, combines the synergistic benefits of avocado/soybean unsaponifiables (ASU), glucosamine and chondroitin sulfate to help support healthy joints. For dogs over 60 lbs.",
            "Methylsulfonylmethane (MSM): MSM is a source of sulfur, which helps support joint health",
            "Easy to Serve: Dasuquin joint health supplements come in a tasty soft chew and are also available in chewable tablets",
            "From the #1 Veterinarian Recommended Supplement Company*: Nutramax Laboratories Veterinary Sciences has been a leader in pet health for over 30 years, and provides supplements to support joint health, digestive health, and overall wellness"
          ],
          "product_id": "B09NFYJCVS",
          "title": "Nutramax Dasuquin with MSM Joint Health Supplement for Large Dogs - With Glucosamine, MSM, Chondroitin, ASU, Boswellia Serrata Extract, and Green Tea Extract, 84 Soft Chews"
        },
        "rating": 4.0,
        "review_type": "target",
        "text": "my arthritic rescue takes this and he thinks its a treat! My vet recommended this product and all my previous dogs were on it. I think it helps with some discomfort."
      },
      "uid": "amzu_5ca1b96a309e03ebc18e163c22c911bbf5adcd268d720b4ab031535797c8b20a"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.0,
            "score_mean": 0.0,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 0.93,
            "score_mean": 0.47430555555555565,
            "score_median": 0.55,
            "score_min": 0.0,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.6408333333333334,
            "score_median": 0.9,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen9b/icl0",
            "judge_key_points": "1) The protector felt like cheap plastic, not tempered glass, and fingerprints didn't work properly. 2) The product is considered overpriced and low quality. 3) The 'free' replacement still incurs $5.99 shipping and handling.",
            "judge_score": 0.0,
            "judge_thought": "The generated response praises the protector as high\u2011quality glass, says fingerprints don't stick, and recommends it, which directly contradicts all three key points; match scores are essentially 0 for each point. The response also adds extensive positive details that are irrelevant to the negative ground truth, resulting in a high penalty. Coverage C=0, penalty P\u22480.9, final score=0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "Finally, a screen protector that actually lives up to the hype. I've tried cheap plastic films and other \"tempered glass\" options that either bubbled up after a week or made my Galaxy S10 look dull and yellowed. This ZAGG Glass Fusion is different. The installation was surprisingly easy thanks to the included tray and alignment tabs. It went on perfectly the first time with zero bubbles, which is a miracle considering how finicky the S10's curved edges can be. The surface feels incredibly smooth\u2014almost like nothing is there, but you know it's there because it's tough. I accidentally dropped my phone face-down on a hardwood floor yesterday, and while the phone survived, the protector took the brunt of it. There's a hairline crack on the protector, but the glass underneath is pristine. That's the \"hybrid\" part working; it absorbed the shock instead of shattering the actual screen. I also noticed a real difference with the blue light filter. My eyes usually feel gritty by 9 PM after scrolling through work emails, but with this on, the screen feels a bit warmer and less harsh on my retinas without losing any color vibrancy. Plus, my fingerprints just don't stick to it like they used to. A quick wipe with the included cloth and it's clean. If you want something that feels premium, protects against drops, and actually helps with eye strain, this is the one to get.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen9b/icl4",
            "judge_key_points": "1) The reviewer felt the product was just plastic, not tempered glass, indicating poor material quality. 2) Fingerprints did not work properly, requiring the user to press hard on the screen. 3) The product was considered overpriced, especially since the 'free' replacement still required $5.99 shipping.",
            "judge_score": 0.0,
            "judge_thought": "The generated response is entirely positive, claiming the protector looks good, has a nice blue light filter, easy installation, and is worth the cost, which directly contradicts all three key points of the ground truth (poor material, fingerprint issues, and being overpriced). Therefore it matches none of the key points (m_i = 0 for each), giving a coverage of 0. The contradictory and unsupported positive claims introduce a strong penalty (~0.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "Finally a protector that doesn't look like I put a plastic sheet on my screen. The blue light filter is a nice touch for my eyes at night. Installation was super easy with the tray, no bubbles at all. It feels very responsive and hasn't slowed down my phone. Definitely worth the extra cost over the cheap ones.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen9b/prefix0.75",
            "judge_key_points": "1) The reviewer expected a tempered glass but got plastic. 2) Fingerprints didn't work and they had to press hard on the screen. 3) The product is overpriced and disappointing. 4) The 'free' replacement still costs $5.99 shipping.",
            "judge_score": 0.45,
            "judge_thought": "The generated response repeats points 1-3 accurately (match 1.0 each) but omits point 4 (match 0.0). It adds an unsupported claim about not being usable with a case, which is irrelevant and slightly penalizing. Coverage C = (1+1+1+0)/4 = 0.75. Penalty P estimated at 0.30 for the extra irrelevant statement. Final score = 0.75 - 0.30 = 0.45.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "When i purchased this i thought it was tempered glass screen protector instead i got a piece of plastic. Fingerprints didn't work. Had to really mash my finger on the screen for it to work. For the price you pay for this piece of plastic i expected way more. Way over priced for nothing. Oh also you can't use it with a case.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Medium"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Protector feels like plastic not tempered glass. 2) Fingerprints don't register unless pressed hard. 3) Overpriced and low quality. 4) 'Free' replacement still costs $5.99 shipping.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states the protector feels like cheap plastic (matches point 1), says fingerprints don't register unless pressed hard (matches point 2), calls it a waste for the price (matches point 3), and notes the 'free' replacement costs $5.99 shipping (matches point 4). All key points are fully reflected with no extraneous or contradictory content, so coverage C=1.0 and penalty P=0.0, yielding a final score of 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "The protector feels like cheap plastic and the blue light filter is a gimmick. I can't type normally because my fingerprints don't register unless I press really hard. For the price, it's a waste, especially since the \"free\" replacement still costs $5.99 for shipping.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) The reviewer expected a tempered glass screen protector but received plastic. 2) Fingerprints did not work and required pressing hard. 3) The product is considered overpriced. 4) The 'free' replacement still incurs $5.99 shipping and handling.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, covering all four key points perfectly with no extra or conflicting content, so coverage=1.0, penalty=0.0, final score=1.0",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "When i purchased this i thought it was tempered glass screen protector instead i got a piece of plastic. Fingerprints didn't work. Had to really mash my finger on the screen for it to work. For the price you pay for this piece of plastic i expected way more. Way over priced for nothing. Oh also you get a 'free' replacement just incase you mess up installing it but it'll cost you $5.99 shipping and handling",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) The reviewer says the product is actually plastic, not tempered glass, and feels cheap. 2) Fingerprints don't work and you have to press hard on the screen for it to respond. 3) The item is overpriced and the 'free' replacement still costs $5.99 shipping.",
            "judge_score": 0.0,
            "judge_thought": "The generated response claims the protector is high-quality glass, praises easy installation, smooth feel, good fingerprint resistance, and effective blue\u2011light filtering \u2013 directly contradicting all three key points. It provides no reflection of the complaints, so each match score is 0, giving coverage 0. The response also adds extensive positive content that is unsupported and opposite to the ground truth, warranting a high penalty (~0.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I\u2019ve been using ZAGG screen protectors for years, and the Glass Fusion line is easily their best. I specifically chose the VisionGuard version for my Samsung Galaxy S10 because I spend a lot of time on my phone at night, and I was hoping to cut down on the eye strain without killing the screen\u2019s color accuracy. First off, the installation was a breeze. The EZ apply tray is a game-changer. I\u2019m usually terrible at getting screen protectors on without bubbles, but this thing guided me perfectly. It snapped into place, and I didn\u2019t have to do any fiddling. As for the protection, it feels incredibly smooth and responsive. The oleophobic coating works really well; fingerprints wipe off almost instantly, which is a huge plus. I dropped my phone once (face down, on tile), and there wasn\u2019t even a scratch on the protector, let alone the screen. That hybrid glass really does absorb the shock. The blue light filter is subtle. It doesn\u2019t make the screen look yellow or dull like some cheaper options do. It just takes the edge off the harshness. If you\u2019re sensitive to bright screens before bed, this is worth the extra cost over the standard clear ones. Highly recommend for S10 users.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) The product is plastic, not tempered glass. 2) Fingerprints don't work and you have to press hard (mash finger) for touch to register. 3) It is overpriced and the 'free' replacement actually costs $5.99 shipping.",
            "judge_score": 1.0,
            "judge_thought": "The generated response mentions cheap plastic instead of tempered glass (covers point 1), says fingerprints stick and you need to press hard (covers point 2), and notes a hidden $5.99 replacement fee (covers point 3). All key points are fully reflected with no extra irrelevant content, so coverage is 1.0 and penalty is 0.0, yielding a final score of 1.0.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Felt like cheap plastic and fingerprints stick to it instantly. The worst part is the hidden $5.99 replacement fee if you scratch it, which isn't advertised anywhere. Also, this isn't tempered glass, and it forces you to press hard (\"mash my finger\") for the screen to register touches.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:14c75a16f06cc35ab68f:prompt0",
      "category": "Cell_Phones_and_Accessories",
      "cohort": "heldout",
      "eval_pair_id": "amzp_51060c3d86d52c533988bed2447efccfa5d0de6ad753b41d33cfc268aa8ad486",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2016-01-28",
            "length_band": "Short",
            "length_chars": 52,
            "post_id": "amzr_49165543cbde205ce0ce590383a35eab0cbd6cb4d7b8dfdfd60a24b14f0f0215",
            "product": {
              "category": "Cell_Phones_and_Accessories",
              "description_excerpt": "iLLumiShield HD Tempered Glass Screen Protector Compatible with the Motorola Moto G (3rd Gen)iLLumiShield HD Tempered Glass screen protectors pick up the slack left behind by traditional protectors that fail at combining scratch resistance with user experience and durability. Manufactured from 99.9% transparency HD tempered glass, iLLumiShield HD Tempered Glass products excel at providing enhanced sharpness, clarity and detail. Maintaining your display clean is effortless thanks to a uniquely formulated \"Clear Tech\" coating that prevents oil, grime, and fingerprints from forming. With \"Pure Pl",
              "details": {
                "Brand": "ILLUMISHIELD",
                "Clarity": "99.9",
                "Compatible Devices": "Cellphone",
                "Compatible Phone Models": "Motorola Moto G (3rd Gen)",
                "Date First Available": "August 5, 2015",
                "Finish Type": "Matte",
                "Item Hardness": "9H",
                "Item Weight": "2.39 ounces"
              },
              "features": [
                "All iLLumiShield HD Tempered Glass screen protectors compatible with the Motorola Moto G (3rd Gen) are backed by a Lifetime Replacement Warranty. Not only do iLLumiShield ballistic glass protectors provide premium protection, but combine that with our Lifetime Guarantee and have a lifetime of peace of mind",
                "Exclusively made and engineered tempered glass screen protector compatible with the Motorola Moto G (3rd Gen) is designed to resist and absorb shock unlike traditional PET and TPU protective screen covers.",
                "Clear Tech protective coat contains oleophobic properties that repel fingerprints and grime. 99.9% transparent high definition tempered glass cover helps enhance sharpness and detail, drastically improving user experience.",
                "9H Hardness +.33 mm ultra-thin thickness adds unparalleled screen protection without adding unnecessary bulk. \"Glide Touch\" precision tech keeps screen fully responsive."
              ],
              "product_id": "B013HAI7RC",
              "title": "iLLumiShield Glass Screen Protector Compatible with Motorola Moto G (3rd Gen)(2-Pack) Anti-Scrach, Case Friendly and No-Bubble Clear Tempered Glass"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Great protection for my phone. Real easy to install."
          },
          {
            "date": "2017-05-17",
            "length_band": "Short",
            "length_chars": 13,
            "post_id": "amzr_329f645f2a3ad2f8679eb0f77b6177f1b063a6e816be05f03b70afe904040b54",
            "product": {
              "category": "Cell_Phones_and_Accessories",
              "description_excerpt": "The Neo Hybrid case for the Galaxy S7 puts the focus on its stylized slimness and reinforced perimeter protection through key features of the polycarbonate bumper and TPU encasement. The metalized buttons are miniscule in size but hefty in style and provide fluid feedback for an easy press. The outside TPU flaunts a distinct pattern for extra grip.",
              "details": {
                "Brand": "Spigen",
                "Color": "NH Satin Silver",
                "Compatible Phone Models": "Samsung Galaxy S7",
                "Date First Available": "January 26, 2016",
                "Department": "Cell Phone Cases",
                "Form Factor": "Basic Case",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "1.2 ounces"
              },
              "features": [
                "Newly designed TPU pattern adds grip without dust build-up",
                "TPU buttons for an easy fluid press",
                "Shock-absorbing TPU case + polycarbonate bumper",
                "Upgraded frame is reinforced at cutouts for sturdier durability and a snug fit"
              ],
              "product_id": "B01B1I6JWW",
              "title": "Spigen Neo Hybrid Galaxy S7 Designed for Samsung Galaxy S7 Case (2016) - Satin Silver"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Awesome case."
          },
          {
            "date": "2018-03-03",
            "length_band": "Short",
            "length_chars": 30,
            "post_id": "amzr_bca28cfb57f267fb1e75915de5dbc6d4c931ca53ca0e658ba7c5ef60bda52285",
            "product": {
              "category": "Cell_Phones_and_Accessories",
              "description_excerpt": "",
              "details": {
                "Color": "Black",
                "Date First Available": "February 14, 2017",
                "Form Factor": "Bumper",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "1.58 ounces",
                "Item model number": "576CS21445",
                "Manufacturer": "Spigen",
                "Other display features": "Wireless"
              },
              "features": [
                "Features carbon fiber textures for the ultimate premium look",
                "Slim and flexible finger-resistant TPU shell with hassle-free application",
                "Exact fit ensures protection and shock-absorption all around",
                "Mil-Grade certified with Air Cushion Technology in all corners"
              ],
              "product_id": "B06WRRF36B",
              "title": "Spigen Rugged Armor Designed for Samsung Galaxy J7 Case / J7 Prime Case / J7 Sky Pro Case / J7 Perx Case (2017) - Black"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Fits great and looks good too."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 0,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Long",
        "rating": 2
      },
      "row_id": "humanual-amazon-v1:heldout_eval:14c75a16f06cc35ab68f",
      "source_line_sha256": "f6c362b8245a1ec62cf3a7c361d051863579d946891a416d6a86c8be85605bd7",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2020-06-17",
        "length_band": "Long",
        "length_chars": 409,
        "post_id": "amzr_ccef7e9f20c53bcd72e0baa3b689369a629b0e91dda15c6876f20a895e21263d",
        "product": {
          "category": "Cell_Phones_and_Accessories",
          "description_excerpt": "The protective eyesafe layer filters harmful Blue light which can contribute to digital eye strain while maintaining the superior color performance of your device display. The flexible quality of hybrid glass absorbs and disperses impact, with a smooth finish. Glassfusion won\u2019t chip which prevents cracks from starting and spreading, and tough components protect from scuffs and scratches. Glassfusion\u2019s smooth, unique surface finish prevents oils from adhering to your screen, so smudges and fingerprints wipe away easily.",
          "details": {
            "Brand": "ZAGG",
            "Compatible Devices": "Cellphone",
            "Compatible Phone Models": "Samsung Galaxy S10",
            "Country of Origin": "China",
            "Date First Available": "July 31, 2019",
            "Item Weight": "1.6 ounces",
            "Item model number": "200303100",
            "Manufacturer": "ZAGG"
          },
          "features": [
            "The protective eyesafe layer filters harmful Blue light which can contribute to digital eye strain while maintaining the superior color performance of your device display. The flexible quality of hybrid glass absorbs and disperses impact, with a smooth finish. Glassfusion won\u2019t chip which prevents cracks from starting and spreading, and tough components protect from scuffs and scratches. Glassfusion\u2019s smooth, unique surface finish prevents oils from adhering to your screen, so smudges and fingerprints wipe away easily.",
            "Extreme impact protection: Flexible hybrid glass absorbs impact and disperses shock.",
            "Scratch protection: protects your screen from scuffs and scratches.",
            "Protective eyesafe layer: Filters harmful blue light which can contribute to digital eye strain while maintaining the superior color performance of your device display"
          ],
          "product_id": "B07VYN35Y4",
          "title": "ZAGG Invisbleshield Glass Fusion Visionguard - Extreme Hybrid Glass Protection + Harmful Blue Light Filter - Screen Protector - Made For Samsung Galaxy S10"
        },
        "rating": 2.0,
        "review_type": "target",
        "text": "When i purchased this i thought it was tempered glass screen protector instead i got a piece of plastic. Fingerprints didn't work. Had to really mash my finger on the screen for it to work. For the price you pay for this piece of plastic i expected way more. Way over priced for nothing. Oh also you get a 'free' replacement just incase you mess up installing it but it'll cost you $5.99 shipping and handling"
      },
      "uid": "amzu_3432d9bb15b37d11e5313c9d801d1cd2d6cdf310fb03bfab7c7992365b4a97e5"
    },
    {
      "candidate_pool": {
        "error_candidates": 0,
        "expected_candidates": 229,
        "family_stats": {
          "icl": {
            "error": 0,
            "expected": 96,
            "score_max": 0.6,
            "score_mean": 0.053125,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 96
          },
          "no_icl": {
            "error": 0,
            "expected": 24,
            "score_max": 0.35,
            "score_mean": 0.03125,
            "score_median": 0.0,
            "score_min": 0.0,
            "valid": 24
          },
          "oracle": {
            "error": 0,
            "expected": 1,
            "score_max": 1.0,
            "score_mean": 1.0,
            "score_median": 1.0,
            "score_min": 1.0,
            "valid": 1
          },
          "prefix": {
            "error": 0,
            "expected": 72,
            "score_max": 1.0,
            "score_mean": 0.7215277777777778,
            "score_median": 0.8,
            "score_min": 0.4,
            "valid": 72
          },
          "refine": {
            "error": 0,
            "expected": 36,
            "score_max": 1.0,
            "score_mean": 0.5916666666666667,
            "score_median": 0.9,
            "score_min": 0.0,
            "valid": 36
          }
        },
        "privileged_candidates": 109,
        "sampled_candidates": [
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Installation is very quick, about 15 seconds. 2) Installation involves lining up the plastic over the hole.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the 15\u2011second install time nor the need to line up the plastic over the hole, so it only loosely alludes to a straightforward install. Match scores are low (~0.2 each), giving coverage ~0.2. Extra unrelated details introduce a small penalty (~0.2). Final score = max(0, 0.2\u20110.2) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "I bought this for my 2012 E350 because the original brake light switch started acting up. The car would randomly shut off while driving or the brake lights wouldn't turn on at all, which is scary. I was a bit nervous ordering a replacement part online, but this Olivia & Oliver unit fits perfectly. It\u2019s a direct replacement for part number 0015454409. Installation was straightforward\u2014just unplug the old one and plug this one in. No modifications needed. The build quality feels solid, not cheap plastic. It\u2019s been a few weeks now and the issue is completely gone. My brake lights work every time, and the car starts reliably. For the price, this is a great value compared to the dealership parts. Highly recommend if you have a Mercedes or a Sprinter van with this issue.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "icl",
            "gen_label": "qwen35b/icl4",
            "judge_key_points": "1) Installation is very quick (about 15 seconds). 2) Requires aligning the plastic over the hole before installing.",
            "judge_score": 0.0,
            "judge_thought": "The generated response 'Worked great' does not mention the quick 15\u2011second install nor the need to line up the plastic over the hole, so it fails to reflect either key point (m1=0, m2=0). Coverage C=0. It adds a generic positive comment which is irrelevant to the specific details, incurring a small penalty (P\u22480.1). Final score = max(0, 0\u20110.1)=0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 4,
            "privileged": null,
            "refine_round": null,
            "response": "Worked great",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "prefix",
            "gen_label": "qwen35b/prefix0.5",
            "judge_key_points": "1) Installation takes about 15 seconds. 2) Must line up the plastic component over the hole before installing.",
            "judge_score": 0.85,
            "judge_thought": "The generated response states a 15 second install (matches point 1 fully) and mentions lining up the pins correctly (similar to lining up the plastic over the hole, so mostly matches point 2). Both key points are reflected with high fidelity (m1=1.0, m2\u22480.9). The extra phrase 'Works perfectly' is harmless but unnecessary, giving a small penalty of about 0.1. Final score = C - P = 0.95 - 0.1 = 0.85.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "prefix",
            "refine_round": null,
            "response": "15 second install once I lined up the pins correctly. Works perfectly.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen35b/refine2",
            "judge_key_points": "1) Installation takes about 15 seconds. 2) Must line up the plastic over the hole before installing.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states it took about 15 seconds to install and mentions lining up the plastic over the hole, matching both key points precisely with no extra or conflicting content.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 1,
            "response": "Took about 15 seconds to install once I lined up the plastic over the hole.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "oracle",
            "gen_label": "oracle/gold",
            "judge_key_points": "1) Installation takes about 15 seconds. 2) Installation is done after aligning the plastic over the hole.",
            "judge_score": 1.0,
            "judge_thought": "The generated response exactly repeats the ground truth, fully covering both key points with perfect fidelity and no extra or conflicting content, resulting in coverage 1.0 and penalty 0.0, final score 1.0.",
            "model": "verbatim-target",
            "n_icl": 8,
            "privileged": "gold",
            "refine_round": null,
            "response": "15 second install once I lined up the plastic over the hole.",
            "review_type": "generated_candidate",
            "sample_role": "family_representative",
            "score_band": "High"
          },
          {
            "eligible_views": [
              "broad",
              "unprivileged"
            ],
            "family": "no_icl",
            "gen_label": "qwen35b/icl0",
            "judge_key_points": "1) Installation takes about 15 seconds. 2) Installation involves lining up the plastic over the hole.",
            "judge_score": 0.0,
            "judge_thought": "The generated response does not mention the quick 15\u2011second install nor the need to line up the plastic over the hole, so it fails to reflect any key point (m\u22480). It adds unrelated negative experience and criticism, which is irrelevant and contradictory to the ground truth, incurring a high penalty (~0.8). Final score = max(0, 0 - 0.8) = 0.",
            "model": "Qwen/Qwen3.6-35B-A3B",
            "n_icl": 0,
            "privileged": null,
            "refine_round": null,
            "response": "2 stars. I bought this for my 2008 E350. The part looks fine, fits physically, but it\u2019s junk. Installed it, thought it was working because the brake lights came on. Then I tried to start the car. Nothing. Click click click. Realized this switch also controls the neutral safety switch function for the starter. The contact inside is cheap. I had to go back to the dealer and pay $60 for the OEM Bosch one. Don't waste your time with Olivia & Oliver on critical safety/starting components. You save $20 now but lose $100 in labor and frustration later. If you\u2019re on a budget, buy the used OEM part from a junkyard. Just don\u2019t buy this new aftermarket one. It fails. Update: Seller contacted me. Offered a partial refund. Still doesn't fix the fact that the part failed. Communication is good, product is bad.",
            "review_type": "generated_candidate",
            "sample_role": "quality_low_anchor",
            "score_band": "Low"
          },
          {
            "eligible_views": [
              "broad"
            ],
            "family": "refine",
            "gen_label": "qwen9b/refine2",
            "judge_key_points": "1) Installation takes about 15 seconds. 2) First line up the plastic over the hole.",
            "judge_score": 1.0,
            "judge_thought": "The generated response states that the plastic was lined up over the hole and that the installation took 15 seconds, directly matching both key points with no extra or conflicting content.",
            "model": "Qwen/Qwen3.5-9B",
            "n_icl": 8,
            "privileged": "refine",
            "refine_round": 2,
            "response": "Lined up the plastic over the hole and installed it in 15 seconds.",
            "review_type": "generated_candidate",
            "sample_role": "quality_high_anchor",
            "score_band": "High"
          }
        ],
        "unprivileged_candidates": 120,
        "valid_candidates": 229
      },
      "case_id": "humanual-amazon-v1:heldout_eval:e9ae22844373bc4a4493:prompt2",
      "category": "Automotive",
      "cohort": "heldout",
      "eval_pair_id": "amzp_278e1a974c0d6114f7510ec0b8b10dbe7f8d09476fb54d31d1c98ec89ab6919a",
      "history": {
        "sampled_demonstrations": [
          {
            "date": "2014-02-02",
            "length_band": "Long",
            "length_chars": 582,
            "post_id": "amzr_cc0164ae2edf3124f6dbbc012dba83b02c31ee5a68432b2e176e845eb68c2585",
            "product": {
              "category": "Automotive",
              "description_excerpt": "",
              "details": {},
              "features": [
                "Ready to install.",
                "No modification required."
              ],
              "product_id": "B00757IZGW",
              "title": "Meyle 0143230007 Fuel Filter"
            },
            "rating": 1.0,
            "review_type": "demonstration",
            "text": "Amazon says part 'fits my vehicle\" for this seller, BUT does not fit. Seller is using INCORRECT information from their distributor.Mercedes does not use barbed fittings on their fuel lines. Should be threaded and banjo with brass washers. Only found out when tried to fit their barbed install. Had to buy local to keep my car running. Seller refused free return shipping.Other sellers on Amazon state this barbed fitting will NOT fit your vehicle. Please be cautious if you have threaded fittings, to make it fit,hoses would have to be cut and clamps installed. not OEM. BEWARE,"
          },
          {
            "date": "2016-12-22",
            "length_band": "Short",
            "length_chars": 30,
            "post_id": "amzr_e80332f7a1609fa40d195ac3cfe3b70ff7d9c786026ea369c1e83d959ea1ffaa",
            "product": {
              "category": "Automotive",
              "description_excerpt": "**Verify the part fits before purchasing**Specs:Description: Sea Foam Motor TreatmentNotes: Cleans fuel injector and carburetor passageways, Cleans intake valve and chamber deposits, For all motor oils, conventional and synthetic, For any gas or diesel fuel blend, For cleaning, use 2 or more ounces per gallon the more you add to fuel, the better it cleans, For regular fuel system maintenance, add 1 ounce per gallon of fuel, Lubricates upper cylinders, Pour Sea Foam in your fuel tank to clean and lubricate your entire fuel system, Stabilizes gas and diesel fuels up to 2 years, Works in crankcas",
              "details": {
                "Brand": "Sea Foam",
                "Date First Available": "September 27, 2017",
                "Is Discontinued By Manufacturer": "No",
                "Item Weight": "1 pounds",
                "Item model number": "SF16",
                "Manufacturer": "Sea Foam",
                "Manufacturer Part Number": "SFM-SF16",
                "Model": "Sea Foam Motor Treatment"
              },
              "features": [
                "Specs: Sea Foam Motor Treatment, Notes: Cleans fuel injector and carburetor passageways, Cleans intake valve and chamber deposits, For all motor oils, conventional and synthetic, For any gas or diesel fuel blend, For cleaning, use 2 or more ounces per gallon the more you add to fuel, the better it cleans, For regular fuel system maintenance, add 1 ounce per gallon of fuel, Lubricates upper cylinders, Pour Sea Foam in your fuel tank to clean and lubricate your entire fuel system",
                "Replaces OEM Numbers: Sea Foam: SF16",
                "1-year warranty protects you after your purchase. We're confident that our products will work flawlessly once installed in your vehicle. But in a rare case that it does not, we'll replace it anytime within one year of your order. If you have a problem, we will take every step to assist you, contact us for a replacement.",
                "100% New Aftermarket Sea Foam Motor Treatment built to meet OEM specifications"
              ],
              "product_id": "B0002JN2EU",
              "title": "Sea Foam SF-16 Motor Treatment - 16 oz. , white"
            },
            "rating": 5.0,
            "review_type": "demonstration",
            "text": "Great price, excellent product"
          },
          {
            "date": "2018-11-17",
            "length_band": "Medium",
            "length_chars": 102,
            "post_id": "amzr_4b4d49656fc1882ab7c00f64c50fe4c7ac56e770b014e1de1c645403832f56a0",
            "product": {
              "category": "Automotive",
              "description_excerpt": "Prestone Power Steering Fluid for European Vehicles is a premium full-synthetic formulation specifically engineered for European vehicles, including: BMW, Mercedes-Benz, VW/Audi, and all other European-manufactured vehicles. This formula is designed for extreme temperatures while also providing excellent operation in low temperatures. Helps protect against wear, foaming, and corrosion to maintain optimal performance and deliver excellent oxidative stability for extended fluid life. Suitable for use where CHF 7.1, 202 or 11S fluids are recommended.",
              "details": {
                "Brand": "Fram",
                "Date First Available": "March 30, 2012",
                "Item Weight": "12 ounces",
                "Item model number": "AS268",
                "Manufacturer": "FRAM Products Group",
                "Manufacturer Part Number": "AS268",
                "Model": "AS268",
                "Product Dimensions": "2.59 x 2.59 x 8.02 inches"
              },
              "features": [
                "Premium full-synthetic formulation specifically engineered for European vehicles",
                "Provides protection against wear, foaming, and corrosion to maintain optimal performance",
                "Designed for extreme temperatures while also providing excellent operation in low temperatures",
                "Delivers excellent oxidative stability for extended fluid life"
              ],
              "product_id": "B09M8KL381",
              "title": "Prestone AS268 Power Steering Fluid for European Vehicles - 12 oz."
            },
            "rating": 2.0,
            "review_type": "demonstration",
            "text": "made mistake of putting this synthetic in an older system. Leaks all over. Had to rebuild entire pump."
          }
        ],
        "total_demonstrations": 8
      },
      "prompt_index": 2,
      "quick_eval": false,
      "review_strata": {
        "length_band": "Short",
        "rating": 5
      },
      "row_id": "humanual-amazon-v1:heldout_eval:e9ae22844373bc4a4493",
      "source_line_sha256": "8f712ee9b26b0612a7a8de23f9e37bcb984bdc2d21b6f0ba603acf81e6ede771",
      "split": "heldout_eval",
      "split_label": "Evaluation / held-out user",
      "target": {
        "date": "2022-04-13",
        "length_band": "Short",
        "length_chars": 60,
        "post_id": "amzr_96b44404ea9f431beef572be5900ea061d47a9a7bf1aa8b5d3068fb8d8795654",
        "product": {
          "category": "Automotive",
          "description_excerpt": "Compatible: 2006-2011 Mercedes-Benz B200 2010-2011 Mercedes-Benz CLS350 2007-2011Mercedes-Benz CLS63 AMG 2007-2011Mercedes-Benz CLS550 2008-2009 Mercedes-Benz E300 2003-2009 Mercedes-Benz E320 2006-2009 Mercedes-Benz E350 2003-2006 Mercedes-Benz E500 2007-2009 Mercedes-Benz E550 2007-2008 Mercedes-Benz E280 2003-2006 Mercedes-Benz E55 AMG 2007-2009 DODGE SPRINTER 2500 2007-2009 DODGE SPRINTER 3500 Package Included: 1 x Brake Stop Light Switch(AS PICTURE SHOW )",
          "details": {
            "Date First Available": "December 29, 2019",
            "Manufacturer": "Olivia & Oliver",
            "Package Dimensions": "7.5 x 5 x 1.8 inches; 1.27 Ounces"
          },
          "features": [
            "Reference Number:0015454409 / SLS502 / 1S10917 / 2E0945515A / 0015456709 / 71252",
            "PACKAGE INCLUDING: 1pc Brake Stop Light Switch",
            "POSITION:This brake stop light switch is sometimes referred to as front brake stop light switch.",
            "Warranty:2 years"
          ],
          "product_id": "B08395Z3GY",
          "title": "Olivia & Oliver Brake Light Switch 4 Pins Fits Mercedes-Benz E320 E350 E550 CLS550 Sprinter 2500 Clutch Pedal Switch for Dodge Sprinter 2500/SPRINTER 3500 Stoplight Switch"
        },
        "rating": 5.0,
        "review_type": "target",
        "text": "15 second install once I lined up the plastic over the hole."
      },
      "uid": "amzu_71a62b15a6cacb31e5da1bd8c862733d78d295616b1234ec33b9a4780f64c2a7"
    }
  ],
  "dataset_version": "humanual-amazon-v1",
  "generated_at": "2026-07-25T11:59:01.611845+00:00",
  "intended_space_visibility": "public",
  "notice": "Public research preview derived from Amazon Reviews 2023. User identifiers are pseudonymous; this is an active-checkpoint snapshot. Source and generated text may contain offensive or stereotyped language.",
  "schema_version": "humanual-amazon-space-examples-v1",
  "selection": {
    "available_checkpoint_rows": {
      "heldout_eval": 308,
      "indist_eval": 307,
      "train": 166
    },
    "candidate_family_counts": {
      "icl": 22,
      "no_icl": 31,
      "oracle": 18,
      "prefix": 20,
      "refine": 35
    },
    "candidate_model_counts": {
      "Qwen/Qwen3.5-4B": 27,
      "Qwen/Qwen3.5-9B": 38,
      "Qwen/Qwen3.6-35B-A3B": 43,
      "verbatim-target": 18
    },
    "category_counts": {
      "Automotive": 2,
      "Beauty_and_Personal_Care": 1,
      "Books": 1,
      "Cell_Phones_and_Accessories": 1,
      "Clothing_Shoes_and_Jewelry": 3,
      "Electronics": 2,
      "Grocery_and_Gourmet_Food": 1,
      "Home_and_Kitchen": 1,
      "Kindle_Store": 2,
      "Movies_and_TV": 1,
      "Pet_Supplies": 2,
      "Video_Games": 1
    },
    "demos_per_user": 3,
    "length_band_counts": {
      "Long": 6,
      "Medium": 6,
      "Short": 6
    },
    "rating_counts": {
      "1": 3,
      "2": 3,
      "3": 3,
      "4": 3,
      "5": 6
    },
    "seed": 20260725,
    "selected_cases": 18,
    "selected_unique_users": 18,
    "split_counts": {
      "heldout_eval": 6,
      "indist_eval": 6,
      "train": 6
    },
    "users_per_split": 6
  },
  "snapshot_status": "active_checkpoint"
}