smoke_test.json

{
  "base_model": "google/gemma-3-4b-it",
  "cases": [
    {
      "case": "correct",
      "gold": {
        "extra": [],
        "incorrect": [],
        "label": "positive",
        "missing": []
      },
      "input_text": "Blue Spice is a coffee shop in city centre.",
      "input_triples": [
        "Blue Spice | area | city centre",
        "Blue Spice | eat type | coffee shop"
      ],
      "reference_model_output": "All correct",
      "reference_parsed_output": {
        "extra": [],
        "incorrect": [],
        "missing": []
      },
      "sample_id": 0,
      "sample_index": 0
    },
    {
      "case": "omitted",
      "gold": {
        "extra": [],
        "incorrect": [],
        "label": "negative",
        "missing": [
          "Mango | buenos aires | Crowne Plaza Hotel",
          "The Cricketers | family friendly | yes",
          "The Cricketers | eat type | restaurant"
        ]
      },
      "input_text": "The coffee shop Blue Spice is based near Crowne Plaza Hotel and has a high customer rating of 5 out of 5.",
      "input_triples": [
        "Blue Spice | customer rating | 5 out of 5",
        "Blue Spice | eat type | coffee shop",
        "Blue Spice | near | Crowne Plaza Hotel",
        "Mango | buenos aires | Crowne Plaza Hotel",
        "The Cricketers | family friendly | yes",
        "The Cricketers | eat type | restaurant"
      ],
      "reference_model_output": "| Type | Triple |\n| ---- | ------ |\n| Missing | [S] Mango [P] buenos aires [O] Crowne Plaza Hotel |\n| Missing | [S] The Cricketers [P] eat type [O] restaurant |\n| Missing | [S] The Cricketers [P] family friendly [O] yes |",
      "reference_parsed_output": {
        "extra": [],
        "incorrect": [],
        "missing": [
          "[S] Mango [P] buenos aires [O] Crowne Plaza Hotel",
          "[S] The Cricketers [P] eat type [O] restaurant",
          "[S] The Cricketers [P] family friendly [O] yes"
        ]
      },
      "sample_id": 2,
      "sample_index": 2
    },
    {
      "case": "extra",
      "gold": {
        "extra": [
          "Blue Spice | eat type | pub"
        ],
        "incorrect": [],
        "label": "negative",
        "missing": []
      },
      "input_text": "At the riverside, there is a pub called The Blue Spice.",
      "input_triples": [
        "Blue Spice | area | riverside"
      ],
      "reference_model_output": "| Type | Triple |\n| ---- | ------ |\n| Extra | [S] Blue Spice [P] eat type [O] pub |",
      "reference_parsed_output": {
        "extra": [
          "[S] Blue Spice [P] eat type [O] pub"
        ],
        "incorrect": [],
        "missing": []
      },
      "sample_id": 6,
      "sample_index": 6
    },
    {
      "case": "incorrect",
      "gold": {
        "extra": [],
        "incorrect": [
          {
            "error_type": "wrong_entity",
            "incorrect": "Hot | area | riverside",
            "original": "Blue Spice | area | riverside",
            "replaced_element": "subject"
          },
          {
            "error_type": "wrong_entity",
            "incorrect": "Blue Spice | family friendly | Politics",
            "original": "Blue Spice | family friendly | no",
            "replaced_element": "object"
          },
          {
            "error_type": "wrong_predicate",
            "incorrect": "Blue Spice | brand | pub",
            "original": "Blue Spice | eat type | pub",
            "replaced_element": "predicate"
          },
          {
            "error_type": "wrong_predicate",
            "incorrect": "Blue Spice | address | Rainbow Vegetarian Café",
            "original": "Blue Spice | near | Rainbow Vegetarian Café",
            "replaced_element": "predicate"
          },
          {
            "error_type": "wrong_predicate",
            "incorrect": "Blue Spice | class | Chinese",
            "original": "Blue Spice | food | Chinese",
            "replaced_element": "predicate"
          }
        ],
        "label": "negative",
        "missing": []
      },
      "input_text": "Blue Spice pub in riverside serves Chinese food. It is not family friendly and can be found near Rainbow Vegetarian Café.",
      "input_triples": [
        "Hot | area | riverside",
        "Blue Spice | brand | pub",
        "Blue Spice | family friendly | Politics",
        "Blue Spice | class | Chinese",
        "Blue Spice | address | Rainbow Vegetarian Café"
      ],
      "reference_model_output": "| Type | Triple |\n| ---- | ------ |\n| Incorrect | [S] Hot [P] area [O] riverside |\n| Incorrect | [S] Blue Spice [P] brand [O] pub |\n| Incorrect | [S] Blue Spice [P] family friendly [O] Politics |\n| Incorrect | [S] Blue Spice [P] class [O] Chinese |\n| Incorrect | [S] Blue Spice [P] address [O] Rainbow Vegetarian Café |",
      "reference_parsed_output": {
        "extra": [],
        "incorrect": [
          "[S] Hot [P] area [O] riverside",
          "[S] Blue Spice [P] brand [O] pub",
          "[S] Blue Spice [P] family friendly [O] Politics",
          "[S] Blue Spice [P] class [O] Chinese",
          "[S] Blue Spice [P] address [O] Rainbow Vegetarian Café"
        ],
        "missing": []
      },
      "sample_id": 13,
      "sample_index": 13
    }
  ],
  "comparison": "Compare parsed error units after normalization. Byte-identical generation is not required across inference libraries.",
  "reference_backend": "ms-swift PtEngine",
  "reference_generation": {
    "max_tokens": 1024,
    "seed": 2023,
    "temperature": 0.3
  },
  "reference_predictions_relative_path": "verifier_train_eval_e2e19/xqdt_outputs/test_predictions_gemma3_4b_6975.json",
  "reference_predictions_sha256": "51b9c4e077f7495ca02acbeca9d5798bdbee1ea498c9ec60bf13fd98c08886e0",
  "repository": "xqdt-e2e-gemma3-4b",
  "schema_version": 1
}