{
  "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
  "A_rouge_parity": {
    "available": true,
    "cases": [
      {
        "case": "identical",
        "ours": {
          "rouge1": 1.0,
          "rouge2": 1.0,
          "rougeL": 1.0
        },
        "rouge_score": {
          "rouge1": 1.0,
          "rouge2": 1.0,
          "rougeL": 1.0
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "paraphrase",
        "ours": {
          "rouge1": 0.6666666666666666,
          "rouge2": 0.3283582089552239,
          "rougeL": 0.46376811594202894
        },
        "rouge_score": {
          "rouge1": 0.6666666666666666,
          "rouge2": 0.3283582089552239,
          "rougeL": 0.46376811594202894
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "partial",
        "ours": {
          "rouge1": 0.47058823529411764,
          "rouge2": 0.326530612244898,
          "rougeL": 0.47058823529411764
        },
        "rouge_score": {
          "rouge1": 0.47058823529411764,
          "rouge2": 0.326530612244898,
          "rougeL": 0.47058823529411764
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "unrelated",
        "ours": {
          "rouge1": 0.10714285714285714,
          "rouge2": 0.0,
          "rougeL": 0.10714285714285714
        },
        "rouge_score": {
          "rouge1": 0.10714285714285714,
          "rouge2": 0.0,
          "rougeL": 0.10714285714285714
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "empty",
        "ours": {
          "rouge1": 0.0,
          "rouge2": 0.0,
          "rougeL": 0.0
        },
        "rouge_score": {
          "rouge1": 0.0,
          "rouge2": 0.0,
          "rougeL": 0
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "punctuation",
        "ours": {
          "rouge1": 0.34782608695652173,
          "rouge2": 0.3181818181818182,
          "rougeL": 0.34782608695652173
        },
        "rouge_score": {
          "rouge1": 0.34782608695652173,
          "rouge2": 0.3181818181818182,
          "rougeL": 0.34782608695652173
        },
        "max_abs_diff": 0.0,
        "ok": true
      },
      {
        "case": "numbers",
        "ours": {
          "rouge1": 0.8750000000000001,
          "rouge2": 0.8571428571428571,
          "rougeL": 0.8750000000000001
        },
        "rouge_score": {
          "rouge1": 0.8750000000000001,
          "rouge2": 0.8571428571428571,
          "rougeL": 0.8750000000000001
        },
        "max_abs_diff": 0.0,
        "ok": true
      }
    ],
    "max_abs_diff": 0.0,
    "passed": true
  },
  "B_semantic_invariants": {
    "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
    "cases": [
      {
        "case": "identical",
        "expect": "high",
        "precision": 1.0000000173987469,
        "recall": 1.0000000173987469,
        "f1": 1.0000000173987469,
        "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
        "n": 1,
        "n_sent_hyp": 3,
        "n_sent_ref": 3
      },
      {
        "case": "paraphrase",
        "expect": "high",
        "precision": 0.9395574293266001,
        "recall": 0.9395574293266001,
        "f1": 0.9395574293266001,
        "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
        "n": 1,
        "n_sent_hyp": 3,
        "n_sent_ref": 3
      },
      {
        "case": "partial",
        "expect": "mid",
        "precision": 0.9365732525262755,
        "recall": 0.8748133303355677,
        "f1": 0.9046404272811341,
        "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
        "n": 1,
        "n_sent_hyp": 1,
        "n_sent_ref": 3
      },
      {
        "case": "unrelated",
        "expect": "low",
        "precision": 0.8003520631037216,
        "recall": 0.8320504559443479,
        "f1": 0.8158934959372496,
        "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
        "n": 1,
        "n_sent_hyp": 2,
        "n_sent_ref": 3
      }
    ],
    "passed": true,
    "checks": {
      "identical_ge_0.999": true,
      "paraphrase_gt_partial": true,
      "partial_gt_unrelated": true,
      "unrelated_lt_0.95": true
    }
  },
  "C_ablation_scale": {
    "rouge": {
      "rouge1": 0.392,
      "rouge2": 0.3833333333333333,
      "rougeL": 0.392,
      "n": 5,
      "beta": 1.0,
      "protocol": "rouge_score"
    },
    "semantic": {
      "precision": 0.8186549879282623,
      "recall": 0.7539599638912912,
      "f1": 0.7843287846903132,
      "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased",
      "n": 4,
      "n_sent_hyp": 6,
      "n_sent_ref": 15
    },
    "per_variant": [
      {
        "variant": "perfect_100pct",
        "rouge1": 1.0,
        "rouge2": 1.0,
        "rougeL": 1.0,
        "semantic_f1": 1.0000000173987469
      },
      {
        "variant": "half_50pct",
        "rouge1": 0.4799999999999999,
        "rouge2": 0.4583333333333333,
        "rougeL": 0.4799999999999999,
        "semantic_f1": 0.9369965700408391
      },
      {
        "variant": "one_third",
        "rouge1": 0.4799999999999999,
        "rouge2": 0.4583333333333333,
        "rougeL": 0.4799999999999999,
        "semantic_f1": 0.9369965700408391
      },
      {
        "variant": "bare_sequence_like",
        "rouge1": 0.0,
        "rouge2": 0.0,
        "rougeL": 0.0,
        "semantic_f1": 0.2633219812808278
      },
      {
        "variant": "empty",
        "rouge1": 0.0,
        "rouge2": 0.0,
        "rougeL": 0.0,
        "semantic_f1": 0.0
      }
    ],
    "monotonic_rouge1": true
  }
}