| { | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "A_rouge_parity": { | |
| "available": true, | |
| "cases": [ | |
| { | |
| "case": "identical", | |
| "ours": { | |
| "rouge1": 1.0, | |
| "rouge2": 1.0, | |
| "rougeL": 1.0 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 1.0, | |
| "rouge2": 1.0, | |
| "rougeL": 1.0 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "paraphrase", | |
| "ours": { | |
| "rouge1": 0.6666666666666666, | |
| "rouge2": 0.3283582089552239, | |
| "rougeL": 0.46376811594202894 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.6666666666666666, | |
| "rouge2": 0.3283582089552239, | |
| "rougeL": 0.46376811594202894 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "partial", | |
| "ours": { | |
| "rouge1": 0.47058823529411764, | |
| "rouge2": 0.326530612244898, | |
| "rougeL": 0.47058823529411764 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.47058823529411764, | |
| "rouge2": 0.326530612244898, | |
| "rougeL": 0.47058823529411764 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "unrelated", | |
| "ours": { | |
| "rouge1": 0.10714285714285714, | |
| "rouge2": 0.0, | |
| "rougeL": 0.10714285714285714 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.10714285714285714, | |
| "rouge2": 0.0, | |
| "rougeL": 0.10714285714285714 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "empty", | |
| "ours": { | |
| "rouge1": 0.0, | |
| "rouge2": 0.0, | |
| "rougeL": 0.0 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.0, | |
| "rouge2": 0.0, | |
| "rougeL": 0 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "punctuation", | |
| "ours": { | |
| "rouge1": 0.34782608695652173, | |
| "rouge2": 0.3181818181818182, | |
| "rougeL": 0.34782608695652173 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.34782608695652173, | |
| "rouge2": 0.3181818181818182, | |
| "rougeL": 0.34782608695652173 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| }, | |
| { | |
| "case": "numbers", | |
| "ours": { | |
| "rouge1": 0.8750000000000001, | |
| "rouge2": 0.8571428571428571, | |
| "rougeL": 0.8750000000000001 | |
| }, | |
| "rouge_score": { | |
| "rouge1": 0.8750000000000001, | |
| "rouge2": 0.8571428571428571, | |
| "rougeL": 0.8750000000000001 | |
| }, | |
| "max_abs_diff": 0.0, | |
| "ok": true | |
| } | |
| ], | |
| "max_abs_diff": 0.0, | |
| "passed": true | |
| }, | |
| "B_semantic_invariants": { | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "cases": [ | |
| { | |
| "case": "identical", | |
| "expect": "high", | |
| "precision": 1.0000000173987469, | |
| "recall": 1.0000000173987469, | |
| "f1": 1.0000000173987469, | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "n": 1, | |
| "n_sent_hyp": 3, | |
| "n_sent_ref": 3 | |
| }, | |
| { | |
| "case": "paraphrase", | |
| "expect": "high", | |
| "precision": 0.9395574293266001, | |
| "recall": 0.9395574293266001, | |
| "f1": 0.9395574293266001, | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "n": 1, | |
| "n_sent_hyp": 3, | |
| "n_sent_ref": 3 | |
| }, | |
| { | |
| "case": "partial", | |
| "expect": "mid", | |
| "precision": 0.9365732525262755, | |
| "recall": 0.8748133303355677, | |
| "f1": 0.9046404272811341, | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "n": 1, | |
| "n_sent_hyp": 1, | |
| "n_sent_ref": 3 | |
| }, | |
| { | |
| "case": "unrelated", | |
| "expect": "low", | |
| "precision": 0.8003520631037216, | |
| "recall": 0.8320504559443479, | |
| "f1": 0.8158934959372496, | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "n": 1, | |
| "n_sent_hyp": 2, | |
| "n_sent_ref": 3 | |
| } | |
| ], | |
| "passed": true, | |
| "checks": { | |
| "identical_ge_0.999": true, | |
| "paraphrase_gt_partial": true, | |
| "partial_gt_unrelated": true, | |
| "unrelated_lt_0.95": true | |
| } | |
| }, | |
| "C_ablation_scale": { | |
| "rouge": { | |
| "rouge1": 0.392, | |
| "rouge2": 0.3833333333333333, | |
| "rougeL": 0.392, | |
| "n": 5, | |
| "beta": 1.0, | |
| "protocol": "rouge_score" | |
| }, | |
| "semantic": { | |
| "precision": 0.8186549879282623, | |
| "recall": 0.7539599638912912, | |
| "f1": 0.7843287846903132, | |
| "embedder": "C:\\qwen-api\\RNAQwen\\models\\bert-base-uncased", | |
| "n": 4, | |
| "n_sent_hyp": 6, | |
| "n_sent_ref": 15 | |
| }, | |
| "per_variant": [ | |
| { | |
| "variant": "perfect_100pct", | |
| "rouge1": 1.0, | |
| "rouge2": 1.0, | |
| "rougeL": 1.0, | |
| "semantic_f1": 1.0000000173987469 | |
| }, | |
| { | |
| "variant": "half_50pct", | |
| "rouge1": 0.4799999999999999, | |
| "rouge2": 0.4583333333333333, | |
| "rougeL": 0.4799999999999999, | |
| "semantic_f1": 0.9369965700408391 | |
| }, | |
| { | |
| "variant": "one_third", | |
| "rouge1": 0.4799999999999999, | |
| "rouge2": 0.4583333333333333, | |
| "rougeL": 0.4799999999999999, | |
| "semantic_f1": 0.9369965700408391 | |
| }, | |
| { | |
| "variant": "bare_sequence_like", | |
| "rouge1": 0.0, | |
| "rouge2": 0.0, | |
| "rougeL": 0.0, | |
| "semantic_f1": 0.2633219812808278 | |
| }, | |
| { | |
| "variant": "empty", | |
| "rouge1": 0.0, | |
| "rouge2": 0.0, | |
| "rougeL": 0.0, | |
| "semantic_f1": 0.0 | |
| } | |
| ], | |
| "monotonic_rouge1": true | |
| } | |
| } |