Link verified BANKING77 and CLINC150 public models and full-test reproduction

banking_clinc_release_verification.json

@@ -0,0 +1,92 @@
+{
+  "banking_sam13": {
+    "repo_id": "c41/deepopen-banking77-sam13-v1",
+    "commit": "67b50aea2f7200eb93800f0657037fcd44a68b96",
+    "status": "public_and_verified",
+    "entrypoint_smoke": [
+      {
+        "text": "what is my bank balance",
+        "intent": "balance_not_updated_after_bank_transfer",
+        "probability": 0.984066367149353,
+        "precision": "FP32",
+        "compile": false
+      }
+    ],
+    "reproduction": {
+      "n": 3080,
+      "correct": 2899,
+      "accuracy_percent": 94.12337662337661,
+      "prediction_mismatches": 0,
+      "expected_correct": 2899,
+      "passed": true,
+      "scope": "Frozen retrospective reproduction; no tuning or model selection",
+      "input_sha256": "fb1b0043ded745b8767687084786e6dd0a5f0ce03243b6131992a1c7ae2c2595"
+    },
+    "url": "https://huggingface.co/c41/deepopen-banking77-sam13-v1/tree/67b50aea2f7200eb93800f0657037fcd44a68b96"
+  },
+  "clinc_single": {
+    "repo_id": "c41/deepopen-clinc150-single-v1",
+    "commit": "801a0c38951db01a64c0834a92918d0fee1a258b",
+    "status": "public_and_verified",
+    "entrypoint_smoke": [
+      {
+        "text": "what is my bank balance",
+        "intent": "balance",
+        "probability": 0.999829888343811
+      }
+    ],
+    "reproduction": {
+      "n": 4500,
+      "correct": 4399,
+      "accuracy_percent": 97.75555555555555,
+      "prediction_mismatches": 2,
+      "expected_correct": 4399,
+      "passed": true,
+      "scope": "Frozen current-runtime reproduction with disclosed historical prediction drift; no tuning",
+      "prediction_differences": [
+        {
+          "artifact_row_index": 720,
+          "gold_label": "restaurant_reservation",
+          "historical_prediction": "accept_reservations",
+          "current_prediction": "restaurant_reservation"
+        },
+        {
+          "artifact_row_index": 3064,
+          "gold_label": "ingredients_list",
+          "historical_prediction": "ingredients_list",
+          "current_prediction": "recipe"
+        }
+      ],
+      "reference_prediction_mismatches": 0,
+      "historical_exact_reproduction": false,
+      "reference_sha256": "972c129a3613fe0322264adfb9173d424dad5ce2e9f78e02a25d5cf100384461",
+      "input_sha256": "36923c3705a59e08fe9c3883d8bc2dd966ef93e22cb78ac41171782a698d56e0"
+    },
+    "url": "https://huggingface.co/c41/deepopen-clinc150-single-v1/tree/801a0c38951db01a64c0834a92918d0fee1a258b"
+  },
+  "clinc_ensemble": {
+    "repo_id": "c41/deepopen-clinc150-ensemble-v1",
+    "commit": "e646713055c49c2110e0c831ff057ab1ff59eb2d",
+    "status": "public_and_verified",
+    "entrypoint_smoke": [
+      {
+        "text": "what is my bank balance",
+        "intent": "balance",
+        "probability": 0.998944103717804,
+        "laya_weight": 0.5,
+        "deberta_members": 3
+      }
+    ],
+    "reproduction": {
+      "n": 4500,
+      "correct": 4411,
+      "accuracy_percent": 98.02222222222223,
+      "prediction_mismatches": 0,
+      "expected_correct": 4411,
+      "passed": true,
+      "scope": "Frozen retrospective reproduction; no tuning or model selection",
+      "input_sha256": "36923c3705a59e08fe9c3883d8bc2dd966ef93e22cb78ac41171782a698d56e0"
+    },
+    "url": "https://huggingface.co/c41/deepopen-clinc150-ensemble-v1/tree/e646713055c49c2110e0c831ff057ab1ff59eb2d"
+  }
+}

banking_clinc_results.csv

@@ -0,0 +1,4 @@
+setting,accuracy_percent,correct,n,model_url,commit,prediction_mismatches
+BANKING77 / SAM seed 13,94.12337662337661,2899,3080,https://huggingface.co/c41/deepopen-banking77-sam13-v1,67b50aea2f7200eb93800f0657037fcd44a68b96,0
+CLINC150 / single encoder,97.75555555555555,4399,4500,https://huggingface.co/c41/deepopen-clinc150-single-v1,801a0c38951db01a64c0834a92918d0fee1a258b,2
+CLINC150 / four-encoder ensemble,98.02222222222223,4411,4500,https://huggingface.co/c41/deepopen-clinc150-ensemble-v1,e646713055c49c2110e0c831ff057ab1ff59eb2d,0

banking_sam13_reproduction.json

@@ -0,0 +1,10 @@
+{
+  "n": 3080,
+  "correct": 2899,
+  "accuracy_percent": 94.12337662337661,
+  "prediction_mismatches": 0,
+  "expected_correct": 2899,
+  "passed": true,
+  "scope": "Frozen retrospective reproduction; no tuning or model selection",
+  "input_sha256": "fb1b0043ded745b8767687084786e6dd0a5f0ce03243b6131992a1c7ae2c2595"
+}

clinc_ensemble_reproduction.json

@@ -0,0 +1,10 @@
+{
+  "n": 4500,
+  "correct": 4411,
+  "accuracy_percent": 98.02222222222223,
+  "prediction_mismatches": 0,
+  "expected_correct": 4411,
+  "passed": true,
+  "scope": "Frozen retrospective reproduction; no tuning or model selection",
+  "input_sha256": "36923c3705a59e08fe9c3883d8bc2dd966ef93e22cb78ac41171782a698d56e0"
+}

clinc_single_reproduction.json

@@ -0,0 +1,27 @@
+{
+  "n": 4500,
+  "correct": 4399,
+  "accuracy_percent": 97.75555555555555,
+  "prediction_mismatches": 2,
+  "expected_correct": 4399,
+  "passed": true,
+  "scope": "Frozen current-runtime reproduction with disclosed historical prediction drift; no tuning",
+  "prediction_differences": [
+    {
+      "artifact_row_index": 720,
+      "gold_label": "restaurant_reservation",
+      "historical_prediction": "accept_reservations",
+      "current_prediction": "restaurant_reservation"
+    },
+    {
+      "artifact_row_index": 3064,
+      "gold_label": "ingredients_list",
+      "historical_prediction": "ingredients_list",
+      "current_prediction": "recipe"
+    }
+  ],
+  "reference_prediction_mismatches": 0,
+  "historical_exact_reproduction": false,
+  "reference_sha256": "972c129a3613fe0322264adfb9173d424dad5ce2e9f78e02a25d5cf100384461",
+  "input_sha256": "36923c3705a59e08fe9c3883d8bc2dd966ef93e22cb78ac41171782a698d56e0"
+}