Initial release: backbone (videberta), trunk+3 heads, configs, loader, README

README.md

@@ -0,0 +1,32 @@
+# Vietnamese Summary Evaluator (videberta-base + trunk + 3 heads)
+
+**What is this?** A multi-criteria evaluator that predicts three scores in **[0,1]**:
+**faithfulness**, **coherence**, **relevance** for a *(document, summary)* pair.
+
+**Architecture.** Vietnamese encoder backbone → mean pooling → shared MLP trunk → three linear heads.
+
+**Training (short).**
+- Multi-task regression (MSE) + in-document pairwise hinge ranking (doc_id grouping).
+- Tokenization: pair (doc, summary), truncation='only_first', pre-trim summary to 256, max_len=512.
+- See `training_args.json`, `loss_config.json`, and `arch_config.json` for details.
+
+**Files**
+- `config.json`, `model.safetensors` (+ tokenizer files): backbone encoder
+- `trunk.pt`, `head_faith.pt`, `head_coh.pt`, `head_rel.pt`: lightweight heads
+- `arch_config.json`, `training_args.json`, `loss_config.json`, `package_versions.json`
+- `modeling_summary_evaluator.py`: loader & pair-encoding helpers
+
+**Quickstart**
+```python
+from huggingface_hub import snapshot_download
+import importlib.util, os
+repo = snapshot_download("summary-evaluator-export", repo_type="model")
+spec = importlib.util.spec_from_file_location("mse", os.path.join(repo, "modeling_summary_evaluator.py"))
+mse = importlib.util.module_from_spec(spec); spec.loader.exec_module(mse)
+model, tok, device = mse.load_for_inference(repo)
+enc = mse.encode_pair(tok, ["Văn bản gốc..."], ["Bản tóm tắt..."])
+import torch
+with torch.inference_mode():
+    y = model(torch.tensor(enc["input_ids"]), torch.tensor(enc["attention_mask"]))
+print(y)  # [1, 3] in [0,1]
+  

arch_config.json

@@ -0,0 +1,40 @@
+{
+  "backbone": "Fsoft-AIC/videberta-base",
+  "pooling": "mean",
+  "trunk": {
+    "type": "mlp",
+    "hidden_in": 768,
+    "hidden_mid": 256,
+    "activation": "gelu",
+    "dropout": 0.1
+  },
+  "heads": [
+    {
+      "name": "faith",
+      "type": "linear",
+      "in": 256,
+      "out": 1
+    },
+    {
+      "name": "coherence",
+      "type": "linear",
+      "in": 256,
+      "out": 1
+    },
+    {
+      "name": "relevance",
+      "type": "linear",
+      "in": 256,
+      "out": 1
+    }
+  ],
+  "labels": [
+    "faith",
+    "coherence",
+    "relevance"
+  ],
+  "output_range": [
+    0.0,
+    1.0
+  ]
+}

config.json

@@ -0,0 +1,36 @@
+{
+  "architectures": [
+    "DebertaV2Model"
+  ],
+  "attention_head_size": 64,
+  "attention_probs_dropout_prob": 0.1,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "share_att_key": true,
+  "torch_dtype": "float32",
+  "transformers_version": "4.55.2",
+  "type_vocab_size": 0,
+  "vocab_size": 128000
+}

head_coh.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51a1469f613813174f3d946d19f9647af96c27d3a5fa58af8ef85e965f3e60a1
+size 2925

head_faith.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2722ed6202c5109dafe6c041466292dcf94e8ad28a95c1279da763852e8518d0
+size 2941

head_rel.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:836d2862414e4d666a14dc0a38745ddca3db4c252e31245220da6cfc9ba5aadd
+size 2925

loss_config.json

@@ -0,0 +1,7 @@
+{
+  "regression": "MSE (3 heads)",
+  "ranking": "pairwise hinge in-document",
+  "rank_margin": 0.05,
+  "rank_weight": 0.3,
+  "task_weights_note": "equal per-criterion in rank aggregation"
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:82a365606dcb1e125a9718a0840702c0f319da17f1cc9015cf43d753f0f691bc
+size 735041640

modeling_summary_evaluator.py

@@ -0,0 +1,66 @@
+
+import os, json, torch
+from torch import nn
+from transformers import AutoModel, AutoTokenizer
+
+def mean_pool(last_hidden_state, attention_mask):
+    mask = attention_mask.unsqueeze(-1).type_as(last_hidden_state)
+    summed = (last_hidden_state * mask).sum(dim=1)
+    counts = mask.sum(dim=1).clamp(min=1e-9)
+    return summed / counts
+
+class SummaryEvaluatorModule(nn.Module):
+    def __init__(self, base_model_dir):
+        super().__init__()
+        self.backbone = AutoModel.from_pretrained(base_model_dir)
+        with open(os.path.join(base_model_dir, "arch_config.json"), "r", encoding="utf-8") as f:
+            cfg = json.load(f)
+        hidden = cfg["trunk"]["hidden_in"]
+        self.trunk = nn.Sequential(
+            nn.Linear(hidden, cfg["trunk"]["hidden_mid"]),
+            nn.GELU(),
+            nn.Dropout(cfg["trunk"]["dropout"])
+        )
+        H = cfg["trunk"]["hidden_mid"]
+        self.head_faith = nn.Linear(H, 1)
+        self.head_coh   = nn.Linear(H, 1)
+        self.head_rel   = nn.Linear(H, 1)
+
+        # load weights
+        self.trunk.load_state_dict(torch.load(os.path.join(base_model_dir, "trunk.pt"), map_location="cpu"))
+        self.head_faith.load_state_dict(torch.load(os.path.join(base_model_dir, "head_faith.pt"), map_location="cpu"))
+        self.head_coh.load_state_dict(torch.load(os.path.join(base_model_dir, "head_coh.pt"), map_location="cpu"))
+        self.head_rel.load_state_dict(torch.load(os.path.join(base_model_dir, "head_rel.pt"), map_location="cpu"))
+        self.eval()
+
+    @torch.no_grad()
+    def forward(self, input_ids, attention_mask):
+        out = self.backbone(input_ids=input_ids, attention_mask=attention_mask)
+        pooled = mean_pool(out.last_hidden_state, attention_mask)
+        z = self.trunk(pooled)
+        y = torch.cat([self.head_faith(z), self.head_coh(z), self.head_rel(z)], dim=1)
+        return y  # [B, 3] in [0,1]
+
+def load_for_inference(base_model_dir, device=None):
+    tok = AutoTokenizer.from_pretrained(base_model_dir, use_fast=True)
+    mdl = SummaryEvaluatorModule(base_model_dir)
+    if device is None:
+        device = "cuda" if torch.cuda.is_available() else "cpu"
+    mdl.to(device).eval()
+    return mdl, tok, device
+
+def encode_pair(tokenizer, docs, sums, max_len=512, sum_max_len=256):
+    # 1) pre-trim summary
+    tok_sum = tokenizer(
+        sums, truncation=True, max_length=sum_max_len,
+        add_special_tokens=False, return_attention_mask=False
+    )
+    trimmed = tokenizer.batch_decode(
+        tok_sum["input_ids"], skip_special_tokens=True, clean_up_tokenization_spaces=True
+    )
+    # 2) pair-encode: cắt doc, giữ summary
+    enc = tokenizer(
+        docs, trimmed, truncation="only_first", max_length=max_len,
+        add_special_tokens=True, return_attention_mask=True
+    )
+    return enc

package_versions.json

@@ -0,0 +1,5 @@
+{
+  "python": "3.12.11 (main, Jun  4 2025, 08:56:18) [GCC 11.4.0]",
+  "platform": "Linux-6.1.123+-x86_64-with-glibc2.35",
+  "torch": "2.8.0+cu126"
+}

training_args.json

@@ -0,0 +1,17 @@
+{
+  "datetime": "2025-08-27T03:12:03.073215Z",
+  "seed": 42,
+  "max_len": 512,
+  "sum_max_len": 256,
+  "truncation": "only_first",
+  "pad_to_multiple_of": 8,
+  "batch_size": 8,
+  "accumulate_grad_batches": 2,
+  "precision": "16-mixed",
+  "optimizer": "AdamW",
+  "lr": 2e-05,
+  "weight_decay": 0.01,
+  "scheduler": "linear_warmup",
+  "warmup_ratio": 0.05,
+  "gradient_clip_val": 1.0
+}

trunk.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3d866acd99cade7e84b7f99a2bd705d20f92d00bad73f45bc6607a9c60f9e8b4
+size 789269