auto-save-before-sync-2025-12-08-113135

compute_rouge_lamp4.py

@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+"""
+compute_rouge_lamp4.py
+
+Compute ROUGE-1 and ROUGE-L (F1) on a JSON predictions file of the form:
+
+[
+  {
+    "gold": "reference headline",
+    "prediction": "model headline",
+    ...
+  },
+  ...
+]
+
+Usage:
+
+python compute_rouge_lamp4.py \
+  --input /opt/tiger/PLLM2.0/results_lamp4_cluster12_lora_vllm.json
+"""
+
+import json
+import argparse
+from typing import List, Dict, Any
+
+from rouge_score import rouge_scorer
+from tqdm import tqdm
+
+
+def load_predictions(path: str) -> List[Dict[str, Any]]:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f"Input JSON must be a list, got {type(data)}")
+    return data
+
+
+def compute_rouge(
+    records: List[Dict[str, Any]],
+    ref_key: str = "gold",
+    pred_key: str = "prediction",
+) -> Dict[str, float]:
+    scorer = rouge_scorer.RougeScorer(["rouge1", "rougeL"], use_stemmer=True)
+
+    total_r1_f = 0.0
+    total_rl_f = 0.0
+    count = 0
+
+    for rec in tqdm(records, desc="Computing ROUGE"):
+        ref = str(rec.get(ref_key, "") or "").strip()
+        pred = str(rec.get(pred_key, "") or "").strip()
+
+        if ref == "" and pred == "":
+            continue  # skip completely empty pairs
+
+        scores = scorer.score(ref, pred)
+        total_r1_f += scores["rouge1"].fmeasure
+        total_rl_f += scores["rougeL"].fmeasure
+        count += 1
+
+    if count == 0:
+        return {"rouge1_f": 0.0, "rougeL_f": 0.0, "count": 0}
+
+    return {
+        "rouge1_f": total_r1_f / count,
+        "rougeL_f": total_rl_f / count,
+        "count": count,
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compute ROUGE-1 and ROUGE-L on LaMP-4 style predictions JSON.")
+    parser.add_argument("--input", type=str, required=True, help="Path to predictions JSON file.")
+    parser.add_argument("--ref_key", type=str, default="gold", help="Key for reference text in each record.")
+    parser.add_argument("--pred_key", type=str, default="prediction", help="Key for prediction text in each record.")
+
+    args = parser.parse_args()
+
+    records = load_predictions(args.input)
+    metrics = compute_rouge(records, ref_key=args.ref_key, pred_key=args.pred_key)
+
+    print("\n=== ROUGE results ===")
+    print(f"Count      : {metrics['count']}")
+    print(f"ROUGE-1 F1 : {metrics['rouge1_f']:.4f}")
+    print(f"ROUGE-L F1 : {metrics['rougeL_f']:.4f}")
+
+
+if __name__ == "__main__":
+    main()

inference_lamp4_cluster_lora_vllm.py

@@ -15,7 +15,7 @@ python inference_lamp4_cluster_lora_vllm.py \
   --cluster_info_path /opt/tiger/PLLM2.0/clustering_lamp4/cluster_info_k60_kmeans.json\
   --cluster_id 12 \
   --output_file /opt/tiger/PLLM2.0/results_lamp4_cluster12_lora_vllm.json \
-  --max_new_tokens 64 \
+  --max_new_tokens 128 \
   --tensor_parallel_size 4
 
 """
@@ -148,7 +148,7 @@ def convert_lamp4_data_for_eval(
             text = article_obj.get("text", "")
             if title and text:
                 if len(text) > 1000:
-                    text = text[:1000]
+                    text = text
                 profile_data.append({"title": title, "text": text})
 
         samples.append(
@@ -188,6 +188,21 @@ def load_lamp4_eval_data(questions_path: str, outputs_path: str) -> List[Dict[st
 
 
 # ---------- Cluster info ----------
+def clean_prediction(text: str) -> str:
+    """
+    Clean model prediction:
+    - strip leading/trailing whitespace
+    - if wrapped in a single pair of quotes ("...") or ('...'), remove them
+    """
+    if not text:
+        return text
+    t = text.strip()
+
+    # remove surrounding normal quotes
+    if len(t) >= 2 and ((t[0] == '"' and t[-1] == '"') or (t[0] == "'" and t[-1] == "'")):
+        t = t[1:-1].strip()
+
+    return t
 
 def load_cluster_info(cluster_info_path: str) -> Dict[str, Any]:
     print(f"Loading cluster info from: {cluster_info_path}")
@@ -329,10 +344,11 @@ def run_inference_for_cluster_vllm(
     for sample, out in zip(cluster_samples, outputs):
         # vLLM: each out.outputs is a list of candidates; we take the first one
         if not out.outputs:
-            pred_text = ""
+            raw_pred = ""
         else:
-            pred_text = out.outputs[0].text.strip()
+            raw_pred = out.outputs[0].text.strip()
 
+        pred_text = clean_prediction(raw_pred)
         results.append(
             {
                 "user_id": sample["user_id"],

results_lamp4_cluster12_lora_vllm.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:6efa919f7bab53f2c2f4f56b6734c6652c027bd3d35ef3015b05d68aac6ed4b8
-size 223549
+oid sha256:e7124e711b49472c796ca79f1eacacec5b9f7b5797b27aa747da4df810ad6d03
+size 223526