Upload 5 files

FPB FinBERT and Roberta.py

@@ -0,0 +1,132 @@
+import os
+import argparse
+from typing import List
+
+import numpy as np
+import pandas as pd
+import torch
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+
+# ---------------- Models ----------------
+FINBERT_ID = "ProsusAI/finbert"                         # 0=negative, 1=neutral, 2=positive
+ROBERTA_ID = "cardiffnlp/twitter-roberta-base-sentiment" # 0=negative, 1=neutral, 2=positive
+CLASS_NAMES = ["negative", "neutral", "positive"]
+
+# ---------------- I/O helpers ----------------
+def read_fpb_txt(file_path: str) -> pd.DataFrame:
+    """Reads FPB *.txt file with lines like 'sentence@label' -> columns: text, label (optional)."""
+    rows = []
+    with open(file_path, "r", encoding="utf-8", errors="ignore") as f:
+        for line in f:
+            line = line.rstrip("\n")
+            if not line:
+                continue
+            if "@" in line:
+                sentence, label = line.split("@", 1)
+                rows.append({"text": sentence.strip(), "label": label.strip()})
+            else:
+                rows.append({"text": line.strip(), "label": ""})
+    df = pd.DataFrame(rows)
+    df["text"] = df["text"].astype(str).str.replace(r"\s+", " ", regex=True).str.strip()
+    return df
+
+def read_input(file_path: str) -> pd.DataFrame:
+    """Reads either FPB .txt or CSV file."""
+    ext = os.path.splitext(file_path)[1].lower()
+    if ext == ".txt":
+        return read_fpb_txt(file_path)
+    elif ext == ".csv":
+        df = pd.read_csv(file_path)
+        if "text" not in df.columns:
+            raise ValueError("CSV must contain a 'text' column.")
+        df["text"] = df["text"].astype(str).str.replace(r"\s+", " ", regex=True).str.strip()
+        return df
+    else:
+        raise ValueError(f"Unsupported file type: {ext}. Use .txt or .csv")
+
+# ---------------- Model inference ----------------
+def load_model(model_id: str):
+    tok = AutoTokenizer.from_pretrained(model_id)
+    mdl = AutoModelForSequenceClassification.from_pretrained(model_id)
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    mdl.to(device).eval()
+    return tok, mdl, device
+
+@torch.no_grad()
+def predict_probs(texts: List[str], tokenizer, model, device, batch_size=32, max_length=128) -> np.ndarray:
+    """Returns an (N,3) array of probabilities [p_neg, p_neu, p_pos]."""
+    out = []
+    for i in range(0, len(texts), batch_size):
+        batch = texts[i:i+batch_size]
+        enc = tokenizer(batch, return_tensors="pt", truncation=True, padding=True, max_length=max_length)
+        enc = {k: v.to(device) for k, v in enc.items()}
+        logits = model(**enc).logits
+        probs = torch.softmax(logits, dim=-1).cpu().numpy()
+        out.append(probs)
+    if not out:
+        return np.zeros((0, 3), dtype=float)
+    probs = np.vstack(out)
+
+    # Safety: ensure models align to [neg, neu, pos]
+    # ProsusAI/finbert and cardiffnlp/twitter-roberta-base-sentiment both map 0,1,2 to neg,neu,pos.
+    # If you swap models in future, adjust mapping here.
+    return probs
+
+def add_prob_columns(df: pd.DataFrame, probs: np.ndarray, prefix: str) -> pd.DataFrame:
+    """Adds p_neg/p_neu/p_pos, {prefix}_label, {prefix}_score columns for one model."""
+    result = df.copy()
+    result[f"{prefix}_p_neg"] = probs[:, 0]
+    result[f"{prefix}_p_neu"] = probs[:, 1]
+    result[f"{prefix}_p_pos"] = probs[:, 2]
+    top = probs.argmax(axis=1)
+    result[f"{prefix}_label"] = [CLASS_NAMES[i] for i in top]
+    result[f"{prefix}_score"] = probs[np.arange(len(probs)), top]
+    return result
+
+# ---------------- Main ----------------
+def parse_args():
+    p = argparse.ArgumentParser(description="Apply FinBERT + RoBERTa to FPB subset.")
+    p.add_argument("--input", required=True, help="Input FPB file (.txt with 'sentence@label' or .csv with 'text')")
+    p.add_argument("--dataset", required=True, help="Dataset tag for the output file (e.g., 50Agree, 66Agree, 75Agree, AllAgree)")
+    p.add_argument("--batch_size", type=int, default=32)
+    p.add_argument("--max_length", type=int, default=128)
+    p.add_argument("--out_dir", default="outputs", help="Directory to save results")
+    p.add_argument("--out_subdir", default=None, help="Optional subdirectory under out_dir (e.g. 'FinBERT and RoBERTa raw probs')")
+    return p.parse_args()
+
+def main():
+    args = parse_args()
+
+    # Load data
+    df = read_input(args.input)
+    if "doc_id" not in df.columns:
+        df.insert(0, "doc_id", np.arange(len(df), dtype=int))
+    print(f"Loaded {len(df)} rows from {args.input}")
+
+    # FinBERT
+    print("Running FinBERT (ProsusAI/finbert)...")
+    fin_tok, fin_mdl, fin_dev = load_model(FINBERT_ID)
+    fin_probs = predict_probs(df["text"].tolist(), fin_tok, fin_mdl, fin_dev, args.batch_size, args.max_length)
+    df = add_prob_columns(df, fin_probs, "fin")
+
+    # RoBERTa
+    print("Running RoBERTa (cardiffnlp/twitter-roberta-base-sentiment)...")
+    rob_tok, rob_mdl, rob_dev = load_model(ROBERTA_ID)
+    rob_probs = predict_probs(df["text"].tolist(), rob_tok, rob_mdl, rob_dev, args.batch_size, args.max_length)
+    df = add_prob_columns(df, rob_probs, "rob")
+
+    # Save (optionally into a named subdirectory)
+    save_dir = args.out_dir
+    if args.out_subdir:
+        # create a subdirectory under out_dir
+        save_dir = os.path.join(args.out_dir, args.out_subdir)
+    os.makedirs(save_dir, exist_ok=True)
+    out_path = os.path.join(save_dir, f"FinSent_{args.dataset}_raw_probs.csv")
+    df.to_csv(out_path, index=False)
+    print(f"Saved to: {out_path}")
+    print("Columns:")
+    print("  fin_p_neg/neu/pos, fin_label, fin_score")
+    print("  rob_p_neg/neu/pos, rob_label, rob_score")
+
+if __name__ == "__main__":
+    main()

FPB Meta Classifier.py

@@ -0,0 +1,820 @@
+"""Meta-classifier for Financial PhraseBank (FPB) datasets.
+
+The script expects the following pre-computed artifacts inside ``outputs/``
+(or custom paths can be supplied):
+
+* ``FinSent_<split>_raw_probs_prob_features.csv`` – base probabilities and
+  probability-derived features for FinBERT/RoBERTa
+* ``FPB_MultiLLM_<split>.csv`` – expert-signal metrics (KL, L1, agreement)
+* ``Sentences_<split>_semantics.csv`` – structured semantics flags
+
+Example command (50Agree subset)::
+
+    python "FPB Meta Classifier.py" \\
+        --dataset 50Agree \\
+        --folds 5 \\
+        --models logreg xgboost \\
+        --artifact_prefix outputs/FinSent_50Agree_meta \\
+        --save_predictions --save_models --verbose
+
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from dataclasses import dataclass
+from typing import Dict, Iterable, List, Optional
+
+import joblib
+import numpy as np
+import pandas as pd
+from sklearn.compose import ColumnTransformer
+from sklearn.linear_model import LogisticRegression
+from sklearn.metrics import classification_report, confusion_matrix
+from sklearn.model_selection import StratifiedKFold, cross_val_predict, cross_validate
+from sklearn.pipeline import Pipeline
+from sklearn.preprocessing import OneHotEncoder, StandardScaler
+
+# tqdm用于进度条显示
+try:
+    from tqdm import tqdm
+except ImportError:
+    tqdm = None
+
+try:
+    from xgboost import XGBClassifier
+except ImportError:  # pragma: no cover - handled at runtime
+    XGBClassifier = None  # type: ignore
+
+try:
+    import torch
+    from transformers import AutoTokenizer, AutoModelForSequenceClassification
+    from scipy.stats import entropy
+    TRANSFORMERS_AVAILABLE = True
+except ImportError:
+    TRANSFORMERS_AVAILABLE = False
+    print("[!] transformers or torch not available. FinSentLLM feature engineering will be disabled.")
+
+from sklearn.base import BaseEstimator, TransformerMixin
+
+
+# ---------------------------------------------------------------------------
+# Data loading
+# ---------------------------------------------------------------------------
+
+@dataclass
+class DatasetPaths:
+    dataset: str
+    prob_features_csv: str
+    multi_llm_csv: str
+    semantics_csv: str
+
+
+def infer_paths(dataset: str, base_dir: str = "outputs") -> DatasetPaths:
+    dtag = dataset.strip()
+    prob_csv = os.path.join(base_dir, "prob features", f"FinSent_{dtag}_raw_probs_prob_features.csv")
+    multi_csv = os.path.join(base_dir, "MultiLLM", f"FPB_MultiLLM_{dtag}.csv")
+    # 修正语义文件路径 - 实际文件在 Structures Financial Semantics 子目录下
+    sem_csv = os.path.join(base_dir, "Structures Financial Semantics", f"Sentences_{dtag}_semantics.csv")
+    return DatasetPaths(dataset=dtag, prob_features_csv=prob_csv, multi_llm_csv=multi_csv, semantics_csv=sem_csv)
+
+
+def _merge_features(left: pd.DataFrame, right: pd.DataFrame, key: str = "doc_id") -> pd.DataFrame:
+    """Merge two DataFrames on ``doc_id`` while dropping duplicate feature columns."""
+    overlap = [c for c in right.columns if c in left.columns and c != key]
+    right_clean = right.drop(columns=overlap, errors="ignore")
+    merged = left.merge(right_clean, on=key, how="left", validate="one_to_one")
+    return merged
+
+
+def load_feature_table(paths: DatasetPaths) -> pd.DataFrame:
+    if not os.path.exists(paths.multi_llm_csv):
+        raise FileNotFoundError(f"Missing Multi-LLM feature CSV: {paths.multi_llm_csv}")
+    base = pd.read_csv(paths.multi_llm_csv)
+
+    # Ensure `doc_id` present for alignment.
+    if "doc_id" not in base.columns:
+        raise KeyError("Expected 'doc_id' column in Multi-LLM CSV. Re-run stage 3 feature extraction.")
+
+    # Merge optional probability features if available (guards against missing engineered columns).
+    if os.path.exists(paths.prob_features_csv):
+        prob = pd.read_csv(paths.prob_features_csv)
+        if "doc_id" not in prob.columns:
+            raise KeyError("Probability features CSV must contain 'doc_id'.")
+        base = _merge_features(base, prob, key="doc_id")
+    else:
+        print(f"[!] Probability feature CSV not found ({paths.prob_features_csv}); proceeding without extra columns.")
+
+    # Merge structured semantics.
+    if not os.path.exists(paths.semantics_csv):
+        raise FileNotFoundError(f"Missing semantics CSV: {paths.semantics_csv}")
+    sem = pd.read_csv(paths.semantics_csv)
+    if "doc_id" not in sem.columns:
+        if "id" in sem.columns:
+            sem = sem.rename(columns={"id": "doc_id"})
+        else:
+            raise KeyError("Semantics CSV must contain 'doc_id' or 'id' column.")
+
+    sem = sem.drop(columns=[c for c in ["label", "sentence", "text"] if c in sem.columns], errors="ignore")
+    merged = _merge_features(base, sem, key="doc_id")
+
+    # Check for missing semantics flags.
+    sem_cols = [c for c in merged.columns if c.startswith("sem_")]
+    if sem_cols:
+        missing_sem = merged[sem_cols].isna().any(axis=1)
+        if missing_sem.any():
+            raise ValueError(
+                f"{int(missing_sem.sum())} rows lack structured semantics after merging. Make sure the semantics file"
+                " matches the dataset split."
+            )
+
+    return merged
+
+
+def load_best_iterations(results_dir: str = "results") -> Dict[str, int]:
+    """Load previously computed best iterations for XGBoost models.
+    
+    Returns:
+        Dictionary mapping dataset names to best iteration counts.
+        Returns empty dict if file not found.
+    """
+    best_iters_file = os.path.join(results_dir, "xgb_meta_best_iterations.csv")
+    
+    if not os.path.exists(best_iters_file):
+        print(f"[!] Best iterations file not found: {best_iters_file}")
+        return {}
+    
+    try:
+        df = pd.read_csv(best_iters_file)
+        # Create mapping from dataset name to best iteration
+        best_iters = {}
+        for _, row in df.iterrows():
+            dataset = row["meta"]  # e.g., "50Agree"
+            best_iter = int(row["best_iteration"])
+            best_iters[dataset] = best_iter
+        
+        print(f"[✓] Loaded best iterations for {len(best_iters)} datasets:")
+        for dataset, iter_count in best_iters.items():
+            print(f"    {dataset}: {iter_count} iterations")
+        
+        return best_iters
+    except Exception as e:
+        print(f"[!] Error loading best iterations: {e}")
+        return {}
+
+
+# ---------------------------------------------------------------------------
+# FinSentLLM Feature Engineering Pipeline
+# ---------------------------------------------------------------------------
+
+class FinSentLLMFeatureEngineering(BaseEstimator, TransformerMixin):
+    """
+    端到端特征工程转换器，将原始文本转换为FinSentLLM的36个特征。
+    包括FinBERT/RoBERTa推理、概率工程、MultiLLM特征和语义特征。
+    """
+    
+    def __init__(self, 
+                 finbert_model_id="ProsusAI/finbert",
+                 roberta_model_id="cardiffnlp/twitter-roberta-base-sentiment",
+                 batch_size=16,
+                 max_length=128,
+                 device=None):
+        self.finbert_model_id = finbert_model_id
+        self.roberta_model_id = roberta_model_id
+        self.batch_size = batch_size
+        self.max_length = max_length
+        self.device = device
+        self.class_names = ["negative", "neutral", "positive"]
+        
+        # 模型组件将在fit时初始化
+        self.finbert_tokenizer = None
+        self.finbert_model = None
+        self.roberta_tokenizer = None
+        self.roberta_model = None
+        self._device = None
+    
+    def _load_models(self):
+        """加载FinBERT和RoBERTa模型"""
+        if not TRANSFORMERS_AVAILABLE:
+            raise ImportError("transformers and torch are required for FinSentLLM feature engineering")
+        
+        print("[📥] Loading FinBERT and RoBERTa models...")
+        
+        # 设置设备
+        if self.device is None:
+            self._device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+        else:
+            self._device = torch.device(self.device)
+        
+        # 加载FinBERT
+        self.finbert_tokenizer = AutoTokenizer.from_pretrained(self.finbert_model_id)
+        self.finbert_model = AutoModelForSequenceClassification.from_pretrained(self.finbert_model_id)
+        self.finbert_model.to(self._device).eval()
+        
+        # 加载RoBERTa
+        self.roberta_tokenizer = AutoTokenizer.from_pretrained(self.roberta_model_id)
+        self.roberta_model = AutoModelForSequenceClassification.from_pretrained(self.roberta_model_id)
+        self.roberta_model.to(self._device).eval()
+        
+        print(f"[✅] Models loaded on {self._device}")
+    
+    @torch.no_grad()
+    def _get_probabilities(self, texts, tokenizer, model):
+        """获取模型的概率预测，带tqdm进度条"""
+        all_probs = []
+        total = len(texts)
+        batch_iter = range(0, total, self.batch_size)
+        use_tqdm = tqdm is not None and total > self.batch_size
+        iterator = tqdm(batch_iter, desc="[tqdm] Encoding & inference", unit="batch") if use_tqdm else batch_iter
+        for i in iterator:
+            batch = texts[i:i + self.batch_size]
+            # 编码
+            encoding = tokenizer(
+                batch, 
+                return_tensors="pt", 
+                truncation=True, 
+                padding=True, 
+                max_length=self.max_length
+            )
+            # 移动到设备
+            encoding = {k: v.to(self._device) for k, v in encoding.items()}
+            # 推理
+            logits = model(**encoding).logits
+            probs = torch.softmax(logits, dim=-1).cpu().numpy()
+            all_probs.append(probs)
+        return np.vstack(all_probs)
+    
+    def _build_features(self, finbert_probs, roberta_probs):
+        """构建完整的36个特征"""
+        eps = 1e-12
+        features = {}
+        n_samples = len(finbert_probs)
+        
+        # 1. 基础概率特征 (8个)
+        for i, cls in enumerate(self.class_names):
+            features[f"fin_p_{cls[:3]}"] = finbert_probs[:, i]
+            features[f"rob_p_{cls[:3]}"] = roberta_probs[:, i]
+        
+        features["fin_score"] = finbert_probs.max(axis=1)
+        features["rob_score"] = roberta_probs.max(axis=1)
+        
+        # 2. 标签特征 (2个)
+        features["fin_label"] = finbert_probs.argmax(axis=1)
+        features["rob_label"] = roberta_probs.argmax(axis=1)
+        
+        # 3. 工程概率特征 (12个)
+        # Logits
+        for i, cls in enumerate(self.class_names):
+            features[f"fin_logit_{cls[:3]}"] = np.log((finbert_probs[:, i] + eps) / (1 - finbert_probs[:, i] + eps))
+            features[f"rob_logit_{cls[:3]}"] = np.log((roberta_probs[:, i] + eps) / (1 - roberta_probs[:, i] + eps))
+        
+        # 最大概率
+        features["fin_max_prob"] = finbert_probs.max(axis=1)
+        features["rob_max_prob"] = roberta_probs.max(axis=1)
+        
+        # 边际 (最高 - 第二高概率)
+        fin_sorted = np.sort(finbert_probs, axis=1)
+        rob_sorted = np.sort(roberta_probs, axis=1)
+        features["fin_margin"] = fin_sorted[:, -1] - fin_sorted[:, -2]
+        features["rob_margin"] = rob_sorted[:, -1] - rob_sorted[:, -2]
+        
+        # 熵
+        features["fin_entropy"] = entropy(finbert_probs.T)
+        features["rob_entropy"] = entropy(roberta_probs.T)
+        
+        # 4. MultiLLM特征 (5个)
+        # L1距离和相似性
+        l1_dist = np.abs(finbert_probs - roberta_probs).sum(axis=1)
+        features["MultiLLM_L1_distance"] = l1_dist
+        features["MultiLLM_L1_similarity"] = 1 / (1 + l1_dist)
+        
+        # KL散度
+        features["MultiLLM_KL_F_to_R"] = entropy(finbert_probs.T, roberta_probs.T)
+        features["MultiLLM_KL_R_to_F"] = entropy(roberta_probs.T, finbert_probs.T)
+        
+        # 一致性
+        fin_pred = finbert_probs.argmax(axis=1)
+        rob_pred = roberta_probs.argmax(axis=1)
+        features["MultiLLM_agree"] = (fin_pred == rob_pred).astype(int)
+        
+        # 5. 结构化语义特征 (9个) - 简化版本，实际使用中应该基于NLP规则
+        # 这里使用基于概率的启发式规则
+        features["sem_compared"] = ((finbert_probs[:, 1] > 0.4) & (roberta_probs[:, 1] > 0.4)).astype(int)
+        features["sem_loss_improve"] = ((finbert_probs[:, 2] > 0.6) & (roberta_probs[:, 2] > 0.5)).astype(int)
+        features["sem_loss_worsen"] = ((finbert_probs[:, 0] > 0.6) & (roberta_probs[:, 0] > 0.5)).astype(int)
+        features["sem_profit_up"] = ((finbert_probs[:, 2] > 0.7) & (l1_dist < 0.3)).astype(int)
+        features["sem_cost_down"] = ((finbert_probs[:, 2] > 0.5) & (features["MultiLLM_agree"] == 1)).astype(int)
+        features["sem_contract_fin"] = ((finbert_probs[:, 1] > 0.8)).astype(int)
+        features["sem_uncertainty"] = ((features["fin_entropy"] > 1.0) | (features["rob_entropy"] > 1.0)).astype(int)
+        features["sem_stable_guidance"] = ((l1_dist < 0.2) & (finbert_probs[:, 1] > 0.5)).astype(int)
+        features["sem_operational"] = ((finbert_probs[:, 1] > 0.3) & (roberta_probs[:, 1] > 0.3)).astype(int)
+        
+        return pd.DataFrame(features)
+    
+    def fit(self, X, y=None):
+        """训练阶段 - 加载模型"""
+        self._load_models()
+        return self
+    
+    def transform(self, X):
+        """转换阶段 - 将文本转换为特征"""
+        if self.finbert_model is None:
+            raise RuntimeError("Models not loaded. Call fit() first.")
+        
+        # 处理输入
+        if isinstance(X, pd.DataFrame):
+            if 'text' in X.columns:
+                texts = X['text'].tolist()
+            elif len(X.columns) == 1:
+                texts = X.iloc[:, 0].tolist()
+            else:
+                raise ValueError("DataFrame must have 'text' column or single column")
+        elif isinstance(X, (list, np.ndarray)):
+            texts = list(X)
+        else:
+            raise ValueError("X must be DataFrame, list, or array")
+        
+        print(f"[🔮] Processing {len(texts)} texts...")
+        
+        # 获取概率
+        finbert_probs = self._get_probabilities(texts, self.finbert_tokenizer, self.finbert_model)
+        roberta_probs = self._get_probabilities(texts, self.roberta_tokenizer, self.roberta_model)
+        
+        # 构建特征
+        features_df = self._build_features(finbert_probs, roberta_probs)
+        
+        print(f"[✅] Generated {len(features_df.columns)} features")
+        return features_df
+
+
+# ---------------------------------------------------------------------------
+# Modeling utilities
+# ---------------------------------------------------------------------------
+
+def build_preprocessor(numeric_cols: List[str], categorical_cols: List[str]) -> ColumnTransformer:
+    transformers = []
+    if numeric_cols:
+        transformers.append(("num", StandardScaler(), numeric_cols))
+    if categorical_cols:
+        transformers.append(("cat", OneHotEncoder(handle_unknown="ignore", sparse_output=False), categorical_cols))
+    if not transformers:
+        raise ValueError("No feature columns selected – check your dataset.")
+    return ColumnTransformer(transformers=transformers, remainder="drop")
+
+
+def build_pipelines(
+    numeric_cols: List[str],
+    categorical_cols: List[str],
+    num_classes: int,
+    random_state: int,
+    models_requested: Iterable[str],
+    dataset: str = "",
+    best_iterations: Dict[str, int] = None,
+    include_feature_engineering: bool = False,
+) -> Dict[str, Pipeline]:
+    pipelines: Dict[str, Pipeline] = {}
+
+
+    # 定义 end-to-end 特征名
+    end2end_categorical_features = ["fin_label", "rob_label"]
+    end2end_numeric_features = [
+        'fin_p_neg', 'fin_p_neu', 'fin_p_pos', 'fin_score',
+        'rob_p_neg', 'rob_p_neu', 'rob_p_pos', 'rob_score',
+        'fin_logit_neg', 'fin_logit_neu', 'fin_logit_pos',
+        'rob_logit_neg', 'rob_logit_neu', 'rob_logit_pos',
+        'fin_max_prob', 'rob_max_prob', 'fin_margin', 'rob_margin',
+        'fin_entropy', 'rob_entropy',
+        'MultiLLM_L1_distance', 'MultiLLM_L1_similarity',
+        'MultiLLM_KL_F_to_R', 'MultiLLM_KL_R_to_F', 'MultiLLM_agree',
+        'sem_compared', 'sem_loss_improve', 'sem_loss_worsen',
+        'sem_profit_up', 'sem_cost_down', 'sem_contract_fin',
+        'sem_uncertainty', 'sem_stable_guidance', 'sem_operational'
+    ]
+
+    if "logreg" in models_requested:
+        logreg = LogisticRegression(max_iter=1000, solver="lbfgs")
+        if include_feature_engineering:
+            preprocessor = build_preprocessor(end2end_numeric_features, end2end_categorical_features)
+            pipelines["logreg"] = Pipeline([
+                ("feature_engineering", FinSentLLMFeatureEngineering()),
+                ("preprocess", preprocessor),
+                ("clf", logreg),
+            ])
+        else:
+            preprocessor = build_preprocessor(numeric_cols, categorical_cols)
+            pipelines["logreg"] = Pipeline([
+                ("preprocess", preprocessor),
+                ("clf", logreg),
+            ])
+
+    if "xgboost" in models_requested:
+        if XGBClassifier is None:
+            raise ImportError(
+                "xgboost is not installed. Install it with 'pip install xgboost' or remove 'xgboost' from --models."
+            )
+        if best_iterations and dataset in best_iterations:
+            n_estimators = best_iterations[dataset]
+            print(f"[✓] Using pre-computed best iterations for {dataset}: {n_estimators}")
+        else:
+            n_estimators = 1000
+            print(f"[!] No pre-computed iterations found for {dataset}, using default: {n_estimators}")
+        xgb = XGBClassifier(
+            objective="multi:softprob",
+            num_class=num_classes,
+            learning_rate=0.05,
+            max_depth=6,
+            subsample=0.8,
+            colsample_bytree=0.8,
+            n_estimators=n_estimators,
+            min_child_weight=2,
+            reg_lambda=1.0,
+            reg_alpha=0.0,
+            tree_method="hist",
+            eval_metric="mlogloss",
+            random_state=random_state,
+            n_jobs=0,
+            verbosity=0,
+        )
+        if include_feature_engineering:
+            feature_preprocessor = build_preprocessor(end2end_numeric_features, end2end_categorical_features)
+            pipelines["xgboost"] = Pipeline([
+                ("feature_engineering", FinSentLLMFeatureEngineering()),
+                ("preprocess", feature_preprocessor),
+                ("clf", xgb),
+            ])
+            print(f"[🤖] Created end-to-end XGBoost pipeline with feature engineering")
+        else:
+            preprocessor = build_preprocessor(numeric_cols, categorical_cols)
+            pipelines["xgboost"] = Pipeline([
+                ("preprocess", preprocessor),
+                ("clf", xgb),
+            ])
+
+    return pipelines
+
+    if "logreg" in models_requested:
+        preprocessor = build_preprocessor(numeric_cols, categorical_cols)
+        logreg = LogisticRegression(max_iter=1000, solver="lbfgs")
+        pipelines["logreg"] = Pipeline([
+            ("preprocess", preprocessor),
+            ("clf", logreg),
+        ])
+
+    if "xgboost" in models_requested:
+        if XGBClassifier is None:
+            raise ImportError(
+                "xgboost is not installed. Install it with 'pip install xgboost' or remove 'xgboost' from --models."
+            )
+        preprocessor = build_preprocessor(numeric_cols, categorical_cols)
+        
+        # 使用预存的最优轮数或默认值
+        if best_iterations and dataset in best_iterations:
+            n_estimators = best_iterations[dataset]
+            print(f"[✓] Using pre-computed best iterations for {dataset}: {n_estimators}")
+        else:
+            n_estimators = 1000  # 默认值
+            print(f"[!] No pre-computed iterations found for {dataset}, using default: {n_estimators}")
+        
+        xgb = XGBClassifier(
+            objective="multi:softprob",
+            num_class=num_classes,
+            learning_rate=0.05,
+            max_depth=6,
+            subsample=0.8,
+            colsample_bytree=0.8,
+            n_estimators=n_estimators,  # 使用预存的最优轮数
+            min_child_weight=2,
+            reg_lambda=1.0,
+            reg_alpha=0.0,
+            tree_method="hist",
+            eval_metric="mlogloss",
+            random_state=random_state,
+            n_jobs=0,
+            verbosity=0,
+        )
+        pipelines["xgboost"] = Pipeline([
+            ("preprocess", preprocessor),
+            ("clf", xgb),
+        ])
+
+    return pipelines
+
+
+def evaluate_model(
+    name: str,
+    pipeline: Pipeline,
+    X: pd.DataFrame,
+    y_train: pd.Series,
+    y_eval: pd.Series,
+    cv: StratifiedKFold,
+    class_labels: List[str],
+    label_decoder: Optional[Dict[int, str]] = None,
+) -> Dict[str, object]:
+    scoring = {"accuracy": "accuracy", "macro_f1": "f1_macro"}
+
+    scores = cross_validate(
+        pipeline,
+        X,
+        y_train,
+        scoring=scoring,
+        cv=cv,
+        n_jobs=None,
+        return_estimator=False,
+    )
+
+    preds = cross_val_predict(pipeline, X, y_train, cv=cv, method="predict")
+    probas = cross_val_predict(pipeline, X, y_train, cv=cv, method="predict_proba")
+
+    # 直接训练模型（已经使用了预存的最优轮数）
+    fitted = pipeline.fit(X, y_train)
+
+    clf_raw_classes = list(fitted.named_steps["clf"].classes_)
+
+    if label_decoder:
+        preds_decoded = np.array([label_decoder[int(p)] for p in preds])
+        proba_labels = [label_decoder[int(c)] for c in clf_raw_classes]
+    else:
+        preds_decoded = preds
+        proba_labels = [str(c) for c in clf_raw_classes]
+
+    if proba_labels != class_labels:
+        reorder_idx = [proba_labels.index(lbl) for lbl in class_labels]
+        probas = probas[:, reorder_idx]
+        proba_labels = class_labels
+
+    y_eval_array = y_eval.to_numpy()
+
+    report = classification_report(y_eval_array, preds_decoded, labels=class_labels, digits=4)
+    cm = confusion_matrix(y_eval_array, preds_decoded, labels=class_labels)
+
+    metrics = {
+        "name": name,
+        "accuracy_mean": float(np.mean(scores["test_accuracy"])),
+        "accuracy_std": float(np.std(scores["test_accuracy"])),
+        "macro_f1_mean": float(np.mean(scores["test_macro_f1"])),
+        "macro_f1_std": float(np.std(scores["test_macro_f1"])),
+        "classification_report": report,
+        "confusion_matrix": cm,
+        "classes": class_labels,
+        "preds": preds_decoded,
+        "probas": probas,
+        "final_model": fitted,
+    }
+    
+    # 为XGBoost添加best_iteration信息
+    if name == "xgboost":
+        if hasattr(fitted.named_steps["clf"], "best_iteration"):
+            metrics["best_iteration"] = fitted.named_steps["clf"].best_iteration
+        elif hasattr(fitted.named_steps["clf"], "n_estimators"):
+            metrics["best_iteration"] = fitted.named_steps["clf"].n_estimators
+        metrics["best_ntree_limit"] = metrics.get("best_iteration", 0) + 1
+    
+    return metrics
+
+
+def print_metrics(metrics: Dict[str, object], verbose: bool = False) -> None:
+    name = metrics["name"]
+    print(f"\n=== {name.upper()} meta-classifier ===")
+    print(
+        f"Accuracy: {metrics['accuracy_mean']*100:.2f}% ± {metrics['accuracy_std']*100:.2f}%\n"
+        f"Macro-F1: {metrics['macro_f1_mean']*100:.2f}% ± {metrics['macro_f1_std']*100:.2f}%"
+    )
+    if verbose:
+        print("\nClassification report:\n", metrics["classification_report"], sep="")
+        print("Confusion matrix (rows=true, cols=pred):")
+        classes = metrics["classes"]
+        header = "    " + "  ".join(f"{c[:7]:>7}" for c in classes)
+        print(header)
+        for c, row in zip(classes, metrics["confusion_matrix"]):
+            row_fmt = "  ".join(f"{int(v):>7}" for v in row)
+            print(f"{c[:7]:>4} {row_fmt}")
+
+
+def save_predictions(base: pd.DataFrame, metrics: Dict[str, object], path: str) -> None:
+    pred_df = base[["doc_id"]].copy()
+    pred_df["true_label"] = base["label"]
+    pred_df["meta_pred"] = metrics["preds"]
+    for idx, cls in enumerate(metrics["classes"]):
+        pred_df[f"meta_proba_{cls}"] = metrics["probas"][:, idx]
+    pred_df.to_csv(path, index=False)
+    print(f"Saved predictions: {path}")
+
+
+# ---------------------------------------------------------------------------
+# CLI
+# ---------------------------------------------------------------------------
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Train FinSentLLM meta-classifiers (LogReg/XGBoost).")
+    parser.add_argument("--dataset", required=True, help="Dataset tag, e.g. 50Agree | 66Agree | 75Agree | AllAgree")
+    parser.add_argument("--prob_features_csv", help="Override path to probability feature CSV")
+    parser.add_argument("--multi_llm_csv", help="Override path to Multi-LLM feature CSV")
+    parser.add_argument("--semantics_csv", help="Override path to structured semantics CSV")
+    parser.add_argument("--folds", type=int, default=5, help="Number of stratified CV folds (default: 5)")
+    parser.add_argument("--seed", type=int, default=7, help="Random seed for CV shuffling (default: 7)")
+    parser.add_argument(
+        "--models",
+        nargs="+",
+        default=["logreg", "xgboost"],
+        choices=["logreg", "xgboost"],
+        help="Which meta-models to evaluate (default: both)",
+    )
+    parser.add_argument("--artifact_prefix", help="If set, saves artifacts using this filepath prefix")
+    parser.add_argument("--out_dir", default="outputs", help="Base output directory")
+    parser.add_argument("--meta_xgb_dir", default="Meta-Classifier_XG_boost_es_optimized", help="Subdir for xgboost artifacts")
+    parser.add_argument("--meta_logreg_dir", default="Meta-Classifier-log_regression", help="Subdir for logreg artifacts")
+    parser.add_argument("--save_predictions", action="store_true", help="Write out-of-fold predictions per model")
+    parser.add_argument("--save_models", action="store_true", help="Persist fitted pipelines per model")
+    parser.add_argument("--verbose", action="store_true", help="Print full reports and confusion matrices")
+    # 默认不使用 end-to-end，直接用预处理特征，速度快
+    parser.add_argument("--end_to_end", action="store_true", default=False, help="[慢] 用大模型重新生成特征 (不建议，除非你需要全流程推理)")
+    return parser.parse_args()
+
+
+def main() -> None:
+
+    args = parse_args()
+
+    # 加载预存的最优轮数
+    best_iterations = load_best_iterations()
+
+    # 如果用户同时指定了 --end_to_end 和特征文件路径，给出警告
+    if args.end_to_end and (args.prob_features_csv or args.multi_llm_csv or args.semantics_csv):
+        print("[警告] --end_to_end 模式下会忽略所有预处理特征文件，全部重新推理，速度极慢！")
+
+    if args.end_to_end:
+        print("[🤖] Creating end-to-end pipelines with feature engineering... (速度极慢，仅用于全流程推理)")
+        if not TRANSFORMERS_AVAILABLE:
+            raise ImportError("transformers and torch are required for end-to-end feature engineering. Install with: pip install transformers torch")
+
+        # 对于端到端pipeline，我们需要原始文本数据
+        paths = infer_paths(args.dataset)
+        data = load_feature_table(paths)
+
+        # 检查是否有文本列
+        text_col = None
+        for col in ['text', 'sentence', 'content']:
+            if col in data.columns:
+                text_col = col
+                break
+
+        if text_col is None:
+            raise ValueError("End-to-end mode requires text data, but no text column found in dataset")
+
+        X_text = data[[text_col]]  # 原始文本
+        target_col = "label"
+        if target_col not in data.columns:
+            raise KeyError("Target column 'label' not found after merging.")
+
+        y = data[target_col].astype(str)
+
+        default_order = ["negative", "neutral", "positive"]
+        observed = list(pd.unique(y))
+        class_labels = [lbl for lbl in default_order if lbl in observed]
+        class_labels += [lbl for lbl in observed if lbl not in class_labels]
+
+        label_to_int = {lbl: idx for idx, lbl in enumerate(class_labels)}
+        int_to_label = {idx: lbl for lbl, idx in label_to_int.items()}
+        y_encoded = y.map(label_to_int).astype(int)
+
+        pipelines = build_pipelines(
+            numeric_cols=[],
+            categorical_cols=[],
+            num_classes=len(class_labels),
+            random_state=args.seed,
+            models_requested=args.models,
+            dataset=args.dataset,
+            best_iterations=best_iterations,
+            include_feature_engineering=True,
+        )
+        X = X_text
+    else:
+        # 默认推荐：直接用预处理特征，速度快
+        paths = infer_paths(args.dataset)
+        if args.prob_features_csv:
+            paths.prob_features_csv = args.prob_features_csv
+        if args.multi_llm_csv:
+            paths.multi_llm_csv = args.multi_llm_csv
+        if args.semantics_csv:
+            paths.semantics_csv = args.semantics_csv
+
+        data = load_feature_table(paths)
+
+        target_col = "label"
+        if target_col not in data.columns:
+            raise KeyError("Target column 'label' not found after merging.")
+
+        categorical_cols = [c for c in ["fin_label", "rob_label"] if c in data.columns]
+        numeric_cols = [
+            c for c in data.select_dtypes(include=[np.number]).columns
+            if c not in {"doc_id"}
+        ]
+
+        X = data[numeric_cols + categorical_cols]
+        y = data[target_col].astype(str)
+
+        default_order = ["negative", "neutral", "positive"]
+        observed = list(pd.unique(y))
+        class_labels = [lbl for lbl in default_order if lbl in observed]
+        class_labels += [lbl for lbl in observed if lbl not in class_labels]
+
+        label_to_int = {lbl: idx for idx, lbl in enumerate(class_labels)}
+        int_to_label = {idx: lbl for lbl, idx in label_to_int.items()}
+        y_encoded = y.map(label_to_int).astype(int)
+
+        pipelines = build_pipelines(
+            numeric_cols=numeric_cols,
+            categorical_cols=categorical_cols,
+            num_classes=len(class_labels),
+            random_state=args.seed,
+            models_requested=args.models,
+            dataset=args.dataset,
+            best_iterations=best_iterations,
+            include_feature_engineering=False,
+        )
+
+    cv = StratifiedKFold(n_splits=args.folds, shuffle=True, random_state=args.seed)
+
+    results = {}
+    for name, pipeline in pipelines.items():
+        if name == "xgboost":
+            metrics = evaluate_model(
+                name,
+                pipeline,
+                X,
+                y_encoded,
+                y,
+                cv=cv,
+                class_labels=class_labels,
+                label_decoder=int_to_label,
+            )
+        else:
+            metrics = evaluate_model(
+                name,
+                pipeline,
+                X,
+                y,
+                y,
+                cv=cv,
+                class_labels=class_labels,
+            )
+        print_metrics(metrics, verbose=args.verbose)
+        results[name] = metrics
+
+        if args.artifact_prefix and args.save_predictions:
+            pred_path = f"{args.artifact_prefix}_{name}_predictions.csv"
+            save_predictions(data, metrics, pred_path)
+
+        if args.artifact_prefix and args.save_models:
+            model_path = f"{args.artifact_prefix}_{name}_model.joblib"
+            # 为XGBoost保存完整的模型字典
+            if name == "xgboost":
+                model_dict = {
+                    "pipeline": metrics["final_model"],
+                    "feature_columns": list(X.columns),
+                    "label_map": label_to_int,
+                    "labels": class_labels,
+                    "best_iteration": metrics.get("best_iteration", 0),
+                    "best_ntree_limit": metrics.get("best_ntree_limit", 1),
+                }
+                joblib.dump(model_dict, model_path)
+            else:
+                joblib.dump(metrics["final_model"], model_path)
+            print(f"Saved model: {model_path}")
+
+        # If no artifact_prefix is provided but user asked to save, route to default meta subfolders
+        if not args.artifact_prefix and args.save_predictions:
+            if name == "xgboost":
+                save_dir = os.path.join(args.out_dir, args.meta_xgb_dir)
+            else:
+                save_dir = os.path.join(args.out_dir, args.meta_logreg_dir)
+            os.makedirs(save_dir, exist_ok=True)
+            pred_path = os.path.join(save_dir, f"FinSent_{args.dataset}_meta_{name}_predictions.csv")
+            save_predictions(data, metrics, pred_path)
+
+        if not args.artifact_prefix and args.save_models:
+            if name == "xgboost":
+                save_dir = os.path.join(args.out_dir, args.meta_xgb_dir)
+            else:
+                save_dir = os.path.join(args.out_dir, args.meta_logreg_dir)
+            os.makedirs(save_dir, exist_ok=True)
+            model_path = os.path.join(save_dir, f"FinSent_{args.dataset}_meta_{name}_model.joblib")
+            # 为XGBoost保存完整的模型字典
+            if name == "xgboost":
+                model_dict = {
+                    "pipeline": metrics["final_model"],
+                    "feature_columns": list(X.columns),
+                    "label_map": label_to_int,
+                    "labels": class_labels,
+                    "best_iteration": metrics.get("best_iteration", 0),
+                    "best_ntree_limit": metrics.get("best_ntree_limit", 1),
+                }
+                joblib.dump(model_dict, model_path)
+            else:
+                joblib.dump(metrics["final_model"], model_path)
+            print(f"Saved model: {model_path}")
+
+
+if __name__ == "__main__":
+    main()

FPB Multi-LLM .py

@@ -0,0 +1,70 @@
+# FPB Multi-LLM .py
+# Compute expert-signal features from multiple LLMs (FinBERT, RoBERT
+import argparse
+import os
+import numpy as np
+import pandas as pd
+from scipy.special import rel_entr
+
+EPS = 1e-12
+
+def row_normalize(a: np.ndarray) -> np.ndarray:
+    s = a.sum(axis=1, keepdims=True)
+    s[s == 0.0] = 1.0
+    return a / s
+
+def kl_divergence(P: np.ndarray, Q: np.ndarray) -> np.ndarray:
+    P = np.clip(P, EPS, 1.0)
+    Q = np.clip(Q, EPS, 1.0)
+    return np.sum(rel_entr(P, Q), axis=1)  # Σ p * log(p/q)
+
+def l1_distance(P: np.ndarray, Q: np.ndarray) -> np.ndarray:
+    return 0.5 * np.sum(np.abs(P - Q), axis=1)  # ∈ [0,1]
+
+def load_probs(df: pd.DataFrame, cols: list, label: str) -> np.ndarray:
+    missing = [c for c in cols if c not in df.columns]
+    if missing:
+        raise ValueError(f"[{label}] Missing columns: {missing}")
+    arr = df[cols].to_numpy(dtype=float)
+    return row_normalize(arr)
+
+def main():
+    ap = argparse.ArgumentParser(description="Compute Multi-LLM expert signals (Stage 3 only).")
+    ap.add_argument("--input", required=True, help="CSV with per-model probabilities")
+    ap.add_argument("--dataset", required=True, help="Dataset tag, e.g., 50Agree | AllAgree")
+    ap.add_argument("--out_dir", default=".", help="Output directory")
+    ap.add_argument("--out_subdir", default="MultiLLM", help="Subdirectory under out_dir to save multi-llm features")
+    # Column names: default to common names; override if your headers differ
+    ap.add_argument("--fin_cols", nargs=3, default=["fin_p_neg","fin_p_neu","fin_p_pos"],
+                    help="FinBERT prob columns [neg neu pos]")
+    ap.add_argument("--rob_cols", nargs=3, default=["rob_p_neg","rob_p_neu","rob_p_pos"],
+                    help="RoBERTa prob columns [neg neu pos]")
+    args = ap.parse_args()
+
+    df = pd.read_csv(args.input)
+
+    # Load & normalize probabilities
+    P = load_probs(df, args.fin_cols, "FinBERT")
+    Q = load_probs(df, args.rob_cols, "RoBERTa")
+
+    # Expert-signal features (paper)
+    df["MultiLLM_L1_distance"]   = l1_distance(P, Q)
+    df["MultiLLM_L1_similarity"] = 1.0 - df["MultiLLM_L1_distance"]
+    df["MultiLLM_KL_F_to_R"]     = kl_divergence(P, Q)
+    df["MultiLLM_KL_R_to_F"]     = kl_divergence(Q, P)
+
+    # Optional: simple agreement flag (same argmax class)
+    df["MultiLLM_agree"] = (np.argmax(P, axis=1) == np.argmax(Q, axis=1)).astype(int)
+
+    save_dir = args.out_dir
+    if args.out_subdir:
+        save_dir = os.path.join(args.out_dir, args.out_subdir)
+    os.makedirs(save_dir, exist_ok=True)
+    out_path = os.path.join(save_dir, f"FPB_MultiLLM_{args.dataset}.csv")
+    df.to_csv(out_path, index=False)
+    print(f"✅ Saved expert-signal features to: {out_path}")
+    print("   Added columns: MultiLLM_L1_distance, MultiLLM_L1_similarity, "
+          "MultiLLM_KL_F_to_R, MultiLLM_KL_R_to_F, MultiLLM_agree")
+
+if __name__ == "__main__":
+    main()

FPB Prob Features.py

@@ -0,0 +1,84 @@
+import argparse
+import os
+#!/usr/bin/env python3
+import argparse
+import os
+import numpy as np
+import pandas as pd
+
+EPS = 1e-12
+
+# ---------- Helper functions ----------
+def logit(p: np.ndarray) -> np.ndarray:
+    """Compute logit(p) = log(p / (1 - p))."""
+    p = np.clip(p, EPS, 1.0 - EPS)
+    return np.log(p / (1.0 - p))
+
+def entropy(p: np.ndarray) -> np.ndarray:
+    """Shannon entropy: H(p) = -Σ p log p."""
+    p = np.clip(p, EPS, 1.0)
+    return -np.sum(p * np.log(p), axis=1)
+
+def top2_margin(p: np.ndarray) -> np.ndarray:
+    """Margin = top1(p) - top2(p)."""
+    s = np.sort(p, axis=1)
+    return s[:, -1] - s[:, -2]
+
+# ---------- Main ----------
+def main():
+    ap = argparse.ArgumentParser(description="Compute probability-derived features (logit, max prob, margin, entropy).")
+    ap.add_argument("--input", required=True,
+                    help="Path to FinSent_*_raw_probs.csv from FinBERT/RoBERTa step.")
+    ap.add_argument("--out_file", default=None,
+                    help="Output CSV (default: adds _prob_features to filename).")
+    ap.add_argument("--out_dir", default="outputs", help="Base output directory")
+    ap.add_argument("--out_subdir", default="prob features", help="Subdirectory under out_dir to save prob features")
+    args = ap.parse_args()
+
+    df = pd.read_csv(args.input)
+
+    # Check probability columns
+    req = [
+        "fin_p_neg","fin_p_neu","fin_p_pos",
+        "rob_p_neg","rob_p_neu","rob_p_pos"
+    ]
+    missing = [c for c in req if c not in df.columns]
+    if missing:
+        raise ValueError(f"Missing columns: {missing}")
+
+    # Prepare arrays
+    p_fin = df[["fin_p_neg","fin_p_neu","fin_p_pos"]].to_numpy(dtype=float)
+    p_rob = df[["rob_p_neg","rob_p_neu","rob_p_pos"]].to_numpy(dtype=float)
+
+    # ---- FinBERT features ----
+    fin_logit = logit(p_fin)
+    for i, cls in enumerate(["neg","neu","pos"]):
+        df[f"fin_logit_{cls}"] = fin_logit[:, i]
+    df["fin_max_prob"] = np.max(p_fin, axis=1)
+    df["fin_margin"] = top2_margin(p_fin)
+    df["fin_entropy"] = entropy(p_fin)
+
+    # ---- RoBERTa features ----
+    rob_logit = logit(p_rob)
+    for i, cls in enumerate(["neg","neu","pos"]):
+        df[f"rob_logit_{cls}"] = rob_logit[:, i]
+    df["rob_max_prob"] = np.max(p_rob, axis=1)
+    df["rob_margin"] = top2_margin(p_rob)
+    df["rob_entropy"] = entropy(p_rob)
+
+    # Save
+    root, ext = os.path.splitext(args.input)
+    if args.out_file:
+        out_path = args.out_file
+    else:
+        # save into outputs/<out_subdir>/ by default
+        save_dir = os.path.join(args.out_dir, args.out_subdir) if args.out_subdir else args.out_dir
+        os.makedirs(save_dir, exist_ok=True)
+        base = os.path.basename(root)
+        out_path = os.path.join(save_dir, f"{base}_prob_features.csv")
+    df.to_csv(out_path, index=False)
+    print(f"[✓] Saved probability-derived features to: {out_path}")
+    print("Added columns: fin/rob logits, max_prob, margin, entropy")
+
+if __name__ == "__main__":
+    main()

FPB_Structured_Financial_Semantics.py

@@ -0,0 +1,120 @@
+
+import re
+from typing import Dict, List
+
+def _compile(patterns: List[str], flags=re.IGNORECASE):
+    return [re.compile(p, flags=flags) for p in patterns]
+
+def _any_match(text: str, regs) -> bool:
+    return any(r.search(text) for r in regs)
+
+# Operators per FinSentLLM Table 1
+_COMPARATIVE = _compile([
+    r"\bcompared\s+to\b",
+    r"\bcompared\s+with\b",
+    r"\bversus\b",
+    r"\bvs\.?\b",
+    r"\bfrom\s+[-+]?\d+(?:\.\d+)?\s*(?:%|percent|percentage|[A-Za-z]+)?\s+to\s+[-+]?\d+(?:\.\d+)?\s*(?:%|percent|percentage|[A-Za-z]+)?\b",
+    r"\bfrom\s+[A-Za-z0-9\.,%-]+\s+to\s+[A-Za-z0-9\.,%-]+\b",
+])
+
+_LOSS_IMPROVE = _compile([
+    r"\bloss(?:es)?\s+(?:narrowed|shr[aou]nk|decreased|fell|reduced)\b",
+    r"\bturn(?:ed)?\s+to\s+(?:profit|black)\b",
+])
+_LOSS_WORSEN = _compile([
+    r"\bloss(?:es)?\s+(?:widened|grew|increased|rose|deepened)\b",
+    r"\bturn(?:ed)?\s+to\s+(?:loss|red)\b",
+])
+
+_PROFIT_UP = _compile([
+    r"\b(profit|profits|net\s+income|earnings|ebit|ebitda|eps|roe|roi|return(?:s)?(?:\s+on\s+equity)?)\b.*\b(rose|grew|increased|up|higher|improved|jumped|surged|soared)\b",
+    r"\b(rose|grew|increased|up|higher|improved|jumped|surged|soared)\b.*\b(profit|profits|net\s+income|earnings|ebit|ebitda|eps|roe|roi|return(?:s)?(?:\s+on\s+equity)?)\b",
+])
+
+_COST_DOWN = _compile([
+    r"\b(cost|costs|expenses|opex|operating\s+expense(?:s)?)\b.*\b(fell|declined|decreased|lower|reduced|down)\b",
+    r"\b(fell|declined|decreased|lower|reduced|down)\b.*\b(cost|costs|expenses|opex|operating\s+expense(?:s)?)\b",
+])
+
+_CONTRACT_FIN = _compile([
+    r"\b(agreement|deal|contract|order|purchase\s+order|framework\s+agreement)\b",
+    r"\b(bond|notes?|debenture|convertible|placement|issuance|issue|offering|ipo|follow-?on)\b",
+    r"\b(loan|credit\s+facility|credit\s+line|revolver|revolving\s+credit|financing)\b",
+])
+
+_UNCERTAIN = _compile([
+    r"\b(uncertain|uncertainty|cannot\s+be\s+determined|not\s+clear|unknown|unpredictable)\b",
+    r"\b(impairment|write-?down|one-?off|exceptional\s+(?:item|charge)|non-?recurring)\b",
+    r"\b(outlook\s+(?:uncertain|cloudy|cautious))\b",
+])
+
+_STABLE_GUIDE = _compile([
+    r"\b(expects?|expected|expects\s+to|guidance|forecast|outlook)\b.*\b(remain(?:s|ed|ing)?\s+(?:stable|unchanged)|in[-\s]?line)\b",
+    r"\b(reiterated|maintained)\s+(?:its\s+)?(guidance|forecast|outlook)\b",
+])
+
+_OPERATIONAL = _compile([
+    r"\b(restructuring|reorganization|spin-?off|divest(?:iture)?|asset\s+sale)\b",
+    r"\b(ban|suspension|halted|blocked|prohibited)\b",
+    r"\b(recall|probe|investigation|lawsuit|litigation|settlement)\b",
+    r"\b(layoffs?|headcount\s+reduction|cut\s+jobs|hiring\s+freeze)\b",
+])
+
+def extract_semantic_flags(text: str) -> Dict[str, int]:
+    t = text.strip().lower()
+
+    flags = {
+        "sem_compared":       int(_any_match(t, _COMPARATIVE)),
+        "sem_loss_improve":   int(_any_match(t, _LOSS_IMPROVE)),
+        "sem_loss_worsen":    int(_any_match(t, _LOSS_WORSEN)),
+        "sem_profit_up":      int(_any_match(t, _PROFIT_UP)),
+        "sem_cost_down":      int(_any_match(t, _COST_DOWN)),
+        "sem_contract_fin":   int(_any_match(t, _CONTRACT_FIN)),
+        "sem_uncertainty":    int(_any_match(t, _UNCERTAIN)),
+        "sem_stable_guidance":int(_any_match(t, _STABLE_GUIDE)),
+        "sem_operational":    int(_any_match(t, _OPERATIONAL)),
+    }
+    return flags
+
+# ============================================================
+# Run directly from terminal
+# ============================================================
+if __name__ == "__main__":
+    import argparse, pandas as pd
+    from pathlib import Path
+
+    parser = argparse.ArgumentParser(description="Extract Structured Financial Semantics from FPB text file.")
+    parser.add_argument("--input", required=True, help="Path to Sentences_*.txt or a CSV with text column.")
+    parser.add_argument("--out", required=True, help="Output CSV path.")
+    parser.add_argument("--text_col", default="sentence", help="Column name if input is CSV.")
+    args = parser.parse_args()
+
+    def parse_fpb_line(line):
+        if "@positive" in line:
+            return line.rsplit("@positive", 1)[0].strip(), "positive"
+        elif "@negative" in line:
+            return line.rsplit("@negative", 1)[0].strip(), "negative"
+        elif "@neutral" in line:
+            return line.rsplit("@neutral", 1)[0].strip(), "neutral"
+        else:
+            return line.strip(), ""
+
+    path = Path(args.input)
+    rows = []
+    if path.suffix.lower() == ".txt":
+        with open(path, "r", encoding="utf-8", errors="ignore") as f:
+            for i, line in enumerate(f):
+                text, label = parse_fpb_line(line)
+                if text:
+                    rows.append({"id": i, args.text_col: text, "label": label})
+        df = pd.DataFrame(rows)
+    else:
+        df = pd.read_csv(path)
+
+    # Apply semantic extraction
+    df_feats = df[args.text_col].astype(str).apply(extract_semantic_flags).apply(pd.Series)
+    df_out = pd.concat([df, df_feats], axis=1)
+    df_out.to_csv(args.out, index=False)
+    print(f"Saved structured semantics to: {args.out}")
+    print("Columns:", [c for c in df_out.columns if c.startswith('sem_')])