Add files using upload-large-folder tool

assignment_llm_1/assignment_text/code/c1.py

@@ -0,0 +1,685 @@
+import os
+from datetime import datetime
+
+# Configure CUDA visibility (set this as appropriate for your environment).
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = "2"
+
+import math
+import random
+import re
+
+import numpy as np
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from collections import Counter
+from sklearn.metrics import accuracy_score, precision_recall_fscore_support
+from sklearn.model_selection import train_test_split
+from torch.utils.data import DataLoader, Dataset
+from datasets import load_dataset
+
+"""
+Homework 1 (Part I) – Transformer-based sentiment analysis on the IMDB dataset.
+
+This script implements:
+- Data loading and preprocessing for the IMDB movie review dataset
+- A Transformer-based text classification model
+- Training and evaluation loops for binary sentiment analysis
+- Saving of the trained model together with vocabulary and configuration
+
+The code is organized into clearly separated sections:
+1) Data preparation and tokenization
+2) Transformer components (building blocks)
+3) Full Transformer classifier
+4) Training and evaluation logic
+5) Execution example using a train/validation split of IMDB
+
+Model Analysis and Improvement:
+1. After evaluation, delve into analyzing your model's behavior to identify
+   areas for improvement and fine-tuning.
+2. Analyze translation errors (if applicable): Examine specific translation
+   examples where the model performs poorly and try to understand the reasons
+   behind these errors. Are there issues with handling rare words or
+   idiomatic expressions?
+3. Explore the impact of model size: Experiment with different Transformer
+   model sizes (e.g., small, medium, large) to understand how model
+   complexity affects performance.
+"""
+
+# ==========================================
+# 1. Data Preparation & Tokenization
+# ==========================================
+
+def tokenize(text):
+    """
+    Tokenize a raw review string into a list of normalized word tokens.
+
+    Steps:
+    - Convert to lowercase
+    - Remove HTML line breaks
+    - Remove non-alphanumeric characters (except whitespace)
+    - Split on whitespace
+
+    Args:
+        text (str): Raw review text.
+
+    Returns:
+        List[str]: List of token strings.
+    """
+    text = text.lower()
+    text = re.sub(r"<br />", " ", text) # Remove HTML line breaks
+    text = re.sub(r"[^a-zA-Z0-9\s]", "", text)
+    return text.split()
+
+class IMDBDataset(Dataset):
+    """
+    Torch Dataset wrapper for IMDB sequences and labels.
+
+    Each item corresponds to:
+    - a fixed-length sequence of token IDs
+    - a sentiment label (0 = negative, 1 = positive)
+    """
+    def __init__(self, sequences, labels):
+        self.sequences = torch.tensor(sequences, dtype=torch.long)
+        self.labels = torch.tensor(labels, dtype=torch.long)
+
+    def __len__(self):
+        return len(self.labels)
+
+    def __getitem__(self, idx):
+        return self.sequences[idx], self.labels[idx]
+
+def build_vocab(texts, max_vocab_size=10000):
+    """
+    Build a word-to-index vocabulary from a collection of texts.
+
+    The vocabulary is constructed using token frequency counts from the
+    training set only to avoid information leakage. Two special tokens
+    are always included:
+    - "<PAD>" mapped to index 0
+    - "<UNK>" mapped to index 1
+
+    The remaining (max_vocab_size - 2) most frequent tokens are added.
+
+    Args:
+        texts (Iterable[str]): Training texts.
+        max_vocab_size (int): Maximum size of the vocabulary.
+
+    Returns:
+        Dict[str, int]: Mapping from token string to integer index.
+    """
+    counter = Counter()
+    for text in texts:
+        counter.update(tokenize(text))
+    
+    # Reserve 0 for padding and 1 for unknown tokens
+    vocab = {"<PAD>": 0, "<UNK>": 1}
+    common_words = counter.most_common(max_vocab_size - 2)
+    for word, _ in common_words:
+        vocab[word] = len(vocab)
+    return vocab
+
+def preprocess_data(texts, vocab, max_len=128):
+    """
+    Convert raw texts into padded/truncated sequences of token IDs.
+
+    Steps:
+    - Tokenize each text
+    - Map tokens to vocabulary indices (using <UNK> for OOV tokens)
+    - Truncate to max_len or pad with <PAD> to reach max_len
+
+    Args:
+        texts (Iterable[str]): Input texts (reviews).
+        vocab (Dict[str, int]): Token-to-index mapping.
+        max_len (int): Maximum sequence length in tokens.
+
+    Returns:
+        np.ndarray: Array of shape (num_examples, max_len) with dtype int.
+    """
+    sequences = []
+    for text in texts:
+        tokens = tokenize(text)
+        token_ids = [vocab.get(token, vocab["<UNK>"]) for token in tokens]
+        # Pad or Truncate
+        if len(token_ids) < max_len:
+            token_ids += [vocab["<PAD>"]] * (max_len - len(token_ids))
+        else:
+            token_ids = token_ids[:max_len]
+        sequences.append(token_ids)
+    return np.array(sequences)
+
+# ==========================================
+# 2. Transformer Components
+# ==========================================
+
+class PositionalEncoding(nn.Module):
+    """
+    Sinusoidal positional encoding module.
+
+    Implements the deterministic positional encoding from the original
+    Transformer paper ("Attention is All You Need"), which is added to
+    token embeddings to inject information about token positions.
+    """
+    def __init__(self, d_model, max_len=5000):
+        super().__init__()
+        pe = torch.zeros(max_len, d_model)
+        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
+        div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
+        
+        pe[:, 0::2] = torch.sin(position * div_term)
+        pe[:, 1::2] = torch.cos(position * div_term)
+        
+        self.register_buffer('pe', pe.unsqueeze(0))
+
+    def forward(self, x):
+        """
+        Add positional encodings to input embeddings.
+
+        Args:
+            x (Tensor): Input tensor of shape [batch_size, seq_len, d_model].
+
+        Returns:
+            Tensor: Positionally encoded representations with same shape as x.
+        """
+        return x + self.pe[:, :x.size(1)]
+
+class MultiHeadAttention(nn.Module):
+    """
+    Multi-head self-attention mechanism.
+
+    For each token, attention is computed over all tokens in the sequence
+    (including itself) using multiple attention heads. Each head operates
+    in its own subspace and the outputs are concatenated.
+    """
+    def __init__(self, d_model, num_heads):
+        super().__init__()
+        assert d_model % num_heads == 0
+        self.d_model = d_model
+        self.num_heads = num_heads
+        self.d_k = d_model // num_heads
+        
+        self.W_q = nn.Linear(d_model, d_model)
+        self.W_k = nn.Linear(d_model, d_model)
+        self.W_v = nn.Linear(d_model, d_model)
+        self.W_o = nn.Linear(d_model, d_model)
+
+    def forward(self, x, mask=None):
+        """
+        Apply multi-head self-attention to the input sequence.
+
+        Args:
+            x (Tensor): Input tensor of shape [batch_size, seq_len, d_model].
+            mask (Tensor, optional): Attention mask of shape
+                [batch_size, 1, 1, seq_len] or broadcastable equivalent,
+                where positions with 0 are masked out.
+
+        Returns:
+            Tensor: Output tensor of shape [batch_size, seq_len, d_model].
+        """
+        batch_size, seq_len, _ = x.shape
+        
+        # Linear projections
+        Q = self.W_q(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)
+        K = self.W_k(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)
+        V = self.W_v(x).view(batch_size, seq_len, self.num_heads, self.d_k).transpose(1, 2)
+        
+        # Scaled Dot-Product Attention
+        scores = torch.matmul(Q, K.transpose(-2, -1)) / math.sqrt(self.d_k)
+        if mask is not None:
+            scores = scores.masked_fill(mask == 0, -1e9)
+        
+        attn = torch.softmax(scores, dim=-1)
+        context = torch.matmul(attn, V)
+        
+        # Concatenate heads
+        context = context.transpose(1, 2).contiguous().view(batch_size, seq_len, self.d_model)
+        return self.W_o(context)
+
+class TransformerEncoderBlock(nn.Module):
+    """
+    Single Transformer encoder block consisting of:
+    - multi-head self-attention sublayer (with residual + layer norm)
+    - position-wise feed-forward sublayer (with residual + layer norm)
+    """
+    def __init__(self, d_model, num_heads, d_ff, dropout=0.1):
+        super().__init__()
+        self.mha = MultiHeadAttention(d_model, num_heads)
+        self.ffn = nn.Sequential(
+            nn.Linear(d_model, d_ff),
+            nn.ReLU(),
+            nn.Linear(d_ff, d_model)
+        )
+        self.layernorm1 = nn.LayerNorm(d_model)
+        self.layernorm2 = nn.LayerNorm(d_model)
+        self.dropout = nn.Dropout(dropout)
+
+    def forward(self, x, mask=None):
+        """
+        Forward pass through one encoder block.
+
+        Args:
+            x (Tensor): Input tensor of shape [batch_size, seq_len, d_model].
+            mask (Tensor, optional): Attention mask (see MultiHeadAttention).
+
+        Returns:
+            Tensor: Output tensor of shape [batch_size, seq_len, d_model].
+        """
+        # Sublayer 1: self-attention with residual connection
+        attn_out = self.mha(x, mask)
+        x = self.layernorm1(x + self.dropout(attn_out))
+        # Sublayer 2: position-wise feed-forward network with residual
+        ffn_out = self.ffn(x)
+        x = self.layernorm2(x + self.dropout(ffn_out))
+        return x
+
+# ==========================================
+# 3. Full Transformer Classifier
+# ==========================================
+
+class TransformerClassifier(nn.Module):
+    """
+    Transformer-based text classifier for IMDB sentiment analysis.
+
+    Architecture:
+    - Token embedding layer
+    - Sinusoidal positional encoding
+    - Stack of Transformer encoder blocks
+    - Global average pooling over sequence dimension
+    - Linear classification head to predict sentiment label
+    """
+    def __init__(self, vocab_size, d_model, num_heads, num_layers, d_ff, max_len, num_classes=2, dropout=0.1):
+        super().__init__()
+        self.embedding = nn.Embedding(vocab_size, d_model)
+        self.pos_encoding = PositionalEncoding(d_model, max_len)
+        
+        self.encoder_layers = nn.ModuleList([
+            TransformerEncoderBlock(d_model, num_heads, d_ff, dropout) 
+            for _ in range(num_layers)
+        ])
+        
+        self.dropout = nn.Dropout(dropout)
+        # Classification Head: Flatten or Global Pool
+        self.classifier = nn.Linear(d_model, num_classes)
+
+    def forward(self, x, mask=None):
+        """
+        Forward pass for the classifier.
+
+        Args:
+            x (Tensor): Input tensor of token IDs
+                with shape [batch_size, seq_len].
+            mask (Tensor, optional): Attention mask (not used in this script).
+
+        Returns:
+            Tensor: Logits of shape [batch_size, num_classes].
+        """
+        x = self.dropout(self.pos_encoding(self.embedding(x)))
+        
+        for layer in self.encoder_layers:
+            x = layer(x, mask)
+        
+        # Global Average Pooling across the sequence dimension
+        x = x.mean(dim=1) 
+        return self.classifier(x)
+
+# ==========================================
+# 4. Training and Evaluation Logic
+# ==========================================
+
+def train_model(model, train_loader, val_loader, epochs, lr, device):
+    """
+    Train the Transformer classifier on the IMDB training split.
+
+    Args:
+        model (nn.Module): TransformerClassifier instance.
+        train_loader (DataLoader): Batches of (sequence, label) for training.
+        val_loader (DataLoader): Batches for validation.
+        epochs (int): Number of full passes through the training set.
+        lr (float): Initial learning rate for Adam optimizer.
+        device (torch.device): Device on which to run training.
+
+    Uses:
+        - CrossEntropyLoss for binary sentiment classification.
+        - Adam optimizer with StepLR scheduler (gamma=0.5 every 2 epochs).
+    """
+    criterion = nn.CrossEntropyLoss()
+    optimizer = optim.Adam(model.parameters(), lr=lr)
+    scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=2, gamma=0.5)
+    
+    model.to(device)
+    
+    for epoch in range(epochs):
+        model.train()
+        total_loss = 0
+        for batch_seq, batch_lab in train_loader:
+            batch_seq, batch_lab = batch_seq.to(device), batch_lab.to(device)
+            
+            optimizer.zero_grad()
+            outputs = model(batch_seq)
+            loss = criterion(outputs, batch_lab)
+            loss.backward()
+            optimizer.step()
+            total_loss += loss.item()
+        
+        scheduler.step()
+        val_metrics = evaluate_model(model, val_loader, device)
+        val_acc = val_metrics["accuracy"]
+        val_p = val_metrics["precision"]
+        val_r = val_metrics["recall"]
+        val_f1 = val_metrics["f1"]
+        print(
+            f"Epoch {epoch+1}/{epochs} | "
+            f"Loss: {total_loss/len(train_loader):.4f} | "
+            f"Val Acc: {val_acc:.4f} | "
+            f"Val P: {val_p:.4f} | Val R: {val_r:.4f} | Val F1: {val_f1:.4f}"
+        )
+
+def evaluate_model(model, loader, device):
+    """
+    Evaluate the model on a dataset.
+
+    Args:
+        model (nn.Module): Trained (or partially trained) classifier.
+        loader (DataLoader): DataLoader for validation or test data.
+        device (torch.device): Device on which to perform evaluation.
+
+    Returns:
+        Dict[str, float]: Dictionary with accuracy, precision, recall, and F1.
+    """
+    model.eval()
+    all_preds = []
+    all_labels = []
+    
+    with torch.no_grad():
+        for batch_seq, batch_lab in loader:
+            batch_seq, batch_lab = batch_seq.to(device), batch_lab.to(device)
+            outputs = model(batch_seq)
+            preds = torch.argmax(outputs, dim=1)
+            all_preds.extend(preds.cpu().numpy())
+            all_labels.extend(batch_lab.cpu().numpy())
+            
+    acc = accuracy_score(all_labels, all_preds)
+    p, r, f1, _ = precision_recall_fscore_support(all_labels, all_preds, average='binary')
+    return {"accuracy": acc, "precision": p, "recall": r, "f1": f1}
+
+def count_trainable_parameters(model):
+    """
+    Count the number of trainable parameters in a model.
+
+    Args:
+        model (nn.Module): Model whose parameters should be counted.
+
+    Returns:
+        int: Number of trainable parameters.
+    """
+    return sum(p.numel() for p in model.parameters() if p.requires_grad)
+
+def write_experiment_report_md(
+    report_path,
+    results,
+    best_result,
+    device,
+    train_size,
+    val_size,
+):
+    """
+    Write a Markdown report summarizing model-size experiment results.
+
+    Args:
+        report_path (str): Output Markdown file path.
+        results (List[Dict]): Per-model experiment outputs.
+        best_result (Dict): Best-performing entry from `results`.
+        device (torch.device): Device used during training.
+        train_size (int): Number of training samples.
+        val_size (int): Number of validation samples.
+    """
+    lines = []
+    lines.append("# IMDB Transformer Model-Size Experiment Report")
+    lines.append("")
+    lines.append(f"- Generated at: `{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}`")
+    lines.append(f"- Device: `{device}`")
+    lines.append(f"- Training samples: `{train_size}`")
+    lines.append(f"- Validation samples: `{val_size}`")
+    lines.append(f"- Max vocab size: `{MAX_VOCAB}`")
+    lines.append(f"- Max sequence length: `{MAX_LEN}`")
+    lines.append(f"- Batch size: `{BATCH_SIZE}`")
+    lines.append(f"- Epochs: `{EPOCHS}`")
+    lines.append(f"- Learning rate: `{LR}`")
+    lines.append("")
+
+    lines.append("## Overall Comparison")
+    lines.append("")
+    lines.append("| Model Size | Trainable Params | Accuracy | Precision | Recall | F1 | Checkpoint |")
+    lines.append("|---|---:|---:|---:|---:|---:|---|")
+    for item in results:
+        metrics = item["metrics"]
+        lines.append(
+            f"| {item['size']} | {item['params']:,} | "
+            f"{metrics['accuracy']:.4f} | {metrics['precision']:.4f} | "
+            f"{metrics['recall']:.4f} | {metrics['f1']:.4f} | "
+            f"`{item['checkpoint_path']}` |"
+        )
+    lines.append("")
+
+    lines.append("## Best Model")
+    lines.append("")
+    lines.append(f"- Best size by validation F1: `{best_result['size']}`")
+    lines.append(f"- Checkpoint: `{best_result['checkpoint_path']}`")
+    lines.append(f"- Trainable parameters: `{best_result['params']:,}`")
+    lines.append("- Metrics:")
+    lines.append(f"  - Accuracy: `{best_result['metrics']['accuracy']:.4f}`")
+    lines.append(f"  - Precision: `{best_result['metrics']['precision']:.4f}`")
+    lines.append(f"  - Recall: `{best_result['metrics']['recall']:.4f}`")
+    lines.append(f"  - F1: `{best_result['metrics']['f1']:.4f}`")
+    lines.append("")
+
+    lines.append("## Per-Model Details")
+    lines.append("")
+    for item in results:
+        cfg = item["config"]
+        metrics = item["metrics"]
+        lines.append(f"### {item['size'].capitalize()} model")
+        lines.append("")
+        lines.append("- Architecture:")
+        lines.append(f"  - `d_model`: `{cfg['d_model']}`")
+        lines.append(f"  - `num_heads`: `{cfg['num_heads']}`")
+        lines.append(f"  - `num_layers`: `{cfg['num_layers']}`")
+        lines.append(f"  - `d_ff`: `{cfg['d_ff']}`")
+        lines.append(f"- Trainable params: `{item['params']:,}`")
+        lines.append(f"- Checkpoint: `{item['checkpoint_path']}`")
+        lines.append("- Validation metrics:")
+        lines.append(f"  - Accuracy: `{metrics['accuracy']:.4f}`")
+        lines.append(f"  - Precision: `{metrics['precision']:.4f}`")
+        lines.append(f"  - Recall: `{metrics['recall']:.4f}`")
+        lines.append(f"  - F1: `{metrics['f1']:.4f}`")
+        lines.append("")
+
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write("\n".join(lines))
+
+# ==========================================
+# 5. Execution Example (Subset of IMDB)
+# ==========================================
+
+# Dataset loading using the real IMDB dataset via HuggingFace datasets.
+# Data source:
+#   HuggingFace Datasets – "imdb" configuration, which originates from the
+#   Large Movie Review Dataset (Maas et al., 2011).
+def load_imdb_texts(split: str = "train"):
+    """
+    Load IMDB dataset texts and labels using `datasets.load_dataset`.
+
+    Args:
+        split (str): Dataset split, e.g. "train" or "test".
+
+    Returns:
+        Tuple[List[str], List[int]]: List of review texts and sentiment labels,
+        where labels are integers 0 (negative) and 1 (positive).
+    """
+    ds = load_dataset("imdb", split=split)
+    texts = ds["text"]
+    labels = ds["label"]
+    return texts, labels
+
+# ===========================
+# Hyperparameters
+# ===========================
+# MAX_VOCAB: upper bound on vocabulary size. Larger values can capture more
+# rare words but increase model size and memory usage.
+MAX_VOCAB = 5000
+# MAX_LEN: maximum number of tokens per review. Longer sequences capture
+# more context but are more expensive to process; here we use 64 for speed.
+MAX_LEN = 64
+# BATCH_SIZE: number of examples per optimization step. Larger batches yield
+# smoother gradients but require more memory.
+BATCH_SIZE = 32
+# EPOCHS: number of full passes through the training dataset.
+EPOCHS = 5
+# LR: initial learning rate for the Adam optimizer.
+LR = 0.001
+
+# Transformer size presets for model-complexity experiments.
+# Each preset controls hidden size, attention heads, number of layers,
+# and feed-forward dimension.
+MODEL_SIZES = {
+    "small": {"d_model": 64, "num_heads": 4, "num_layers": 1, "d_ff": 128},
+    "medium": {"d_model": 128, "num_heads": 8, "num_layers": 2, "d_ff": 256},
+    "large": {"d_model": 256, "num_heads": 8, "num_layers": 4, "d_ff": 512},
+}
+
+# Directory to save trained model and related artifacts (checkpoint, vocab,
+# and configuration dictionary for reproducibility).
+# Keep output paths relative to the current working directory.
+SAVE_DIR = os.path.join(".", "saved_model")
+os.makedirs(SAVE_DIR, exist_ok=True)
+MODEL_PATH = os.path.join(SAVE_DIR, "transformer_imdb.pt")
+REPORT_PATH = os.path.join(SAVE_DIR, "transformer_imdb_experiment_report.md")
+
+def main():
+    """
+    Train a Transformer-based sentiment classifier on IMDB and save the model,
+    vocabulary, and configuration to disk.
+    """
+    # 1) Load IMDB training split and then create train/validation split.
+    all_train_texts, all_train_labels = load_imdb_texts(split="train")
+
+    train_texts, val_texts, train_labels, val_labels = train_test_split(
+        all_train_texts,
+        all_train_labels,
+        test_size=0.2,
+        random_state=42,
+        stratify=all_train_labels,
+    )
+
+    # 2) Build vocabulary using training texts only (avoid validation leakage).
+    vocab = build_vocab(train_texts, MAX_VOCAB)
+
+    # 3) Preprocess train and validation data into fixed-length ID sequences.
+    train_sequences = preprocess_data(train_texts, vocab, MAX_LEN)
+    val_sequences = preprocess_data(val_texts, vocab, MAX_LEN)
+
+    train_dataset = IMDBDataset(train_sequences, train_labels)
+    val_dataset = IMDBDataset(val_sequences, val_labels)
+
+    # DataLoaders for mini-batch training and validation.
+    train_loader = DataLoader(train_dataset, batch_size=BATCH_SIZE, shuffle=True)
+    val_loader = DataLoader(val_dataset, batch_size=BATCH_SIZE, shuffle=False)
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+    results = []
+
+    # Train and evaluate multiple model sizes to analyze how complexity
+    # changes sentiment-classification performance.
+    for size_name, size_cfg in MODEL_SIZES.items():
+        print("\n" + "=" * 72)
+        print(f"Training {size_name.upper()} model with config: {size_cfg}")
+        print("=" * 72)
+
+        model = TransformerClassifier(
+            len(vocab),
+            size_cfg["d_model"],
+            size_cfg["num_heads"],
+            size_cfg["num_layers"],
+            size_cfg["d_ff"],
+            MAX_LEN,
+        )
+        param_count = count_trainable_parameters(model)
+        print(f"Trainable parameters ({size_name}): {param_count:,}")
+
+        train_model(model, train_loader, val_loader, EPOCHS, LR, device)
+        val_metrics = evaluate_model(model, val_loader, device)
+        size_model_path = os.path.join(SAVE_DIR, f"transformer_imdb_{size_name}.pt")
+        results.append(
+            {
+                "size": size_name,
+                "params": param_count,
+                "config": size_cfg,
+                "metrics": val_metrics,
+                "checkpoint_path": size_model_path,
+            }
+        )
+
+        # Save each trained size-specific model.
+        torch.save(
+            {
+                "model_state_dict": model.state_dict(),
+                "vocab": vocab,
+                "config": {
+                    "max_vocab": MAX_VOCAB,
+                    "max_len": MAX_LEN,
+                    "batch_size": BATCH_SIZE,
+                    "epochs": EPOCHS,
+                    "lr": LR,
+                    "size_name": size_name,
+                    **size_cfg,
+                },
+                "val_metrics": val_metrics,
+            },
+            size_model_path,
+        )
+        print(f"Saved {size_name} model to {size_model_path}")
+
+    # Print a concise comparison table at the end.
+    print("\n" + "#" * 72)
+    print("Model Size Impact Summary (Validation Set)")
+    print("#" * 72)
+    print(f"{'Size':<10} {'Params':>12} {'Acc':>8} {'Precision':>10} {'Recall':>8} {'F1':>8}")
+    for item in results:
+        m = item["metrics"]
+        print(
+            f"{item['size']:<10} "
+            f"{item['params']:>12,} "
+            f"{m['accuracy']:>8.4f} "
+            f"{m['precision']:>10.4f} "
+            f"{m['recall']:>8.4f} "
+            f"{m['f1']:>8.4f}"
+        )
+
+    # Keep a compatibility checkpoint name for the best model by validation F1.
+    best_result = max(results, key=lambda x: x["metrics"]["f1"])
+    best_model_path = os.path.join(SAVE_DIR, f"transformer_imdb_{best_result['size']}.pt")
+    torch.save(
+        {
+            "best_size": best_result["size"],
+            "best_model_path": best_model_path,
+            "all_results": results,
+        },
+        MODEL_PATH,
+    )
+    print(f"\nBest model by Val F1: {best_result['size']} -> {best_model_path}")
+    print(f"Experiment summary saved to {MODEL_PATH}")
+
+    write_experiment_report_md(
+        REPORT_PATH,
+        results,
+        best_result,
+        device,
+        train_size=len(train_texts),
+        val_size=len(val_texts),
+    )
+    print(f"Markdown report saved to {REPORT_PATH}")
+
+
+if __name__ == "__main__":
+    main()

assignment_llm_1/assignment_text/code/c1_analysis.py

@@ -0,0 +1,253 @@
+"""
+Evaluation and qualitative error analysis helpers for the IMDB Transformer model.
+
+This module is separate from `c1.py` and focuses only on:
+- Loading a previously trained model from disk.
+- Evaluating it on an IMDB split.
+- Inspecting misclassified examples for qualitative error analysis.
+"""
+
+from typing import Dict, List, Tuple
+
+import argparse
+import os
+
+import torch
+import torch.nn.functional as F
+from torch.utils.data import DataLoader
+
+from c1 import (
+    IMDBDataset,
+    TransformerClassifier,
+    preprocess_data,
+    evaluate_model,
+    load_imdb_texts,
+)
+
+# Keep output/checkpoint paths relative to the current working directory.
+SAVE_DIR = os.path.join(".", "saved_model")
+MODEL_PATH = os.path.join(SAVE_DIR, "transformer_imdb.pt")
+
+
+def analyze_misclassifications_on_texts(
+    model: torch.nn.Module,
+    texts: List[str],
+    labels: List[int],
+    vocab: Dict[str, int],
+    max_len: int,
+    device: torch.device,
+    num_examples: int = 5,
+) -> None:
+    """
+    Inspect concrete examples where the model makes mistakes to understand
+    *why* it fails and how to improve it.
+
+    How to read the output (practical guidance):
+    - Start with the true vs. predicted label:
+      - For each misclassified review, ask whether the ground-truth label
+        actually matches the human-intuitive sentiment. Occasional noisy
+        labels are common in IMDB-style datasets.
+    - Look at the confidence vector:
+      - Very confident but wrong predictions often indicate *systematic bias*
+        (e.g., the model over-trusts certain keywords like "great", "worst").
+      - Low-confidence errors may simply reflect inherently ambiguous reviews.
+    - Scan the text content:
+      - Check for **rare or domain-specific words** (brand names, slang,
+        technical jargon) that might not appear often enough in training.
+      - Look for **negation patterns** ("not good", "hardly bad", "no longer
+        terrible") where bag-of-words style cues can mislead attention.
+      - Notice **mixed sentiment** or **topic vs. opinion** separation
+        (e.g., long plot summary plus a brief opinion at the end).
+      - Pay attention to **sarcasm and irony**, which are notoriously hard
+        for models relying mostly on local lexical cues.
+    - Compare several misclassified examples:
+      - If you see many errors with long reviews, consider increasing MAX_LEN
+        or using a deeper model.
+      - If errors cluster around subtle, low-intensity sentiment, you may need
+        more expressive capacity (higher d_model / more layers) or additional
+        training data.
+
+    Based on these observations you can propose targeted improvements, such as:
+    - Expanding the vocabulary or switching to subword tokenization.
+    - Adjusting hyperparameters (sequence length, model size).
+    - Incorporating pre-trained language models for richer semantics.
+    """
+    model.eval()
+    sequences = preprocess_data(texts, vocab, max_len)
+    dataset = IMDBDataset(sequences, labels)
+    loader = DataLoader(dataset, batch_size=64, shuffle=False)
+
+    printed = 0
+    with torch.no_grad():
+        for batch_idx, (batch_seq, batch_lab) in enumerate(loader):
+            batch_seq, batch_lab = batch_seq.to(device), batch_lab.to(device)
+            logits = model(batch_seq)
+            probs = F.softmax(logits, dim=1)
+            preds = torch.argmax(probs, dim=1)
+
+            start = batch_idx * loader.batch_size
+            end = start + batch_seq.size(0)
+            batch_texts = texts[start:end]
+
+            for text, true_y, pred_y, prob_vec in zip(
+                batch_texts,
+                batch_lab.cpu().numpy(),
+                preds.cpu().numpy(),
+                probs.cpu().numpy(),
+            ):
+                if true_y != pred_y:
+                    printed += 1
+                    print("=" * 80)
+                    print(f"Misclassified example #{printed}")
+                    print(f"True label     : {true_y} (0=neg, 1=pos)")
+                    print(f"Predicted label: {pred_y}")
+                    print(f"Model confidence (class 0, class 1): {prob_vec}")
+
+                    if printed >= num_examples:
+                        print("=" * 80)
+                        print(
+                            f"Displayed the first {num_examples} misclassified "
+                            "examples on this split."
+                        )
+                        return
+
+    if printed == 0:
+        print("No misclassified examples found on this split (perfect accuracy).")
+
+
+def load_trained_model_from_checkpoint(
+    checkpoint_path: str = MODEL_PATH,
+    device: torch.device | None = None,
+) -> Tuple[torch.nn.Module, Dict[str, int], Dict]:
+    """
+    Load a previously trained Transformer model, along with its vocabulary
+    and configuration, from the checkpoint saved by `c1.py`.
+
+    Returns:
+        model: Loaded TransformerClassifier on the requested device.
+        vocab: Token-to-index mapping used during training.
+        config: Hyperparameter/config dictionary saved in the checkpoint.
+    """
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    ckpt = torch.load(checkpoint_path, map_location=device)
+    vocab: Dict[str, int] = ckpt["vocab"]
+    config: Dict = ckpt["config"]
+
+    model = TransformerClassifier(
+        vocab_size=len(vocab),
+        d_model=config["d_model"],
+        num_heads=config["num_heads"],
+        num_layers=config["num_layers"],
+        d_ff=config["d_ff"],
+        max_len=config["max_len"],
+    ).to(device)
+    model.load_state_dict(ckpt["model_state_dict"])
+    model.eval()
+
+    return model, vocab, config
+
+
+def evaluate_and_analyze_saved_model(
+    split: str = "test",
+    checkpoint_path: str | None = None,
+    model_size: str = "medium",
+    num_examples: int = 5,
+    device: torch.device | None = None,
+) -> None:
+    """
+    High-level helper that:
+    1) Loads the trained model/vocab/config from disk.
+    2) Evaluates it on the requested IMDB split.
+    3) Runs qualitative error analysis on that split.
+    """
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    if checkpoint_path is None:
+        checkpoint_path = os.path.join(SAVE_DIR, f"transformer_imdb_{model_size}.pt")
+
+    print(f"Loading trained model from: {checkpoint_path}")
+    model, vocab, config = load_trained_model_from_checkpoint(
+        checkpoint_path=checkpoint_path,
+        device=device,
+    )
+
+    print(f"Evaluating on IMDB '{split}' split...")
+    texts, labels = load_imdb_texts(split=split)
+    sequences = preprocess_data(texts, vocab, config["max_len"])
+    dataset = IMDBDataset(sequences, labels)
+    loader = DataLoader(dataset, batch_size=config["batch_size"], shuffle=False)
+
+    metrics = evaluate_model(model, loader, device)
+    print("Evaluation metrics:", metrics)
+
+    print("\nRunning qualitative error analysis...")
+    analyze_misclassifications_on_texts(
+        model=model,
+        texts=texts,
+        labels=labels,
+        vocab=vocab,
+        max_len=config["max_len"],
+        device=device,
+        num_examples=num_examples,
+    )
+
+
+def main():
+    """
+    Command-line interface for evaluation and analysis utilities.
+
+    Example:
+        # Evaluate medium model on IMDB test split and show 5 errors
+        python c1_analysis.py --split test --model_size medium --num_examples 5
+    """
+    parser = argparse.ArgumentParser(description="IMDB Transformer evaluation and analysis utilities")
+    parser.add_argument(
+        "--split",
+        type=str,
+        default="test",
+        help="IMDB split to evaluate on (e.g., 'test', 'train').",
+    )
+    parser.add_argument(
+        "--checkpoint",
+        type=str,
+        default=None,
+        help=(
+            "Optional explicit checkpoint path. If provided, this overrides "
+            "--model_size."
+        ),
+    )
+    parser.add_argument(
+        "--model_size",
+        type=str,
+        choices=["small", "medium", "large"],
+        default="medium",
+        help=(
+            "Model size to load from saved checkpoints. Used when --checkpoint "
+            "is not provided."
+        ),
+    )
+    parser.add_argument(
+        "--num_examples",
+        type=int,
+        default=5,
+        help="Number of misclassified examples to print in error analysis.",
+    )
+    args = parser.parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    evaluate_and_analyze_saved_model(
+        split=args.split,
+        checkpoint_path=args.checkpoint,
+        model_size=args.model_size,
+        num_examples=args.num_examples,
+        device=device,
+    )
+
+
+if __name__ == "__main__":
+    main()
+

assignment_llm_1/assignment_text/code/explanation_creation.py

@@ -0,0 +1,174 @@
+import argparse
+import json
+import torch
+import torch.nn.functional as F
+from typing import Dict, List, Tuple
+from torch.utils.data import DataLoader
+
+# Assuming these are in your c1.py
+from c1 import (
+    IMDBDataset,
+    TransformerClassifier,
+    preprocess_data,
+    evaluate_model,
+    load_imdb_texts,
+    MODEL_PATH,
+)
+
+# You would need to install openai: pip install openai
+from openai import OpenAI
+api_file = "/home/mshahidul/api_new.json"
+with open(api_file, "r") as f:
+    api_keys = json.load(f)
+openai_api_key = api_keys["openai"]
+
+client = OpenAI(api_key=openai_api_key)
+# Initialize your client (ensure your API key is in your environment variables)
+
+def get_llm_explanation(review_text: str, true_y: int, pred_y: int) -> str:
+    """
+    Uses an LLM to perform qualitative reasoning on why the model failed.
+    """
+    sentiment = {0: "Negative", 1: "Positive"}
+    
+    prompt = f"""
+    A Transformer model misclassified the following movie review.
+    
+    REVIEW: "{review_text[:1000]}" 
+    TRUE LABEL: {sentiment[true_y]}
+    MODEL PREDICTED: {sentiment[pred_y]}
+    
+    Task: Provide a concise (2-3 sentence) explanation of why a machine learning 
+    model might have struggled with this specific text. Mention linguistic 
+    features like sarcasm, double negatives, mixed sentiment, or specific keywords.
+    """
+
+    try:
+        response = client.chat.completions.create(
+            model="gpt-4o-mini",  # Using 4o-mini as a high-performance proxy for "mini" models
+            messages=[{"role": "user", "content": prompt}],
+            temperature=0.2
+        )
+        return response.choices[0].message.content.strip()
+    except Exception as e:
+        return f"LLM Analysis failed: {str(e)}"
+
+def analyze_misclassifications_on_texts(
+    model: torch.nn.Module,
+    texts: List[str],
+    labels: List[int],
+    vocab: Dict[str, int],
+    max_len: int,
+    device: torch.device,
+    num_examples: int = 10,
+) -> List[Dict]:
+    """
+    Identifies errors, generates LLM explanations, and returns structured results.
+    """
+    model.eval()
+    sequences = preprocess_data(texts, vocab, max_len)
+    dataset = IMDBDataset(sequences, labels)
+    loader = DataLoader(dataset, batch_size=64, shuffle=False)
+
+    error_results = []
+    printed = 0
+    
+    with torch.no_grad():
+        for batch_idx, (batch_seq, batch_lab) in enumerate(loader):
+            batch_seq, batch_lab = batch_seq.to(device), batch_lab.to(device)
+            logits = model(batch_seq)
+            probs = F.softmax(logits, dim=1)
+            preds = torch.argmax(probs, dim=1)
+
+            start = batch_idx * loader.batch_size
+            batch_texts = texts[start:start + batch_seq.size(0)]
+
+            for text, true_y, pred_y, prob_vec in zip(
+                batch_texts,
+                batch_lab.cpu().numpy(),
+                preds.cpu().numpy(),
+                probs.cpu().numpy(),
+            ):
+                if true_y != pred_y:
+                    printed += 1
+                    
+                    print(f"Analyzing error #{printed} with LLM...")
+                    explanation = get_llm_explanation(text, true_y, pred_y)
+                    
+                    error_entry = {
+                        "example_id": printed,
+                        "true_label": int(true_y),
+                        "predicted_label": int(pred_y),
+                        "confidence_neg": float(prob_vec[0]),
+                        "confidence_pos": float(prob_vec[1]),
+                        "text": text,
+                        "explanation": explanation
+                    }
+                    error_results.append(error_entry)
+
+                    # Print to console for immediate feedback
+                    print("=" * 80)
+                    print(f"True: {true_y} | Pred: {pred_y}")
+                    print(f"Reasoning: {explanation}")
+                    print("=" * 80)
+
+                    if printed >= num_examples:
+                        return error_results
+
+    return error_results
+
+def load_trained_model_from_checkpoint(
+    checkpoint_path: str = MODEL_PATH,
+    device: torch.device | None = None,
+) -> Tuple[torch.nn.Module, Dict[str, int], Dict]:
+    if device is None:
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    ckpt = torch.load(checkpoint_path, map_location=device)
+    vocab = ckpt["vocab"]
+    config = ckpt["config"]
+
+    model = TransformerClassifier(
+        vocab_size=len(vocab),
+        d_model=config["d_model"],
+        num_heads=config["num_heads"],
+        num_layers=config["num_layers"],
+        d_ff=config["d_ff"],
+        max_len=config["max_len"],
+    ).to(device)
+    model.load_state_dict(ckpt["model_state_dict"])
+    return model, vocab, config
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--split", type=str, default="test")
+    parser.add_argument("--num_examples", type=int, default=10)
+    parser.add_argument("--output", type=str, default="error_analysis.json")
+    args = parser.parse_args()
+
+    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+    # 1. Load Model
+    model, vocab, config = load_trained_model_from_checkpoint(device=device)
+
+    # 2. Load Data
+    texts, labels = load_imdb_texts(split=args.split)
+
+    # 3. Analyze
+    errors = analyze_misclassifications_on_texts(
+        model=model,
+        texts=texts,
+        labels=labels,
+        vocab=vocab,
+        max_len=config["max_len"],
+        device=device,
+        num_examples=args.num_examples
+    )
+
+    # 4. Save Results
+    with open(args.output, "w") as f:
+        json.dump(errors, f, indent=4)
+    print(f"\nAnalysis complete. Results saved to {args.output}")
+
+if __name__ == "__main__":
+    main()

assignment_llm_1/assignment_text/documentation/different_model_size_and_performance.md

@@ -0,0 +1,77 @@
+# IMDB Transformer Model-Size Experiment Report
+
+- Generated at: `2026-02-10 09:52:58`
+- Device: `cuda`
+- Training samples: `20000`
+- Validation samples: `5000`
+- Max vocab size: `5000`
+- Max sequence length: `64`
+- Batch size: `32`
+- Epochs: `5`
+- Learning rate: `0.001`
+
+## Overall Comparison
+
+| Model Size | Trainable Params | Accuracy | Precision | Recall | F1 | Checkpoint |
+|---|---:|---:|---:|---:|---:|---|
+| small | 353,602 | 0.7816 | 0.7832 | 0.7788 | 0.7810 | `assignment_llm_1/saved_model/transformer_imdb_small.pt` |
+| medium | 905,218 | 0.7874 | 0.7948 | 0.7748 | 0.7847 | `assignment_llm_1/saved_model/transformer_imdb_medium.pt` |
+| large | 3,388,930 | 0.7374 | 0.7392 | 0.7336 | 0.7364 | `assignment_llm_1/saved_model/transformer_imdb_large.pt` |
+
+## Best Model
+
+- Best size by validation F1: `medium`
+- Checkpoint: `assignment_llm_1/saved_model/transformer_imdb_medium.pt`
+- Trainable parameters: `905,218`
+- Metrics:
+  - Accuracy: `0.7874`
+  - Precision: `0.7948`
+  - Recall: `0.7748`
+  - F1: `0.7847`
+
+## Per-Model Details
+
+### Small model
+
+- Architecture:
+  - `d_model`: `64`
+  - `num_heads`: `4`
+  - `num_layers`: `1`
+  - `d_ff`: `128`
+- Trainable params: `353,602`
+- Checkpoint: `/assignment_llm_1/saved_model/transformer_imdb_small.pt`
+- Validation metrics:
+  - Accuracy: `0.7816`
+  - Precision: `0.7832`
+  - Recall: `0.7788`
+  - F1: `0.7810`
+
+### Medium model
+
+- Architecture:
+  - `d_model`: `128`
+  - `num_heads`: `8`
+  - `num_layers`: `2`
+  - `d_ff`: `256`
+- Trainable params: `905,218`
+- Checkpoint: `/assignment_llm_1/saved_model/transformer_imdb_medium.pt`
+- Validation metrics:
+  - Accuracy: `0.7874`
+  - Precision: `0.7948`
+  - Recall: `0.7748`
+  - F1: `0.7847`
+
+### Large model
+
+- Architecture:
+  - `d_model`: `256`
+  - `num_heads`: `8`
+  - `num_layers`: `4`
+  - `d_ff`: `512`
+- Trainable params: `3,388,930`
+- Checkpoint: `/assignment_llm_1/saved_model/transformer_imdb_large.pt`
+- Validation metrics:
+  - Accuracy: `0.7374`
+  - Precision: `0.7392`
+  - Recall: `0.7336`
+  - F1: `0.7364`

assignment_llm_1/assignment_text/documentation/documentation.md

@@ -0,0 +1,142 @@
+## Homework 1 (Part I) – Transformer-Based Sentiment Analysis on IMDB
+
+### 1. Introduction
+
+This project implements a Transformer-based neural network for binary sentiment analysis on movie reviews. Given a raw text review from the IMDB dataset, the model predicts whether the review expresses a positive or negative sentiment. Sentiment analysis is a fundamental task in natural language processing (NLP) with applications in opinion mining, recommendation systems, and social media monitoring, where understanding users’ attitudes at scale is essential.
+
+The Transformer architecture is well suited to this task because it relies on self-attention rather than recurrent connections. Self-attention allows the model to capture long-range dependencies between words regardless of their distance in the sequence and to focus on sentiment-bearing tokens (for example, “not”, “excellent”, “boring”) even when they appear far apart. This makes Transformers more effective and easier to train in parallel than traditional RNN or LSTM models for document-level classification.
+
+### 2. Dataset Description
+
+The model is trained and validated on the IMDB movie reviews dataset, loaded via the HuggingFace `datasets` library using the `"imdb"` configuration. This dataset corresponds to the Large Movie Review Dataset introduced by Maas et al. (2011), which contains 50,000 labeled movie reviews (25,000 train and 25,000 test) plus additional unlabeled reviews. The original dataset is available from the Stanford AI Lab at `http://ai.stanford.edu/~amaas/data/sentiment/`. Each labeled review is annotated with a binary label: 0 for negative and 1 for positive sentiment.
+
+In this implementation, I use the labeled training split exposed by the HuggingFace wrapper and perform an additional 80/20 train/validation split with stratification over the labels. This results in approximately 20,000 reviews for training and 5,000 reviews for validation, preserving the original class balance. The official test split and the unlabeled reviews from the Large Movie Review Dataset are not used in this assignment. The labels are integers taking values in {0, 1}, corresponding to negative and positive sentiment respectively.
+
+The raw text consists of HTML-formatted movie reviews. As a minimal preprocessing step, HTML line breaks (`<br />`) are removed and all non-alphanumeric characters (except whitespace) are filtered out. No additional filtering or subsetting (such as length-based pruning) is performed; instead, sequence length is controlled later during token-to-index mapping and padding.
+
+When using this dataset, the canonical citation requested by the authors is:
+
+Andrew L. Maas, Raymond E. Daly, Peter T. Pham, Dan Huang, Andrew Y. Ng, and Christopher Potts. 2011. Learning Word Vectors for Sentiment Analysis. In Proceedings of the 49th Annual Meeting of the Association for Computational Linguistics: Human Language Technologies, pages 142–150.
+
+### 3. Data Preprocessing
+
+Preprocessing is implemented in three main stages: tokenization, vocabulary construction, and sequence padding.
+
+Tokenization uses a simple regular expression–based tokenizer. The text is converted to lowercase, HTML line breaks are replaced by spaces, and all characters that are not letters, digits, or whitespace are removed. The cleaned string is then split on whitespace to obtain a list of tokens. This approach yields a plain word-level tokenization that is easy to interpret and sufficient for this assignment.
+
+Vocabulary construction builds a word-to-index mapping from the training texts only, which avoids information leakage from the validation set. A `Counter` is used to accumulate token frequencies over all training reviews. Two special tokens are reserved: `<PAD>` with index 0 and `<UNK>` with index 1. The remaining slots (up to `MAX_VOCAB = 5000`) are filled with the most frequent words in the training corpus. Any word outside this vocabulary is mapped to the `<UNK>` index at inference time.
+
+For sequence padding and truncation, each review is converted to a fixed-length sequence of token IDs with `MAX_LEN = 64`. Tokens are mapped to their integer indices via the vocabulary; if a token is not present, it is mapped to `<UNK>`. If a sequence is shorter than 64 tokens, it is padded on the right with `<PAD>` tokens until length 64. If it is longer, it is truncated to the first 64 tokens. This produces a dense matrix of shape (number_of_examples, 64), which is used as input to the model.
+
+### 4. Model Architecture
+
+At a high level, the model is a Transformer encoder stacked on top of learned token embeddings and sinusoidal positional encodings, followed by a global pooling operation and a linear classification head. It is a pure encoder architecture tailored for sequence-level classification rather than sequence-to-sequence generation.
+
+First, input token IDs are passed through an embedding layer that maps each vocabulary index to a continuous vector of dimension `d_model = 128`. These token embeddings capture distributional semantics learned during training. Since embeddings alone do not encode the order of tokens, the model adds positional encodings. A standard sinusoidal positional encoding is precomputed for positions up to a maximum length and added elementwise to the token embeddings. This provides the model with information about the absolute position of each token in the sequence.
+
+The resulting sequence of encoded token representations is processed by a stack of Transformer encoder layers. In this implementation, there are `NUM_LAYERS = 2` such layers, each consisting of a multi-head self-attention sublayer followed by a position-wise feed-forward network, both wrapped in residual connections and layer normalization.
+
+Within each encoder block, multi-head self-attention projects the input sequence into queries, keys, and values of dimension `d_k = d_model / num_heads`, where `num_heads = 8`. For each head, attention weights are computed as scaled dot products between queries and keys, divided by the square root of `d_k` to ensure stable gradients. A softmax is then applied to obtain a probability distribution over positions, and a weighted sum of the value vectors is taken. The outputs of all heads are concatenated and passed through a final linear projection back to `d_model`. Self-attention lets each token representation attend to all other tokens in the sequence, enabling the model to capture context such as negation and long-distance sentiment cues.
+
+The output of the attention sublayer is added back to the original input via a residual connection and normalized using layer normalization. A feed-forward sublayer then applies a two-layer MLP with hidden dimension `d_ff = 256` and ReLU activation, followed again by residual addition and layer normalization. This combination allows the model to perform both contextual mixing (via attention) and local non-linear transformations (via the feed-forward network).
+
+After the final encoder layer, the model applies global average pooling over the sequence dimension, computing the mean of the hidden states across all time steps. This yields a single `d_model`-dimensional vector that summarizes the entire review. A dropout layer with dropout probability 0.1 is applied during training to reduce overfitting. Finally, a linear classification head maps this pooled representation to `num_classes = 2` logits, corresponding to negative and positive sentiment. The predicted class is obtained via an argmax over these logits.
+
+In summary, self-attention in this model works by dynamically weighting each token’s contribution to every other token’s representation based on learned similarity scores. Tokens with stronger semantic or syntactic relevance to a given position receive higher attention weights, enabling the model to focus on the most informative parts of the review for sentiment prediction.
+
+### 5. Training Pipeline
+
+The training loop is implemented around a standard supervised learning pipeline for PyTorch models. The loss function used is cross-entropy loss (`nn.CrossEntropyLoss`), which is appropriate for multi-class (here, binary) classification with mutually exclusive classes. It directly optimizes the log-likelihood of the correct label given the model’s logits.
+
+Optimization is performed using the Adam optimizer (`torch.optim.Adam`) with an initial learning rate of `LR = 0.001`. Adam is chosen for its robustness to gradient scaling and its ability to adaptively tune learning rates per parameter, which is beneficial when training Transformer-style models. To encourage better convergence, a learning rate scheduler (`StepLR`) is applied: the learning rate is multiplied by `gamma = 0.5` every `step_size = 2` epochs. This gradually reduces the learning rate as training progresses, helping the model to fine-tune around a local optimum.
+
+Batches are constructed using `DataLoader` objects with a batch size of `BATCH_SIZE = 32`. The training loader shuffles the data at each epoch, while the validation loader does not. The model is trained for `EPOCHS = 5` full passes over the training set. Dropout with probability 0.1 is applied to the embeddings and intermediate representations, providing regularization by randomly masking components during training. No explicit early stopping is implemented in code; instead, performance is monitored on the validation set after each epoch via evaluation metrics. If desired, early stopping could be implemented externally by tracking the best validation F1 score and halting when it stops improving.
+
+### 6. Evaluation Metrics
+
+The evaluation function computes four metrics on the validation (or test) data: accuracy, precision, recall, and F1-score. Predictions are obtained by taking the argmax over the model’s logits for each example.
+
+Accuracy measures the overall proportion of correctly classified reviews and is a natural baseline metric for binary sentiment classification. However, accuracy alone can be misleading if the dataset is imbalanced or if different error types have different costs. Therefore, precision, recall, and F1-score (with binary averaging) are also reported. Precision captures the fraction of predicted positive reviews that are truly positive, recall captures the fraction of true positive reviews that are correctly identified, and the F1-score is the harmonic mean of precision and recall. For sentiment analysis where both false positives and false negatives are important, F1-score provides a more informative single-number summary than accuracy alone.
+
+### 7. Experimental Results (Updated with Model-Size Study)
+
+The training script now explicitly explores model complexity by training three Transformer sizes:
+
+- `small`: `d_model=64`, `num_heads=4`, `num_layers=1`, `d_ff=128`
+- `medium`: `d_model=128`, `num_heads=8`, `num_layers=2`, `d_ff=256`
+- `large`: `d_model=256`, `num_heads=8`, `num_layers=4`, `d_ff=512`
+
+For each size, the code reports training loss and validation metrics (accuracy, precision, recall, F1-score). It also computes trainable parameter counts and prints a final comparison table so performance can be compared against model complexity in a single run.
+
+In addition to console logging, the script saves a human-readable report:
+
+- `assignment_text/saved_model/transformer_imdb_experiment_report.md`
+
+This report includes run metadata, per-size architecture details, checkpoint paths, parameter counts, and metric summaries, making the experiment easy to share and inspect.
+
+### 8. Model Analysis and Error Inspection
+
+The main strength of this model is its ability to exploit contextual information across the entire review via self-attention. Unlike bag-of-words or simple n-gram models, it can represent interactions between distant tokens, such as handling negation (“not good”) or nuanced phrases that influence sentiment only in context. Additionally, the use of positional encoding preserves word order, which is crucial for distinguishing, for example, “not only good but great” from superficially similar but differently ordered phrases.
+
+However, several weaknesses are apparent. First, the tokenizer is very simple and does not handle subword units, morphology, or out-of-vocabulary words in a sophisticated way. Rare words are collapsed into the `<UNK>` token, which can obscure important sentiment cues. Second, reviews are truncated to 64 tokens; for very long reviews, important information appearing later in the text may be discarded, potentially harming performance. Third, the model capacity (two encoder layers with `d_model = 128`) is modest compared to modern large-scale Transformers and may limit performance on more challenging or nuanced reviews.
+
+From an overfitting perspective, validation metrics track training performance reasonably well, and the use of dropout and a relatively small model size helps keep overfitting under control. That said, with only 5 epochs of training, there is also a risk of underfitting: the model might not fully exploit the available training data. Adding more epochs with careful learning rate scheduling or early stopping based on validation F1 could further improve performance.
+
+Potential improvements include increasing the maximum sequence length to capture more of each review, adopting a more advanced tokenizer such as Byte-Pair Encoding (BPE) or WordPiece to better handle rare words and subwords, and scaling up the model (more layers, larger `d_model`, larger `d_ff`) if computational resources allow. Pretraining on a large unlabeled corpus and then fine-tuning on IMDB would also likely yield significant gains, following the standard transfer learning paradigm for NLP.
+
+For concrete examples, error-analysis instances are stored in:
+
+- `assignment_text/documentation/error_analysis.json`
+
+Based on the current file (`10` sampled misclassified reviews), the dominant failure mode is clear: all listed mistakes are **false positives** where the true label is negative (`0`) but the model predicts positive (`1`). This indicates the model is comparatively weaker at recognizing subtle or mixed negative sentiment.
+
+Common patterns observed in these errors include:
+
+- **Mixed sentiment with mild praise + stronger criticism**: e.g., "worth a rental" or "semi-alright action" appears early, but overall judgment is negative.
+- **Sarcasm and irony**: phrases such as "only thing good..." or "top of the garbage list" are difficult for the model to interpret reliably.
+- **Contrastive structure**: reviews that begin with positive setup and then pivot to criticism are often classified as positive.
+- **Soft negative wording**: "average", "passable", "not really worth watching again" can be semantically negative but lexically less explicit.
+- **Calibration issue on borderline negatives**: several wrong predictions have moderate confidence for the positive class, suggesting uncertain decision boundaries around nuanced negative reviews.
+
+This error profile suggests improvements should prioritize better handling of nuanced negative language (especially sarcasm and contrastive discourse), for example via richer tokenization (subword methods), longer context windows, and additional hard-negative examples during training.
+
+### 9. Conclusion
+
+This implementation demonstrates that a relatively small Transformer encoder can perform effective sentiment analysis on the IMDB dataset using only word-level tokenization and simple preprocessing. Through self-attention and positional encoding, the model captures long-range dependencies and focuses on sentiment-relevant parts of each review, achieving strong validation performance compared with simple baselines.
+
+From this assignment, the key takeaways are that Transformer-based models are flexible and powerful for text classification tasks and that even a compact architecture benefits from core Transformer ideas such as multi-head self-attention, residual connections, and normalization. At the same time, overall performance is sensitive to choices in tokenization, sequence length, and model capacity, highlighting the importance of careful design and experimentation when applying Transformers to NLP problems.
+
+### 10. Reproducibility, Code Organization, and Key Implementation Details
+
+The project is organized around core scripts and a directory for saved artifacts:
+
+- `assignment_text/code/c1.py`: End-to-end training/evaluation script with multi-size experiments (`small`, `medium`, `large`). It saves per-size checkpoints and a markdown experiment report.
+- `assignment_text/code/c1_analysis.py`: Evaluation and qualitative analysis script for loading a trained checkpoint and inspecting misclassifications. It supports `--model_size` to select `small`, `medium`, or `large`.
+- `assignment_text/saved_model/transformer_imdb_small.pt`, `assignment_text/saved_model/transformer_imdb_medium.pt`, `assignment_text/saved_model/transformer_imdb_large.pt`: Size-specific checkpoints.
+- `assignment_text/saved_model/transformer_imdb_experiment_report.md`: Experiment summary report generated after training.
+- `assignment_text/saved_model/transformer_imdb.pt`: Compatibility summary file containing best-model metadata and all results.
+
+Within `c1.py`, the data preprocessing pipeline is implemented by `tokenize`, `build_vocab`, and `preprocess_data`. The `tokenize` function normalizes and splits raw review text into lowercase word tokens, `build_vocab` constructs a frequency-based mapping from tokens to integer indices using only the training texts (reserving indices 0 and 1 for padding and unknown tokens), and `preprocess_data` converts each review into a fixed-length sequence of token IDs via truncation and padding. The `IMDBDataset` class wraps these sequences and labels into a PyTorch Dataset, and `DataLoader` objects provide mini-batches for training and validation.
+
+The model components are built in a modular fashion. `PositionalEncoding` implements sinusoidal position encodings that are added to token embeddings, `MultiHeadAttention` performs scaled dot-product self-attention over the sequence across multiple heads, and `TransformerEncoderBlock` combines multi-head attention, a position-wise feed-forward network, residual connections, and layer normalization into a single encoder layer. The `TransformerClassifier` class composes an embedding layer, positional encoding, a stack of encoder blocks, global average pooling over the sequence dimension, and a final linear layer that outputs logits for the two sentiment classes.
+
+The training and evaluation logic is encapsulated in `train_model` and `evaluate_model`. `train_model` iterates over the training DataLoader for a specified number of epochs, computes the cross-entropy loss, performs backpropagation and optimizer updates (Adam with an initial learning rate of 0.001), applies a StepLR scheduler that halves the learning rate every two epochs, and reports validation accuracy, precision, recall, and F1-score at the end of each epoch. `evaluate_model` runs the model in evaluation mode on a given DataLoader and aggregates predictions and labels to compute the same metrics. The function `load_imdb_texts` serves as a thin wrapper around the HuggingFace `datasets.load_dataset` API and clearly documents that the underlying data originates from the Large Movie Review Dataset (Maas et al., 2011).
+
+To reproduce the experiments, the main dependencies are PyTorch, scikit-learn, NumPy, and the HuggingFace `datasets` library. The script is designed to use a CUDA GPU if available, otherwise it falls back to CPU.
+
+Global hyperparameters are defined near the end of `c1.py`:
+
+- `MAX_VOCAB = 5000` (maximum vocabulary size)
+- `MAX_LEN = 64` (maximum sequence length in tokens)
+- `BATCH_SIZE = 32` (mini-batch size)
+- `EPOCHS = 5` (number of training epochs)
+- `LR = 0.001` (initial learning rate for Adam)
+
+Model-size-specific hyperparameters are defined in `MODEL_SIZES`:
+
+- `small`: `d_model=64`, `num_heads=4`, `num_layers=1`, `d_ff=128`
+- `medium`: `d_model=128`, `num_heads=8`, `num_layers=2`, `d_ff=256`
+- `large`: `d_model=256`, `num_heads=8`, `num_layers=4`, `d_ff=512`
+
+Running `c1.py` end-to-end will (1) download and preprocess the IMDB training split, (2) train all three model sizes on the training portion, (3) evaluate each model on the validation portion, and (4) save checkpoints plus an experiment report to the `assignment_text/saved_model` directory. Running `c1_analysis.py` with `--model_size` then allows targeted evaluation and qualitative error analysis for the selected size. With these components documented, the experiment is fully reproducible and can be extended for further study.
+

assignment_llm_1/assignment_text/documentation/error_analysis.json

@@ -0,0 +1,92 @@
+[
+    {
+        "example_id": 1,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.13079692423343658,
+        "confidence_pos": 0.8692030906677246,
+        "text": "Worth the entertainment value of a rental, especially if you like action movies. This one features the usual car chases, fights with the great Van Damme kick style, shooting battles with the 40 shell load shotgun, and even terrorist style bombs. All of this is entertaining and competently handled but there is nothing that really blows you away if you've seen your share before.<br /><br />The plot is made interesting by the inclusion of a rabbit, which is clever but hardly profound. Many of the characters are heavily stereotyped -- the angry veterans, the terrified illegal aliens, the crooked cops, the indifferent feds, the bitchy tough lady station head, the crooked politician, the fat federale who looks like he was typecast as the Mexican in a Hollywood movie from the 1940s. All passably acted but again nothing special.<br /><br />I thought the main villains were pretty well done and fairly well acted. By the end of the movie you certainly knew who the good guys were and weren't. There was an emotional lift as the really bad ones got their just deserts. Very simplistic, but then you weren't expecting Hamlet, right? The only thing I found really annoying was the constant cuts to VDs daughter during the last fight scene.<br /><br />Not bad. Not good. Passable 4.",
+        "explanation": "The model likely struggled with this review due to its mixed sentiment, where the reviewer acknowledges some entertainment value while simultaneously critiquing the film's lack of originality and reliance on stereotypes. Phrases like \"worth the entertainment value of a rental\" may have been interpreted positively, overshadowing the underlying negative sentiment expressed through descriptors like \"nothing special\" and \"hardly profound.\" Additionally, the nuanced critique of character portrayals and plot elements may have led to misclassification, as the model may not have effectively captured the sarcasm or the overall negative tone."
+    },
+    {
+        "example_id": 2,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.19945980608463287,
+        "confidence_pos": 0.8005402088165283,
+        "text": "its a totally average film with a few semi-alright action sequences that make the plot seem a little better and remind the viewer of the classic van dam films. parts of the plot don't make sense and seem to be added in to use up time. the end plot is that of a very basic type that doesn't leave the viewer guessing and any twists are obvious from the beginning. the end scene with the flask backs don't make sense as they are added in and seem to have little relevance to the history of van dam's character. not really worth watching again, bit disappointed in the end production, even though it is apparent it was shot on a low budget certain shots and sections in the film are of poor directed quality",
+        "explanation": "The model likely struggled with this review due to the presence of mixed sentiment and subtle linguistic cues that indicate negativity, such as phrases like \"totally average,\" \"not really worth watching again,\" and \"bit disappointed.\" Additionally, the use of qualifiers like \"semi-alright\" and the overall tone may have led the model to misinterpret the review as more positive than intended, as it could have focused on the mention of \"action sequences\" without fully grasping the critical context surrounding them."
+    },
+    {
+        "example_id": 3,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.07963180541992188,
+        "confidence_pos": 0.9203682541847229,
+        "text": "First off let me say, If you haven't enjoyed a Van Damme movie since bloodsport, you probably will not like this movie. Most of these movies may not have the best plots or best actors but I enjoy these kinds of movies for what they are. This movie is much better than any of the movies the other action guys (Segal and Dolph) have thought about putting out the past few years. Van Damme is good in the movie, the movie is only worth watching to Van Damme fans. It is not as good as Wake of Death (which i highly recommend to anyone of likes Van Damme) or In hell but, in my opinion it's worth watching. It has the same type of feel to it as Nowhere to Run. Good fun stuff!",
+        "explanation": "The model likely struggled with this review due to its mixed sentiment and nuanced language. While the reviewer expresses enjoyment of Van Damme's movies, they also imply that the film may not appeal to a broader audience, which can create confusion. Additionally, phrases like \"not as good as\" and \"only worth watching to Van Damme fans\" introduce a negative sentiment that may have been overshadowed by the overall positive tone, leading to the misclassification."
+    },
+    {
+        "example_id": 4,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.04191816598176956,
+        "confidence_pos": 0.958081841468811,
+        "text": "Isaac Florentine has made some of the best western Martial Arts action movies ever produced. In particular US Seals 2, Cold Harvest, Special Forces and Undisputed 2 are all action classics. You can tell Isaac has a real passion for the genre and his films are always eventful, creative and sharp affairs, with some of the best fight sequences an action fan could hope for. In particular he has found a muse with Scott Adkins, as talented an actor and action performer as you could hope for. This is borne out with Special Forces and Undisputed 2, but unfortunately The Shepherd just doesn't live up to their abilities.<br /><br />There is no doubt that JCVD looks better here fight-wise than he has done in years, especially in the fight he has (for pretty much no reason) in a prison cell, and in the final showdown with Scott, but look in his eyes. JCVD seems to be dead inside. There's nothing in his eyes at all. It's like he just doesn't care about anything throughout the whole film. And this is the leading man.<br /><br />There are other dodgy aspects to the film, script-wise and visually, but the main problem is that you are utterly unable to empathise with the hero of the film. A genuine shame as I know we all wanted this film to be as special as it genuinely could have been. There are some good bits, mostly the action scenes themselves. This film had a terrific director and action choreographer, and an awesome opponent for JCVD to face down. This could have been the one to bring the veteran action star back up to scratch in the balls-out action movie stakes.<br /><br />Sincerely a shame that this didn't happen.",
+        "explanation": "The model likely struggled with this review due to its mixed sentiment and nuanced language. While the reviewer praises Isaac Florentine's work and highlights positive aspects of the films, the overall sentiment shifts negatively when discussing JCVD's performance, particularly with phrases like \"dead inside\" and \"doesn't care.\" This complexity, combined with the presence of both positive and negative keywords, may have led the model to misinterpret the overall sentiment as positive."
+    },
+    {
+        "example_id": 5,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.45393702387809753,
+        "confidence_pos": 0.5460630059242249,
+        "text": "A group of heirs to a mysterious old mansion find out that they have to live in it as part of a clause in the will or be disinherited, but they soon find out of its history of everybody whom had lived there before them having either died in weird accidents or having had killed each other.<br /><br />You've seen it all before, and this one is too low-budget and slow paced to be scary, and doesn't have any real surprises in the climax. No special effects or gore to speak of, in fact the only really amusing thing about the whole film is the quality of the English dubbing, which at times is as bad as a cheap martial arts movie.<br /><br />3 out of 10, pretty low in the pecking order of 80's haunted house movies.",
+        "explanation": "The machine learning model likely struggled with this review due to the presence of mixed sentiment and subtle sarcasm. Phrases like \"you've seen it all before\" and \"the only really amusing thing\" can be interpreted as positive, while the overall context and explicit negative ratings (e.g., \"3 out of 10\") indicate dissatisfaction. Additionally, the model may have misinterpreted the lack of special effects and the critique of dubbing as neutral or positive elements, leading to an incorrect classification."
+    },
+    {
+        "example_id": 6,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.34184929728507996,
+        "confidence_pos": 0.6581506729125977,
+        "text": "The Forgotten (AKA: Don't Look In The Basement) is a very cheaply made and very old looking horror movie.<br /><br />The story is very slow and never really reaches anything worth getting excited about.<br /><br />The patients at the asylum are embarrassingly funny especially Sam and the old woman who always quotes an old saying to everyone. (Look out for the bit when she gets close to the camera, tell me you can watch without laughing!).<br /><br />Now the gore is very poor looking, with the blood looking pink in many scenes so it doesn't really deserve its place on the video nasties list!.<br /><br />Overall if you aren't looking for a fantastic horror film and have some time to spare then it's worth a watch.",
+        "explanation": "The model likely struggled with this review due to its mixed sentiment and the presence of sarcasm. Phrases like \"embarrassingly funny\" and \"worth a watch\" can be interpreted positively, despite the overall negative tone conveyed by descriptors like \"cheaply made\" and \"very slow.\" Additionally, the use of humor in describing the film's shortcomings may have misled the model into classifying the review as positive."
+    },
+    {
+        "example_id": 7,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.1487887054681778,
+        "confidence_pos": 0.8512113094329834,
+        "text": "I of course saw the previews for this at the beginning of some other Lion's Gate extravaganza, so of course it was only the best parts and therefore looked intriguing. And it is, to a point. A young college student (Sarah)is finding riddles all over the place and is becoming obsessed with answering them, and in doing so she's unwittingly becoming involved in some game. Now that's fairly intriguing right there but unfortunately it all gets rather muddled and becomes so complicated that the viewer (like myself) will most likely become frustrated. Characters appear with little introduction and you're not really sure who they are or why Sarah knows them or is hanging out with them. All of this has something to do with this woman who tried to drown a young boy years ago and her reason for that was that it's \"all part of the design\". In reality, it's all part of the \"very sketchy script\" and when the film is over you'll find yourself feeling that you've lost about an hour and a half of your life that you want back for more productive uses of your time, like cleaning the bathroom, for instance. 4 out of 10.",
+        "explanation": "The model likely struggled with this review due to the presence of mixed sentiment and nuanced language. Phrases like \"fairly intriguing\" and \"best parts\" may have been interpreted positively, while the overall context reveals frustration with the plot's complexity and a \"very sketchy script.\" Additionally, the use of sarcasm and the negative conclusion about losing time could have been overlooked by the model, leading to an incorrect positive classification."
+    },
+    {
+        "example_id": 8,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.32372772693634033,
+        "confidence_pos": 0.6762722134590149,
+        "text": "Four things intrigued me as to this film - firstly, it stars Carly Pope (of \"Popular\" fame), who is always a pleasure to watch. Secdonly, it features brilliant New Zealand actress Rena Owen. Thirdly, it is filmed in association with the New Zealand Film Commission. Fourthly, a friend recommended it to me. However, I was utterly disappointed. The whole storyline is absurd and complicated, with very little resolution. Pope's acting is fine, but Owen is unfortunately under-used. The other actors and actresses are all okay, but I am unfamiliar with them all. Aside from the nice riddles which are littered throughout the movie (and Pope and Owen), this film isn't very good. So the moral of the story is...don't watch it unless you really want to.",
+        "explanation": "The model likely struggled with this review due to the presence of mixed sentiment and subtle sarcasm. While the reviewer mentions positive aspects such as the cast and riddles, the overall tone is negative, particularly in phrases like \"utterly disappointed\" and \"this film isn't very good,\" which may have been overshadowed by the initial positive keywords. Additionally, the complexity of the review's structure may have led the model to misinterpret the sentiment conveyed."
+    },
+    {
+        "example_id": 9,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.35207340121269226,
+        "confidence_pos": 0.6479266285896301,
+        "text": "<br /><br />Never ever take a film just for its good looking title.<br /><br />Although it all starts well, the film suffers the same imperfections you see in B-films. Its like at a certain moment the writer does not any more how to end the film, so he ends it in a way nobody suspects it thinking this way he is ingenious.<br /><br />A film to be listed on top of the garbage list.<br /><br />",
+        "explanation": "The model likely struggled with the review due to the presence of sarcasm and mixed sentiment, particularly in phrases like \"good looking title\" and \"thinking this way he is ingenious,\" which could be misinterpreted as positive. Additionally, the use of phrases like \"top of the garbage list\" may not have been weighted heavily enough by the model, leading to an incorrect positive classification despite the overall negative sentiment expressed in the review."
+    },
+    {
+        "example_id": 10,
+        "true_label": 0,
+        "predicted_label": 1,
+        "confidence_neg": 0.42465338110923767,
+        "confidence_pos": 0.5753466486930847,
+        "text": "Lowe returns to the nest after, yet another, failed relationship, to find he's been assigned to jury duty. It's in the plans to, somehow, get out of it, when he realizes the defendant is the girl he's had a serious crush on since the first grade.<br /><br />Through living in the past by telling other people about his feelings towards this girl (played by Camp), Lowe remembers those feelings and does everything in his power to clear Camp of attempted murder, while staying away from the real bad guys at the same time, and succeeding in creating a successful film at the same time.<br /><br />I've heard that St Augustine is the oldest city in the US, and I also know it has some ties to Ponce de Leon, so the backdrop is a good place to start. Unfortunately, it's the only thing good about this movie. The local police are inept, the judge is an idiot, and the defense counsel does everything in her power to make herself look like Joanie Cunningham! I don't know whether to blame the director for poor direction, or for just letting the cast put in such a hapless effort.<br /><br />In short, this movie was so boring, I could not even sleep through it! 1 out of 10 stars!",
+        "explanation": "The model likely struggled with this review due to the presence of mixed sentiment and sarcasm. While the reviewer mentions some positive elements, such as the interesting backdrop of St. Augustine, the overall tone is negative, highlighted by phrases like \"the only thing good about this movie\" and criticisms of the characters and plot. The model may have misinterpreted the positive keywords and failed to recognize the underlying negative sentiment conveyed through the reviewer's frustration and sarcasm."
+    }
+]

assignment_llm_1/assignment_text/saved_model/Untitled

@@ -0,0 +1 @@
+saved_model

assignment_llm_1/assignment_text/saved_model/transformer_imdb_experiment_report.md

@@ -0,0 +1,77 @@
+# IMDB Transformer Model-Size Experiment Report
+
+- Generated at: `2026-02-12 01:50:15`
+- Device: `cuda`
+- Training samples: `20000`
+- Validation samples: `5000`
+- Max vocab size: `5000`
+- Max sequence length: `64`
+- Batch size: `32`
+- Epochs: `5`
+- Learning rate: `0.001`
+
+## Overall Comparison
+
+| Model Size | Trainable Params | Accuracy | Precision | Recall | F1 | Checkpoint |
+|---|---:|---:|---:|---:|---:|---|
+| small | 353,602 | 0.7742 | 0.7627 | 0.7960 | 0.7790 | `./saved_model/transformer_imdb_small.pt` |
+| medium | 905,218 | 0.7804 | 0.7674 | 0.8048 | 0.7856 | `./saved_model/transformer_imdb_medium.pt` |
+| large | 3,388,930 | 0.5190 | 0.5098 | 0.9880 | 0.6726 | `./saved_model/transformer_imdb_large.pt` |
+
+## Best Model
+
+- Best size by validation F1: `medium`
+- Checkpoint: `./saved_model/transformer_imdb_medium.pt`
+- Trainable parameters: `905,218`
+- Metrics:
+  - Accuracy: `0.7804`
+  - Precision: `0.7674`
+  - Recall: `0.8048`
+  - F1: `0.7856`
+
+## Per-Model Details
+
+### Small model
+
+- Architecture:
+  - `d_model`: `64`
+  - `num_heads`: `4`
+  - `num_layers`: `1`
+  - `d_ff`: `128`
+- Trainable params: `353,602`
+- Checkpoint: `./saved_model/transformer_imdb_small.pt`
+- Validation metrics:
+  - Accuracy: `0.7742`
+  - Precision: `0.7627`
+  - Recall: `0.7960`
+  - F1: `0.7790`
+
+### Medium model
+
+- Architecture:
+  - `d_model`: `128`
+  - `num_heads`: `8`
+  - `num_layers`: `2`
+  - `d_ff`: `256`
+- Trainable params: `905,218`
+- Checkpoint: `./saved_model/transformer_imdb_medium.pt`
+- Validation metrics:
+  - Accuracy: `0.7804`
+  - Precision: `0.7674`
+  - Recall: `0.8048`
+  - F1: `0.7856`
+
+### Large model
+
+- Architecture:
+  - `d_model`: `256`
+  - `num_heads`: `8`
+  - `num_layers`: `4`
+  - `d_ff`: `512`
+- Trainable params: `3,388,930`
+- Checkpoint: `./saved_model/transformer_imdb_large.pt`
+- Validation metrics:
+  - Accuracy: `0.5190`
+  - Precision: `0.5098`
+  - Recall: `0.9880`
+  - F1: `0.6726`

assignment_llm_1/data/cifar-10-batches-py/readme.html

@@ -0,0 +1 @@
+<meta HTTP-EQUIV="REFRESH" content="0; url=http://www.cs.toronto.edu/~kriz/cifar.html">

code/RL_model/inference_data/inference_20260213_002423.jsonl

@@ -0,0 +1,2 @@
+{"row_index": 0, "data_source": "multiclinsum", "ability": "summarization", "prediction": "{\n  \"low_health_literacy\": \"A 35-year-old woman had a lump on her cheek side (parotid gland) that grew over 2 years. At first, it looked like a slow-growing tumor, but later it got bigger and caused her face to droop. Doctors saw a hard, fixed lump that spread into nearby muscles. An MRI showed the lump was growing deep inside and had spread beyond the gland. She had surgery to remove the whole lump and nearby lymph nodes, keeping her facial nerve safe. Path tests confirmed it was a rare cancer called PNET, which often starts in nerves and has a special genetic change in 60% of the cells. After surgery, she got chemotherapy (four rounds of drugs: vincristine, doxorubicin, cyclophosphamide, ifosfamide, and etoposide) and then radiation to the area. Her body handled the treatment well, with only mild side effects. She checks in every 3 months with scans, and so far, no signs of cancer coming back.\"\n}"}
+{"row_index": 1, "data_source": "multiclinsum", "ability": "summarization", "prediction": "{\n    \"intermediate_health_literacy\": \"A 35-year-old woman developed a lump in her left parotid gland two years earlier. Initially, a small biopsy showed a non-cancerous mixed salivary gland tumor, and she had surgery that removed the mass completely. Six months later, a new, firm, painful lump appeared near the surgical site, and she began to lose movement on her left side—this is known as facial paralysis. An MRI scan showed a growing mass measuring 42 mm that had spread into nearby muscles and was not clearly defined. After further testing, doctors found it was a rare cancer called extra-skeletal Ewing’s sarcoma, a type of primary peripheral neuroectodermal tumor (PNET). This tumor has a specific genetic change (EWING/PNET-type translocation) found in 60% of the cells. She had surgery to remove the entire left parotid gland and nearby lymph nodes, with care taken to preserve her facial nerve. Following surgery, she received chemotherapy using a combination of drugs—vincristine, doxorubicin, cyclophosphamide, ifosfamide, and etoposide—for four cycles, followed by external radiation therapy. The treatment was well tolerated, with only mild side effects like mouth sores and skin irritation. At 10 months after treatment, there was no sign of the cancer coming back locally or elsewhere in her body.\"\n}"}

code/RL_model/inference_data/vllm_inference_20260213_003845_meta.json

@@ -0,0 +1,10 @@
+{
+  "model_path_for_tokenizer": "/home/mshahidul/readctrl/code/RL_model/converted_model/v1",
+  "dataset_path": "/home/mshahidul/readctrl/code/RL_model/verl/verl_train/dataset/test.parquet",
+  "base_url": "http://127.0.0.1:8001/v1",
+  "served_model_name": "inference",
+  "batch_size": 8,
+  "num_samples": 510,
+  "output_jsonl": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_20260213_003845.jsonl",
+  "output_parquet": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_20260213_003845.parquet"
+}

code/RL_model/inference_data/vllm_inference_20260213_170937_meta.json

@@ -0,0 +1,10 @@
+{
+  "model_path_for_tokenizer": "/home/mshahidul/readctrl/code/RL_model/models/converted_model/v1",
+  "dataset_path": "/home/mshahidul/readctrl/code/text_classifier/data/verified_combined_0-80_clean200.json",
+  "base_url": "http://127.0.0.1:8001/v1",
+  "served_model_name": "inference",
+  "batch_size": 8,
+  "num_samples": 200,
+  "output_jsonl": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_20260213_170937.jsonl",
+  "output_parquet": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_20260213_170937.parquet"
+}

code/RL_model/inference_data/vllm_inference_qwen-qwen3-4b-instruct-2507_20260213_173334_meta.json

@@ -0,0 +1,10 @@
+{
+  "model_path_for_tokenizer": "Qwen/Qwen3-4B-Instruct-2507",
+  "dataset_path": "/home/mshahidul/readctrl/code/text_classifier/data/verified_combined_0-80_clean200.json",
+  "base_url": "http://127.0.0.1:8001/v1",
+  "served_model_name": "inference",
+  "batch_size": 8,
+  "num_samples": 200,
+  "output_jsonl": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_qwen-qwen3-4b-instruct-2507_20260213_173334.jsonl",
+  "output_parquet": "/home/mshahidul/readctrl/code/RL_model/inference_data/vllm_inference_qwen-qwen3-4b-instruct-2507_20260213_173334.parquet"
+}

code/RL_model/unsloth_rl/RL_code.py

@@ -0,0 +1,165 @@
+import os
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = "3"
+from unsloth import FastLanguageModel
+import torch
+from health_classifier import classifier
+max_seq_length = 8192 
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = "/home/mshahidul/readctrl_model/RL_model/readability_sft_lora_model",
+    max_seq_length = max_seq_length,
+    load_in_4bit = False, # Set to False if you have enough VRAM
+    fast_inference = False,
+)
+
+# Simply enable gradient checkpointing and prepare for training
+model = FastLanguageModel.for_training(model)
+
+# /home/mshahidul/readctrl/data/extracting_subclaim/extracted_subclaims_multiclinsum_test_en_full.json
+with open("/home/mshahidul/readctrl/data/extracting_subclaim/extracted_subclaims_multiclinsum_test_en_full.json", "r") as f:
+    import json
+    data = json.load(f)
+from datasets import Dataset
+dataset = Dataset.from_list(data)
+with open('/home/mshahidul/readctrl/code/RL_model/prompt', 'r') as f:
+    prompt_template = f.read()
+dataset = dataset.map(lambda x: {
+    "prompt" : [
+        {"role": "system", "content": prompt_template},
+        {"role": "user",   "content": f'''
+- Input Language: English
+- Gold Summary (the anchor reference summary): {x['summary']}
+- Source Text (detailed content): {x['fulltext']}
+'''},
+    ],
+    "answer": {
+        "fulltext_subclaims": x['fulltext_subclaims'],
+        "summary_subclaims": x['summary_subclaims'],
+    },
+})
+import requests
+import json
+import re
+
+from claim_verifier import MedicalClaimVerifier
+
+verifier = MedicalClaimVerifier()
+
+def claim_reward_func(prompts, completions, answer, **kwargs):
+    # import ipdb; ipdb.set_trace()
+    """
+    GRPO reward function. 
+    Expects 'summary_subclaims' and 'fulltext_subclaims' to be in the dataset.
+    """
+    rewards = []
+    # We loop through the group of completions
+    for i in range(len(completions)):
+        reward = verifier.get_reward_score(
+            completions[i], 
+            answer[i]["summary_subclaims"], 
+            answer[i]["fulltext_subclaims"]
+        )
+        rewards.append(reward)
+    return rewards
+
+
+# def format_reward_func(completions, **kwargs):
+#     required_keys = ["low_health_literacy", "intermediate_health_literacy", "proficient_health_literacy"]
+#     scores = []
+#     for completion in completions:
+#         try:
+#             match = re.search(r"<SOLUTION>(.*?)</SOLUTION>", completion, re.DOTALL)
+#             content = match.group(1) if match else completion
+#             data = json.loads(content)
+#             if all(k in data for k in required_keys):
+#                 scores.append(2.0)
+#             else:
+#                 scores.append(-1.0)
+#         except:
+#             scores.append(-2.0)
+#     return scores
+
+
+import json
+
+def literacy_classifier_reward_func(completions, **kwargs):
+    scores = []
+    for completion in completions:
+        try:
+            # 1. Clean up potential Markdown formatting
+            cleaned_content = completion[0]['content'].strip()
+            if cleaned_content.startswith("```"):
+                # Removes leading ```json or ``` and trailing ```
+                cleaned_content = cleaned_content.split("```")[1]
+                if cleaned_content.startswith("json"):
+                    cleaned_content = cleaned_content[4:]
+            
+            # 2. Parse the JSON
+            data = json.loads(cleaned_content.strip())
+            
+            alignment_score = 0.0
+            target_labels = ["low", "intermediate", "proficient"]
+            
+            for label in target_labels:
+                key = f"{label}_health_literacy"
+                text_to_test = data.get(key, "")
+        
+                
+                if text_to_test:
+                    # Run the DSPy classifier
+                    result = classifier(summary_text=text_to_test)
+                    predicted = result.label # Expected format: "low_health_literacy"
+                    # import ipdb; ipdb.set_trace()
+                    
+                    if predicted == key:
+                        alignment_score += 1.0
+                    else:
+                        # Soft penalty for misclassification
+                        alignment_score -= 0.5
+                else:
+                    # Penalty if a specific literacy level is missing from the JSON
+                    alignment_score -= 0.3
+            
+            scores.append(alignment_score)
+            
+        except (json.JSONDecodeError, Exception):
+            # Significant penalty for malformed JSON or failed processing
+            scores.append(-1.0)
+            
+    return scores
+
+
+from trl import GRPOConfig, GRPOTrainer
+
+training_args = GRPOConfig(
+    learning_rate = 5e-6,
+    lr_scheduler_type = "cosine",
+    weight_decay = 0.1,
+    max_prompt_length = 8192,
+    max_completion_length = 4096,
+    # num_of_epochs = 10,
+    num_generations = 4, # GRPO group size
+    per_device_train_batch_size = 4,
+    gradient_accumulation_steps = 4,
+    max_steps = 500,
+    bf16 = True,
+    output_dir = "medical_grpo_outputs",
+)
+
+trainer = GRPOTrainer(
+    model = model,
+    reward_funcs = [
+        claim_reward_func,
+        # format_reward_func,
+        literacy_classifier_reward_func
+    ],
+    args = training_args,
+    train_dataset = dataset, # Use the same dataset from your SFT prep
+    tokenizer = tokenizer,
+)
+
+trainer.train()
+
+model.save_pretrained("/home/mshahidul/readctrl_model/readability_GRPO_model_v1")
+tokenizer.save_pretrained("/home/mshahidul/readctrl_model/readability_GRPO_model_v1")

code/RL_model/unsloth_rl/RL_training.ipynb

@@ -0,0 +1,475 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "8a790cb6",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from unsloth import FastLanguageModel\n",
+    "import torch\n",
+    "max_seq_length = 2048 # Can increase for longer reasoning traces\n",
+    "lora_rank = 32 # Larger rank = smarter, but slower\n",
+    "\n",
+    "model, tokenizer = FastLanguageModel.from_pretrained(\n",
+    "    model_name = \"unsloth/Qwen3-4B-Base\",\n",
+    "    max_seq_length = max_seq_length,\n",
+    "    load_in_4bit = False, # False for LoRA 16bit\n",
+    "    fast_inference = True, # Enable vLLM fast inference\n",
+    "    max_lora_rank = lora_rank,\n",
+    "    gpu_memory_utilization = 0.9, # Reduce if out of memory\n",
+    ")\n",
+    "\n",
+    "model = FastLanguageModel.get_peft_model(\n",
+    "    model,\n",
+    "    r = lora_rank, # Choose any number > 0 ! Suggested 8, 16, 32, 64, 128\n",
+    "    target_modules = [\n",
+    "        \"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
+    "        \"gate_proj\", \"up_proj\", \"down_proj\",\n",
+    "    ],\n",
+    "    lora_alpha = lora_rank*2, # *2 speeds up training\n",
+    "    use_gradient_checkpointing = \"unsloth\", # Reduces memory usage\n",
+    "    random_state = 3407,\n",
+    ")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ba056efa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# /home/mshahidul/readctrl/data/extracting_subclaim/extracted_subclaims_multiclinsum_test_en_full.json\n",
+    "with open('/home/mshahidul/readctrl/data/extracting_subclaim/extracted_subclaims_multiclinsum_test_en_full.json', 'r') as f:\n",
+    "    synthetic_data_with_gs_summary_en = json.load(f)\n",
+    "from datasets import Dataset\n",
+    "dataset = Dataset.from_list(synthetic_data_with_gs_summary_en)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fa285d3f",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ad059247",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# /home/mshahidul/readctrl/code/RL_model/prompt\n",
+    "with open('/home/mshahidul/readctrl/code/RL_model/prompt', 'r') as f:\n",
+    "    prompt_template = f.read()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "f74cbfda",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "dataset = dataset.map(lambda x: {\n",
+    "    \"prompt\" : [\n",
+    "        {\"role\": \"system\", \"content\": prompt_template},\n",
+    "        {\"role\": \"user\",   \"content\": f'''\n",
+    "- Input Language: English\n",
+    "- Gold Summary (the anchor reference summary): {x['summary']}\n",
+    "- Source Text (detailed content): {x['fulltext']}\n",
+    "'''},\n",
+    "    ],\n",
+    "    \"answer\": {\n",
+    "        \"fulltext_subclaims\": x['fulltext_subclaims'],\n",
+    "        \"summary_subclaims\": x['summary_subclaims'],\n",
+    "    },\n",
+    "})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "0dd615f4",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# /home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_20_67.json\n",
+    "import json\n",
+    "with open('/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full.json', 'r') as f:\n",
+    "    synthetic_data_diff_labels_en = json.load(f)\n",
+    "full_data=[]\n",
+    "# print((synthetic_data_diff_labels_en)[0].keys())\n",
+    "for item in synthetic_data_diff_labels_en:\n",
+    "    texts=item['diff_label_texts']\n",
+    "    for label in texts:\n",
+    "        full_data.append({\n",
+    "            \"index\": item['index'],\n",
+    "            'label': label,\n",
+    "            \"original_text\": item['fulltext'],\n",
+    "            \"generated_summary\": texts[label]\n",
+    "        })\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "3ba2a6cf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('/home/mshahidul/readctrl/data/data_annotator_data/syn_data_diff_labels_en_0_80.json', 'w') as f:\n",
+    "    json.dump(full_data, f, indent=4)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7cddc461",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# /home/mshahidul/readctrl/data/translated_data/translation_english2bangla_v1.json\n",
+    "import json\n",
+    "with open('/home/mshahidul/readctrl/data/testing_data_gs/multiclinsum_gs_train_en.json', 'r', encoding='utf-8') as f:\n",
+    "    dataset = json.load(f)\n",
+    "print(dataset[0].keys())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "id": "2b3f2a96",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "0_low_health_literacy\n",
+      "0_intermediate_health_literacy\n",
+      "0_proficient_health_literacy\n",
+      "1_low_health_literacy\n",
+      "1_intermediate_health_literacy\n",
+      "1_proficient_health_literacy\n",
+      "2_low_health_literacy\n",
+      "2_intermediate_health_literacy\n",
+      "2_proficient_health_literacy\n",
+      "3_low_health_literacy\n",
+      "3_intermediate_health_literacy\n",
+      "3_proficient_health_literacy\n",
+      "4_low_health_literacy\n",
+      "4_intermediate_health_literacy\n",
+      "4_proficient_health_literacy\n",
+      "5_low_health_literacy\n",
+      "5_intermediate_health_literacy\n",
+      "5_proficient_health_literacy\n",
+      "6_low_health_literacy\n",
+      "6_intermediate_health_literacy\n",
+      "6_proficient_health_literacy\n",
+      "7_low_health_literacy\n",
+      "7_intermediate_health_literacy\n",
+      "7_proficient_health_literacy\n",
+      "8_low_health_literacy\n",
+      "8_intermediate_health_literacy\n",
+      "8_proficient_health_literacy\n",
+      "9_low_health_literacy\n",
+      "9_intermediate_health_literacy\n",
+      "9_proficient_health_literacy\n",
+      "10_low_health_literacy\n",
+      "10_intermediate_health_literacy\n",
+      "10_proficient_health_literacy\n",
+      "11_low_health_literacy\n",
+      "11_intermediate_health_literacy\n",
+      "11_proficient_health_literacy\n",
+      "12_low_health_literacy\n",
+      "12_intermediate_health_literacy\n",
+      "12_proficient_health_literacy\n",
+      "13_low_health_literacy\n",
+      "13_intermediate_health_literacy\n",
+      "13_proficient_health_literacy\n",
+      "14_low_health_literacy\n",
+      "14_intermediate_health_literacy\n",
+      "14_proficient_health_literacy\n",
+      "15_low_health_literacy\n",
+      "15_intermediate_health_literacy\n",
+      "15_proficient_health_literacy\n",
+      "16_low_health_literacy\n",
+      "16_intermediate_health_literacy\n",
+      "16_proficient_health_literacy\n",
+      "17_low_health_literacy\n",
+      "17_intermediate_health_literacy\n",
+      "17_proficient_health_literacy\n",
+      "18_low_health_literacy\n",
+      "18_intermediate_health_literacy\n",
+      "18_proficient_health_literacy\n",
+      "19_low_health_literacy\n",
+      "19_intermediate_health_literacy\n",
+      "19_proficient_health_literacy\n",
+      "20_low_health_literacy\n",
+      "20_intermediate_health_literacy\n",
+      "20_proficient_health_literacy\n",
+      "21_low_health_literacy\n",
+      "21_intermediate_health_literacy\n",
+      "21_proficient_health_literacy\n",
+      "22_low_health_literacy\n",
+      "22_intermediate_health_literacy\n",
+      "22_proficient_health_literacy\n",
+      "23_low_health_literacy\n",
+      "23_intermediate_health_literacy\n",
+      "23_proficient_health_literacy\n",
+      "24_low_health_literacy\n",
+      "24_intermediate_health_literacy\n",
+      "24_proficient_health_literacy\n",
+      "25_low_health_literacy\n",
+      "25_intermediate_health_literacy\n",
+      "25_proficient_health_literacy\n",
+      "26_low_health_literacy\n",
+      "26_intermediate_health_literacy\n",
+      "26_proficient_health_literacy\n",
+      "27_low_health_literacy\n",
+      "27_intermediate_health_literacy\n",
+      "27_proficient_health_literacy\n",
+      "28_low_health_literacy\n",
+      "28_intermediate_health_literacy\n",
+      "28_proficient_health_literacy\n",
+      "29_low_health_literacy\n",
+      "29_intermediate_health_literacy\n",
+      "29_proficient_health_literacy\n",
+      "30_low_health_literacy\n",
+      "30_intermediate_health_literacy\n",
+      "30_proficient_health_literacy\n",
+      "31_low_health_literacy\n",
+      "31_intermediate_health_literacy\n",
+      "31_proficient_health_literacy\n",
+      "32_low_health_literacy\n",
+      "32_intermediate_health_literacy\n",
+      "32_proficient_health_literacy\n",
+      "33_low_health_literacy\n",
+      "33_intermediate_health_literacy\n",
+      "33_proficient_health_literacy\n",
+      "34_low_health_literacy\n",
+      "34_intermediate_health_literacy\n",
+      "34_proficient_health_literacy\n",
+      "35_low_health_literacy\n",
+      "35_intermediate_health_literacy\n",
+      "35_proficient_health_literacy\n",
+      "36_low_health_literacy\n",
+      "36_intermediate_health_literacy\n",
+      "36_proficient_health_literacy\n",
+      "37_low_health_literacy\n",
+      "37_intermediate_health_literacy\n",
+      "37_proficient_health_literacy\n",
+      "38_low_health_literacy\n",
+      "38_intermediate_health_literacy\n",
+      "38_proficient_health_literacy\n",
+      "39_low_health_literacy\n",
+      "39_intermediate_health_literacy\n",
+      "39_proficient_health_literacy\n",
+      "40_low_health_literacy\n",
+      "40_intermediate_health_literacy\n",
+      "40_proficient_health_literacy\n",
+      "41_low_health_literacy\n",
+      "41_intermediate_health_literacy\n",
+      "41_proficient_health_literacy\n",
+      "42_low_health_literacy\n",
+      "42_intermediate_health_literacy\n",
+      "42_proficient_health_literacy\n",
+      "43_low_health_literacy\n",
+      "43_intermediate_health_literacy\n",
+      "43_proficient_health_literacy\n",
+      "44_low_health_literacy\n",
+      "44_intermediate_health_literacy\n",
+      "44_proficient_health_literacy\n",
+      "45_low_health_literacy\n",
+      "45_intermediate_health_literacy\n",
+      "45_proficient_health_literacy\n",
+      "46_low_health_literacy\n",
+      "46_intermediate_health_literacy\n",
+      "46_proficient_health_literacy\n",
+      "47_low_health_literacy\n",
+      "47_intermediate_health_literacy\n",
+      "47_proficient_health_literacy\n",
+      "48_low_health_literacy\n",
+      "48_intermediate_health_literacy\n",
+      "48_proficient_health_literacy\n",
+      "49_low_health_literacy\n",
+      "49_intermediate_health_literacy\n",
+      "49_proficient_health_literacy\n",
+      "50_low_health_literacy\n",
+      "50_intermediate_health_literacy\n",
+      "50_proficient_health_literacy\n",
+      "51_low_health_literacy\n",
+      "51_intermediate_health_literacy\n",
+      "51_proficient_health_literacy\n",
+      "52_low_health_literacy\n",
+      "52_intermediate_health_literacy\n",
+      "52_proficient_health_literacy\n",
+      "53_low_health_literacy\n",
+      "53_intermediate_health_literacy\n",
+      "53_proficient_health_literacy\n",
+      "54_low_health_literacy\n",
+      "54_intermediate_health_literacy\n",
+      "54_proficient_health_literacy\n",
+      "55_low_health_literacy\n",
+      "55_intermediate_health_literacy\n",
+      "55_proficient_health_literacy\n",
+      "56_low_health_literacy\n",
+      "56_intermediate_health_literacy\n",
+      "56_proficient_health_literacy\n",
+      "57_low_health_literacy\n",
+      "57_intermediate_health_literacy\n",
+      "57_proficient_health_literacy\n",
+      "58_low_health_literacy\n",
+      "58_intermediate_health_literacy\n",
+      "58_proficient_health_literacy\n",
+      "59_low_health_literacy\n",
+      "59_intermediate_health_literacy\n",
+      "59_proficient_health_literacy\n",
+      "60_low_health_literacy\n",
+      "60_intermediate_health_literacy\n",
+      "60_proficient_health_literacy\n",
+      "61_low_health_literacy\n",
+      "61_intermediate_health_literacy\n",
+      "61_proficient_health_literacy\n",
+      "62_low_health_literacy\n",
+      "62_intermediate_health_literacy\n",
+      "62_proficient_health_literacy\n",
+      "63_low_health_literacy\n",
+      "63_intermediate_health_literacy\n",
+      "63_proficient_health_literacy\n",
+      "64_low_health_literacy\n",
+      "64_intermediate_health_literacy\n",
+      "64_proficient_health_literacy\n",
+      "65_low_health_literacy\n",
+      "65_intermediate_health_literacy\n",
+      "65_proficient_health_literacy\n",
+      "66_low_health_literacy\n",
+      "66_intermediate_health_literacy\n",
+      "66_proficient_health_literacy\n",
+      "67_low_health_literacy\n",
+      "67_intermediate_health_literacy\n",
+      "67_proficient_health_literacy\n",
+      "68_low_health_literacy\n",
+      "68_intermediate_health_literacy\n",
+      "68_proficient_health_literacy\n",
+      "69_low_health_literacy\n",
+      "69_intermediate_health_literacy\n",
+      "69_proficient_health_literacy\n",
+      "70_low_health_literacy\n",
+      "70_intermediate_health_literacy\n",
+      "70_proficient_health_literacy\n",
+      "71_low_health_literacy\n",
+      "71_intermediate_health_literacy\n",
+      "71_proficient_health_literacy\n",
+      "72_low_health_literacy\n",
+      "72_intermediate_health_literacy\n",
+      "72_proficient_health_literacy\n",
+      "73_low_health_literacy\n",
+      "73_intermediate_health_literacy\n",
+      "73_proficient_health_literacy\n",
+      "74_low_health_literacy\n",
+      "74_intermediate_health_literacy\n",
+      "74_proficient_health_literacy\n",
+      "75_low_health_literacy\n",
+      "75_intermediate_health_literacy\n",
+      "75_proficient_health_literacy\n",
+      "76_low_health_literacy\n",
+      "76_intermediate_health_literacy\n",
+      "76_proficient_health_literacy\n",
+      "77_low_health_literacy\n",
+      "77_intermediate_health_literacy\n",
+      "77_proficient_health_literacy\n",
+      "78_low_health_literacy\n",
+      "78_intermediate_health_literacy\n",
+      "78_proficient_health_literacy\n",
+      "79_low_health_literacy\n",
+      "79_intermediate_health_literacy\n",
+      "79_proficient_health_literacy\n"
+     ]
+    }
+   ],
+   "source": [
+    "# /home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full_updated.json\n",
+    "with open('/home/mshahidul/readctrl/data/synthetic_dataset_diff_labels/syn_data_diff_labels_en_0_80_full_updated.json', 'r') as f:\n",
+    "    syn_data_diff_labels_en_0_80_full_updated = json.load(f)\n",
+    "map_data={}\n",
+    "for item in syn_data_diff_labels_en_0_80_full_updated:\n",
+    "    for label in list(item['diff_label_texts'].keys()):\n",
+    "        key=f\"{item['index']}_{label}\"\n",
+    "        print(key)\n",
+    "        map_data[key]={\n",
+    "            'doc_id':item['index'],\n",
+    "            'label':label,\n",
+    "            'fulltext':item['fulltext'],\n",
+    "            \"diff_label_texts\":item['diff_label_texts'][label],\n",
+    "            'summary':item['summary']\n",
+    "        }\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "id": "c52e96ab",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# /home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/combine/consolidated_ratings_0-20(not_all_category).json\n",
+    "with open('/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/combine/consolidated_ratings_0-20(not_all_category).json', 'r') as f:\n",
+    "    consolidated_ratings_0_20 = json.load(f)\n",
+    "new_data=[]\n",
+    "for item in consolidated_ratings_0_20:\n",
+    "    key=f\"{item['doc_id']}_{item['health_literacy_label']}\"\n",
+    "    new_data.append({\n",
+    "        **map_data[key],\n",
+    "    })\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "id": "bfd6cf96",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open('/home/mshahidul/readctrl/data/annotators_validate_data_(20_80)/combine/verified_data_0-20.json', 'w') as f:\n",
+    "    json.dump(new_data, f, indent=4)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "cf797af6",
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "un",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.14"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

code/RL_model/unsloth_rl/claim_verifier.py

@@ -0,0 +1,175 @@
+import json
+import re
+import concurrent.futures
+from openai import OpenAI
+
+class MedicalClaimVerifier:
+    def __init__(self):
+        # OpenAI API configuration
+        api_file = "/home/mshahidul/api_new.json"
+        with open(api_file, "r") as f:
+            api_keys = json.load(f)
+        self.api_key = api_keys["openai"]
+        self.model_name = "gpt-5-mini"
+        self.client = OpenAI(api_key=self.api_key)
+
+        # Literacy ranges (IQR after outlier removal) from paper summary
+        # comp = completeness vs gold summary; cov = source_coverage vs full text
+        self.threshold_ranges = {
+            "low": {"comp": (0.9600, 1.0000), "cov": (0.1765, 0.3226)},
+            "intermediate": {"comp": (0.9393, 1.0000), "cov": (0.1818, 0.4091)},
+            "proficient": {"comp": (0.9231, 1.0000), "cov": (0.7725, 0.9347)},
+        }
+
+        # Minimum required information (upper bound of IQR)
+        self.thresholds = {
+            "low": {"comp": 1.0, "cov": 0.3226},
+            "intermediate": {"comp": 1.0, "cov": 0.4091},
+            "proficient": {"comp": 1.0, "cov": 0.9347},
+        }
+
+    def get_prompt(self,context,claim):
+        prompt = f"""
+        CONTEXT:
+        {context}
+
+        CLAIM TO VERIFY:
+        {claim}
+
+        INSTRUCTION:
+        Does the CONTEXT above provide enough evidence to support the CLAIM? 
+        - Answer 'supported' if the claim is explicitly stated or logically followable.
+        - Answer 'not_supported' if the claim is missing, contradicts the text, or requires outside info.
+
+        Output only one word: 'supported' or 'not_supported'.
+        """
+        return prompt
+
+    def check_support_api(self, prompt):
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            res = response.choices[0].message.content.strip().lower()
+            # print("API Response:", res)
+            return 1.0 if "supported" in res and "not_supported" not in res else 0.0
+        except Exception as e:
+            print(f"API call error: {e}")
+            return 0.0
+
+    def evaluate_level(self, gen_text, gold_subs, full_subs, level_key):
+        """Calculates scores for a single literacy level."""
+        if not gen_text: return 0.0, 0.0
+
+        # Run API calls in parallel to save time during RL
+        try:
+            with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+                # Completeness check (vs Gold Summary Subclaims)
+                comp_prompts = [self.get_prompt(gen_text, s) for s in gold_subs]
+                comp_results = list(executor.map(self.check_support_api, comp_prompts))
+                comp_score = sum(comp_results) / len(comp_results) if comp_results else 0.0
+
+
+                # Coverage check (vs Full Text Subclaims)
+                cov_prompts = [self.get_prompt(gen_text, s) for s in full_subs]
+                cov_results = list(executor.map(self.check_support_api, cov_prompts))
+                cov_score = sum(cov_results) / len(cov_results) if cov_results else 0.0
+            # print(f"Comp Score: {comp_score}, Cov Score: {cov_score} for {level_key}")
+        except Exception as e:
+            print(f"Parallel API call error: {e}")
+            return 0.0, 0.0
+    
+        return comp_score, cov_score
+
+    import json
+
+    def get_reward_score(self, completion, gold_subs, full_subs):
+        data = None
+        
+        # 1. Robust JSON Extraction
+        try:
+            # Clean potential markdown or whitespace
+            text = completion[0]['content'].strip().replace("```json", "").replace("```", "").strip()                
+            data = json.loads(text)
+        except (json.JSONDecodeError, IndexError, ValueError) as e:
+            print("JSON Parsing Error in Reward Calculation")
+            # If all extraction attempts fail
+            return -5.0
+
+        # 2. Schema Validation
+        levels = ["low", "intermediate", "proficient"]
+        # Check if any required keys are missing
+        if not all(f"{lvl}_health_literacy" in data for lvl in levels):
+            return -2.0  # Slightly smaller penalty for partial formatting success
+
+        # 3. Scoring Logic
+        try:
+            total_reward = 0.0
+            pass_reward = 1.0
+            fail_penalty = -1.0
+            for lvl in levels:
+                gen_text = data.get(f"{lvl}_health_literacy", "")
+                
+                # Skip scoring if text is empty
+                if not gen_text:
+                    total_reward += fail_penalty
+                    continue
+                    
+                comp_score, cov_score = self.evaluate_level(gen_text, gold_subs, full_subs, lvl)
+                
+                # Apply Thresholds
+                total_reward += pass_reward if comp_score >= self.thresholds[lvl]["comp"] else fail_penalty
+                total_reward += pass_reward if cov_score >= self.thresholds[lvl]["cov"] else fail_penalty
+                
+            return total_reward
+        except Exception:
+            return -5.0
+
+
+# 1. Ground Truth Subclaims (Extracted from a medical paper on Hypertension)
+gold_summary_subclaims = [
+    "Hypertension is defined as blood pressure above 140/90 mmHg.",
+    "Lifestyle changes like low salt intake can reduce blood pressure.",
+    "Diuretics are often the first line of pharmacological treatment."
+]
+
+full_text_subclaims = [
+    "Hypertension is defined as blood pressure above 140/90 mmHg.",
+    "Lifestyle changes like low salt intake can reduce blood pressure.",
+    "Diuretics are often the first line of pharmacological treatment.",
+    "The DASH diet emphasizes fruits, vegetables, and low-fat dairy.",
+    "Chronic hypertension increases the risk of stroke and myocardial infarction.",
+    "ACE inhibitors are contraindicated during pregnancy.",
+    "Secondary hypertension can be caused by renal artery stenosis."
+]
+
+# 2. Mock Model Completion (The output being evaluated)
+# This mimics the format your RL environment would pass to the reward function
+mock_completion = [{
+    'content': """
+    {
+        "low_health_literacy": "High blood pressure is when your blood is too strong for your veins. You should eat less salt to help stay healthy.",
+        "intermediate_health_literacy": "Hypertension is blood pressure over 140/90. You can lower it by eating less salt and taking water pills (diuretics) if your doctor says so.",
+        "proficient_health_literacy": "Hypertension (BP > 140/90 mmHg) is managed via lifestyle modifications like the DASH diet and salt restriction. Pharmacological interventions include diuretics as first-line therapy, though risks like stroke or heart attack persist if untreated. Secondary causes like renal artery stenosis should be screened, and ACE inhibitors must be avoided in pregnancy."
+    }
+    """
+}]
+
+# Initialize your verifier
+verifier = MedicalClaimVerifier()
+
+# Test the reward calculation
+reward = verifier.get_reward_score(
+    completion=mock_completion, 
+    gold_subs=gold_summary_subclaims, 
+    full_subs=full_text_subclaims
+)
+
+print(f"--- Evaluation Result ---")
+print(f"Total Reward Score: {reward}")
+
+# Logic Explanation:
+# - Low: Likely fails 'comp' (missing 140/90 info), but might pass 'cov' (low threshold).
+# - Intermediate: Likely passes 'comp' and 'cov'.
+# - Proficient: Needs to cover almost all 7 subclaims to pass the 0.77 coverage threshold.

code/RL_model/unsloth_rl/finetune.py

@@ -0,0 +1,91 @@
+import os
+# Set GPU environment variables
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = "2"
+import torch
+from unsloth import FastLanguageModel
+from datasets import load_dataset
+from trl import SFTTrainer, SFTConfig
+from unsloth.chat_templates import get_chat_template, standardize_data_formats, train_on_responses_only
+
+# 1. Configuration
+model_name = "unsloth/Qwen3-4B-Instruct-2507"
+max_seq_length = 8192
+dataset_path = "/home/mshahidul/readctrl/data/finetuning_data/training_data_readability_data_generation.json"
+output_dir = "/home/mshahidul/readctrl_model/RL_model/readability_sft_lora_model"
+
+# 2. Load Model and Tokenizer
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = model_name,
+    max_seq_length = max_seq_length,
+    load_in_4bit = True,
+)
+
+# 3. Add LoRA Adapters
+model = FastLanguageModel.get_peft_model(
+    model,
+    r = 32,
+    target_modules = ["q_proj", "k_proj", "v_proj", "o_proj",
+                      "gate_proj", "up_proj", "down_proj",],
+    lora_alpha = 32,
+    lora_dropout = 0,
+    bias = "none",
+    use_gradient_checkpointing = "unsloth",
+    random_state = 3407,
+)
+
+# 4. Data Preparation
+tokenizer = get_chat_template(
+    tokenizer,
+    chat_template = "qwen3-instruct",
+)
+
+dataset = load_dataset("json", data_files = dataset_path, split = "train")
+dataset = standardize_data_formats(dataset)
+
+def formatting_prompts_func(examples):
+    convos = examples["conversations"]
+    texts = [tokenizer.apply_chat_template(convo, tokenize = False, add_generation_prompt = False) for convo in convos]
+    return { "text" : texts, }
+
+dataset = dataset.map(formatting_prompts_func, batched = True)
+
+# 5. Training Setup
+trainer = SFTTrainer(
+    model = model,
+    tokenizer = tokenizer,
+    train_dataset = dataset,
+    dataset_text_field = "text",
+    max_seq_length = max_seq_length,
+    args = SFTConfig(
+        per_device_train_batch_size = 2,
+        gradient_accumulation_steps = 4,
+        warmup_steps = 5,
+        # max_steps = 60, # Adjust as needed for your dataset size
+        num_train_epochs = 3,
+        learning_rate = 2e-4,
+        fp16 = not torch.cuda.is_bf16_supported(),
+        bf16 = torch.cuda.is_bf16_supported(),
+        logging_steps = 1,
+        optim = "adamw_8bit",
+        weight_decay = 0.01,
+        lr_scheduler_type = "linear",
+        seed = 3407,
+        output_dir = "outputs",
+    ),
+)
+
+# Train only on assistant responses
+trainer = train_on_responses_only(
+    trainer,
+    instruction_part = "<|im_start|>user\n",
+    response_part = "<|im_start|>assistant\n",
+)
+
+# 6. Train and Save
+trainer.train()
+
+model.save_pretrained(output_dir)
+tokenizer.save_pretrained(output_dir)
+
+print(f"Model saved to {output_dir}")

code/RL_model/unsloth_rl/health_classifier.py

@@ -0,0 +1,42 @@
+import dspy
+import json
+from typing import Literal
+
+# --- 1. LLM Configuration ---
+def setup_dspy_classifier(save_path, api_key_path):
+    with open(api_key_path, "r") as f:
+        api_keys = json.load(f)
+
+    # Configure the LM
+    # Note: 'gpt-5-mini' is used per your configuration; ensure this matches your provider
+    openai_model = dspy.LM(model='gpt-5-mini', api_key=api_keys["openai"])
+    dspy.configure(lm=openai_model)
+
+    class HealthLiteracySignature(dspy.Signature):
+        """
+        Judge the health literacy level of a generated medical summary.
+        Identify if the language is suitable for a layperson (low) or requires medical expertise (proficient).
+        """
+        summary_text: str = dspy.InputField(desc="The generated medical summary to be analyzed.")
+        reasoning: str = dspy.OutputField(desc="Analysis of jargon, acronyms, and sentence complexity.")
+        label: Literal["low_health_literacy", "intermediate_health_literacy", "proficient_health_literacy"] = dspy.OutputField()
+
+    class HealthLiteracyClassifier(dspy.Module):
+        def __init__(self):
+            super().__init__()
+            self.predictor = dspy.ChainOfThought(HealthLiteracySignature)
+
+        def forward(self, summary_text):
+            return self.predictor(summary_text=summary_text)
+
+    # Initialize and load weights
+    classifier_instance = HealthLiteracyClassifier()
+    classifier_instance.load(save_path)
+    return classifier_instance
+
+# Global instantiation (optional, or you can call setup in your main script)
+API_FILE = "/home/mshahidul/api_new.json"
+SAVE_PATH = "/home/mshahidul/readctrl/data/new_exp/optimized_health_classifier_gpt5-mini_v2.json"
+
+# Create the instance to be imported
+classifier = setup_dspy_classifier(SAVE_PATH, API_FILE)

code/RL_model/unsloth_rl/highlighter.py

@@ -0,0 +1,103 @@
+import gradio as gr
+from transformers import AutoModel
+import torch
+
+# 1. Load the model (ensure you have transformers and torch installed)
+print("Loading model... This may take a moment.")
+model = AutoModel.from_pretrained(
+    "zilliz/semantic-highlight-bilingual-v1",
+    trust_remote_code=True
+)
+
+def process_and_highlight(question, context, threshold):
+    if not question or not context:
+        return "Please provide both a question and context."
+
+    # 2. Run the model inference
+    result = model.process(
+        question=question,
+        context=context,
+        threshold=threshold,
+        return_sentence_metrics=True
+    )
+
+    highlighted_sentences = result.get("highlighted_sentences", [])
+    
+    # 3. Create the highlighted HTML output
+    # We iterate through the context and wrap highlighted sentences in HTML tags
+    output_html = context
+    
+    # Sort highlighted sentences by length (descending) to avoid partial 
+    # matching issues if one sentence is a substring of another
+    highlighted_sentences.sort(key=len, reverse=True)
+
+    for sent in highlighted_sentences:
+        # Use a bright yellow highlight style
+        style = "background-color: #fff176; color: #000; padding: 2px; border-radius: 3px; font-weight: 500;"
+        highlighted_tag = f'<span style="{style}">{sent}</span>'
+        output_html = output_html.replace(sent, highlighted_tag)
+
+    # Wrap in a container for better typography
+    final_output = f"""
+    <div style="font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif; line-height: 1.8; font-size: 16px; color: #333;">
+        {output_html}
+    </div>
+    """
+    
+    # 4. Format metrics for the display
+    metrics_str = "No specific probabilities returned."
+    if "sentence_probabilities" in result:
+        metrics_str = "\n".join([f"• {p:.4f}" for p in result["sentence_probabilities"]])
+
+    return final_output, metrics_str
+
+# 5. Build the Gradio UI
+with gr.Blocks(theme=gr.themes.Soft(), title="Semantic Highlighter") as demo:
+    gr.Markdown("# 🔍 Semantic Highlight Explorer")
+    gr.Markdown("Identify and highlight parts of a text that answer a specific question using the Zilliz bilingual model.")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            question_input = gr.Textbox(
+                label="Question", 
+                placeholder="e.g., What are the symptoms of dehydration?",
+                lines=2
+            )
+            context_input = gr.Textbox(
+                label="Context / Full Text", 
+                placeholder="Paste the document text here...",
+                lines=10
+            )
+            threshold_slider = gr.Slider(
+                minimum=0.1, maximum=1.0, value=0.5, step=0.05, 
+                label="Confidence Threshold"
+            )
+            submit_btn = gr.Button("Analyze & Highlight", variant="primary")
+            
+        with gr.Column(scale=1):
+            gr.Label("Highlighted Result")
+            output_display = gr.HTML()
+            
+            with gr.Accordion("Sentence Metrics", open=False):
+                metrics_display = gr.Textbox(label="Probabilities", lines=5)
+
+    # Add example from your snippet
+    gr.Examples(
+        examples=[
+            [
+                "What are the symptoms of dehydration?",
+                "Dehydration occurs when your body loses more fluid than you take in. Common signs include feeling thirsty and having a dry mouth. The human body is composed of about 60% water. Dark yellow urine and infrequent urination are warning signs. Water is essential for many bodily functions. Dizziness, fatigue, and headaches can indicate severe dehydration.",
+                0.5
+            ]
+        ],
+        inputs=[question_input, context_input, threshold_slider]
+    )
+
+    submit_btn.click(
+        fn=process_and_highlight,
+        inputs=[question_input, context_input, threshold_slider],
+        outputs=[output_display, metrics_display]
+    )
+
+if __name__ == "__main__":
+    demo.launch(share=True)

code/RL_model/unsloth_rl/inference.py

@@ -0,0 +1,120 @@
+import json
+import os
+# Set GPU environment variables
+os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
+os.environ["CUDA_VISIBLE_DEVICES"] = "2"
+import torch
+from unsloth import FastLanguageModel
+from transformers import TextStreamer
+
+# 1. Configuration
+model_path = "/home/mshahidul/readctrl_model/RL_model/readability_sft_lora_model"
+max_seq_length = 8192
+
+# 2. Load the Fine-tuned Model and Tokenizer
+# Unsloth automatically reloads the base Qwen3 model and attaches your adapters.
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name = model_path,
+    max_seq_length = max_seq_length,
+    load_in_4bit = False,
+)
+
+# 3. Enable Fast Inference
+# This activates Unsloth's optimized inference kernels for a 2x speedup.
+FastLanguageModel.for_inference(model)
+
+# 4. Prepare your Test Data
+# Replace these with actual values from your evaluation set
+gold_summary = "A 34-year-old pregnant woman presents with seizures and dysarthria and is urgently referred for a cranial MRI. The classic ‘Medusa head’ sign is seen and the diagnosis is made as a venous anomaly of development with peripheral partial thrombosis and proximal slow flow.\n"
+fulltext = "We present the case of a 34-year-old woman, eight weeks pregnant with no other personal history of interest, who presents to the emergency department with generalized convulsions with dysarthria in the postcritical period, which resolve progressively in less than two hours. On physical examination, she is conscious, oriented, with no language or motor or sensory deficits. Only signs of a right lateral tongue bite are observed.\n\nThe complementary tests, such as blood tests or the electrocardiogram, are normal. Given that the episode corresponds with a first epileptic seizure and the patient is pregnant, an urgent magnetic resonance of the skull is requested.\n\nThe usual protocol was performed and 3D T1 sequences without and with intravenous contrast were obtained in axial, coronal and sagital planes, axial FLAIR, axial T2, VEN BOLD and magnetic susceptibility sequences, as well as axial diffusion and apparent diffusion coefficient map. The MRI identified multiple venous cortico-medullary vascular structures converging centripetally to a large central venous structure draining through the inferior anastomotic vein into the left transverse sinus, forming the classic ‘Medusa head’ sign. In the T1 sequences, the drainage vein was seen to be increased in signal with central hyphocaptation after contrast administration, suggesting partial thrombosis versus slow flow. In addition, in T2 and FLAIR sequences, the brain tissue surrounding the drainage vein was seen to be hyperintense, without diffusion restriction and compatible with edema.\n\nThese findings are suggestive of a venous anomaly of development with signs of partial peripheral thrombosis and slow flow more proximal, which cause edema of the surrounding tissue. She is started on clexane 60 mg/12 hours and levetiracetam 500 mg/12 hours and the patient shows improvement and symptomatic stability after one week.\n"
+       
+
+# Define your exact system prompt
+system_prompt = f"""
+    **System Role:**
+
+    You are an expert medical editor and Health Literacy specialist. Your task is to transform complex medical text into three distinct versions based on the reader's health literacy level. You must maintain the source language of the input while adjusting the linguistic complexity. Use the provided Gold Summary as the factual anchor to ensure the simplified versions remain accurate and focused on the most important information.
+
+    **User Prompt:**
+
+    Please process the following medical Source Text and its corresponding Gold Summary to generate three versions tailored to different health literacy levels.
+    ### Instructions for Each Level:
+
+    1. Level: Low Health Literacy (High Readability)
+
+    Target: Individuals needing the simplest terms for immediate action.
+
+    Linguistic Goal: Use "living room" language. Replace all medical jargon with functional descriptions (e.g., "renal" becomes "kidney").
+
+    Information Density: Focus strictly on the "need-to-know" info found in the Gold Summary.
+
+    Strategy: High paraphrasing using analogies. One idea per sentence.
+
+    Faithfulness: Must align perfectly with the Gold Summary.
+
+    2. Level: Intermediate Health Literacy (Medium Readability)
+
+    Target: The general public (news-reading level).
+
+    Linguistic Goal: Standard vocabulary. Common medical terms are okay, but technical "doctor-speak" must be simplified.
+
+    Information Density: Balanced. Use the Gold Summary as the lead, supplemented by necessary context from the Source Text.
+
+    Strategy: Moderate paraphrasing. Remove minor technical details to avoid information overload.
+
+    Faithfulness: Maintains the main narrative of the Gold Summary.
+
+    3. Level: Proficient Health Literacy (Low Readability)
+
+    Target: Researchers, clinicians, or highly informed patients.
+
+    Linguistic Goal: Technical and academic language. Prioritize clinical nuance and medical accuracy.
+
+    Information Density: High. Use the Full Source Text to include data, physiological mechanisms, and statistics.
+
+    Strategy: Minimal paraphrasing. Retain all original technical terminology.
+
+    Faithfulness: Adhere to the Source Text; you may add related subclaims that provide deeper scientific context.
+
+    Input Language: English
+    Gold Summary (The Anchor): 
+    {gold_summary}
+    Source Text (The Detail): 
+    {fulltext}
+
+    **Output Format (JSON only):**
+    {{
+    "low_health_literacy": "...",
+    "intermediate_health_literacy": "...",
+    "proficient_health_literacy": "..."
+    }}
+"""
+
+# Format for Qwen-3 instruction template
+messages = [
+    {"role": "user", "content": system_prompt}
+]
+
+input_text = tokenizer.apply_chat_template(
+    messages,
+    tokenize = False,
+    add_generation_prompt = True,
+)
+
+inputs = tokenizer([input_text], return_tensors = "pt").to("cuda")
+
+# 5. Run Generation
+# Using recommended sampling parameters for Qwen3 non-thinking mode.
+text_streamer = TextStreamer(tokenizer, skip_prompt = True,skip_special_tokens = True)
+
+print("--- Model Response ---")
+_ = model.generate(
+    **inputs,
+    streamer = text_streamer,
+    max_new_tokens = 2048,
+    temperature = 0.7, 
+    top_p = 0.8, 
+    top_k = 20,
+    repetition_penalty = 1.05,
+    use_cache = True,
+)

code/RL_model/unsloth_rl/prompt

@@ -0,0 +1,58 @@
+**System Role:**
+
+You are an expert medical editor and Health Literacy specialist. Your task is to transform complex medical text into three distinct versions based on the reader's health literacy level. You must maintain the source language of the input while adjusting the linguistic complexity. Use the provided Gold Summary as the factual anchor to ensure the simplified versions remain accurate and focused on the most important information.
+
+**User Prompt:**
+
+Please process the following medical Source Text and its corresponding Gold Summary to generate three versions tailored to different health literacy levels.
+### Instructions for Each Level:
+
+1. Level: Low Health Literacy (High Readability)
+
+Target: Individuals needing the simplest terms for immediate action.
+
+Linguistic Goal: Use "living room" language. Replace all medical jargon with functional descriptions (e.g., "renal" becomes "kidney").
+
+Information Density: Focus strictly on the "need-to-know" info found in the Gold Summary.
+
+Strategy: High paraphrasing using analogies. One idea per sentence.
+
+Faithfulness: Must align perfectly with the Gold Summary.
+
+2. Level: Intermediate Health Literacy (Medium Readability)
+
+Target: The general public (news-reading level).
+
+Linguistic Goal: Standard vocabulary. Common medical terms are okay, but technical "doctor-speak" must be simplified.
+
+Information Density: Balanced. Use the Gold Summary as the lead, supplemented by necessary context from the Source Text.
+
+Strategy: Moderate paraphrasing. Remove minor technical details to avoid information overload.
+
+Faithfulness: Maintains the main narrative of the Gold Summary.
+
+3. Level: Proficient Health Literacy (Low Readability)
+
+Target: Researchers, clinicians, or highly informed patients.
+
+Linguistic Goal: Technical and academic language. Prioritize clinical nuance and medical accuracy.
+
+Information Density: High. Use the Full Source Text to include data, physiological mechanisms, and statistics.
+
+Strategy: Minimal paraphrasing. Retain all original technical terminology.
+
+Faithfulness: Adhere to the Source Text; you may add related subclaims that provide deeper scientific context.
+
+
+I will provide the following information:
+
+- Input Language: <<<SOURCE_LANGUAGE>>>
+- Gold Summary (the anchor reference summary): <<<GOLD_SUMMARY>>>
+- Source Text (detailed content): <<<FULL_TEXT>>>
+
+**Output Format (JSON only):**
+    {{
+    "low_health_literacy": "...",
+    "intermediate_health_literacy": "...",
+    "proficient_health_literacy": "..."
+    }}

code/RL_model/unsloth_rl/reward_mock.py

@@ -0,0 +1,127 @@
+import os
+import json
+import re
+import concurrent.futures
+from openai import OpenAI
+
+class MedicalClaimVerifier:
+    def __init__(self):
+        # Implementation remains similar, but with safer error handling
+        api_file = "/home/mshahidul/api_new.json"
+        with open(api_file, "r") as f:
+            api_keys = json.load(f)
+        self.api_key = api_keys["openai"]
+        # Note: Ensure gpt-5-nano is actually available in your tier
+        self.model_name = "gpt-5-nano" 
+        self.client = OpenAI(api_key=self.api_key)
+
+        self.thresholds = {
+            "low": {"comp": 1.0, "cov": 0.3226},
+            "intermediate": {"comp": 1.0, "cov": 0.4091},
+            "proficient": {"comp": 1.0, "cov": 0.9347},
+        }
+
+    def get_prompt(self,context,claim):
+        prompt = f"""
+        CONTEXT:
+        {context}
+
+        CLAIM TO VERIFY:
+        {claim}
+
+        INSTRUCTION:
+        Does the CONTEXT above provide enough evidence to support the CLAIM? 
+        - Answer 'supported' if the claim is explicitly stated or logically followable.
+        - Answer 'not_supported' if the claim is missing, contradicts the text, or requires outside info.
+
+        Output only one word: 'supported' or 'not_supported'.
+        """
+        return prompt
+
+    def check_support_api(self, prompt):
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+            )
+            res = response.choices[0].message.content.strip().lower()
+            return 1.0 if "supported" in res and "not_supported" not in res else 0.0
+        except Exception:
+            return 0.0
+
+    def evaluate_level(self, gen_text, gold_subs, full_subs):
+        if not gen_text or not gold_subs or not full_subs:
+            return 0.0, 0.0
+        
+        # Combining calls to reduce overhead
+        all_claims = gold_subs + full_subs
+        with concurrent.futures.ThreadPoolExecutor(max_workers=10) as executor:
+            results = list(executor.map(self.check_support_api, [self.get_prompt(gen_text, s) for s in all_claims]))
+        
+        comp_results = results[:len(gold_subs)]
+        cov_results = results[len(gold_subs):]
+        
+        comp_score = sum(comp_results) / len(gold_subs)
+        cov_score = sum(cov_results) / len(full_subs)
+        return comp_score, cov_score
+
+verifier = MedicalClaimVerifier()
+
+def compute_score(data_source, solution_str, ground_truth, extra_info=None):  
+    gold_subs = ground_truth.get('summary_subclaims', [])
+    full_subs = ground_truth.get('fulltext_subclaims', [])
+    
+    if not gold_subs or not full_subs:
+        return 0.0
+
+    # 1. Parsing with fallback
+    try:
+        cleaned_str = solution_str.strip()
+        if "```json" in cleaned_str:
+            cleaned_str = cleaned_str.split("```json")[1].split("```")[0].strip()
+        elif "```" in cleaned_str:
+            cleaned_str = cleaned_str.split("```")[1].split("```")[0].strip()
+        data = json.loads(cleaned_str)
+    except Exception:
+        return -5.0 
+
+    levels = ["low", "intermediate", "proficient"]
+    scores = {}
+    
+    # 2. Score Calculation
+    for lvl in levels:
+        gen_text = data.get(f"{lvl}_health_literacy", "")
+        if not gen_text:
+            scores[lvl] = {"comp": 0.0, "cov": 0.0, "missing": True}
+        else:
+            comp, cov = verifier.evaluate_level(gen_text, gold_subs, full_subs)
+            scores[lvl] = {"comp": comp, "cov": cov, "missing": False}
+
+    # 3. Reward Shaping Logic
+    total_reward = 0.0
+    
+    low_cov = scores["low"]["cov"]
+    int_cov = scores["intermediate"]["cov"]
+    pro_cov = scores["proficient"]["cov"]
+
+    # Soft Hierarchy Check: Reward progression, penalize stagnation
+    # Instead of -2.0 exit, we subtract if the order is wrong
+    hierarchy_penalty = 0.0
+    if not (low_cov <= int_cov <= pro_cov):
+        hierarchy_penalty = -2.0 
+
+    for lvl in levels:
+        if scores[lvl]["missing"]:
+            total_reward -= 1.0 # Penalty per missing field
+            continue
+
+        comp_s = scores[lvl]["comp"]
+        cov_s = scores[lvl]["cov"]
+        thresh = verifier.thresholds[lvl]
+
+        # Continuous Reward: (Actual - Threshold) 
+        # This tells the model "You're 10% away" vs "You failed"
+        total_reward += (comp_s - thresh["comp"]) 
+        total_reward += (cov_s - thresh["cov"])
+
+    return total_reward + hierarchy_penalty

code/RL_model/unsloth_rl/test_reward_mock_unittest.py

@@ -0,0 +1,139 @@
+"""Minimal, offline tests for reward_mock.py.
+
+Run:
+  python code/RL_model/unsloth_rl/test_reward_mock_unittest.py
+
+These tests avoid real OpenAI calls by:
+- mocking the API key file read
+- stubbing OpenAI client construction
+- overriding verifier.evaluate_level to deterministic outputs
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import sys
+import types
+import unittest
+from pathlib import Path
+from unittest.mock import mock_open, patch
+
+
+THIS_DIR = Path(__file__).resolve().parent
+REWARD_MOCK_PATH = THIS_DIR / "reward_mock.py"
+
+
+class FakeOpenAI:
+    def __init__(self, api_key: str | None = None, **_kwargs):
+        self.api_key = api_key
+
+
+def load_reward_mock_module():
+    """Load reward_mock.py from its file path under test-friendly patches."""
+    module_name = "reward_mock_under_test"
+    if module_name in sys.modules:
+        del sys.modules[module_name]
+
+    spec = importlib.util.spec_from_file_location(module_name, str(REWARD_MOCK_PATH))
+    if spec is None or spec.loader is None:
+        raise RuntimeError(f"Failed to create import spec for {REWARD_MOCK_PATH}")
+
+    module = importlib.util.module_from_spec(spec)
+
+    # Ensure 'openai' import is available and OpenAI ctor is patched.
+    # reward_mock does: `from openai import OpenAI`
+    with patch("builtins.open", mock_open(read_data='{"openai": "sk-test"}')):
+        with patch("openai.OpenAI", FakeOpenAI):
+            spec.loader.exec_module(module)
+
+    sys.modules[module_name] = module
+    return module
+
+
+class TestRewardMockComputeScore(unittest.TestCase):
+    def test_valid_json_progression_no_hierarchy_penalty(self):
+        rm = load_reward_mock_module()
+
+        def fake_evaluate_level(gen_text, gold_subs, full_subs):
+            # Return (comp, cov) deterministically based on the generated text.
+            if gen_text == "LOW":
+                return 1.0, 0.3000
+            if gen_text == "INTER":
+                return 1.0, 0.4000
+            if gen_text == "PRO":
+                return 1.0, 0.9500
+            return 0.0, 0.0
+
+        rm.verifier.evaluate_level = fake_evaluate_level
+
+        solution_str = """```json
+        {
+          "low_health_literacy": "LOW",
+          "intermediate_health_literacy": "INTER",
+          "proficient_health_literacy": "PRO"
+        }
+        ```"""
+
+        ground_truth = {
+            "summary_subclaims": ["a", "b"],
+            "fulltext_subclaims": ["x", "y", "z"],
+        }
+
+        score = rm.compute_score(data_source=None, solution_str=solution_str, ground_truth=ground_truth)
+
+        # comp thresholds are 1.0 -> comp deltas = 0
+        # cov deltas: (0.3000-0.3226) + (0.4000-0.4091) + (0.9500-0.9347) = -0.0164
+        self.assertAlmostEqual(score, -0.0164, places=4)
+
+    def test_missing_field_penalizes_and_triggers_hierarchy_penalty(self):
+        rm = load_reward_mock_module()
+
+        def fake_evaluate_level(gen_text, gold_subs, full_subs):
+            if gen_text == "LOW":
+                return 1.0, 0.3000
+            if gen_text == "PRO":
+                return 1.0, 0.9500
+            return 0.0, 0.0
+
+        rm.verifier.evaluate_level = fake_evaluate_level
+
+        # intermediate is missing => -1.0
+        # BUT its cov will be 0.0 for the hierarchy check, so low_cov(0.3) <= int_cov(0.0) fails => -2.0
+        solution_str = '{"low_health_literacy": "LOW", "proficient_health_literacy": "PRO"}'
+
+        ground_truth = {
+            "summary_subclaims": ["a"],
+            "fulltext_subclaims": ["x"],
+        }
+
+        score = rm.compute_score(data_source=None, solution_str=solution_str, ground_truth=ground_truth)
+        expected = (0.3000 - 0.3226) + (0.9500 - 0.9347) - 1.0 - 2.0
+        self.assertAlmostEqual(score, expected, places=4)
+
+    def test_invalid_json_returns_minus_five(self):
+        rm = load_reward_mock_module()
+
+        ground_truth = {
+            "summary_subclaims": ["a"],
+            "fulltext_subclaims": ["x"],
+        }
+
+        score = rm.compute_score(data_source=None, solution_str="not a json", ground_truth=ground_truth)
+        self.assertEqual(score, -5.0)
+
+    def test_missing_claims_returns_zero(self):
+        rm = load_reward_mock_module()
+
+        solution_str = '{"low_health_literacy": "LOW", "intermediate_health_literacy": "INTER", "proficient_health_literacy": "PRO"}'
+
+        # Missing subclaims => early return 0.0
+        score = rm.compute_score(
+            data_source=None,
+            solution_str=solution_str,
+            ground_truth={"summary_subclaims": [], "fulltext_subclaims": ["x"]},
+        )
+        self.assertEqual(score, 0.0)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

code/RL_model/unsloth_rl/testing.py

@@ -0,0 +1,215 @@
+import json
+import concurrent.futures
+from unittest.mock import MagicMock
+
+# --- The Class (Modified slightly for standalone demo) ---
+
+class MedicalClaimVerifier:
+    def __init__(self, mock_mode=False):
+        self.thresholds = {
+            "low": {"comp": 0.6107, "cov": 0.3723},
+            "intermediate": {"comp": 0.8199, "cov": 0.6611},
+            "proficient": {"comp": 0.9569, "cov": 0.9069}
+        }
+        self.mock_mode = mock_mode
+        
+        if not mock_mode:
+            from openai import OpenAI
+            self.api_url = "http://172.16.34.29:8004/v1"
+            self.client = OpenAI(base_url=self.api_url, api_key="EMPTY")
+            self.model_name = "qwen3-32b-readctrl"
+
+    def get_audit_prompt(self, literacy_level):
+        level_guidelines = {
+            "low_health_literacy": """
+            Level: Low Health Literacy (High Readability)
+            Target: Individuals needing simple terms.
+            Goal: 'Living room' language. Replace jargon (e.g., 'renal' -> 'kidney').
+            Density: Strictly 'need-to-know' info from Gold Summary.
+            Strategy: High paraphrasing, analogies, one idea per sentence.
+            Faithfulness: Must align with Gold Summary.""",
+            
+            "intermediate_health_literacy": """
+            Level: Intermediate Health Literacy (Medium Readability)
+            Target: General public.
+            Goal: Standard vocabulary. Common medical terms okay; technical speak simplified.
+            Density: Balanced. Use Gold Summary as lead, supplemented by context from Source.
+            Strategy: Moderate paraphrasing. Remove minor technical details.
+            Faithfulness: Maintain main narrative of Gold Summary.""",
+            
+            "proficient_health_literacy": """
+            Level: Proficient Health Literacy (Low Readability)
+            Target: Researchers/Clinicians.
+            Goal: Technical/Academic. Prioritize clinical nuance and accuracy.
+            Density: High. Include data, physiological mechanisms, and statistics from Source.
+            Strategy: Minimal paraphrasing. Retain original technical terminology.
+            Faithfulness: Adhere to Source Text; add deeper scientific context."""
+        }
+
+        guidelines = level_guidelines.get(literacy_level, "Follow standard medical audit practices.")
+        level_desc = literacy_level.replace("_", " ")
+
+        base_instructions = f"""
+        ### Literacy Level Context:
+        {guidelines}
+
+        ### Task Instructions:"""
+        return base_instructions
+
+    def get_completeness_prompt(self, generated_text, source_subclaim, literacy_level):
+        base_instructions = self.get_audit_prompt(literacy_level)
+        level_desc = literacy_level.replace("_", " ")
+        return f"""{base_instructions}
+            1. Determine whether this Fact from the Gold Standard is covered in the {level_desc} summary.
+            2. Mark 'supported' ONLY IF:
+            - The fact is explicitly stated in the summary, OR
+            - The fact is clearly paraphrased or simplified in a way that preserves its meaning.
+            3. Do NOT mark 'supported' based solely on omission.
+            - Absence of mention does NOT imply intentional exclusion.
+            - Negative or exclusionary facts (e.g., "no complications," "no family history," "no systemic signs") must be explicitly conveyed.
+            4. Mark 'not_supported' if:
+            - The fact is completely omitted, OR
+            - The summary discusses related information but does not confirm the specific fact.
+            5. Literacy-based simplification is allowed, but factual meaning must be preserved.
+
+            SUMMARY: {generated_text}
+            FACT: {source_subclaim}
+
+            output: 'supported' or 'not_supported'.
+            """
+
+    def get_source_coverage_prompt(self, generated_text, source_subclaim, literacy_level):
+        base_instructions = self.get_audit_prompt(literacy_level)
+        level_desc = literacy_level.replace("_", " ")
+        return f"""{base_instructions}
+        1. Check whether the following Fact from the ORIGINAL Source Text is explicitly covered in the generated {level_desc} summary.
+        2. Mark 'supported' ONLY IF:
+        - The summary clearly states the fact, OR
+        - The fact is conveyed through an explicit paraphrase or simplification that preserves its meaning.
+        3. Do NOT infer support from silence or omission.
+        - Absence of mention does NOT count as support.
+        - Especially for negative or exclusionary facts (e.g., "no family history," "no extra-renal signs," "no complications"), the summary must explicitly indicate absence.
+        4. Mark 'not_supported' if:
+        - The summary omits the fact entirely, OR
+        - The summary discusses related topics but does not clearly confirm the specific fact.
+        5. Simplification for literacy level is allowed, but factual meaning must be preserved.
+
+        GENERATED SUMMARY: {generated_text}
+        SOURCE FACT: {source_subclaim}
+
+        output: 'supported' or 'not_supported'."""
+
+    def check_support_api(self, prompt):
+        # print(f"Prompt Sent:\n{prompt}\n")
+        
+        # Real logic
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model_name,
+                messages=[{"role": "user", "content": prompt}],
+                max_tokens=300, temperature=0.1,
+            )
+            res = response.choices[0].message.content.strip().lower()
+            print(f"Response Received:\n{res}\n")
+            return 1.0 if "supported" in res and "not_supported" not in res else 0.0
+        except:
+            return 0.0
+
+    def evaluate_level(self, gen_text, gold_subs, full_subs, level_key):
+        if not gen_text: return 0.0, 0.0
+        
+        # Using 2 workers for demo to avoid overhead
+        with concurrent.futures.ThreadPoolExecutor(max_workers=2) as executor:
+            comp_prompts = [self.get_completeness_prompt(gen_text, s, level_key) for s in gold_subs]
+            comp_results = list(executor.map(self.check_support_api, comp_prompts))
+            comp_score = sum(comp_results) / len(comp_results) if comp_results else 0.0
+
+            cov_prompts = [self.get_source_coverage_prompt(gen_text, s, level_key) for s in full_subs]
+            cov_results = list(executor.map(self.check_support_api, cov_prompts))
+            cov_score = sum(cov_results) / len(cov_results) if cov_results else 0.0
+
+        return comp_score, cov_score
+
+    def get_reward_score(self, completion, gold_subs, full_subs):
+        data = None
+        try:
+            # completion[0]['content'] structure as expected by RL frameworks
+            text = completion[0]['content'].strip()
+            
+            if "```json" in text:
+                text = text.split("```json")[-1].split("```")[0].strip()
+            elif "```" in text:
+                text = text.split("```")[-1].split("```")[0].strip()
+            
+            if "<SOLUTION>" in text:
+                text = text.split("<SOLUTION>")[-1].split("</SOLUTION>")[0].strip()
+                
+            data = json.loads(text)
+        except Exception as e:
+            print(f"JSON Parse Error: {e}")
+            return -5.0
+
+        levels = ["low", "intermediate", "proficient"]
+        if not all(f"{lvl}_health_literacy" in data for lvl in levels):
+            return -2.0
+
+        try:
+            total_reward = 0.0
+            print("\n--- Evaluation Breakdown ---")
+            for lvl in levels:
+                gen_text = data.get(f"{lvl}_health_literacy", "")
+                comp_score, cov_score = self.evaluate_level(gen_text, gold_subs, full_subs, f"{lvl}_health_literacy")
+                
+                # Logic check
+                comp_passed = comp_score >= self.thresholds[lvl]["comp"]
+                cov_passed = cov_score >= self.thresholds[lvl]["cov"]
+                
+                total_reward += 1.0 if comp_passed else -0.5
+                total_reward += 1.0 if cov_passed else -0.5
+                
+                print(f"[{lvl.upper()}] Comp: {comp_score:.2f} ({comp_passed}), Cov: {cov_score:.2f} ({cov_passed})")
+                
+            return total_reward
+        except Exception as e:
+            print(f"Scoring Error: {e}")
+            return -5.0
+
+# --- Execution Block ---
+
+if __name__ == "__main__":
+    verifier = MedicalClaimVerifier(mock_mode=False)
+
+    # 1. Mock Input Data (what the model generated)
+    pass_completion = [{
+    "content": """
+        <SOLUTION>
+        {
+            "low_health_literacy": "This medicine makes it easier for your heart to pump and relaxes your blood tubes. You might feel dizzy if you stand up too fast.",
+            "intermediate_health_literacy": "ACE inhibitors like Lisinopril relax blood vessels to improve flow and lower heart attack risk. Side effects include low blood pressure.",
+            "proficient_health_literacy": "ACE inhibitors attenuate the effects of stress hormones on the myocardium while inducing vasodilation to reduce afterload and prevent myocardial infarction."
+        }
+        </SOLUTION>
+        """
+    }]
+
+        # Completeness (Essential findings from a Gold Summary)
+    gold_subs = [
+        "ACE inhibitors help the heart pump better.",
+        "These medicines relax blood vessels.",
+        "Common side effects include dizziness and low blood pressure."
+    ]
+
+    # Source Coverage (Detailed facts from the original Full Text)
+    full_subs = [
+        "Lisinopril is an example of an ACE inhibitor.",
+        "ACE inhibitors lower the risk of a heart attack.",
+        "The medication prevents stress hormones from damaging the heart.",
+        "Patients should stand up slowly to avoid dizziness."
+    ]
+
+    # 3. Run Demo
+    print("Starting Demo Run...")
+    final_reward = verifier.get_reward_score(pass_completion, gold_subs, full_subs)
+    
+    print("-" * 30)
+    print(f"FINAL REWARD SCORE: {final_reward}")

code/RL_model/unsloth_rl/testing_v2.py

@@ -0,0 +1,138 @@
+import json
+import concurrent.futures
+from openai import OpenAI
+
+class FactualityBenchmarker:
+    def __init__(self, api_url="http://172.16.34.29:8004/v1", model="qwen3-32b-readctrl"):
+        self.client = OpenAI(base_url=api_url, api_key="EMPTY")
+        self.model = model
+
+    def verify_claim(self, context, claim):
+        """
+        Asks the model to determine if the context supports the claim.
+        """
+        prompt = f"""
+        CONTEXT:
+        {context}
+
+        CLAIM TO VERIFY:
+        {claim}
+
+        INSTRUCTION:
+        Does the CONTEXT above provide enough evidence to support the CLAIM? 
+        - Answer 'supported' if the claim is explicitly stated or logically followable.
+        - Answer 'not_supported' if the claim is missing, contradicts the text, or requires outside info.
+
+        Output only one word: 'supported' or 'not_supported'.
+        """
+        
+        try:
+            response = self.client.chat.completions.create(
+                model=self.model,
+                messages=[{"role": "user", "content": prompt}],
+                temperature=0.0, # Zero temp for consistency in benchmarks
+                max_tokens=10
+            )
+            result = response.choices[0].message.content.strip().lower()
+            return "supported" if "supported" in result and "not_supported" not in result else "not_supported"
+        except Exception as e:
+            print(f"Error: {e}")
+            return "not_supported"
+
+    def run_evaluation(self, test_cases):
+        """
+        Runs the benchmark over a list of test cases.
+        Each test case: {"context": "...", "claims": [{"text": "...", "label": 1.0/0.0}]}
+        """
+        total_claims = 0
+        correct_predictions = 0
+
+        print(f"--- Starting Evaluation on {self.model} ---")
+        
+        for i, case in enumerate(test_cases):
+            context = case["context"]
+            print(f"\nTest Case {i+1}:")
+            
+            for claim_data in case["claims"]:
+                claim_text = claim_data["text"]
+                expected = claim_data["expected"]
+                
+                # Model Prediction
+                prediction = self.verify_claim(context, claim_text)
+                
+                is_correct = (prediction == expected)
+                if is_correct:
+                    correct_predictions += 1
+                total_claims += 1
+                
+                status = "PASS" if is_correct else "FAIL"
+                print(f"  [{status}] Claim: {claim_text[:60]}... (Expected: {expected}, Got: {prediction})")
+
+        accuracy = (correct_predictions / total_claims) * 100 if total_claims > 0 else 0
+        print(f"\n" + "="*30)
+        print(f"FINAL ACCURACY: {accuracy:.2f}% ({correct_predictions}/{total_claims})")
+        print("="*30)
+
+# --- Define your test data here ---
+test_data = [
+    {
+        "context": """CASE PRESENTATION:
+A 64-year-old male with a 15-year history of Type 2 Diabetes Mellitus and stage 3 chronic kidney disease (CKD) 
+presented to the emergency department with acute shortness of breath and peripheral edema. On physical 
+examination, the patient was hypertensive (175/95 mmHg) and tachycardic (110 bpm). Lung auscultation revealed 
+bilateral crackles in the lower lobes, consistent with pulmonary congestion. Notable laboratory findings 
+included a Serum Creatinine of 2.8 mg/dL (baseline 1.9 mg/dL) and a Brain Natriuretic Peptide (BNP) of 1,250 pg/mL. 
+
+Crucially, the patient reported no history of tobacco use and denied any chest pain or radiating pain to the 
+left arm. An EKG showed sinus tachycardia but no ST-segment elevation or T-wave inversion. The medical team 
+initiated a regimen of intravenous furosemide (40mg bolus) and transitioned the patient from his home 
+medication (Metformin) to insulin glargine to manage blood glucose during the acute episode, citing concerns 
+over lactic acidosis risk given the acute kidney injury. After 48 hours, the patient's oxygen saturation 
+improved from 89% on room air to 95%, and his weight decreased by 3.2 kg due to successful diuresis. 
+The discharge summary noted that despite the respiratory distress, there were no signs of systemic infection 
+or fever during the entire 4-day hospital stay.""",
+        "claims":[
+    # 1. Literal Extraction
+    {"text": "The patient has had Type 2 Diabetes for 15 years.", "expected": "supported"},
+    
+    # 2. Medical Paraphrasing (Reading Control)
+    {"text": "The patient showed signs of fluid buildup in the lungs.", "expected": "supported"}, # 'bilateral crackles/congestion'
+    
+    # 3. Negative Constraint (Exclusionary fact)
+    {"text": "The patient has a history of smoking.", "expected": "not_supported"}, # Text says 'no history of tobacco'
+    
+    # 4. Mathematical Inference
+    {"text": "The patient's Serum Creatinine increased by 0.9 mg/dL from his baseline.", "expected": "supported"}, # 2.8 - 1.9 = 0.9
+    
+    # 5. Logic: Cause and Effect
+    {"text": "The doctors stopped Metformin because of the risk of lactic acidosis.", "expected": "supported"},
+    
+    # 6. Negative Finding (Testing 'Silence')
+    {"text": "The patient complained of pain moving down his left arm.", "expected": "not_supported"}, # Specifically denied
+    
+    # 7. Vital Sign Interpretation
+    {"text": "The patient was experiencing high blood pressure and a fast heart rate upon arrival.", "expected": "supported"}, # 175/95 and 110bpm
+    
+    # 8. Numerical Recovery
+    {"text": "The patient lost over 3 kilograms during the first two days of treatment.", "expected": "supported"}, # 3.2 kg
+    
+    # 9. Complex Inference (EKG interpretation)
+    {"text": "The EKG provided clear evidence of an active heart attack.", "expected": "not_supported"}, # Text says 'no ST-elevation'
+    
+    # 10. Systemic Health Status
+    {"text": "The patient remained afebrile throughout the hospitalization.", "expected": "supported"} # 'no fever' = afebrile
+]
+    },
+    {
+        "context": "The company reported a 15% increase in revenue, reaching $2 billion this quarter. However, net profit dropped due to high R&D costs.",
+        "claims": [
+            {"text": "Revenue reached $2 billion.", "expected": "supported"},
+            {"text": "Net profit increased this quarter.", "expected": "not_supported"},
+            {"text": "Spending on Research and Development impacted profits.", "expected": "supported"}
+        ]
+    }
+]
+
+if __name__ == "__main__":
+    benchmarker = FactualityBenchmarker()
+    benchmarker.run_evaluation(test_data)

code/RL_model/verl/Search-R1/.gitignore

@@ -0,0 +1,122 @@
+**/*.pt
+**/checkpoints
+**/wget-log
+**/_build/
+**/*.ckpt
+**/outputs
+**/*.tar.gz
+**/playground
+**/wandb
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+dataset/*
+tensorflow/my_graph/*
+.idea/
+# C extensions
+*.so
+data
+sft/output/*
+sft/data/*
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+image_outputs
+
+checkpoints
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+
+# virtualenv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# vscode
+.vscode
+
+# Mac
+.DS_Store
+
+# output logs
+tests/e2e/toy_examples/deepspeed/synchronous/output.txt
+
+# vim
+*.swp
+
+# log*
+log/
+
+**logs

code/RL_model/verl/Search-R1/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

code/RL_model/verl/Search-R1/Notice.txt

@@ -0,0 +1 @@
+Copyright 2023-2024 Bytedance Ltd. and/or its affiliates 

code/RL_model/verl/Search-R1/README.md

@@ -0,0 +1,275 @@
+# Search-R1: Train your LLMs to reason and call a search engine with reinforcement learning
+
+<div align="center">
+  <img src="https://raw.githubusercontent.com/PeterGriffinJin/Search-R1/main/public/logo.png" alt="logo" width="300"/>
+</div>
+
+<p align="center">
+  <a href="https://arxiv.org/abs/2503.09516">
+    <img src="https://img.shields.io/badge/Paper1-blue?style=for-the-badge" alt="Button1"/>
+  </a>
+  <a href="https://arxiv.org/abs/2505.15117">
+    <img src="https://img.shields.io/badge/Paper2-green?style=for-the-badge" alt="Button2"/>
+  </a>
+  <a href="https://huggingface.co/collections/PeterJinGo/search-r1-67d1a021202731cb065740f5">
+    <img src="https://img.shields.io/badge/Resources-orange?style=for-the-badge" alt="Button3"/>
+  </a>
+  <a href="https://x.com/BowenJin13/status/1895544294473109889">
+    <img src="https://img.shields.io/badge/Tweet-red?style=for-the-badge" alt="Button4"/>
+  </a>
+  <a href="https://wandb.ai/peterjin/Search-R1-v0.2">
+    <img src="https://img.shields.io/badge/Logs-purple?style=for-the-badge" alt="Button5"/>
+  </a>
+</p>
+
+
+<!-- <strong>Search-R1</strong> is a reinforcement learning framework for <em>training reasoning and searching (tool-call) interleaved LLMs</em>.  -->
+<!-- We built upon [veRL](https://github.com/volcengine/verl). -->
+**Search-R1** is a reinforcement learning framework designed for training **reasoning-and-searching interleaved LLMs**—language models that learn to reason and make tool calls (e.g., to search engines) in a coordinated manner.
+
+<!-- It can be seen as an extension of <strong>DeepSeek-R1(-Zero)</strong> with interleaved search engine calling and an opensource RL training-based solution for <strong>OpenAI DeepResearch</strong>. -->
+Built upon [veRL](https://github.com/volcengine/verl), Search-R1 extends the ideas of **DeepSeek-R1(-Zero)** by incorporating interleaved search engine access and provides a fully open-source RL training pipeline. It serves as an alternative and open solution to **OpenAI DeepResearch**, enabling research and development in tool-augmented LLM reasoning.
+
+<!-- Through RL (rule-based outcome reward), the 3B **base** LLM (both Qwen2.5-3b-base and Llama3.2-3b-base) develops reasoning and search engine calling abilities all on its own. -->
+
+We support different RL methods (e.g., PPO, GRPO, reinforce), different LLMs (e.g., llama3, Qwen2.5, etc) and different search engines (e.g., local sparse/dense retrievers and online search engines).
+
+Paper: [link1](https://arxiv.org/pdf/2503.09516), [link2](https://arxiv.org/abs/2505.15117); Model and data: [link](https://huggingface.co/collections/PeterJinGo/search-r1-67d1a021202731cb065740f5); Twitter thread: [link](https://x.com/BowenJin13/status/1895544294473109889); Full experiment log: [prelim](https://wandb.ai/peterjin/Search-R1-open); [v0.1](https://wandb.ai/peterjin/Search-R1-nq_hotpotqa_train); [v0.2](https://wandb.ai/peterjin/Search-R1-v0.2); [v0.3](https://wandb.ai/peterjin/Search-R1-v0.3). Details about these logs and methods can be find [here](https://github.com/PeterGriffinJin/Search-R1/blob/main/docs/experiment_log.md).
+
+
+![single-turn](public/main.png)
+
+## News
+
+- [2025.10] Search-R1 is featured by Thinking Machines Lab's first product [Tinker](https://github.com/thinking-machines-lab/tinker-cookbook)! Details: [Document](https://github.com/thinking-machines-lab/tinker-cookbook/tree/main/tinker_cookbook/recipes/tool_use/search).
+- [2025.7] Search-R1 is supported by [SkyRL](https://github.com/NovaSky-AI/SkyRL)! Detailed instructions: [code](https://github.com/NovaSky-AI/SkyRL/tree/main/skyrl-train/examples/search), [Document](https://novasky-ai.notion.site/skyrl-searchr1).
+- [2025.6] Search-R1 is now integrated into the latest version of veRL and can take advantage of its most up-to-date features! Detailed instructions: [veRL](https://verl.readthedocs.io/en/latest/sglang_multiturn/search_tool_example.html), [English Document](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/tool_examples/verl-multiturn-searchR1-like.md), [Chinese Document](https://github.com/zhaochenyang20/Awesome-ML-SYS-Tutorial/blob/main/rlhf/verl/multi-turn/tool_examples/verl-multiturn-searchR1-like_ZH.md).
+- [2025.5] The second [paper](https://arxiv.org/abs/2505.15117) conducting detailed empirical studies is published with logs: [v0.3](https://wandb.ai/peterjin/Search-R1-v0.3). 
+- [2025.4] We support [multinode](https://github.com/PeterGriffinJin/Search-R1/blob/main/docs/multinode.md) training for 30B+ LLMs!
+- [2025.4] We support [different search engines](https://github.com/PeterGriffinJin/Search-R1/blob/main/docs/retriever.md) including sparse local retriever, dense local retriever with ANN indexing and online search engines!
+- [2025.3] The first Search-R1 [paper](https://arxiv.org/pdf/2503.09516) is published with the logs: [v0.1](https://wandb.ai/peterjin/Search-R1-nq_hotpotqa_train); [v0.2](https://wandb.ai/peterjin/Search-R1-v0.2).
+- [2025.2] We opensource Search-R1 codebase with [preliminary results](https://wandb.ai/peterjin/Search-R1-open).
+
+## Links
+
+- [Installation](#installation)
+- [Quick start](#quick-start)
+- [Preliminary results](#preliminary-results)
+- [Inference](#inference)
+- [Use your own dataset](#use-your-own-dataset)
+- [Use your own search engine](#use-your-own-search-engine)
+- [Features](#features)
+- [Ackowledge](#acknowledge)
+- [Citations](#citations)
+
+## Installation
+
+### Search-r1 environment
+```bash
+conda create -n searchr1 python=3.9
+conda activate searchr1
+# install torch [or you can skip this step and let vllm to install the correct version for you]
+pip install torch==2.4.0 --index-url https://download.pytorch.org/whl/cu121
+# install vllm
+pip3 install vllm==0.6.3 # or you can install 0.5.4, 0.4.2 and 0.3.1
+
+# verl
+pip install -e .
+
+# flash attention 2
+pip3 install flash-attn --no-build-isolation
+pip install wandb
+```
+
+### Retriever environment (optional)
+If you would like to call a local retriever as the search engine, you can install the environment as follows. (We recommend using a seperate environment.)
+```bash
+conda create -n retriever python=3.10
+conda activate retriever
+
+# we recommend installing torch with conda for faiss-gpu
+conda install pytorch==2.4.0 torchvision==0.19.0 torchaudio==2.4.0 pytorch-cuda=12.1 -c pytorch -c nvidia
+pip install transformers datasets pyserini
+
+## install the gpu version faiss to guarantee efficient RL rollout
+conda install -c pytorch -c nvidia faiss-gpu=1.8.0
+
+## API function
+pip install uvicorn fastapi
+```
+
+
+## Quick start
+
+Train a reasoning + search LLM on NQ dataset with e5 as the retriever and wikipedia as the corpus.
+
+(1) Download the indexing and corpus.
+```bash
+save_path=/the/path/to/save
+python scripts/download.py --save_path $save_path
+cat $save_path/part_* > $save_path/e5_Flat.index
+gzip -d $save_path/wiki-18.jsonl.gz
+```
+
+(2) Process the NQ dataset.
+```bash
+python scripts/data_process/nq_search.py
+```
+
+(3) Launch a local retrieval server.
+```bash
+conda activate retriever
+bash retrieval_launch.sh
+```
+
+(4) Run RL training (PPO) with Llama-3.2-3b-base.
+```bash
+conda activate searchr1
+bash train_ppo.sh
+```
+
+## Preliminary results
+
+(1) The base model (llama3.2-3b-base) learns to call the search engine and obtain improved performance.
+
+![llama-3b](public/llama32-3b.png)
+
+
+(2) The base model (Qwen2.5-7b-base) can learn to conduct multi-turn search engine calling and reasoning with RL.
+
+![multi-turn](public/multi-turn.png)
+
+## Inference
+#### You can play with the trained Search-R1 model with your own question.
+(1) Launch a local retrieval server.
+```bash
+conda activate retriever
+bash retrieval_launch.sh
+```
+
+(2) Run inference.
+```bash
+conda activate searchr1
+python infer.py
+```
+You can modify the ```question``` on line 7 to something you're interested in.
+
+## Use your own dataset
+
+### QA data
+For each question-answer sample, it should be a dictionary containing the desired content as below:
+
+```
+data = {
+        "data_source": data_source,
+        "prompt": [{
+            "role": "user",
+            "content": question,
+        }],
+        "ability": "fact-reasoning",
+        "reward_model": {
+            "style": "rule",
+            "ground_truth": solution
+        },
+        "extra_info": {
+            'split': split,
+            'index': idx,
+        }
+    }
+```
+
+You can refer to ```scripts/data_process/nq_search.py``` for a concrete data processing example.
+
+### Corpora
+
+It is recommended to make your corpus a jsonl file, where each line (a dictionary with "id" key and "contents" key) corresponds to one passage. You can refer to ```example/corpus.jsonl``` for an example.
+
+The "id" key corresponds to the passage id, while the "contents" key corresponds to the passage content ('"' + title + '"\n' + text).
+For example:
+```
+{"id": "0", "contents": "Evan Morris Evan L. Morris (January 26, 1977 \u2013 July 9, 2015) was a lobbyist for Genentech and its parent corporation Roche in Washington."}
+...
+{"id": "100", "contents": "Three years later, when the United States Exploring Expedition to little-known portions of the globe was organised under Charles Wilkes, Hale was recommended, while yet an undergraduate."}
+...
+```
+
+**Index your corpora (optional).**
+If you would like to use a local retriever as the search engine, you can index your own corpus by:
+```
+bash search_r1/search/build_index.sh
+```
+You can change ```retriever_name``` and ```retriever_model``` to your interested off-the-shelf retriever.
+
+## Use your own search engine
+
+Our codebase supports local sparse retriever (e.g., BM25), local dense retriever (both flat indexing with GPUs and ANN indexing with CPUs) and online search engine (e.g., Google, Bing, etc). More details can be found [here](https://github.com/PeterGriffinJin/Search-R1/tree/main/docs/retriever.md).
+
+The main philosophy is to launch a local or remote search engine server separately from the main RL training pipeline. 
+
+The LLM can call the search engine by calling the search API (e.g., "http://127.0.0.1:8000/retrieve").
+
+You can refer to ```search_r1/search/retriever_server.py``` for an example of launching a local retriever server.
+
+## Features
+- Support local sparse retrievers (e.g., BM25). ✔️
+- Support local dense retrievers (both flat indexing and ANN indexing) ✔️
+- Support google search / bing search / brave search API and others. ✔️
+- Support off-the-shelf neural rerankers. ✔️
+- Support different RL methods (e.g., PPO, GRPO, reinforce). ✔️
+- Support different LLMs (e.g., llama3, Qwen2.5, etc). ✔️
+
+## Acknowledge
+
+The concept of Search-R1 is inspired by [Deepseek-R1](https://github.com/deepseek-ai/DeepSeek-R1) and [TinyZero](https://github.com/Jiayi-Pan/TinyZero/tree/main).
+Its implementation is built upon [veRL](https://github.com/volcengine/verl) and [RAGEN](https://github.com/ZihanWang314/RAGEN/tree/main). 
+We sincerely appreciate the efforts of these teams for their contributions to open-source research and development.
+
+## Awesome work powered or inspired by Search-R1
+
+- [DeepResearcher](https://github.com/GAIR-NLP/DeepResearcher): Scaling Deep Research via Reinforcement Learning in Real-world Environments. [![[code]](https://img.shields.io/github/stars/GAIR-NLP/DeepResearcher)](https://github.com/GAIR-NLP/DeepResearcher)
+- [Multimodal-Search-R1](https://github.com/EvolvingLMMs-Lab/multimodal-search-r1): Incentivizing LMMs to Search. [![[code]](https://img.shields.io/github/stars/EvolvingLMMs-Lab/multimodal-search-r1)](https://github.com/EvolvingLMMs-Lab/multimodal-search-r1)
+- [OTC](https://arxiv.org/pdf/2504.14870): Optimal Tool Calls via Reinforcement Learning.
+- [ZeroSearch](https://github.com/Alibaba-NLP/ZeroSearch): Incentivize the Search Capability of LLMs without Searching. [![[code]](https://img.shields.io/github/stars/Alibaba-NLP/ZeroSearch)](https://github.com/Alibaba-NLP/ZeroSearch)
+- [IKEA](https://github.com/hzy312/knowledge-r1): Reinforced Internal-External Knowledge Synergistic Reasoning for Efficient Adaptive Search Agent. [![[code]](https://img.shields.io/github/stars/hzy312/knowledge-r1)](https://github.com/hzy312/knowledge-r1)
+- [Scent of Knowledge](https://arxiv.org/abs/2505.09316): Optimizing Search-Enhanced Reasoning with Information Foraging.
+- [AutoRefine](https://www.arxiv.org/pdf/2505.11277): Search and Refine During Think. [![[code]](https://img.shields.io/github/stars/syr-cn/AutoRefine)](https://github.com/syr-cn/AutoRefine)
+- [O^2-Searcher](https://arxiv.org/pdf/2505.16582): A Searching-based Agent Model for Open-Domain Open-Ended Question Answering. [![[code]](https://img.shields.io/github/stars/Acade-Mate/O2-Searcher)](https://github.com/Acade-Mate/O2-Searcher)
+- [MaskSearch](https://arxiv.org/pdf/2505.20285): A Universal Pre-Training Framework to Enhance Agentic Search Capability. [![[code]](https://img.shields.io/github/stars/Alibaba-NLP/MaskSearch)](https://github.com/Alibaba-NLP/MaskSearch)
+- [VRAG-RL](https://arxiv.org/abs/2505.22019): Vision-Perception-Based RAG for Visually Rich Information Understanding. [![[code]](https://img.shields.io/github/stars/Alibaba-NLP/VRAG)](https://github.com/Alibaba-NLP/VRAG)
+- [R1-Code-Interpreter](https://arxiv.org/abs/2505.21668): Training LLMs to Reason with Code via SFT and RL. [![[code]](https://img.shields.io/github/stars/yongchao98/R1-Code-Interpreter)](https://github.com/yongchao98/R1-Code-Interpreter)
+- [R-Search](https://arxiv.org/abs/2506.04185): Empowering LLM Reasoning with Search via Multi-Reward Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/QingFei1/R-Search)](https://github.com/QingFei1/R-Search)
+- [StepSearch](https://arxiv.org/pdf/2505.15107): Igniting LLMs Search Ability via Step-Wise Proximal Policy Optimization. [![[code]](https://img.shields.io/github/stars/Zillwang/StepSearch)](https://github.com/Zillwang/StepSearch)
+- [SimpleTIR](https://simpletir.notion.site/report): Stable End-to-End Reinforcement Learning for Multi-Turn Tool-Integrated Reasoning. [![[code]](https://img.shields.io/github/stars/ltzheng/SimpleTIR)](https://github.com/ltzheng/SimpleTIR)
+- [Router-R1](https://arxiv.org/pdf/2506.09033): Teaching LLMs Multi-Round Routing and Aggregation via Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/ulab-uiuc/Router-R1)](https://github.com/ulab-uiuc/Router-R1)
+- [SkyRL](https://skyrl.readthedocs.io/en/latest/): A Modular Full-stack RL Library for LLMs. [![[code]](https://img.shields.io/github/stars/NovaSky-AI/SkyRL)](https://github.com/NovaSky-AI/SkyRL)
+- [ASearcher](https://arxiv.org/abs/2508.07976): Large-Scale RL for Search Agents. [![[code]](https://img.shields.io/github/stars/inclusionAI/ASearcher)](https://github.com/inclusionAI/ASearcher)
+- [ParallelSearch](https://www.arxiv.org/abs/2508.09303): Decompose Query and Search Sub-queries in Parallel with RL. [![[code]](https://img.shields.io/github/stars/Tree-Shu-Zhao/ParallelSearch)](https://github.com/Tree-Shu-Zhao/ParallelSearch)
+- [AutoTIR](https://arxiv.org/pdf/2507.21836): Autonomous Tools Integrated Reasoning via Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/weiyifan1023/AutoTIR)](https://github.com/weiyifan1023/AutoTIR)
+- [verl-tool](https://arxiv.org/pdf/2509.01055): A version of verl to support diverse tool use. [![[code]](https://img.shields.io/github/stars/TIGER-AI-Lab/verl-tool)](https://github.com/TIGER-AI-Lab/verl-tool)
+- [Tree-GRPO](https://arxiv.org/abs/2509.21240): Tree Search for LLM Agent Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/AMAP-ML/Tree-GRPO)](https://github.com/AMAP-ML/Tree-GRPO)
+- [EviNote-RAG](https://arxiv.org/abs/2509.00877): Enhancing RAG Models via Answer-Supportive Evidence Notes. [![[code]](https://img.shields.io/github/stars/Da1yuqin/EviNoteRAG)](https://github.com/Da1yuqin/EviNoteRAG)
+- [GlobalRAG](https://arxiv.org/pdf/2510.20548v1): GlobalRAG: Enhancing Global Reasoning in Multi-hop Question Answering via Reinforcement Learning. [![[code]](https://img.shields.io/github/stars/CarnegieBin/GlobalRAG)](https://github.com/CarnegieBin/GlobalRAG)
+
+
+
+
+
+## Citations
+
+```bibtex
+@article{jin2025search,
+  title={Search-r1: Training llms to reason and leverage search engines with reinforcement learning},
+  author={Jin, Bowen and Zeng, Hansi and Yue, Zhenrui and Yoon, Jinsung and Arik, Sercan and Wang, Dong and Zamani, Hamed and Han, Jiawei},
+  journal={arXiv preprint arXiv:2503.09516},
+  year={2025}
+}
+```
+
+```bibtex
+@article{jin2025empirical,
+  title={An Empirical Study on Reinforcement Learning for Reasoning-Search Interleaved LLM Agents},
+  author={Jin, Bowen and Yoon, Jinsung and Kargupta, Priyanka and Arik, Sercan O and Han, Jiawei},
+  journal={arXiv preprint arXiv:2505.15117},
+  year={2025}
+}
+```

code/RL_model/verl/Search-R1/VERL_README.md

@@ -0,0 +1,103 @@
+<h1 style="text-align: center;">veRL: Volcano Engine Reinforcement Learning for LLM</h1>
+
+veRL is a flexible, efficient and production-ready RL training framework designed for large language models (LLMs). 
+
+veRL is the open-source version of **[HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2)** paper.
+
+veRL is flexible and easy to use with:
+
+- **Easy extension of diverse RL algorithms**: The Hybrid programming model combines the strengths of single-controller and multi-controller paradigms to enable flexible representation and efficient execution of complex Post-Training dataflows. Allowing users to build RL dataflows in a few lines of code.
+
+- **Seamless integration of existing LLM infra with modular APIs**: Decouples computation and data dependencies, enabling seamless integration with existing LLM frameworks, such as PyTorch FSDP, Megatron-LM and vLLM. Moreover, users can easily extend to other LLM training and inference frameworks.
+
+- **Flexible device mapping**: Supports various placement of models onto different sets of GPUs for efficient resource utilization and scalability across different cluster sizes.
+
+- Readily integration with popular HuggingFace models
+
+
+veRL is fast with:
+
+- **State-of-the-art throughput**: By seamlessly integrating existing SOTA LLM training and inference frameworks, veRL achieves high generation and training throughput.
+
+- **Efficient actor model resharding with 3D-HybridEngine**: Eliminates memory redundancy and significantly reduces communication overhead during transitions between training and generation phases.
+
+<p align="center">
+| <a href="https://verl.readthedocs.io/en/latest/index.html"><b>Documentation</b></a> | <a href="https://arxiv.org/abs/2409.19256v2"><b>Paper</b></a> | <a href="https://join.slack.com/t/verlgroup/shared_invite/zt-2w5p9o4c3-yy0x2Q56s_VlGLsJ93A6vA"><b>Slack</b></a> | <a href="https://raw.githubusercontent.com/eric-haibin-lin/verl-community/refs/heads/main/WeChat.JPG"><b>Wechat</b></a> | 
+
+<!-- <a href=""><b>Slides</b></a> | -->
+</p>
+
+## News
+
+- [2024/12] The team presented <a href="https://neurips.cc/Expo/Conferences/2024/workshop/100677">Post-training LLMs: From Algorithms to Infrastructure</a> at NeurIPS 2024. [Slides](https://github.com/eric-haibin-lin/verl-data/tree/neurips) and [video](https://neurips.cc/Expo/Conferences/2024/workshop/100677) available.
+- [2024/10] veRL is presented at Ray Summit. [Youtube video](https://www.youtube.com/watch?v=MrhMcXkXvJU&list=PLzTswPQNepXntmT8jr9WaNfqQ60QwW7-U&index=37) available.
+- [2024/08] HybridFlow (verl) is accepted to EuroSys 2025.
+
+## Key Features
+
+- **FSDP** and **Megatron-LM** for training.
+- **vLLM** and **TGI** for rollout generation, **SGLang** support coming soon.
+- huggingface models support
+- Supervised fine-tuning
+- Reward model training
+- Reinforcement learning from human feedback with PPO
+- flash-attention integration, sequence packing
+- scales up to 70B models and hundreds of GPUs
+- experiment tracking with wandb and mlflow
+
+
+## Getting Started
+
+Checkout this [Jupyter Notebook](https://github.com/volcengine/verl/tree/main/examples/ppo_trainer/verl_getting_started.ipynb) to get started with PPO training with a single 24GB L4 GPU (**FREE** GPU quota provided by [Lighting Studio](https://lightning.ai/hlin-verl/studios/verl-getting-started))!
+
+**Quickstart:**
+- [Installation](https://verl.readthedocs.io/en/latest/start/install.html)
+- [Quickstart](https://verl.readthedocs.io/en/latest/start/quickstart.html)
+
+**Running an PPO example step-by-step:**
+- Data and Reward Preparation
+  - [Prepare Data (Parquet) for Post-Training](https://verl.readthedocs.io/en/latest/preparation/prepare_data.html)
+  - [Implement Reward Function for Dataset](https://verl.readthedocs.io/en/latest/preparation/reward_function.html)
+- Understanding the PPO Example
+  - [PPO Example Architecture](https://verl.readthedocs.io/en/latest/examples/ppo_code_architecture.html)
+  - [Config Explanation](https://verl.readthedocs.io/en/latest/examples/config.html)
+  - [Run GSM8K Example](https://verl.readthedocs.io/en/latest/examples/gsm8k_example.html)
+
+**Reproducible algorithm baselines:**
+- [PPO](https://verl.readthedocs.io/en/latest/experiment/ppo.html)
+
+**For code explanation and advance usage (extension):**
+- PPO Trainer and Workers
+  - [PPO Ray Trainer](https://verl.readthedocs.io/en/latest/workers/ray_trainer.html)
+  - [PyTorch FSDP Backend](https://verl.readthedocs.io/en/latest/workers/fsdp_workers.html)
+  - [Megatron-LM Backend](https://verl.readthedocs.io/en/latest/index.html)
+- Advance Usage and Extension
+  - [Ray API Design Tutorial](https://verl.readthedocs.io/en/latest/advance/placement.html)
+  - [Extend to other RL(HF) algorithms](https://verl.readthedocs.io/en/latest/advance/dpo_extension.html)
+  - [Add models with the FSDP backend](https://verl.readthedocs.io/en/latest/advance/fsdp_extension.html)
+  - [Add models with the Megatron-LM backend](https://verl.readthedocs.io/en/latest/advance/megatron_extension.html)
+
+
+## Citation and acknowledgement
+
+If you find the project helpful, please cite:
+- [HybridFlow: A Flexible and Efficient RLHF Framework](https://arxiv.org/abs/2409.19256v2)
+- [A Framework for Training Large Language Models for Code Generation via Proximal Policy Optimization](https://i.cs.hku.hk/~cwu/papers/gmsheng-NL2Code24.pdf)
+
+```tex
+@article{sheng2024hybridflow,
+  title   = {HybridFlow: A Flexible and Efficient RLHF Framework},
+  author  = {Guangming Sheng and Chi Zhang and Zilingfeng Ye and Xibin Wu and Wang Zhang and Ru Zhang and Yanghua Peng and Haibin Lin and Chuan Wu},
+  year    = {2024},
+  journal = {arXiv preprint arXiv: 2409.19256}
+}
+```
+
+verl is inspired by the design of Nemo-Aligner, Deepspeed-chat and OpenRLHF. The project is adopted and supported by Anyscale, Bytedance, LMSys.org, Shanghai AI Lab, Tsinghua University, UC Berkeley, UCLA, UIUC, and University of Hong Kong.
+
+## Publications Using veRL
+- [Enhancing Multi-Step Reasoning Abilities of Language Models through Direct Q-Function Optimization](https://arxiv.org/abs/2410.09302)
+- [Flaming-hot Initiation with Regular Execution Sampling for Large Language Models](https://arxiv.org/abs/2410.21236)
+- [Process Reinforcement Through Implicit Rewards](https://github.com/PRIME-RL/PRIME/)
+
+We are HIRING! Send us an [email](mailto:haibin.lin@bytedance.com) if you are interested in internship/FTE opportunities in MLSys/LLM reasoning/multimodal alignment.

code/RL_model/verl/Search-R1/infer.py

@@ -0,0 +1,128 @@
+import transformers
+import torch
+import random
+from datasets import load_dataset
+import requests
+
+question = "Mike Barnett negotiated many contracts including which player that went on to become general manager of CSKA Moscow of the Kontinental Hockey League?"
+
+# Model ID and device setup
+model_id = "PeterJinGo/SearchR1-nq_hotpotqa_train-qwen2.5-7b-em-ppo"
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+question = question.strip()
+if question[-1] != '?':
+    question += '?'
+curr_eos = [151645, 151643] # for Qwen2.5 series models
+curr_search_template = '\n\n{output_text}<information>{search_results}</information>\n\n'
+
+# Prepare the message
+prompt = f"""Answer the given question. \
+You must conduct reasoning inside <think> and </think> first every time you get new information. \
+After reasoning, if you find you lack some knowledge, you can call a search engine by <search> query </search> and it will return the top searched results between <information> and </information>. \
+You can search as many times as your want. \
+If you find no further external knowledge needed, you can directly provide the answer inside <answer> and </answer>, without detailed illustrations. For example, <answer> Beijing </answer>. Question: {question}\n"""
+
+# Initialize the tokenizer and model
+tokenizer = transformers.AutoTokenizer.from_pretrained(model_id)
+model = transformers.AutoModelForCausalLM.from_pretrained(model_id, torch_dtype=torch.bfloat16, device_map="auto")
+
+# Define the custom stopping criterion
+class StopOnSequence(transformers.StoppingCriteria):
+    def __init__(self, target_sequences, tokenizer):
+        # Encode the string so we have the exact token-IDs pattern
+        self.target_ids = [tokenizer.encode(target_sequence, add_special_tokens=False) for target_sequence in target_sequences]
+        self.target_lengths = [len(target_id) for target_id in self.target_ids]
+        self._tokenizer = tokenizer
+
+    def __call__(self, input_ids, scores, **kwargs):
+        # Make sure the target IDs are on the same device
+        targets = [torch.as_tensor(target_id, device=input_ids.device) for target_id in self.target_ids]
+
+        if input_ids.shape[1] < min(self.target_lengths):
+            return False
+
+        # Compare the tail of input_ids with our target_ids
+        for i, target in enumerate(targets):
+            if torch.equal(input_ids[0, -self.target_lengths[i]:], target):
+                return True
+
+        return False
+
+def get_query(text):
+    import re
+    pattern = re.compile(r"<search>(.*?)</search>", re.DOTALL)
+    matches = pattern.findall(text)
+    if matches:
+        return matches[-1]
+    else:
+        return None
+
+def search(query: str):
+    payload = {
+            "queries": [query],
+            "topk": 3,
+            "return_scores": True
+        }
+    results = requests.post("http://127.0.0.1:8000/retrieve", json=payload).json()['result']
+                
+    def _passages2string(retrieval_result):
+        format_reference = ''
+        for idx, doc_item in enumerate(retrieval_result):
+                        
+            content = doc_item['document']['contents']
+            title = content.split("\n")[0]
+            text = "\n".join(content.split("\n")[1:])
+            format_reference += f"Doc {idx+1}(Title: {title}) {text}\n"
+        return format_reference
+
+    return _passages2string(results[0])
+
+
+# Initialize the stopping criteria
+target_sequences = ["</search>", " </search>", "</search>\n", " </search>\n", "</search>\n\n", " </search>\n\n"]
+stopping_criteria = transformers.StoppingCriteriaList([StopOnSequence(target_sequences, tokenizer)])
+
+cnt = 0
+
+if tokenizer.chat_template:
+    prompt = tokenizer.apply_chat_template([{"role": "user", "content": prompt}], add_generation_prompt=True, tokenize=False)
+
+print('\n\n################# [Start Reasoning + Searching] ##################\n\n')
+print(prompt)
+# Encode the chat-formatted prompt and move it to the correct device
+while True:
+    input_ids = tokenizer.encode(prompt, return_tensors='pt').to(device)
+    attention_mask = torch.ones_like(input_ids)
+    
+    # Generate text with the stopping criteria
+    outputs = model.generate(
+        input_ids,
+        attention_mask=attention_mask,
+        max_new_tokens=1024,
+        stopping_criteria=stopping_criteria,
+        pad_token_id=tokenizer.eos_token_id,
+        do_sample=True,
+        temperature=0.7
+    )
+
+    if outputs[0][-1].item() in curr_eos:
+        generated_tokens = outputs[0][input_ids.shape[1]:]
+        output_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
+        print(output_text)
+        break
+
+    generated_tokens = outputs[0][input_ids.shape[1]:]
+    output_text = tokenizer.decode(generated_tokens, skip_special_tokens=True)
+    
+    tmp_query = get_query(tokenizer.decode(outputs[0], skip_special_tokens=True))
+    if tmp_query:
+        # print(f'searching "{tmp_query}"...')
+        search_results = search(tmp_query)
+    else:
+        search_results = ''
+
+    search_text = curr_search_template.format(output_text=output_text, search_results=search_results)
+    prompt += search_text
+    cnt += 1
+    print(search_text)

code/RL_model/verl/Search-R1/llm_guard_3B_10k_v2.log

@@ -0,0 +1,180 @@
+/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
+  import pynvml  # type: ignore[import]
+2026-02-01 20:43:15,317	INFO worker.py:2014 -- Started a local Ray instance. View the dashboard at http://127.0.0.1:8301 
+/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/ray/_private/worker.py:2062: FutureWarning: Tip: In future versions of Ray, Ray will no longer override accelerator visible devices env var if num_gpus=0 or num_gpus=None (default). To enable this behavior and turn off this error message, set RAY_ACCEL_ENV_VAR_OVERRIDE_ON_ZERO=0
+  warnings.warn(
+(pid=1646422) /home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/torch/cuda/__init__.py:63: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you.
+(pid=1646422)   import pynvml  # type: ignore[import]
+(main_task pid=1646422) {'actor_rollout_ref': {'actor': {'clip_ratio': 0.2,
+(main_task pid=1646422)                                  'entropy_coeff': 0.001,
+(main_task pid=1646422)                                  'fsdp_config': {'fsdp_size': -1,
+(main_task pid=1646422)                                                  'grad_offload': False,
+(main_task pid=1646422)                                                  'optimizer_offload': True,
+(main_task pid=1646422)                                                  'param_offload': True,
+(main_task pid=1646422)                                                  'wrap_policy': {'min_num_params': 0}},
+(main_task pid=1646422)                                  'grad_clip': 1.0,
+(main_task pid=1646422)                                  'kl_loss_coef': 0.001,
+(main_task pid=1646422)                                  'kl_loss_type': 'low_var_kl',
+(main_task pid=1646422)                                  'optim': {'lr': 1e-06,
+(main_task pid=1646422)                                            'lr_warmup_steps_ratio': 0.0,
+(main_task pid=1646422)                                            'min_lr_ratio': None,
+(main_task pid=1646422)                                            'total_training_steps': -1,
+(main_task pid=1646422)                                            'warmup_style': 'constant'},
+(main_task pid=1646422)                                  'ppo_epochs': 1,
+(main_task pid=1646422)                                  'ppo_max_token_len_per_gpu': 16384,
+(main_task pid=1646422)                                  'ppo_micro_batch_size': 64,
+(main_task pid=1646422)                                  'ppo_micro_batch_size_per_gpu': 16,
+(main_task pid=1646422)                                  'ppo_mini_batch_size': 64,
+(main_task pid=1646422)                                  'shuffle': False,
+(main_task pid=1646422)                                  'state_masking': False,
+(main_task pid=1646422)                                  'strategy': 'fsdp',
+(main_task pid=1646422)                                  'ulysses_sequence_parallel_size': 1,
+(main_task pid=1646422)                                  'use_dynamic_bsz': False,
+(main_task pid=1646422)                                  'use_kl_loss': False},
+(main_task pid=1646422)                        'hybrid_engine': True,
+(main_task pid=1646422)                        'model': {'enable_gradient_checkpointing': True,
+(main_task pid=1646422)                                  'external_lib': None,
+(main_task pid=1646422)                                  'override_config': {},
+(main_task pid=1646422)                                  'path': 'Qwen/Qwen3-4B-Instruct-2507',
+(main_task pid=1646422)                                  'use_remove_padding': False},
+(main_task pid=1646422)                        'ref': {'fsdp_config': {'fsdp_size': -1,
+(main_task pid=1646422)                                                'param_offload': True,
+(main_task pid=1646422)                                                'wrap_policy': {'min_num_params': 0}},
+(main_task pid=1646422)                                'log_prob_max_token_len_per_gpu': 16384,
+(main_task pid=1646422)                                'log_prob_micro_batch_size': 64,
+(main_task pid=1646422)                                'log_prob_use_dynamic_bsz': False,
+(main_task pid=1646422)                                'ulysses_sequence_parallel_size': 1},
+(main_task pid=1646422)                        'rollout': {'do_sample': True,
+(main_task pid=1646422)                                    'dtype': 'bfloat16',
+(main_task pid=1646422)                                    'enforce_eager': True,
+(main_task pid=1646422)                                    'free_cache_engine': True,
+(main_task pid=1646422)                                    'gpu_memory_utilization': 0.4,
+(main_task pid=1646422)                                    'ignore_eos': False,
+(main_task pid=1646422)                                    'load_format': 'dummy_dtensor',
+(main_task pid=1646422)                                    'log_prob_max_token_len_per_gpu': 16384,
+(main_task pid=1646422)                                    'log_prob_micro_batch_size': 64,
+(main_task pid=1646422)                                    'log_prob_use_dynamic_bsz': False,
+(main_task pid=1646422)                                    'max_num_batched_tokens': 8192,
+(main_task pid=1646422)                                    'max_num_seqs': 1024,
+(main_task pid=1646422)                                    'n': 1,
+(main_task pid=1646422)                                    'n_agent': 1,
+(main_task pid=1646422)                                    'name': 'vllm',
+(main_task pid=1646422)                                    'prompt_length': 4096,
+(main_task pid=1646422)                                    'response_length': 1024,
+(main_task pid=1646422)                                    'temperature': 1.0,
+(main_task pid=1646422)                                    'tensor_model_parallel_size': 1,
+(main_task pid=1646422)                                    'top_k': -1,
+(main_task pid=1646422)                                    'top_p': 0.95}},
+(main_task pid=1646422)  'algorithm': {'adv_estimator': 'grpo',
+(main_task pid=1646422)                'gamma': 1.0,
+(main_task pid=1646422)                'kl_ctrl': {'kl_coef': 0.001, 'type': 'fixed'},
+(main_task pid=1646422)                'kl_penalty': 'kl',
+(main_task pid=1646422)                'lam': 1.0,
+(main_task pid=1646422)                'no_think_rl': False,
+(main_task pid=1646422)                'state_masking': {'end_state_marker': '</information>',
+(main_task pid=1646422)                                  'start_state_marker': '<information>'}},
+(main_task pid=1646422)  'critic': {'cliprange_value': 0.5,
+(main_task pid=1646422)             'forward_max_token_len_per_gpu': 32768,
+(main_task pid=1646422)             'forward_micro_batch_size': 64,
+(main_task pid=1646422)             'grad_clip': 1.0,
+(main_task pid=1646422)             'model': {'enable_gradient_checkpointing': False,
+(main_task pid=1646422)                       'external_lib': None,
+(main_task pid=1646422)                       'fsdp_config': {'fsdp_size': -1,
+(main_task pid=1646422)                                       'grad_offload': False,
+(main_task pid=1646422)                                       'optimizer_offload': False,
+(main_task pid=1646422)                                       'param_offload': False,
+(main_task pid=1646422)                                       'wrap_policy': {'min_num_params': 0}},
+(main_task pid=1646422)                       'override_config': {},
+(main_task pid=1646422)                       'path': '~/models/deepseek-llm-7b-chat',
+(main_task pid=1646422)                       'tokenizer_path': 'Qwen/Qwen3-4B-Instruct-2507',
+(main_task pid=1646422)                       'use_remove_padding': False},
+(main_task pid=1646422)             'optim': {'lr': 1e-05,
+(main_task pid=1646422)                       'lr_warmup_steps_ratio': 0.0,
+(main_task pid=1646422)                       'min_lr_ratio': None,
+(main_task pid=1646422)                       'total_training_steps': -1,
+(main_task pid=1646422)                       'warmup_style': 'constant'},
+(main_task pid=1646422)             'ppo_epochs': 1,
+(main_task pid=1646422)             'ppo_max_token_len_per_gpu': 32768,
+(main_task pid=1646422)             'ppo_micro_batch_size': 64,
+(main_task pid=1646422)             'ppo_mini_batch_size': 64,
+(main_task pid=1646422)             'shuffle': False,
+(main_task pid=1646422)             'strategy': 'fsdp',
+(main_task pid=1646422)             'ulysses_sequence_parallel_size': 1,
+(main_task pid=1646422)             'use_dynamic_bsz': False},
+(main_task pid=1646422)  'data': {'max_obs_length': 512,
+(main_task pid=1646422)           'max_prompt_length': 4096,
+(main_task pid=1646422)           'max_response_length': 1024,
+(main_task pid=1646422)           'max_start_length': 256,
+(main_task pid=1646422)           'prompt_key': 'prompt',
+(main_task pid=1646422)           'return_raw_chat': False,
+(main_task pid=1646422)           'return_raw_input_ids': False,
+(main_task pid=1646422)           'shuffle_train_dataloader': True,
+(main_task pid=1646422)           'tokenizer': None,
+(main_task pid=1646422)           'train_batch_size': 128,
+(main_task pid=1646422)           'train_data_num': None,
+(main_task pid=1646422)           'train_files': '/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet',
+(main_task pid=1646422)           'val_batch_size': 64,
+(main_task pid=1646422)           'val_data_num': None,
+(main_task pid=1646422)           'val_files': '/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet'},
+(main_task pid=1646422)  'do_search': False,
+(main_task pid=1646422)  'max_turns': 1,
+(main_task pid=1646422)  'retriever': {'topk': 3, 'url': 'http://127.0.0.1:8000/retrieve'},
+(main_task pid=1646422)  'reward_model': {'enable': False,
+(main_task pid=1646422)                   'final_format_score': 0,
+(main_task pid=1646422)                   'forward_max_token_len_per_gpu': 32768,
+(main_task pid=1646422)                   'max_length': None,
+(main_task pid=1646422)                   'micro_batch_size': 64,
+(main_task pid=1646422)                   'model': {'external_lib': None,
+(main_task pid=1646422)                             'fsdp_config': {'min_num_params': 0,
+(main_task pid=1646422)                                             'param_offload': False},
+(main_task pid=1646422)                             'input_tokenizer': 'Qwen/Qwen3-4B-Instruct-2507',
+(main_task pid=1646422)                             'path': '~/models/FsfairX-LLaMA3-RM-v0.1',
+(main_task pid=1646422)                             'use_remove_padding': False},
+(main_task pid=1646422)                   'retrieval_score': 0,
+(main_task pid=1646422)                   'strategy': 'fsdp',
+(main_task pid=1646422)                   'structure_format_score': 0,
+(main_task pid=1646422)                   'ulysses_sequence_parallel_size': 1,
+(main_task pid=1646422)                   'use_dynamic_bsz': False},
+(main_task pid=1646422)  'trainer': {'critic_warmup': 0,
+(main_task pid=1646422)              'default_hdfs_dir': '~/experiments/gsm8k/ppo/llm_guard_3B_10k_v2',
+(main_task pid=1646422)              'default_local_dir': 'verl_checkpoints/llm_guard_3B_10k_v2',
+(main_task pid=1646422)              'experiment_name': 'llm_guard_3B_10k_v2',
+(main_task pid=1646422)              'logger': ['wandb'],
+(main_task pid=1646422)              'n_gpus_per_node': 2,
+(main_task pid=1646422)              'nnodes': 1,
+(main_task pid=1646422)              'project_name': '',
+(main_task pid=1646422)              'save_freq': 100,
+(main_task pid=1646422)              'test_freq': 50,
+(main_task pid=1646422)              'total_epochs': 15,
+(main_task pid=1646422)              'total_training_steps': 1005}}
+(main_task pid=1646422) W0201 20:43:46.380000 1646422 /data/home_beta/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/torch/utils/cpp_extension.py:117] No CUDA runtime is found, using CUDA_HOME='/usr/local/cuda'
+Error executing job with overrides: ['data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet', 'data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet', 'data.train_batch_size=128', 'data.val_batch_size=64', 'data.max_prompt_length=4096', 'data.max_response_length=1024', 'data.shuffle_train_dataloader=True', 'algorithm.adv_estimator=grpo', 'actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507', 'actor_rollout_ref.model.enable_gradient_checkpointing=true', 'actor_rollout_ref.model.use_remove_padding=False', 'actor_rollout_ref.actor.optim.lr=1e-6', 'actor_rollout_ref.actor.ppo_mini_batch_size=64', '+actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16', 'actor_rollout_ref.actor.fsdp_config.param_offload=true', 'actor_rollout_ref.actor.fsdp_config.optimizer_offload=true', 'actor_rollout_ref.rollout.log_prob_micro_batch_size=64', 'actor_rollout_ref.rollout.tensor_model_parallel_size=1', 'actor_rollout_ref.rollout.name=vllm', 'actor_rollout_ref.rollout.gpu_memory_utilization=0.4', 'actor_rollout_ref.ref.log_prob_micro_batch_size=64', 'actor_rollout_ref.ref.fsdp_config.param_offload=True', 'actor_rollout_ref.actor.kl_loss_coef=0.001', 'trainer.logger=[wandb]', 'trainer.n_gpus_per_node=2', 'trainer.nnodes=1', 'trainer.save_freq=100', 'trainer.test_freq=50', 'trainer.project_name=', 'trainer.experiment_name=llm_guard_3B_10k_v2', 'trainer.total_epochs=15', 'trainer.total_training_steps=1005', 'trainer.default_local_dir=verl_checkpoints/llm_guard_3B_10k_v2', 'do_search=false', 'max_turns=1']
+Traceback (most recent call last):
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/main_ppo.py", line 110, in main
+    ray.get(main_task.remote(config))
+  File "/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/ray/_private/auto_init_hook.py", line 22, in auto_init_wrapper
+    return fn(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^
+  File "/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/ray/_private/client_mode_hook.py", line 104, in wrapper
+    return func(*args, **kwargs)
+           ^^^^^^^^^^^^^^^^^^^^^
+  File "/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/ray/_private/worker.py", line 2972, in get
+    values, debugger_breakpoint = worker.get_objects(
+                                  ^^^^^^^^^^^^^^^^^^^
+  File "/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/ray/_private/worker.py", line 1031, in get_objects
+    raise value.as_instanceof_cause()
+ray.exceptions.RayTaskError(ImportError): ray::main_task() (pid=1646422, ip=172.16.34.29)
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/trainer/main_ppo.py", line 136, in main_task
+    from verl.workers.fsdp_workers import ActorRolloutRefWorker, CriticWorker
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/workers/fsdp_workers.py", line 39, in <module>
+    from verl.workers.sharding_manager.fsdp_ulysses import FSDPUlyssesShardingManager
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/workers/sharding_manager/__init__.py", line 23, in <module>
+    from .megatron_vllm import AllGatherPPModel, MegatronVLLMShardingManager
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/workers/sharding_manager/megatron_vllm.py", line 230, in <module>
+    from verl.third_party.vllm import parallel_state as vllm_ps
+  File "/data/home_beta/mshahidul/readctrl/code/RL_model/verl/Search-R1/verl/third_party/vllm/__init__.py", line 52, in <module>
+    from vllm import LLM, LLMEngine, parallel_state
+ImportError: cannot import name 'parallel_state' from 'vllm' (/home/mshahidul/miniconda3/envs/verl/lib/python3.12/site-packages/vllm/__init__.py)
+
+Set the environment variable HYDRA_FULL_ERROR=1 for a complete stack trace.
+[W201 20:43:50.186179538 AllocatorConfig.cpp:28] Warning: PYTORCH_CUDA_ALLOC_CONF is deprecated, use PYTORCH_ALLOC_CONF instead (function operator())

code/RL_model/verl/Search-R1/pyproject.toml

@@ -0,0 +1,78 @@
+# -------------------------------
+# build-system
+# -------------------------------
+[build-system]
+requires = [
+    "setuptools>=61.0",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"
+
+# -------------------------------
+# project (PEP 621 metadata)
+# -------------------------------
+[project]
+name = "verl"
+# We'll mark the version as "dynamic" because it's read from the file "verl/version/version" 
+# (PEP 621 calls this "dynamic version"). 
+# The actual version is specified in the [tool.setuptools.dynamic] section below.
+dynamic = ["version"]
+
+description = "veRL: Volcano Engine Reinforcement Learning for LLM"
+license = {file = "LICENSE"}  # or "Apache-2.0", if you prefer an SPDX identifier
+readme = {file = "README.md", content-type = "text/markdown"}
+requires-python = ">=3.8"
+
+authors = [
+  { name = "Bytedance - Seed - MLSys", email = "zhangchi.usc1992@bytedance.com" },
+  { name = "Bytedance - Seed - MLSys", email = "gmsheng@connect.hku.hk" },
+]
+
+# Dependencies corresponding to install_requires in setup.py
+dependencies = [
+    "accelerate",
+    "codetiming",
+    "datasets",
+    "dill",
+    "hydra-core",
+    "numpy",
+    "pybind11",
+    "ray",
+    "tensordict",
+    "transformers<4.48",
+    "vllm<=0.6.3",
+]
+
+# Optional dependencies (extras_require in setup.py)
+[project.optional-dependencies]
+test = [
+  "pytest", "yapf"
+]
+
+# URLs
+[project.urls]
+Homepage = "https://github.com/volcengine/verl"
+
+# -------------------------------
+# tool.setuptools - Additional config
+# -------------------------------
+[tool.setuptools]
+# True means `setuptools` will attempt to include all relevant files in package_data automatically.
+# This corresponds to `include_package_data=True` in setup.py.
+include-package-data = true
+
+# We read the version from a file in 'verl/version/version'
+[tool.setuptools.dynamic]
+version = {file = "verl/version/version"}
+
+# If you need to mimic `package_dir={'': '.'}`:
+[tool.setuptools.package-dir]
+"" = "."
+
+# If you need to include specific non-Python data (like YAML files or version file):
+# This is the rough equivalent of package_data={'': ['version/*'], 'verl': ['trainer/config/*.yaml']}
+[tool.setuptools.package-data]
+verl = [
+  "version/*",
+  "trainer/config/*.yaml"
+]

code/RL_model/verl/Search-R1/requirements.txt

@@ -0,0 +1,16 @@
+accelerate
+codetiming
+datasets
+dill
+flash-attn
+hydra-core
+numpy
+pandas
+pybind11
+ray
+tensordict<0.6
+transformers<4.48
+vllm<=0.6.3
+wandb
+IPython
+matplotlib

code/RL_model/verl/Search-R1/retrieval_launch.sh

@@ -0,0 +1,13 @@
+
+file_path=/the/path/you/save/corpus
+index_file=$file_path/e5_Flat.index
+corpus_file=$file_path/wiki-18.jsonl
+retriever_name=e5
+retriever_path=intfloat/e5-base-v2
+
+python search_r1/search/retrieval_server.py --index_path $index_file \
+                                            --corpus_path $corpus_file \
+                                            --topk 3 \
+                                            --retriever_name $retriever_name \
+                                            --retriever_model $retriever_path \
+                                            --faiss_gpu

code/RL_model/verl/Search-R1/setup.py

@@ -0,0 +1,54 @@
+# Copyright 2024 Bytedance Ltd. and/or its affiliates
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# setup.py is the fallback installation script when pyproject.toml does not work
+from setuptools import setup, find_packages
+import os
+
+version_folder = os.path.dirname(os.path.join(os.path.abspath(__file__)))
+
+with open(os.path.join(version_folder, 'verl/version/version')) as f:
+    __version__ = f.read().strip()
+
+
+with open('requirements.txt') as f:
+    required = f.read().splitlines()
+    install_requires = [item.strip() for item in required if item.strip()[0] != '#']
+
+extras_require = {
+    'test': ['pytest', 'yapf']
+}
+
+from pathlib import Path
+this_directory = Path(__file__).parent
+long_description = (this_directory / "README.md").read_text()
+
+setup(
+    name='verl',
+    version=__version__,
+    package_dir={'': '.'},
+    packages=find_packages(where='.'),
+    url='https://github.com/volcengine/verl',
+    license='Apache 2.0',
+    author='Bytedance - Seed - MLSys',
+    author_email='zhangchi.usc1992@bytedance.com, gmsheng@connect.hku.hk',
+    description='veRL: Volcano Engine Reinforcement Learning for LLM',
+    install_requires=install_requires,
+    extras_require=extras_require,
+    package_data={'': ['version/*'],
+                  'verl': ['trainer/config/*.yaml'],},
+    include_package_data=True,
+    long_description=long_description,
+    long_description_content_type='text/markdown'
+)

code/RL_model/verl/Search-R1/train_grpo.sh

@@ -0,0 +1,46 @@
+
+export PYTORCH_CUDA_ALLOC_CONF=""
+export EXPERIMENT_NAME=llm_guard_3B_10k_v2
+export WAND_PROJECT='guard'
+export CUDA_DEVICE_ORDER="PCI_BUS_ID"
+export CUDA_VISIBLE_DEVICES=1,2
+export VLLM_ATTENTION_BACKEND=FLASH_ATTN
+
+
+PYTHONUNBUFFERED=1 NCCL_P2P_DISABLE=1 NCCL_IB_DISABLE=1 python3 -m verl.trainer.main_ppo \
+    data.train_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/train.parquet \
+    data.val_files=/home/mshahidul/readctrl/code/RL_model/verl/Search-R1/dataset/test.parquet \
+    data.train_batch_size=64 \
+    data.val_batch_size=64 \
+    data.max_prompt_length=4096 \
+    data.max_response_length=1024 \
+    data.shuffle_train_dataloader=True \
+    algorithm.adv_estimator=grpo \
+    actor_rollout_ref.model.path=Qwen/Qwen3-4B-Instruct-2507 \
+    actor_rollout_ref.model.enable_gradient_checkpointing=true \
+    actor_rollout_ref.model.use_remove_padding=False \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=64 \
+    +actor_rollout_ref.actor.ppo_micro_batch_size_per_gpu=16 \
+    actor_rollout_ref.actor.fsdp_config.param_offload=true \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=true \
+    actor_rollout_ref.rollout.log_prob_micro_batch_size=64 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.4 \
+    actor_rollout_ref.ref.log_prob_micro_batch_size=64 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    actor_rollout_ref.actor.kl_loss_coef=0.001 \
+    trainer.logger=['wandb'] \
+    trainer.n_gpus_per_node=2 \
+    trainer.nnodes=1 \
+    trainer.save_freq=100 \
+    trainer.test_freq=50 \
+    trainer.project_name=$WANDB_PROJECT \
+    trainer.experiment_name=$EXPERIMENT_NAME \
+    trainer.total_epochs=15 \
+    trainer.total_training_steps=1005 \
+    trainer.default_local_dir=verl_checkpoints/$EXPERIMENT_NAME \
+    do_search=false \
+    max_turns=1 \
+    2>&1 | tee $EXPERIMENT_NAME.log

code/RL_model/verl/Search-R1/train_ppo.sh

@@ -0,0 +1,90 @@
+export CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+export DATA_DIR='data/nq_search'
+
+WAND_PROJECT='Search-R1'
+
+export BASE_MODEL='meta-llama/Llama-3.2-3B'
+export EXPERIMENT_NAME=nq-search-r1-ppo-llama3.2-3b-em
+# export BASE_MODEL='meta-llama/Llama-3.2-3B-Instruct'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-llama3.2-3b-it-em
+# export BASE_MODEL='meta-llama/Llama-3.1-8B'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-llama3.1-8b-em
+# export BASE_MODEL='meta-llama/Llama-3.1-8B-Instruct'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-llama3.1-8b-it-em
+
+# export BASE_MODEL='Qwen/Qwen2.5-3B'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-qwen2.5-3b-em
+# export BASE_MODEL='Qwen/Qwen2.5-3B-Instruct'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-qwen2.5-3b-it-em
+# export BASE_MODEL='Qwen/Qwen2.5-7B'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-qwen2.5-7b-em
+# export BASE_MODEL='Qwen/Qwen2.5-7B-Instruct'
+# export EXPERIMENT_NAME=nq-search-r1-ppo-qwen2.5-7b-it-em
+
+# set -x
+export VLLM_ATTENTION_BACKEND=XFORMERS # vllm + qwen2-7b with flash_attn has some issues
+
+# max_prompt_length = (config['training']['max_start_length'] + config['training']['max_response_length'] * (config['training']['max_turns'] - 1) + config['training']['max_obs_length'] * config['training']['max_turns'])
+
+PYTHONUNBUFFERED=1 python3 -m verl.trainer.main_ppo \
+    data.train_files=$DATA_DIR/train.parquet \
+    data.val_files=$DATA_DIR/test.parquet \
+    data.train_data_num=null \
+    data.val_data_num=null \
+    data.train_batch_size=512 \
+    data.val_batch_size=256 \
+    data.max_prompt_length=4096 \
+    data.max_response_length=500 \
+    data.max_start_length=2048 \
+    data.max_obs_length=500 \
+    data.shuffle_train_dataloader=True \
+    algorithm.adv_estimator=gae \
+    actor_rollout_ref.model.path=$BASE_MODEL \
+    actor_rollout_ref.actor.optim.lr=1e-6 \
+    actor_rollout_ref.model.enable_gradient_checkpointing=true \
+    actor_rollout_ref.model.use_remove_padding=True \
+    actor_rollout_ref.actor.optim.lr_warmup_steps_ratio=0.285 \
+    actor_rollout_ref.actor.ppo_mini_batch_size=256 \
+    actor_rollout_ref.actor.ppo_micro_batch_size=64 \
+    actor_rollout_ref.actor.fsdp_config.param_offload=true \
+    actor_rollout_ref.actor.fsdp_config.grad_offload=true \
+    actor_rollout_ref.actor.fsdp_config.optimizer_offload=true \
+    actor_rollout_ref.rollout.log_prob_micro_batch_size=128 \
+    actor_rollout_ref.rollout.tensor_model_parallel_size=1 \
+    actor_rollout_ref.rollout.name=vllm \
+    actor_rollout_ref.rollout.gpu_memory_utilization=0.6 \
+    actor_rollout_ref.ref.log_prob_micro_batch_size=128 \
+    actor_rollout_ref.ref.fsdp_config.param_offload=True \
+    actor_rollout_ref.rollout.n_agent=1 \
+    actor_rollout_ref.rollout.temperature=1 \
+    actor_rollout_ref.actor.state_masking=true \
+    critic.optim.lr=1e-5 \
+    critic.model.use_remove_padding=True \
+    critic.optim.lr_warmup_steps_ratio=0.015 \
+    critic.model.path=$BASE_MODEL \
+    critic.model.enable_gradient_checkpointing=true \
+    critic.ppo_micro_batch_size=8 \
+    critic.model.fsdp_config.param_offload=true \
+    critic.model.fsdp_config.grad_offload=true \
+    critic.model.fsdp_config.optimizer_offload=true \
+    algorithm.kl_ctrl.kl_coef=0.001 \
+    algorithm.no_think_rl=false \
+    trainer.critic_warmup=0 \
+    trainer.logger=['wandb'] \
+    +trainer.val_only=false \
+    +trainer.val_before_train=true \
+    trainer.default_hdfs_dir=null \
+    trainer.n_gpus_per_node=8 \
+    trainer.nnodes=1 \
+    trainer.save_freq=100 \
+    trainer.test_freq=50 \
+    trainer.project_name=$WAND_PROJECT \
+    trainer.experiment_name=$EXPERIMENT_NAME \
+    trainer.total_epochs=15 \
+    trainer.total_training_steps=1005 \
+    trainer.default_hdfs_dir=null \
+    trainer.default_local_dir=verl_checkpoints/$EXPERIMENT_NAME \
+    max_turns=2 \
+    retriever.url="http://127.0.0.1:8000/retrieve" \
+    retriever.topk=3 \
+    2>&1 | tee $EXPERIMENT_NAME.log

code/RL_model/verl/verl_train/.git-blame-ignore-revs

@@ -0,0 +1,13 @@
+# Local uasge: git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# [dev] feat: immigrate from yapf & pylint to ruff based on pre-commit
+# Changed 268 files, +10k/-9k lines. This is the biggest formatter change.
+b00f77d8559b48d57a33c0132a5ba1c81891a536
+
+# [ci] refactor: reduce ruff line-length from 300 to 120
+# Changed 238 files, +6k/-1k lines. Global formatting change.
+00a10a8ef389556f957a2f36132b2358fd6a109f
+
+# [Lint] fix: linting errors in all files
+# Changed 179 files, +1k/-3k lines. Global lint fix.
+8e5ad4688a13de81727c014a3c2e2fb26324bc20

code/RL_model/verl/verl_train/.gitignore

@@ -0,0 +1,130 @@
+**/*.pt
+**/checkpoints
+**/wget-log
+**/_build/
+**/*.ckpt
+**/outputs
+**/*.tar.gz
+**/playground
+**/wandb
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+dataset/*
+tensorflow/my_graph/*
+.idea/
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+# env/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+tmp/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*,cover
+.hypothesis/
+pytest.ini
+output.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# IPython Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# dotenv
+.env
+
+# virtualenv
+venv/
+.venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+
+# Rope project settings
+.ropeproject
+
+# vscode
+.vscode
+
+# Mac
+.DS_Store
+
+# vim
+*.swp
+
+# emacs
+*~
+
+# ckpt
+*.lock
+
+# data
+*.parquet
+
+
+# local logs
+logs
+log
+outputs
+.history

code/RL_model/verl/verl_train/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "recipe"]
+	path = recipe
+	url = https://github.com/verl-project/verl-recipe.git

code/RL_model/verl/verl_train/.pre-commit-config.yaml

@@ -0,0 +1,45 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: "v0.12.2"
+    hooks:
+      - id: ruff
+        args: ["--fix", "--show-fixes", "--output-format=full"]
+        exclude: ^.*\.(ipynb)$
+      - id: ruff-format
+
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: "v1.17.0"
+    hooks:
+      - id: mypy
+
+  - repo: local
+    hooks:
+      - id: autogen-trainer-cfg
+        name: Generate and verify verl/trainer/config/_generated_*.yaml
+        entry: scripts/generate_trainer_config.sh
+        language: script
+        pass_filenames: false
+
+  - repo: local
+    hooks:
+      - id: check-docstrings
+        name: Check doc string coverage
+        entry: python3 tests/special_sanity/check_docstrings.py
+        language: python
+        pass_filenames: false
+
+  - repo: local
+    hooks:
+      - id: check-license
+        name: Check license
+        entry: python3 tests/special_sanity/check_license.py --directories examples scripts tests verl setup.py
+        language: python
+        pass_filenames: false
+
+  - repo: local
+    hooks:
+      - id: compileall
+        name: Compile all python files
+        entry: sh -c 'PYTHONWARNINGS=error python3 -m compileall -q .'
+        language: python
+        pass_filenames: false

code/RL_model/verl/verl_train/.readthedocs.yaml

@@ -0,0 +1,19 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3.11"
+    rust: "1.70"
+
+sphinx:
+  configuration: docs/conf.py
+
+python:
+  install:
+    - requirements: docs/requirements-docs.txt
+    - method: pip
+      path: .

code/RL_model/verl/verl_train/CONTRIBUTING.md

@@ -0,0 +1,90 @@
+# Contributing to verl
+
+Thank you for considering a contribution to verl! We welcome contributions of any kind - bug fixes, enhancements, documentation improvements, or even just feedback. Whether you're an experienced developer or this is your first open-source project, your help is invaluable.
+
+Your support can take many forms:
+- Report issues or unexpected behaviors.
+- Suggest or implement new features.
+- Improve or expand documentation.
+- Review pull requests and assist other contributors.
+- Spread the word: share verl in blog posts, social media, or give the repo a ⭐.
+
+## Finding Issues to Contribute
+
+Looking for ways to dive in? Check out these issues:
+- [Good first issues](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22good%20first%20issue%22)
+- [Call for contribution](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3A%22call%20for%20contribution%22)
+Furthermore, you can learn the development plan and roadmap via [RFC](https://github.com/volcengine/verl/issues?q=is%3Aissue%20state%3Aopen%20label%3ARFC) and [Roadmap](https://github.com/volcengine/verl/issues?q=state%3Aopen%20label%3A%22roadmap%22).
+
+
+## Developing
+
+- **Python-only**: install verl via `pip install -e .[test,vllm]` or `pip install -e .[test,sglang]` and iterate quickly. For full dependency setup, check out the verl [installation doc](https://verl.readthedocs.io/en/latest/start/install.html).
+
+## Code Linting and Formatting
+
+We rely on pre-commit to keep our code consistent. To set it up:
+
+```bash
+pip install pre-commit
+pre-commit install
+# for staged changes
+pre-commit run
+# for all files in the repo
+pre-commit run --all-files
+# run a specific hook with pre-commit
+# pre-commit run --all-files --show-diff-on-failure --color=always <hood-id>
+pre-commit run --all-files --show-diff-on-failure --color=always ruff
+pre-commit run --all-files --show-diff-on-failure --color=always autogen-trainer-cfg
+```
+
+## Testing
+
+Our test suites run on GitHub Actions. Check these workflows for details:
+- [GPU unit tests](https://github.com/volcengine/verl/blob/main/.github/workflows/gpu_unit_tests.yml)
+- [CPU unit tests](https://github.com/volcengine/verl/blob/main/.github/workflows/cpu_unit_tests.yml)
+- [vLLM tests](https://github.com/volcengine/verl/blob/main/.github/workflows/vllm.yml)
+- [SGLang tests](https://github.com/volcengine/verl/blob/main/.github/workflows/sgl.yml)
+
+### Adding CI tests
+
+If possible, please add CI test(s) for your new feature:
+
+1. Find the most relevant workflow yml file, which usually corresponds to a `hydra` default config (e.g. `ppo_trainer`, `ppo_megatron_trainer`, `sft_trainer`, etc).
+2. Add related path patterns to the `paths` section if not already included.
+3. Minimize the workload of the test script(s) (see existing scripts for examples).
+
+## Building the Docs
+```
+# Ensure verl is on your PYTHONPATH, e.g.:
+pip install -e .[test]
+
+# Install documentation dependencies
+cd docs
+pip install -r requirements-docs.txt
+
+# Generate HTML docs
+make clean
+make html
+
+# Preview locally
+python -m http.server -d _build/html/
+```
+Open your browser at http://localhost:8000 to explore the docs.
+
+## Pull Requests & Code Reviews
+
+Thanks for submitting a PR! To streamline reviews:
+- Follow our Pull Request Template for title format and checklist.
+- Adhere to our pre-commit lint rules and ensure all checks pass.
+- Update docs for any user-facing changes.
+- Add or update tests in the CI workflows, or explain why tests aren't applicable.
+
+## License
+
+See the [LICENSE](https://github.com/volcengine/verl/blob/main/LICENSE) file for full details.
+
+## Thank You
+
+We appreciate your contributions to verl. Your efforts help make the project stronger and more user-friendly. Happy coding!
+