no

app.py

@@ -13,7 +13,7 @@ import gradio as gr
 from datasets import Dataset
 from transformers import AutoTokenizer, AutoModelForCausalLM
 from peft import PeftModel, prepare_model_for_kbit_training, LoraConfig, get_peft_model
-from trl import DPOTrainer, DPOConfig
+from transformers import Trainer, TrainingArguments, DataCollatorForLanguageModeling
 import warnings
 import subprocess
 import gc
@@ -126,7 +126,7 @@ def format_prompt(query, title, content):
     if len(content) > 1000:
         content = content[:1000] + "..."
     
-    return f"""you would get a query and document's title and content and return yes (if the document is relevant to the query)/ or no (if the document is not relevant to the query)
+    return f"""You would get a query and document's title and content and return yes (if the document is relevant to the query) or no (if the document is not relevant to the query).
 Answer only yes / no.
 Document:
 ####DOCUMENT START
@@ -139,9 +139,7 @@ Query:
 {query}
 ####Query END
 
-ANSWER:
-####ANSWER START
-"""
+ANSWER: """
 
 
 def load_model_and_tokenizer(checkpoint_path=None, model_id=None):
@@ -240,7 +238,7 @@ def get_trained_models_list():
         text += f"{i}. **{model['repo']}**\n"
         text += f"   - Accuracy: {model['accuracy']:.2%}\n"
         text += f"   - Predictions: Yes {model['yes_ratio']:.1%}, No {model['no_ratio']:.1%}\n"
-        text += f"   - Beta: {model['beta']}, Model: {model['model_id'].split('/')[-1]}\n"
+        text += f"   - LR: {model.get('lr', 'N/A')}, Model: {model['model_id'].split('/')[-1]}\n"
         text += f"   - Link: https://huggingface.co/{model['repo']}\n\n"
     
     return text
@@ -312,9 +310,9 @@ def collate_fn(batch):
     }
 
 
-def prepare_dpo_dataset(df):
-    """Convert 4-category labels to DPO format with chosen/rejected pairs"""
-    dpo_data = []
+def prepare_finetuning_dataset(df):
+    """Convert 4-category labels to standard fine-tuning format"""
+    ft_data = []
     
     # Map 4 categories to yes/no
     label_mapping = {
@@ -347,28 +345,21 @@ def prepare_dpo_dataset(df):
         original_label = row['label']
         mapped_label = label_mapping.get(original_label, original_label)
         
-        if mapped_label == 'yes':
-            # For 'yes' examples, chosen is "yes" and rejected is "no"
-            dpo_data.append({
-                'prompt': prompt,
-                'chosen': 'yes',
-                'rejected': 'no',
-                'original_label': original_label  # Keep original for analysis
-            })
-        else:
-            # For 'no' examples, chosen is "no" and rejected is "yes"
-            dpo_data.append({
-                'prompt': prompt,
-                'chosen': 'no',
-                'rejected': 'yes',
-                'original_label': original_label  # Keep original for analysis
-            })
-    
-    return pd.DataFrame(dpo_data)
+        # Create the full text with prompt and answer
+        text = prompt + mapped_label
+        
+        ft_data.append({
+            'text': text,
+            'prompt': prompt,
+            'label': mapped_label,
+            'original_label': original_label  # Keep original for analysis
+        })
+    
+    return pd.DataFrame(ft_data)
 
 
-def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=None):
-    """Training with DPO (Direct Preference Optimization)"""
+def train_model(train_df, val_df, epochs=5, batch_size=32, lr=5e-6, max_samples=None):
+    """Standard fine-tuning for document relevance classification"""
     global current_model, current_tokenizer
     
     # Clear GPU memory before training
@@ -385,14 +376,14 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
         train_df = train_df.sample(n=max_samples, random_state=42)
         val_df = val_df.head(min(len(val_df), max_samples // 5))  # Proportional validation set
     
-    # Convert to DPO format
-    logger.info("Converting to DPO format...")
-    dpo_train_df = prepare_dpo_dataset(train_df)
-    dpo_val_df = prepare_dpo_dataset(val_df)
+    # Convert to fine-tuning format
+    logger.info("Preparing fine-tuning dataset...")
+    ft_train_df = prepare_finetuning_dataset(train_df)
+    ft_val_df = prepare_finetuning_dataset(val_df)
     
     # Create datasets
-    train_dataset = Dataset.from_pandas(dpo_train_df)
-    val_dataset = Dataset.from_pandas(dpo_val_df)
+    train_dataset = Dataset.from_pandas(ft_train_df)
+    val_dataset = Dataset.from_pandas(ft_val_df)
     
     # Prepare model for training
     if hasattr(current_model, 'is_loaded_in_4bit') and current_model.is_loaded_in_4bit:
@@ -428,20 +419,19 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
         target_modules=target_modules
     )
     
-    logger.info(f"Starting DPO training with {len(train_df)} train samples, {len(val_df)} val samples")
+    logger.info(f"Starting fine-tuning with {len(train_df)} train samples, {len(val_df)} val samples")
     logger.info(f"Learning rate: {lr}, Effective batch size: {batch_size}, Epochs: {epochs}")
 
     # Create output directory
     os.makedirs(OUTPUT_DIR, exist_ok=True)
     
-    # DPO training configuration optimized for A100
-    # GPT-OSS-20B uses ~16GB with native MXFP4 quantization
-    target_batch_size = 32  # Target effective batch size
+    # Training configuration optimized for standard fine-tuning
+    target_batch_size = batch_size  # Target effective batch size
     
     if current_model_id == "openai/gpt-oss-20b":
         # For GPT-OSS-20B: use smaller per-device batch with gradient accumulation
         actual_batch_size = 2  # Per-device batch size
-        seq_length = 256  # Shorter sequences to save memory
+        seq_length = 512  # Standard sequence length
         grad_accum = target_batch_size // actual_batch_size  # 16 gradient accumulation steps
     else:
         # For smaller models like Phi-3 - can use larger per-device batch
@@ -449,40 +439,76 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
         grad_accum = max(1, target_batch_size // actual_batch_size)  # Accumulate if needed
         seq_length = 512
     
-    training_args = DPOConfig(
+    # Tokenize the datasets
+    def tokenize_function(examples):
+        # Tokenize the full texts (prompt + answer)
+        model_inputs = current_tokenizer(
+            examples['text'],
+            truncation=True,
+            padding="max_length",
+            max_length=seq_length,
+            return_tensors=None
+        )
+        
+        # For causal LM, labels are the same as input_ids
+        model_inputs["labels"] = model_inputs["input_ids"].copy()
+        
+        # Store metadata for evaluation
+        model_inputs["original_labels"] = examples['original_label']
+        model_inputs["mapped_labels"] = examples['label']
+        
+        return model_inputs
+    
+    # Tokenize datasets
+    tokenized_train = train_dataset.map(tokenize_function, batched=True, remove_columns=train_dataset.column_names)
+    tokenized_val = val_dataset.map(tokenize_function, batched=True, remove_columns=val_dataset.column_names)
+    
+    # Standard training arguments
+    training_args = TrainingArguments(
         output_dir=OUTPUT_DIR,
         num_train_epochs=epochs,
         per_device_train_batch_size=actual_batch_size,
         per_device_eval_batch_size=actual_batch_size,
         gradient_accumulation_steps=grad_accum,
-        gradient_checkpointing=True,  # Still use for memory efficiency
+        gradient_checkpointing=True,
         learning_rate=lr,
         lr_scheduler_type="cosine",
-        warmup_steps=100,
+        warmup_steps=500,  # More warmup for standard fine-tuning
         logging_steps=10,
         save_strategy="epoch",
-        eval_strategy="epoch",
-        bf16=True,  # A100 supports bf16 efficiently
+        evaluation_strategy="epoch",
+        bf16=True,
         fp16=False,
-        remove_unused_columns=False,
-        run_name="dpo-relevance-a100-8bit",
+        weight_decay=0.01,
+        optim="adamw_torch",
+        save_total_limit=3,
+        load_best_model_at_end=True,
+        metric_for_best_model="eval_loss",
+        greater_is_better=False,
         report_to=[],
-        max_length=seq_length,
-        max_prompt_length=seq_length,
-        beta=1.0,  # Increased from 0.1 for stronger preference learning
-        optim="adamw_8bit" if current_model_id == "openai/gpt-oss-20b" else "adamw_torch",
-        dataloader_num_workers=2,  # A100 can handle parallel loading
+        run_name="standard-ft-relevance",
+        dataloader_num_workers=2,
+    )
+    
+    # Create data collator
+    data_collator = DataCollatorForLanguageModeling(
+        tokenizer=current_tokenizer,
+        mlm=False,  # Causal LM, not masked LM
+        pad_to_multiple_of=8
     )
     
-    # Create DPO trainer
-    dpo_trainer = DPOTrainer(
+    # Apply LoRA to the model
+    current_model = get_peft_model(current_model, peft_config)
+    current_model.print_trainable_parameters()
+    
+    # Create standard trainer
+    trainer = Trainer(
         model=current_model,
-        ref_model=None,  # Will use the model's initial state as reference
         args=training_args,
-        train_dataset=train_dataset,
-        eval_dataset=val_dataset,
-        processing_class=current_tokenizer,  # Changed from tokenizer
-        peft_config=peft_config,
+        train_dataset=tokenized_train,
+        eval_dataset=tokenized_val,
+        data_collator=data_collator,
+        tokenizer=current_tokenizer,
     )
     
     # Custom logging callback
@@ -501,11 +527,14 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
     
     def compute_accuracy_metrics(trainer, eval_dataset, num_samples=100):
         """Compute accuracy metrics and confusion matrix on a subset of eval data"""
+        # Get the original dataframe for easier access to prompts and labels
+        eval_df = ft_val_df
+        
         # Sample subset for faster evaluation
-        indices = np.random.choice(len(eval_dataset), min(num_samples, len(eval_dataset)), replace=False)
+        sample_size = min(num_samples, len(eval_df))
+        sample_df = eval_df.sample(n=sample_size, random_state=42)
         
         # Initialize confusion matrix counters
-        # Rows: true labels, Cols: predicted labels
         confusion_matrix = {
             'easy_positive': {'yes': 0, 'no': 0},
             'hard_positive': {'yes': 0, 'no': 0},
@@ -517,11 +546,10 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
         predictions_no = 0
         correct = 0
         
-        for idx in indices:
-            item = eval_dataset[int(idx)]
-            prompt = item['prompt']
-            true_choice = item['chosen']  # This is the correct answer (yes/no)
-            original_label = item.get('original_label', None)  # Get original 4-category label
+        for idx, row in sample_df.iterrows():
+            prompt = row['prompt']
+            true_label = row['label']  # This is the mapped label (yes/no)
+            original_label = row['original_label']  # Get original 4-category label
             
             # Tokenize and run inference
             inputs = current_tokenizer(prompt, return_tensors="pt", truncation=True, max_length=512)
@@ -546,7 +574,7 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
                 else:
                     predictions_no += 1
                 
-                if prediction == true_choice:
+                if prediction == true_label:
                     correct += 1
                 
                 # Update confusion matrix if we have original label
@@ -628,23 +656,23 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
                         training_status["progress"] = min(int((state.global_step / total_steps) * 100), 99)
     
     # Add callback with trainer and eval dataset
-    status_callback = StatusCallback(dpo_trainer, val_dataset)
-    dpo_trainer.add_callback(status_callback)
+    status_callback = StatusCallback(trainer, val_dataset)
+    trainer.add_callback(status_callback)
     
     # Train
     try:
-        logger.info("Starting DPO training...")
-        dpo_trainer.train()
+        logger.info("Starting fine-tuning...")
+        trainer.train()
         
         # Save final model
         save_path = os.path.join(OUTPUT_DIR, "final")
-        dpo_trainer.save_model(save_path)
+        trainer.save_model(save_path)
         current_tokenizer.save_pretrained(save_path)
         logger.info(f"Model saved to {save_path}")
         
         # Compute final metrics
         logger.info("Computing final accuracy metrics...")
-        final_metrics = compute_accuracy_metrics(dpo_trainer, val_dataset, num_samples=200)
+        final_metrics = compute_accuracy_metrics(trainer, val_dataset, num_samples=200)
         logger.info(f"Final Accuracy: {final_metrics['accuracy']:.2%}")
         logger.info(f"Final Prediction Distribution - Yes: {final_metrics['yes_ratio']:.1%}, No: {final_metrics['no_ratio']:.1%}")
         
@@ -666,7 +694,7 @@ def train_model(train_df, val_df, epochs=3, batch_size=4, lr=2e-5, max_samples=N
                         )
         
         # Update global model reference
-        current_model = dpo_trainer.model
+        current_model = trainer.model
         current_model.eval()
         
         # Push to hub if token available
@@ -710,11 +738,11 @@ model-index:
 
 # {model_short_name} Document Relevance Classifier
 
-This model was trained using DPO (Direct Preference Optimization) for document relevance classification.
+This model was trained using standard fine-tuning for document relevance classification.
 
 ## Training Configuration
 - Base Model: {current_model_id}
-- DPO Beta: {training_args.beta}
+- Training Type: Standard Fine-tuning
 - Learning Rate: {training_args.learning_rate}
 - Batch Size: {training_args.per_device_train_batch_size * training_args.gradient_accumulation_steps}
 - Epochs: {training_args.num_train_epochs}
@@ -747,7 +775,7 @@ model = PeftModel.from_pretrained(model, "{HF_USERNAME}/{repo_name.split('/')[-1
                 current_model.push_to_hub(
                     repo_name, 
                     use_auth_token=HF_TOKEN,
-                    commit_message=f"DPO training with beta={training_args.beta}, accuracy={final_metrics['accuracy']:.2%}"
+                    commit_message=f"Standard fine-tuning with lr={training_args.learning_rate}, accuracy={final_metrics['accuracy']:.2%}"
                 )
                 current_tokenizer.push_to_hub(repo_name, use_auth_token=HF_TOKEN)
                 
@@ -779,7 +807,7 @@ model = PeftModel.from_pretrained(model, "{HF_USERNAME}/{repo_name.split('/')[-1
                         "accuracy": final_metrics['accuracy'],
                         "yes_ratio": final_metrics['yes_ratio'],
                         "no_ratio": final_metrics['no_ratio'],
-                        "beta": training_args.beta,
+                        "lr": training_args.learning_rate,
                         "model_id": current_model_id
                     })
                     
@@ -892,7 +920,7 @@ def run_training(csv_path, shuffle_flag=False, split_ratio=0.8):
             max_samples = 2000  # Start conservative
         else:
             max_samples = None
-        train_model(train_df, test_df, epochs=3, batch_size=32, lr=2e-5, max_samples=max_samples)
+        train_model(train_df, test_df, epochs=5, batch_size=32, lr=5e-6, max_samples=max_samples)
         
         with training_lock:
             training_status["status"] = "completed"

inference_chatgpt_simple.py

@@ -49,7 +49,7 @@ def main():
     df = pd.read_csv(csv_path)
     # Process each row
     prds = [(str(row['query_text']),str(row['title']),str(row['text'])) for idx, row in df.iterrows()]
-    predictions = ThreadPool(1000).starmap(get_prediction,prds)
+    predictions = ThreadPool(100).starmap(get_prediction,prds)
 
     df['prediction'] = predictions
     conf_matrix = pd.crosstab(
@@ -69,5 +69,12 @@ def main():
     print("\nResults:")
     print(df['prediction'].value_counts())
 
+def make_sample_db():
+    df = pd.read_csv(rf"train_datasets_creation/full_train_dataset.csv")
+    dfs = [df[df['label']==d].sample(100) for d in df['label'].unique()]
+    df = pd.concat(dfs).reset_index()
+    df.to_csv(f"sample_db_{datetime.now().isoformat()}.csv")
+
+
 if __name__ == "__main__":
-    main()
+    make_sample_db()