Upload train_qwen3_telecom.py with huggingface_hub

train_qwen3_telecom.py

@@ -0,0 +1,118 @@
+# /// script
+# dependencies = ["trl>=0.12.0", "peft>=0.7.0", "trackio", "datasets", "transformers", "accelerate", "bitsandbytes"]
+# ///
+
+from datasets import load_dataset
+from peft import LoraConfig
+from trl import SFTTrainer, SFTConfig
+import trackio
+
+# Load and preprocess the TeleQnA dataset
+print('Loading TeleQnA dataset...')
+raw_dataset = load_dataset('netop/TeleQnA', split='train')
+
+def format_for_sft(example):
+    """Convert TeleQnA format to chat messages format"""
+    # Build the question with options
+    options_text = []
+    for i in range(1, 6):  # Handle up to 5 options
+        option_key = f'option {i}'
+        if option_key in example and example[option_key]:
+            options_text.append(f'{i}. {example[option_key]}')
+
+    question_with_options = f"""{example['question']}
+
+Options:
+{chr(10).join(options_text)}"""
+
+    # Build the answer with explanation
+    answer_text = f"""{example['answer']}
+
+Explanation: {example['explanation']}"""
+
+    # Format as chat messages
+    return {
+        'messages': [
+            {'role': 'user', 'content': question_with_options},
+            {'role': 'assistant', 'content': answer_text}
+        ]
+    }
+
+print('Preprocessing dataset...')
+dataset = raw_dataset.map(format_for_sft, remove_columns=raw_dataset.column_names)
+
+# Create train/eval split (90/10)
+print('Creating train/eval split...')
+dataset_split = dataset.train_test_split(test_size=0.1, seed=42)
+
+print(f'Train examples: {len(dataset_split["train"])}')
+print(f'Eval examples: {len(dataset_split["test"])}')
+
+# Configure LoRA for efficient fine-tuning
+peft_config = LoraConfig(
+    r=16,
+    lora_alpha=32,
+    lora_dropout=0.05,
+    target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
+    bias="none",
+    task_type="CAUSAL_LM"
+)
+
+# Configure SFT training
+training_args = SFTConfig(
+    output_dir="qwen3-telecom-finetuned",
+
+    # Training hyperparameters
+    num_train_epochs=3,
+    per_device_train_batch_size=2,
+    per_device_eval_batch_size=2,
+    gradient_accumulation_steps=8,  # Effective batch size = 16
+
+    # Optimization
+    learning_rate=2e-4,
+    lr_scheduler_type="cosine",
+    warmup_ratio=0.1,
+
+    # Evaluation and saving
+    eval_strategy="steps",
+    eval_steps=100,
+    save_strategy="steps",
+    save_steps=200,
+    save_total_limit=3,
+
+    # Logging and monitoring
+    logging_steps=10,
+    report_to="trackio",
+    run_name="qwen3-0.6b-telecom-domain-adaptation",
+    project="telecom-finetuning",
+
+    # Memory optimization
+    gradient_checkpointing=True,
+    bf16=True,
+
+    # Hub integration
+    push_to_hub=True,
+    hub_model_id="wlabchoi/qwen3-0.6b-telecom",
+    hub_strategy="every_save",
+    hub_private_repo=False,
+)
+
+# Initialize trainer
+print('Initializing SFT trainer...')
+trainer = SFTTrainer(
+    model="Qwen/Qwen3-0.6B",
+    train_dataset=dataset_split["train"],
+    eval_dataset=dataset_split["test"],
+    peft_config=peft_config,
+    args=training_args,
+)
+
+# Start training
+print('Starting training...')
+trainer.train()
+
+# Push final model to Hub
+print('Pushing final model to Hub...')
+trainer.push_to_hub(commit_message="Training complete - Qwen3-0.6B fine-tuned on TeleQnA")
+
+print('Training completed successfully!')