Upload train.py with huggingface_hub

train.py

@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+RAYAP-CODER Training Script
+D1337 SOVEREIGN LABS - DO NOT EMBARRASS US
+"""
+
+import os
+import torch
+from datasets import load_dataset
+from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from trl import SFTTrainer, SFTConfig
+from huggingface_hub import login
+
+# ============================================================
+# CONFIG - Token from Space Secrets
+# ============================================================
+HF_TOKEN = os.environ.get("HF_TOKEN")
+if not HF_TOKEN:
+    raise ValueError("HF_TOKEN not set! Add it to Space Secrets.")
+BASE_MODEL = "huihui-ai/Qwen3-30B-A3B-abliterated"
+DATASET = "pacman1337/rayap-coder-dataset"
+OUTPUT = "pacman1337/rayap-coder-30b"
+
+# ============================================================
+# MAIN
+# ============================================================
+def main():
+    print("=" * 60)
+    print("RAYAP-CODER TRAINING")
+    print("D1337 SOVEREIGN LABS")
+    print("Palo Alto | CrowdStrike | SentinelOne | Trend Micro | d1337.ai")
+    print("=" * 60)
+    
+    # Login
+    login(token=HF_TOKEN)
+    
+    # Load dataset
+    print("\n[1/5] Loading dataset...")
+    dataset = load_dataset(DATASET, split="train")
+    print(f"Dataset: {len(dataset)} examples")
+    
+    # Quantization (4-bit for memory)
+    print("\n[2/5] Loading model (4-bit quantized)...")
+    bnb_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_compute_dtype=torch.bfloat16,
+        bnb_4bit_use_double_quant=True
+    )
+    
+    model = AutoModelForCausalLM.from_pretrained(
+        BASE_MODEL,
+        quantization_config=bnb_config,
+        device_map="auto",
+        trust_remote_code=True,
+        torch_dtype=torch.bfloat16,
+        attn_implementation="flash_attention_2"
+    )
+    
+    tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)
+    tokenizer.pad_token = tokenizer.eos_token
+    tokenizer.padding_side = "right"
+    
+    # Prepare for training
+    print("\n[3/5] Preparing LoRA...")
+    model = prepare_model_for_kbit_training(model)
+    
+    lora_config = LoraConfig(
+        r=64,
+        lora_alpha=128,
+        lora_dropout=0.05,
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj", "gate_proj", "up_proj", "down_proj"],
+        bias="none",
+        task_type="CAUSAL_LM"
+    )
+    
+    model = get_peft_model(model, lora_config)
+    model.print_trainable_parameters()
+    
+    # Training args
+    print("\n[4/5] Training...")
+    training_args = SFTConfig(
+        output_dir="./rayap-coder-checkpoints",
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=4,
+        num_train_epochs=3,
+        learning_rate=2e-4,
+        lr_scheduler_type="cosine",
+        warmup_ratio=0.1,
+        bf16=True,
+        gradient_checkpointing=True,
+        max_seq_length=4096,
+        logging_steps=5,
+        save_strategy="epoch",
+        optim="adamw_torch",
+        push_to_hub=True,
+        hub_model_id=OUTPUT,
+        hub_token=HF_TOKEN,
+        report_to="none"
+    )
+    
+    def format_chat(example):
+        return tokenizer.apply_chat_template(example["messages"], tokenize=False)
+    
+    trainer = SFTTrainer(
+        model=model,
+        train_dataset=dataset,
+        args=training_args,
+        formatting_func=format_chat,
+        tokenizer=tokenizer
+    )
+    
+    # TRAIN
+    trainer.train()
+    
+    # Push
+    print("\n[5/5] Pushing to Hub...")
+    trainer.save_model()
+    trainer.push_to_hub()
+    
+    print(f"""
+╔═══════════════════════════════════════════════════════════════╗
+║                    TRAINING COMPLETE!                         ║
+╠═══════════════════════════════════════════════════════════════╣
+║  Model: https://huggingface.co/{OUTPUT}
+║  
+║  D1337 SOVEREIGN LABS
+║  Palo Alto | CrowdStrike | SentinelOne | Trend Micro | d1337.ai
+║
+║  Update endpoint LORA_MODULES:
+║  rayap-coder=pacman1337/rayap-coder-30b
+╚═══════════════════════════════════════════════════════════════╝
+""")
+
+if __name__ == "__main__":
+    main()