Create pipeline.sh

pipeline.sh

@@ -0,0 +1,93 @@
+#!/bin/bash
+set -euo pipefail
+
+# =============================================================================
+# BENGALI-CODE LLM - DEV PIPELINE SCRIPT
+# =============================================================================
+# This script is designed to run in the resource-constrained Hugging Face Space.
+
+echo "🚀 Initializing Dev Pipeline..."
+
+# --- Configuration ---
+VOCAB_SIZE=16000 # Smaller vocab for faster dev run
+PROJECT_DIR="$(pwd)"
+
+# --- Create Directory Structure ---
+mkdir -p {data/{raw,processed},tokenizer,models,checkpoints,results,logs,scripts,configs}
+
+# --- 1. Data Collection (Sample Data) ---
+echo "📚 Step 1: Creating a small sample dataset..."
+cat > data/raw/sample_data.txt <<'EOF'
+আমার সোনার বাংলা, আমি তোমায় ভালোবাসি।
+The quick brown fox jumps over the lazy dog.
+def factorial(n):
+    # This function calculates the factorial of a number
+    if n == 0:
+        return 1
+    else:
+        return n * factorial(n-1)
+import math
+print(math.pi)
+EOF
+echo "✅ Sample dataset created."
+
+# --- 2. Preprocessing & Tokenizer Training ---
+echo "🧹 Step 2: Preprocessing data..."
+cat data/raw/*.txt > data/processed/combined.txt
+head -n 3 data/processed/combined.txt > data/processed/train.txt
+tail -n +4 data/processed/combined.txt > data/processed/validation.txt
+echo "✅ Data preprocessed."
+
+echo "🔤 Step 3: Training tokenizer..."
+python3 << EOF
+import sentencepiece as spm
+import os
+os.makedirs('tokenizer', exist_ok=True)
+spm.SentencePieceTrainer.train(
+    input='data/processed/train.txt',
+    model_prefix='tokenizer/bengali_code_dev',
+    vocab_size=${VOCAB_SIZE},
+    model_type='bpe',
+    pad_id=0, unk_id=1, bos_id=2, eos_id=3
+)
+EOF
+echo "✅ Tokenizer trained."
+
+# --- 3. Model Training (Tiny Dev Model) ---
+echo "🧠 Step 4: Configuring and Training Tiny Model..."
+cat > scripts/train_dev.py << 'EOF'
+import torch, argparse, sentencepiece as spm
+from transformers import AutoConfig, AutoModelForCausalLM, TrainingArguments, Trainer, DataCollatorForLanguageModeling
+from datasets import load_dataset
+
+class Tokenizer:
+    def __init__(self, path): self.sp = spm.SentencePieceProcessor(model_file=path)
+    def __call__(self, t, **k): return {'input_ids': self.sp.encode(t, out_type=int)}
+    def decode(self, ids, **k): return self.sp.decode(ids)
+    @property
+    def vocab_size(self): return self.sp.vocab_size()
+    @property
+    def pad_token_id(self): return self.sp.pad_id()
+
+tokenizer = Tokenizer(path="tokenizer/bengali_code_dev.model")
+dataset = load_dataset("text", data_files={"train": "data/processed/train.txt", "validation": "data/processed/validation.txt"})
+tokenized_ds = dataset.map(lambda e: tokenizer(e["text"]), remove_columns=["text"])
+
+config = AutoConfig.from_pretrained("gpt2", vocab_size=tokenizer.vocab_size, n_layer=2, n_head=2, n_embd=128)
+model = AutoModelForCausalLM.from_config(config)
+print(f"✅ Tiny model created with ~{sum(p.numel() for p in model.parameters())/1e6:.1f}M parameters.")
+
+trainer = Trainer(
+    model=model,
+    args=TrainingArguments(output_dir='./results', num_train_epochs=1, logging_steps=1, report_to="none"),
+    train_dataset=tokenized_ds["train"], eval_dataset=tokenized_ds["validation"],
+    tokenizer=tokenizer, data_collator=DataCollatorForLanguageModeling(tokenizer=tokenizer, mlm=False)
+)
+print("🚀 Starting training...")
+trainer.train()
+print("✅ Training complete.")
+EOF
+
+python3 scripts/train_dev.py
+
+echo "🎉 PIPELINE COMPLETED SUCCESSFULLY!"