| # ============================================================================ | |
| # Requirements for Arabic LLM Training & Inference | |
| # ============================================================================ | |
| # Install: pip install -r requirements.txt | |
| # ============================================================================ | |
| # Core ML Frameworks | |
| torch>=2.1.0 | |
| transformers>=4.45.0 | |
| accelerate>=0.25.0 | |
| datasets>=2.16.0 | |
| # Parameter-Efficient Fine-Tuning | |
| peft>=0.7.0 | |
| # Tokenizers | |
| tokenizers>=0.15.0 | |
| sentencepiece>=0.1.99 | |
| # Data Processing | |
| pandas>=2.0.0 | |
| numpy>=1.24.0 | |
| scikit-learn>=1.3.0 | |
| # Progress & Logging | |
| tqdm>=4.66.0 | |
| # Evaluation Metrics | |
| evaluate>=0.4.0 | |
| seqeval>=1.2.2 | |
| # Optional: Weights & Biases for experiment tracking | |
| # wandb>=0.16.0 | |
| # Optional: Flash Attention 2 (requires CUDA) | |
| # flash-attn>=2.3.0 | |
| # Optional: BitsAndBytes for quantization | |
| # bitsandbytes>=0.41.0 | |
| # Jupyter Support (optional) | |
| # jupyter>=1.0.0 | |
| # ipykernel>=6.25.0 | |
| # Hugging Face Hub | |
| huggingface-hub>=0.19.0 | |
| # SafeTensors for efficient model loading | |
| safetensors>=0.4.0 | |
| # Regex for text preprocessing | |
| regex>=2023.10.0 | |