Upload scripts/config.py with huggingface_hub

scripts/config.py

@@ -0,0 +1,37 @@
+"""Configuration for the Generic vs Semantic classifier project."""
+
+# Paths
+DATA_DIR = "data"
+RAW_DIR = f"{DATA_DIR}/raw"
+PROCESSED_DIR = f"{DATA_DIR}/processed"
+MODELS_DIR = "models"
+SCRIPTS_DIR = "scripts"
+
+# Dataset
+TOTAL_PER_CATEGORY = 3000
+BATCH_SIZE_GEN = 50  # examples per Ollama API call
+MAX_CONCURRENT = 8   # parallel API requests
+
+# Categories
+CATEGORIES = {
+    "en_generic": {"lang": "English", "label": "GENERIC"},
+    "en_semantic": {"lang": "English", "label": "SEMANTIC"},
+    "hi_generic": {"lang": "Hindi", "label": "GENERIC"},
+    "hi_semantic": {"lang": "Hindi", "label": "SEMANTIC"},
+}
+
+# Model
+MODEL_NAME = "distilbert-base-multilingual-cased"
+MAX_SEQ_LEN = 64
+NUM_LABELS = 2
+LABEL_MAP = {"GENERIC": 0, "SEMANTIC": 1}
+
+# Training
+BATCH_SIZE = 32
+LEARNING_RATE = 2e-5
+NUM_EPOCHS = 5
+TEST_SPLIT = 0.15
+
+# Ollama
+OLLAMA_URL = "http://localhost:11434/api/chat"
+OLLAMA_MODEL = "llama3.1:8b-instruct-q4_K_M"