---
license: apache-2.0
datasets:
- fka/awesome-chatgpt-prompts
language:
- hi
- ta
- ml
---
import datasets
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
from datasets import Dataset

# Step 1: Define your colloquial dataset
# Sample conversational data in different languages (adjust based on your task)
data = {
    'text': [
        'kaise ho?',  # informal Hindi greeting
        'kya scene hai?',  # Hindi slang phrase
        'apne kahan jana hai?',  # informal Hindi sentence
        'yentha vara',  # Tamil slang
        'mizhhi pidichu',  # Malayalam slang
        'enthu cheyyumo',  # Malayalam slang
        'uru kuthi',  # Tamil slang
        'ekdam mast',  # Hindi slang
    ],
    'label': [0, 1, 2, 3, 4, 4, 3, 1]  # Example labels for intent or sentiment
}

# Step 2: Convert data into Hugging Face Dataset format
dataset = Dataset.from_dict(data)

# Step 3: Tokenize the data using a multilingual model tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")

# Tokenization function
def tokenize_function(examples):
    return tokenizer(examples['text'], padding="max_length", truncation=True)

# Apply tokenization to the dataset
dataset = dataset.map(tokenize_function, batched=True)

# Step 4: Load a pre-trained model for sequence classification
model = AutoModelForSequenceClassification.from_pretrained("bert-base-multilingual-cased", num_labels=5)

# Step 5: Set up Trainer for fine-tuning the model
training_args = TrainingArguments(
    output_dir='./results',  # Output directory to save model and logs
    evaluation_strategy="epoch",  # Evaluate after each epoch
    per_device_train_batch_size=8,  # Batch size during training
    per_device_eval_batch_size=8,  # Batch size during evaluation
    num_train_epochs=3,  # Number of epochs for training
    logging_dir='./logs',  # Log directory for training details
    logging_steps=10,  # Number of steps to log
)

# Initialize the Trainer
trainer = Trainer(
    model=model,
    args=training_args,
    train_dataset=dataset,
    eval_dataset=dataset,  # Typically, split dataset into training and validation sets
)

# Step 6: Train the model
trainer.train()

# Step 7: Save the trained model and tokenizer
model.save_pretrained("./my_colloquial_model")
tokenizer.save_pretrained("./my_colloquial_model")

# Optional: Upload to Hugging Face
# Uncomment and use Hugging Face CLI to upload the model:
# !huggingface-cli login  # Log in to your Hugging Face account
# model.push_to_hub("my_colloquial_model")

print("Model training and saving complete.")