import datasets import torch from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments from datasets import Dataset
Step 1: Define your colloquial dataset
Sample conversational data in different languages (adjust based on your task)
data = { 'text': [ 'kaise ho?', # informal Hindi greeting 'kya scene hai?', # Hindi slang phrase 'apne kahan jana hai?', # informal Hindi sentence 'yentha vara', # Tamil slang 'mizhhi pidichu', # Malayalam slang 'enthu cheyyumo', # Malayalam slang 'uru kuthi', # Tamil slang 'ekdam mast', # Hindi slang ], 'label': [0, 1, 2, 3, 4, 4, 3, 1] # Example labels for intent or sentiment }
Step 2: Convert data into Hugging Face Dataset format
dataset = Dataset.from_dict(data)
Step 3: Tokenize the data using a multilingual model tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
Tokenization function
def tokenize_function(examples): return tokenizer(examples['text'], padding="max_length", truncation=True)
Apply tokenization to the dataset
dataset = dataset.map(tokenize_function, batched=True)
Step 4: Load a pre-trained model for sequence classification
model = AutoModelForSequenceClassification.from_pretrained("bert-base-multilingual-cased", num_labels=5)
Step 5: Set up Trainer for fine-tuning the model
training_args = TrainingArguments( output_dir='./results', # Output directory to save model and logs evaluation_strategy="epoch", # Evaluate after each epoch per_device_train_batch_size=8, # Batch size during training per_device_eval_batch_size=8, # Batch size during evaluation num_train_epochs=3, # Number of epochs for training logging_dir='./logs', # Log directory for training details logging_steps=10, # Number of steps to log )
Initialize the Trainer
trainer = Trainer( model=model, args=training_args, train_dataset=dataset, eval_dataset=dataset, # Typically, split dataset into training and validation sets )
Step 6: Train the model
trainer.train()
Step 7: Save the trained model and tokenizer
model.save_pretrained("./my_colloquial_model") tokenizer.save_pretrained("./my_colloquial_model")
Optional: Upload to Hugging Face
Uncomment and use Hugging Face CLI to upload the model:
!huggingface-cli login # Log in to your Hugging Face account
model.push_to_hub("my_colloquial_model")
print("Model training and saving complete.")