File size: 2,632 Bytes
257c574 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 |
---
license: apache-2.0
datasets:
- fka/awesome-chatgpt-prompts
language:
- hi
- ta
- ml
---
import datasets
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification, Trainer, TrainingArguments
from datasets import Dataset
# Step 1: Define your colloquial dataset
# Sample conversational data in different languages (adjust based on your task)
data = {
'text': [
'kaise ho?', # informal Hindi greeting
'kya scene hai?', # Hindi slang phrase
'apne kahan jana hai?', # informal Hindi sentence
'yentha vara', # Tamil slang
'mizhhi pidichu', # Malayalam slang
'enthu cheyyumo', # Malayalam slang
'uru kuthi', # Tamil slang
'ekdam mast', # Hindi slang
],
'label': [0, 1, 2, 3, 4, 4, 3, 1] # Example labels for intent or sentiment
}
# Step 2: Convert data into Hugging Face Dataset format
dataset = Dataset.from_dict(data)
# Step 3: Tokenize the data using a multilingual model tokenizer
tokenizer = AutoTokenizer.from_pretrained("bert-base-multilingual-cased")
# Tokenization function
def tokenize_function(examples):
return tokenizer(examples['text'], padding="max_length", truncation=True)
# Apply tokenization to the dataset
dataset = dataset.map(tokenize_function, batched=True)
# Step 4: Load a pre-trained model for sequence classification
model = AutoModelForSequenceClassification.from_pretrained("bert-base-multilingual-cased", num_labels=5)
# Step 5: Set up Trainer for fine-tuning the model
training_args = TrainingArguments(
output_dir='./results', # Output directory to save model and logs
evaluation_strategy="epoch", # Evaluate after each epoch
per_device_train_batch_size=8, # Batch size during training
per_device_eval_batch_size=8, # Batch size during evaluation
num_train_epochs=3, # Number of epochs for training
logging_dir='./logs', # Log directory for training details
logging_steps=10, # Number of steps to log
)
# Initialize the Trainer
trainer = Trainer(
model=model,
args=training_args,
train_dataset=dataset,
eval_dataset=dataset, # Typically, split dataset into training and validation sets
)
# Step 6: Train the model
trainer.train()
# Step 7: Save the trained model and tokenizer
model.save_pretrained("./my_colloquial_model")
tokenizer.save_pretrained("./my_colloquial_model")
# Optional: Upload to Hugging Face
# Uncomment and use Hugging Face CLI to upload the model:
# !huggingface-cli login # Log in to your Hugging Face account
# model.push_to_hub("my_colloquial_model")
print("Model training and saving complete.") |