Delete dataset.py
Browse files- dataset.py +0 -98
dataset.py
DELETED
|
@@ -1,98 +0,0 @@
|
|
| 1 |
-
import torch
|
| 2 |
-
from torch.utils.data import Dataset
|
| 3 |
-
import json
|
| 4 |
-
import random
|
| 5 |
-
|
| 6 |
-
|
| 7 |
-
class MTPDataset(Dataset):
|
| 8 |
-
"""Dataset mejorado con augmentación de datos"""
|
| 9 |
-
|
| 10 |
-
def __init__(self, corpus_path, tokenizer, max_seq_len=512,
|
| 11 |
-
use_augmentation=False, augmentation_prob=0.3):
|
| 12 |
-
self.tokenizer = tokenizer
|
| 13 |
-
self.max_seq_len = max_seq_len
|
| 14 |
-
self.use_augmentation = use_augmentation
|
| 15 |
-
self.augmentation_prob = augmentation_prob
|
| 16 |
-
self.data = []
|
| 17 |
-
|
| 18 |
-
# Load corpus
|
| 19 |
-
with open(corpus_path, 'r', encoding='utf-8') as f:
|
| 20 |
-
for line in f:
|
| 21 |
-
entry = json.loads(line)
|
| 22 |
-
if 'instruction' in entry and 'response' in entry:
|
| 23 |
-
self.data.append(entry)
|
| 24 |
-
|
| 25 |
-
print(f"✓ Loaded {len(self.data)} examples from corpus")
|
| 26 |
-
if use_augmentation:
|
| 27 |
-
print(f"✓ Data augmentation enabled (prob={augmentation_prob})")
|
| 28 |
-
|
| 29 |
-
def __len__(self):
|
| 30 |
-
return len(self.data)
|
| 31 |
-
|
| 32 |
-
def augment_text(self, text):
|
| 33 |
-
"""Augmentación simple de texto"""
|
| 34 |
-
if not self.use_augmentation or random.random() > self.augmentation_prob:
|
| 35 |
-
return text
|
| 36 |
-
|
| 37 |
-
# Variación 1: Agregar espacios aleatorios (simula variaciones en formato)
|
| 38 |
-
if random.random() < 0.3:
|
| 39 |
-
text = text.strip()
|
| 40 |
-
|
| 41 |
-
# Variación 2: Cambiar puntuación final
|
| 42 |
-
if random.random() < 0.2:
|
| 43 |
-
if text.endswith('.'):
|
| 44 |
-
text = text[:-1]
|
| 45 |
-
elif not text.endswith(('.', '!', '?')):
|
| 46 |
-
text = text + '.'
|
| 47 |
-
|
| 48 |
-
return text
|
| 49 |
-
|
| 50 |
-
def __getitem__(self, idx):
|
| 51 |
-
entry = self.data[idx]
|
| 52 |
-
|
| 53 |
-
instruction = entry['instruction']
|
| 54 |
-
response = entry['response']
|
| 55 |
-
|
| 56 |
-
# Aplicar augmentación
|
| 57 |
-
instruction = self.augment_text(instruction)
|
| 58 |
-
response = self.augment_text(response)
|
| 59 |
-
|
| 60 |
-
# Formato mejorado
|
| 61 |
-
full_text = f"### Instrucción:\n{instruction}\n\n### Respuesta:\n{response}"
|
| 62 |
-
|
| 63 |
-
# Tokenize
|
| 64 |
-
tokens = self.tokenizer.encode(full_text)
|
| 65 |
-
|
| 66 |
-
# Add BOS and EOS
|
| 67 |
-
tokens = [self.tokenizer.bos_id()] + tokens + [self.tokenizer.eos_id()]
|
| 68 |
-
|
| 69 |
-
# Truncate if too long
|
| 70 |
-
if len(tokens) > self.max_seq_len:
|
| 71 |
-
# Truncar manteniendo BOS y EOS
|
| 72 |
-
tokens = [tokens[0]] + tokens[1:self.max_seq_len-1] + [self.tokenizer.eos_id()]
|
| 73 |
-
|
| 74 |
-
# Convert to tensor
|
| 75 |
-
input_ids = torch.tensor(tokens[:-1], dtype=torch.long)
|
| 76 |
-
target_ids = torch.tensor(tokens[1:], dtype=torch.long)
|
| 77 |
-
|
| 78 |
-
return input_ids, target_ids
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
def collate_fn(batch, pad_id=0):
|
| 82 |
-
"""Custom collate function con padding inteligente"""
|
| 83 |
-
input_ids = [item[0] for item in batch]
|
| 84 |
-
target_ids = [item[1] for item in batch]
|
| 85 |
-
|
| 86 |
-
# Find max length in batch
|
| 87 |
-
max_len = max(len(ids) for ids in input_ids)
|
| 88 |
-
|
| 89 |
-
# Pad sequences
|
| 90 |
-
input_ids_padded = []
|
| 91 |
-
target_ids_padded = []
|
| 92 |
-
|
| 93 |
-
for inp, tgt in zip(input_ids, target_ids):
|
| 94 |
-
pad_len = max_len - len(inp)
|
| 95 |
-
input_ids_padded.append(torch.cat([inp, torch.full((pad_len,), pad_id, dtype=torch.long)]))
|
| 96 |
-
target_ids_padded.append(torch.cat([tgt, torch.full((pad_len,), pad_id, dtype=torch.long)]))
|
| 97 |
-
|
| 98 |
-
return torch.stack(input_ids_padded), torch.stack(target_ids_padded)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|