teszenofficial commited on
Commit
347efa6
·
verified ·
1 Parent(s): 2c9e103

Delete dataset.py

Browse files
Files changed (1) hide show
  1. dataset.py +0 -98
dataset.py DELETED
@@ -1,98 +0,0 @@
1
- import torch
2
- from torch.utils.data import Dataset
3
- import json
4
- import random
5
-
6
-
7
- class MTPDataset(Dataset):
8
- """Dataset mejorado con augmentación de datos"""
9
-
10
- def __init__(self, corpus_path, tokenizer, max_seq_len=512,
11
- use_augmentation=False, augmentation_prob=0.3):
12
- self.tokenizer = tokenizer
13
- self.max_seq_len = max_seq_len
14
- self.use_augmentation = use_augmentation
15
- self.augmentation_prob = augmentation_prob
16
- self.data = []
17
-
18
- # Load corpus
19
- with open(corpus_path, 'r', encoding='utf-8') as f:
20
- for line in f:
21
- entry = json.loads(line)
22
- if 'instruction' in entry and 'response' in entry:
23
- self.data.append(entry)
24
-
25
- print(f"✓ Loaded {len(self.data)} examples from corpus")
26
- if use_augmentation:
27
- print(f"✓ Data augmentation enabled (prob={augmentation_prob})")
28
-
29
- def __len__(self):
30
- return len(self.data)
31
-
32
- def augment_text(self, text):
33
- """Augmentación simple de texto"""
34
- if not self.use_augmentation or random.random() > self.augmentation_prob:
35
- return text
36
-
37
- # Variación 1: Agregar espacios aleatorios (simula variaciones en formato)
38
- if random.random() < 0.3:
39
- text = text.strip()
40
-
41
- # Variación 2: Cambiar puntuación final
42
- if random.random() < 0.2:
43
- if text.endswith('.'):
44
- text = text[:-1]
45
- elif not text.endswith(('.', '!', '?')):
46
- text = text + '.'
47
-
48
- return text
49
-
50
- def __getitem__(self, idx):
51
- entry = self.data[idx]
52
-
53
- instruction = entry['instruction']
54
- response = entry['response']
55
-
56
- # Aplicar augmentación
57
- instruction = self.augment_text(instruction)
58
- response = self.augment_text(response)
59
-
60
- # Formato mejorado
61
- full_text = f"### Instrucción:\n{instruction}\n\n### Respuesta:\n{response}"
62
-
63
- # Tokenize
64
- tokens = self.tokenizer.encode(full_text)
65
-
66
- # Add BOS and EOS
67
- tokens = [self.tokenizer.bos_id()] + tokens + [self.tokenizer.eos_id()]
68
-
69
- # Truncate if too long
70
- if len(tokens) > self.max_seq_len:
71
- # Truncar manteniendo BOS y EOS
72
- tokens = [tokens[0]] + tokens[1:self.max_seq_len-1] + [self.tokenizer.eos_id()]
73
-
74
- # Convert to tensor
75
- input_ids = torch.tensor(tokens[:-1], dtype=torch.long)
76
- target_ids = torch.tensor(tokens[1:], dtype=torch.long)
77
-
78
- return input_ids, target_ids
79
-
80
-
81
- def collate_fn(batch, pad_id=0):
82
- """Custom collate function con padding inteligente"""
83
- input_ids = [item[0] for item in batch]
84
- target_ids = [item[1] for item in batch]
85
-
86
- # Find max length in batch
87
- max_len = max(len(ids) for ids in input_ids)
88
-
89
- # Pad sequences
90
- input_ids_padded = []
91
- target_ids_padded = []
92
-
93
- for inp, tgt in zip(input_ids, target_ids):
94
- pad_len = max_len - len(inp)
95
- input_ids_padded.append(torch.cat([inp, torch.full((pad_len,), pad_id, dtype=torch.long)]))
96
- target_ids_padded.append(torch.cat([tgt, torch.full((pad_len,), pad_id, dtype=torch.long)]))
97
-
98
- return torch.stack(input_ids_padded), torch.stack(target_ids_padded)