teszenofficial commited on
Commit
3f23748
路
verified 路
1 Parent(s): 685a587

Upload config.yaml with huggingface_hub

Browse files
Files changed (1) hide show
  1. config.yaml +64 -0
config.yaml ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # MTP Mini - Configuraci贸n Mejorada para Generaci贸n Coherente
2
+
3
+ model:
4
+ vocab_size: 4000
5
+ d_model: 512 # Aumentado para m谩s capacidad
6
+ n_layers: 8 # M谩s capas
7
+ n_heads: 8 # M谩s cabezas de atenci贸n
8
+ d_ff: 2048 # 4x d_model
9
+ max_seq_len: 512 # Contexto m谩s largo
10
+ dropout: 0.2 # M谩s dropout para evitar overfitting
11
+ use_swiglu: true # Activaci贸n mejorada
12
+
13
+ training:
14
+ batch_size: 4 # Batch m谩s peque帽o para corpus peque帽o
15
+ accumulation_steps: 4 # Effective batch = 16
16
+ epochs: 20 # MENOS 茅pocas
17
+ learning_rate: 0.0003 # LR m谩s alto para convergencia r谩pida
18
+ min_lr: 0.00001
19
+ weight_decay: 0.1 # M脕S weight decay para regularizaci贸n
20
+ max_grad_norm: 1.0
21
+ num_threads: 4
22
+ save_every: 5
23
+
24
+ # Early stopping
25
+ patience: 5 # Parar si no mejora en 5 epochs
26
+ min_delta: 0.001 # Mejora m铆nima requerida
27
+
28
+ # Learning rate schedule
29
+ warmup_steps: 100
30
+ use_lr_scheduler: true
31
+
32
+ # Regularizaci贸n adicional
33
+ label_smoothing: 0.1
34
+ use_eos_loss_weight: true # Dar m谩s peso al token EOS
35
+
36
+ data:
37
+ corpus_path: corpus/mtp_mini_corpus.jsonl
38
+ min_text_length: 50 # Textos m谩s largos
39
+ max_text_length: 2000 # Permitir respuestas largas
40
+ validation_split: 0.15
41
+
42
+ # Augmentaci贸n de datos
43
+ use_augmentation: true
44
+ augmentation_prob: 0.3
45
+
46
+ generation:
47
+ # Par谩metros de generaci贸n mejorados
48
+ default_max_tokens: 150
49
+ default_temperature: 0.7
50
+ default_top_k: 40
51
+ default_top_p: 0.92
52
+ default_repetition_penalty: 1.15
53
+ min_response_length: 20
54
+ use_length_penalty: true
55
+
56
+ # Control de coherencia
57
+ use_perplexity_filter: true
58
+ max_perplexity: 100.0
59
+
60
+ # Stop sequences
61
+ stop_sequences:
62
+ - "###"
63
+ - "\n\n\n"
64
+ - "Instrucci贸n:"