Upload 9 files

transformer/GQA/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a03700f4a6dcb8daeccacc4e6722a721dd37e345f3002e212d078141a222ac
+size 115478760

transformer/GQA/config.yaml

@@ -0,0 +1,44 @@
+# ------------- 分词器 -----------------
+tokenizer: tokenizer.NltkTokenizer     # 自定义分词器
+
+# ------------- 模型结构 (针对 100K 数据 & 8G 显存优化) ----------------
+model:
+  type: transformer
+  enc_layers: 3          # Transformer Encoder 层数
+  dec_layers: 3          # Transformer Decoder 层数
+  emb_size: 256          # 词向量 / 隐层维度
+  nhead: 4               # Multi-Head Attention 头数,每个头维度为 64 (256/4)
+  ffn_dim: 1024           # Feed-Forward 隐层
+  dropout: 0.05           # Dropout 概率
+  # 新参数，用于attention
+  attn_type: gqa   # 代码中逻辑：走 GroupedQueryAttention
+  num_kv_heads: 2  # 2 个 Q 头共享 1 个 KV 头 (4/2=2)
+
+# ------------- 训练超参 ----------------
+train:
+  batch_size: 64
+  epochs: 15             # batchsize减少，epoch增加
+  lr: 0.0005             #
+  weight_decay: 0.0001
+  lr_step: 8             # batchsize减少，epoch增加，lr衰减适当减少
+  lr_gamma: 0.5          # 衰减系数
+  save_dir: runs/train/Nltk_100k
+  num_workers: 0
+
+# ------------- 数据路径 ----------------
+data:
+  raw_train:      data/train_100k.jsonl # 输入文件不变
+  raw_val:        data/valid.jsonl
+  raw_test:       data/test.jsonl
+
+  processed_dir:  data/processed_nltk_100k
+  train_processed: data/processed_nltk_100k/train.jsonl
+  val_processed:   data/processed_nltk_100k/val.jsonl
+  test_processed:  data/processed_nltk_100k/test.jsonl
+
+  src_vocab:      data/processed_nltk_100k/src_vocab.pkl
+  tgt_vocab:      data/processed_nltk_100k/tgt_vocab.pkl
+  min_freq: 2
+
+# ------------- 其余 --------------------
+seed: 3407          # 固定随机种子，保证可复现

transformer/GQA/log.txt

@@ -0,0 +1,94 @@
+[Info] Logging started. Output will be saved to runs\train\Nltk_100k\transformer\20251226_1433\log.txt
+[Info] Model Architecture: TRANSFORMER
+Seq2SeqTransformer(
+  (encoder): Encoder(
+    (embed): Embedding(34122, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x EncoderLayer(
+        (self_attn): GroupedQueryAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=128, bias=False)
+          (v_proj): Linear(in_features=256, out_features=128, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (decoder): Decoder(
+    (embed): Embedding(28647, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x DecoderLayer(
+        (self_attn): GroupedQueryAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=128, bias=False)
+          (v_proj): Linear(in_features=256, out_features=128, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (cross_attn): MultiHeadAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=256, bias=False)
+          (v_proj): Linear(in_features=256, out_features=256, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (proj): Linear(in_features=256, out_features=28647, bias=False)
+)
+[Info] Total Parameters: 28,532,992
+[Info] Trainable Parameters: 28,532,992
+[Info] Model Size (approx): 108.84 MB
+===== Epoch 1/10 =====
+Epoch 01 | Time: 2m28s | train loss: 6.1915 | val loss: 6.0763
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 1
+===== Epoch 2/10 =====
+Epoch 02 | Time: 2m24s | train loss: 5.3152 | val loss: 5.6670
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 2
+===== Epoch 3/10 =====
+Epoch 03 | Time: 2m26s | train loss: 4.7037 | val loss: 5.3431
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 3
+===== Epoch 4/10 =====
+Epoch 04 | Time: 2m25s | train loss: 4.2414 | val loss: 5.1238
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 4
+===== Epoch 5/10 =====
+Epoch 05 | Time: 2m26s | train loss: 3.9414 | val loss: 5.0703
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 5
+===== Epoch 6/10 =====
+Epoch 06 | Time: 2m26s | train loss: 3.7310 | val loss: 5.0379
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1433\best_model.pt, best epoch: 6
+===== Epoch 7/10 =====
+Epoch 07 | Time: 2m26s | train loss: 3.5739 | val loss: 5.0417
+===== Epoch 8/10 =====
+Epoch 08 | Time: 2m26s | train loss: 3.4508 | val loss: 5.0557
+===== Epoch 9/10 =====
+Epoch 09 | Time: 2m25s | train loss: 3.1906 | val loss: 5.0585
+===== Epoch 10/10 =====
+Epoch 10 | Time: 2m26s | train loss: 3.0900 | val loss: 5.1495
+
+Training finished! Total Epochs: 10 Total Time: 24m22s

transformer/MHA/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:93644665a99dbe03250abee7b3798d4108f8f483479abcceae91e6030b8b784b
+size 117051688

transformer/MHA/config_nltk_transformer_100k.yaml

@@ -0,0 +1,44 @@
+# ------------- 分词器 -----------------
+tokenizer: tokenizer.NltkTokenizer     # 自定义分词器
+
+# ------------- 模型结构 (针对 100K 数据 & 8G 显存优化) ----------------
+model:
+  type: transformer
+  enc_layers: 3          # Transformer Encoder 层数
+  dec_layers: 3          # Transformer Decoder 层数
+  emb_size: 256          # 词向量 / 隐层维度
+  nhead: 4               # Multi-Head Attention 头数,每个头维度为 64 (256/4)
+  ffn_dim: 1024           # Feed-Forward 隐层
+  dropout: 0.05           # best
+  # 新参数，用于attention
+  attn_type: mha          # 选项: 'mha' (默认), 'mqa', 'gqa', 'sparse'
+  num_kv_heads: 4         # 用于 MQA/GQA。MHA时等于nhead(4); MQA时为1; GQA时为2
+
+# ------------- 训练超参 ----------------
+train:
+  batch_size: 64
+  epochs: 10
+  lr: 0.0005             #
+  weight_decay: 0.0001
+  lr_step: 8             # batchsize减少，epoch增加，lr衰减适当减少
+  lr_gamma: 0.5          # 衰减系数
+  save_dir: runs/train/Nltk_100k
+  num_workers: 0
+
+# ------------- 数据路径 ----------------
+data:
+  raw_train:      data/train_100k.jsonl # 输入文件不变
+  raw_val:        data/valid.jsonl
+  raw_test:       data/test.jsonl
+
+  processed_dir:  data/processed_nltk_100k
+  train_processed: data/processed_nltk_100k/train.jsonl
+  val_processed:   data/processed_nltk_100k/val.jsonl
+  test_processed:  data/processed_nltk_100k/test.jsonl
+
+  src_vocab:      data/processed_nltk_100k/src_vocab.pkl
+  tgt_vocab:      data/processed_nltk_100k/tgt_vocab.pkl
+  min_freq: 2
+
+# ------------- 其余 --------------------
+seed: 3407          # 固定随机种子，保证可复现

transformer/MHA/log.txt

@@ -0,0 +1,95 @@
+[Info] Logging started. Output will be saved to runs\train\Nltk_100k\transformer\20251227_1329\log.txt
+[Info] Model Architecture: TRANSFORMER
+Seq2SeqTransformer(
+  (encoder): Encoder(
+    (embed): Embedding(34122, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x EncoderLayer(
+        (self_attn): MultiHeadAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=256, bias=False)
+          (v_proj): Linear(in_features=256, out_features=256, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (decoder): Decoder(
+    (embed): Embedding(28647, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x DecoderLayer(
+        (self_attn): MultiHeadAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=256, bias=False)
+          (v_proj): Linear(in_features=256, out_features=256, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (cross_attn): MultiHeadAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=256, bias=False)
+          (v_proj): Linear(in_features=256, out_features=256, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (proj): Linear(in_features=256, out_features=28647, bias=False)
+)
+[Info] Total Parameters: 28,926,208
+[Info] Trainable Parameters: 28,926,208
+[Info] Model Size (approx): 110.34 MB
+===== Epoch 1/10 =====
+Epoch 01 | Time: 2m21s | train loss: 6.1264 | val loss: 6.0301
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 1
+===== Epoch 2/10 =====
+Epoch 02 | Time: 2m8s | train loss: 5.1221 | val loss: 5.4822
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 2
+===== Epoch 3/10 =====
+Epoch 03 | Time: 2m9s | train loss: 4.4576 | val loss: 5.1111
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 3
+===== Epoch 4/10 =====
+Epoch 04 | Time: 2m7s | train loss: 4.0223 | val loss: 5.0020
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 4
+===== Epoch 5/10 =====
+Epoch 05 | Time: 2m7s | train loss: 3.7541 | val loss: 4.9490
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 5
+===== Epoch 6/10 =====
+Epoch 06 | Time: 2m7s | train loss: 3.5682 | val loss: 4.9301
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 6
+===== Epoch 7/10 =====
+Epoch 07 | Time: 2m7s | train loss: 3.4254 | val loss: 4.9166
+New best model saved to runs\train\Nltk_100k\transformer\20251227_1329\best_model.pt, best epoch: 7
+===== Epoch 8/10 =====
+Epoch 08 | Time: 2m8s | train loss: 3.3132 | val loss: 4.9628
+===== Epoch 9/10 =====
+Epoch 09 | Time: 2m7s | train loss: 3.0561 | val loss: 4.9778
+===== Epoch 10/10 =====
+Epoch 10 | Time: 2m7s | train loss: 2.9551 | val loss: 5.0848
+
+Training finished! Total Epochs: 10 Total Time: 21m35s

transformer/MQA/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:44390b04f4e756dc99e1983207bd16fc58046e4aadd4ecaf890b96dea9c98c9f
+size 114692328

transformer/MQA/config.yaml

@@ -0,0 +1,43 @@
+# ------------- 分词器 -----------------
+tokenizer: tokenizer.NltkTokenizer     # 自定义分词器
+
+# ------------- 模型结构 (针对 100K 数据 & 8G 显存优化) ----------------
+model:
+  type: transformer
+  enc_layers: 3          # Transformer Encoder 层数
+  dec_layers: 3          # Transformer Decoder 层数
+  emb_size: 256          # 词向量 / 隐层维度
+  nhead: 4               # Multi-Head Attention 头数,每个头维度为 64 (256/4)
+  ffn_dim: 1024           # Feed-Forward 隐层
+  dropout: 0.05           # Dropout 概率
+  # 新参数，用于attention
+  attn_type: mqa   # 代码中逻辑：走 GroupedQueryAttention
+  num_kv_heads: 1  # 只有 1 个 KV 头
+# ------------- 训练超参 ----------------
+train:
+  batch_size: 64
+  epochs: 10             # batchsize减少，epoch增加
+  lr: 0.0005             #
+  weight_decay: 0.0001
+  lr_step: 8             # batchsize减少，epoch增加，lr衰减适当减少
+  lr_gamma: 0.5          # 衰减系数
+  save_dir: runs/train/Nltk_100k
+  num_workers: 0
+
+# ------------- 数据路径 ----------------
+data:
+  raw_train:      data/train_100k.jsonl # 输入文件不变
+  raw_val:        data/valid.jsonl
+  raw_test:       data/test.jsonl
+
+  processed_dir:  data/processed_nltk_100k
+  train_processed: data/processed_nltk_100k/train.jsonl
+  val_processed:   data/processed_nltk_100k/val.jsonl
+  test_processed:  data/processed_nltk_100k/test.jsonl
+
+  src_vocab:      data/processed_nltk_100k/src_vocab.pkl
+  tgt_vocab:      data/processed_nltk_100k/tgt_vocab.pkl
+  min_freq: 2
+
+# ------------- 其余 --------------------
+seed: 3407          # 固定随机种子，保证可复现

transformer/MQA/log.txt

@@ -0,0 +1,96 @@
+[Info] Logging started. Output will be saved to runs\train\Nltk_100k\transformer\20251226_1357\log.txt
+[Info] Model Architecture: TRANSFORMER
+Seq2SeqTransformer(
+  (encoder): Encoder(
+    (embed): Embedding(34122, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x EncoderLayer(
+        (self_attn): GroupedQueryAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=64, bias=False)
+          (v_proj): Linear(in_features=256, out_features=64, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (decoder): Decoder(
+    (embed): Embedding(28647, 256, padding_idx=0)
+    (pe): PositionalEncoding()
+    (layers): ModuleList(
+      (0-2): 3 x DecoderLayer(
+        (self_attn): GroupedQueryAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=64, bias=False)
+          (v_proj): Linear(in_features=256, out_features=64, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (cross_attn): MultiHeadAttention(
+          (q_proj): Linear(in_features=256, out_features=256, bias=False)
+          (k_proj): Linear(in_features=256, out_features=256, bias=False)
+          (v_proj): Linear(in_features=256, out_features=256, bias=False)
+          (out_proj): Linear(in_features=256, out_features=256, bias=True)
+          (dropout): Dropout(p=0.05, inplace=False)
+        )
+        (ffn): FeedForward(
+          (linear1): Linear(in_features=256, out_features=1024, bias=True)
+          (activation): ReLU()
+          (dropout): Dropout(p=0.05, inplace=False)
+          (linear2): Linear(in_features=1024, out_features=256, bias=True)
+        )
+        (norm1): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm2): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (norm3): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+        (dropout): Dropout(p=0.05, inplace=False)
+      )
+    )
+    (norm): LayerNorm((256,), eps=1e-05, elementwise_affine=True)
+  )
+  (proj): Linear(in_features=256, out_features=28647, bias=False)
+)
+[Info] Total Parameters: 28,336,384
+[Info] Trainable Parameters: 28,336,384
+[Info] Model Size (approx): 108.09 MB
+===== Epoch 1/10 =====
+Epoch 01 | Time: 3m34s | train loss: 6.2330 | val loss: 6.0593
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 1
+===== Epoch 2/10 =====
+Epoch 02 | Time: 3m37s | train loss: 5.2751 | val loss: 5.6802
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 2
+===== Epoch 3/10 =====
+Epoch 03 | Time: 3m28s | train loss: 4.7900 | val loss: 5.4412
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 3
+===== Epoch 4/10 =====
+Epoch 04 | Time: 3m32s | train loss: 4.4403 | val loss: 5.2774
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 4
+===== Epoch 5/10 =====
+Epoch 05 | Time: 3m34s | train loss: 4.1581 | val loss: 5.2082
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 5
+===== Epoch 6/10 =====
+Epoch 06 | Time: 3m33s | train loss: 3.9421 | val loss: 5.1301
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 6
+===== Epoch 7/10 =====
+Epoch 07 | Time: 3m36s | train loss: 3.7749 | val loss: 5.0828
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 7
+===== Epoch 8/10 =====
+Epoch 08 | Time: 3m33s | train loss: 3.6390 | val loss: 5.0811
+New best model saved to runs\train\Nltk_100k\transformer\20251226_1357\best_model.pt, best epoch: 8
+===== Epoch 9/10 =====
+Epoch 09 | Time: 3m37s | train loss: 3.3762 | val loss: 5.0981
+===== Epoch 10/10 =====
+Epoch 10 | Time: 3m32s | train loss: 3.2740 | val loss: 5.1790
+
+Training finished! Total Epochs: 10 Total Time: 35m43s