Hugging Face's logo

khanhvinh9
/
lmc-code

Model card Files
xet
 Community
lmc-code
File size: 12,848 Bytes
a20151e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
#LEARNABLE
# WANDB_MODE=online CUDA_VISIBLE_DEVICES=7  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8
# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-learnable-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_60000 \
#     --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

#ROPE
WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 8 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 4 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 0 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 20 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
    --model-path /mnt/data/vinhbk/weights/enwik8/lr0.00025-rope-step60000-warm0-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_56000 \
    --seed 40 --tgt_len 512 --mem_len 512 --eval_tgt_len 128  --n_head 16 --lmc-layer-indices 0 \
    --learning-rate 0.00025 --batch-size 24 --max_step 60000 --warmup_step 0 --dataset enwik8 \
    --wandb-project LMC-Attention --wandb-group "GPT2-Enwik8-FFN" --wandb-entity "vinh-bui0512-hcmut"\
    --model-save-dir /mnt/data/vinhbk/weights/enwik8/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/enwik8

# WANDB_MODE=online CUDA_VISIBLE_DEVICES=7  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/wt103/lr0.00025-learnable-step60000-warm2000-size64-layer12-embd192-heads3-shared1-routed0-topk0/best_60000 \
#     --seed 0 --tgt_len 256 --mem_len 256 --eval_tgt_len 256 --n_head 3 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 64 --max_step 60000 --warmup_step 2000 --dataset wt103 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Wikitext103-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/wt103/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/wt103
# WANDB_MODE=online CUDA_VISIBLE_DEVICES=6  python src/lgmodeling/finetune.py \
#     --model-path /mnt/data/vinhbk/weights/wt103/lr0.00025-rope-step60000-warm2000-size64-layer12-embd192-heads3-shared1-routed0-topk0/best_60000 \
#     --seed 40 --tgt_len 256 --mem_len 256 --eval_tgt_len 256 --n_head 4 --lmc-layer-indices 0 \
#     --learning-rate 0.00025 --batch-size 64 --max_step 60000 --warmup_step 2000 --dataset wt103 \
#     --wandb-project LMC-Attention --wandb-group "GPT2-Wikitext103-FFN" --wandb-entity "vinh-bui0512-hcmut"\
#     --model-save-dir /mnt/data/vinhbk/weights/wt103/gpt2-finetune --data-path /mnt/data/vinhbk/datasets/wt103