Hugging Face's logo

NewEden-Forge
/
C

Model card Files
xet
 Community
C
File size: 3,062 Bytes
72e253e
 
 
 
 
 
 
 
 
 
 
 
f8418eb
 
72e253e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f8418eb
72e253e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
f8418eb
 
 
72e253e
f8418eb
 
 
 
 
72e253e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
## model
base_model: /raid/Mango/axolotl/24B-KTO/merged
model_type: AutoModelForCausalLM
tokenizer_type: AutoTokenizer

## qlora
load_in_8bit: false
load_in_4bit: false
strict: false

## data 
datasets:
  - path: PocketDoc/Dans-Codemaxx-LeetCode
    type: dan-chat-advanced
  - path: Nitral-AI/ARES-ShareGPT
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Logicmaxx-FI-VeriMed
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Prosemaxx-Cowriter-3-XS
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Toolmaxx-Functions-apigen
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Toolmaxx-Functions-Toolbench
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Logicmaxx-SAT-AP
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Logicmaxx-Skunkworks
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Taskmaxx-ConcurrentQA-Reworked
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Toolmaxx-Agent
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Toolmaxx-ShellCommands
    type: dan-chat-advanced
  - path: PocketDoc/Dans-MemoryCore-CoreCurriculum-Small
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Codemaxx-CodeFeedback-Conversations
    type: dan-chat-advanced
  - path: Delta-Vector/Orion-Praxis-Co-Writer
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Prosemaxx-Instructwriter-Long
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Prosemaxx-Cowriter-3-XS
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Prosemaxx-InstructWriter-ZeroShot
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Benchmaxx-COT
    type: dan-chat-advanced
  - path: PocketDoc/Dans-Benchmaxx
    type: dan-chat-advanced
  - path: NewEden/xlam-function-calling-60k-shareGPT
    type: dan-chat-advanced
shuffle_merged_datasets: true
dataset_prepared_path: dataset_prepared
val_set_size: 0.0020
output_dir: ./MS3.2-SFT-V2
## Liger + CCE
plugins:
  - axolotl.integrations.liger.LigerPlugin
  - axolotl.integrations.cut_cross_entropy.CutCrossEntropyPlugin
liger_rope: true
liger_rms_norm: true
liger_layer_norm: true
liger_glu_activation: true
liger_fused_linear_cross_entropy: false
cut_cross_entropy: true

## CTX settings
sequence_len: 16000
sample_packing: true
eval_sample_packing: false
pad_to_sequence_len: true

## max grad norm
max_grad_norm: 0.001


## WandB
wandb_project: Training-A100
wandb_entity:
wandb_watch:
wandb_name: SFT-KTO-V2
wandb_log_model:

## evals
evals_per_epoch: 4
eval_table_size: 
eval_max_new_tokens: 128

## hparams
gradient_accumulation_steps: 6
micro_batch_size: 6
num_epochs: 4
optimizer: paged_ademamix_8bit
optim_args: "beta1=0.9,beta2=0.999,beta3=0.999,alpha=5"
lr_scheduler: rex
learning_rate: 1e-6
warmup_ratio: 0.1
weight_decay: 0.0

train_on_inputs: false
group_by_length: false
bf16: auto
fp16:
tf32: false

gradient_checkpointing: true
early_stopping_patience:
resume_from_checkpoint:
local_rank:
logging_steps: 1
xformers_attention:
flash_attention: true
s2_attention:
saves_per_epoch: 2
debug:
deepspeed: ./deepspeed_configs/zero3_bf16.json
fsdp:
fsdp_config:
special_tokens:
  pad_token: <pad>